From d81a72d8fae5fc5e00248c2a3236e682f72a96a5 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Fri, 24 May 2024 12:25:05 -0400 Subject: [PATCH 01/10] rebase PR to new organization; continue making updates --- src/pyflask/apis/__init__.py | 1 + src/pyflask/apis/_neurosift.py | 68 ++++++++ src/pyflask/apis/data.py | 24 ++- src/pyflask/apis/neuroconv.py | 65 +++----- src/pyflask/apis/startup.py | 6 +- src/pyflask/app.py | 146 +++++++++--------- src/pyflask/errorHandlers/__init__.py | 1 - .../errorHandlers/notBadRequestException.py | 5 - src/pyflask/manageNeuroconv/info/__init__.py | 1 - src/pyflask/manageNeuroconv/info/urls.py | 11 -- src/pyflask/tests/conftest.py | 4 +- src/pyflask/utils/__init__.py | 7 + src/pyflask/utils/_flask_errors.py | 36 +++++ 13 files changed, 227 insertions(+), 148 deletions(-) create mode 100644 src/pyflask/apis/_neurosift.py delete mode 100644 src/pyflask/errorHandlers/__init__.py delete mode 100644 src/pyflask/errorHandlers/notBadRequestException.py create mode 100644 src/pyflask/utils/__init__.py create mode 100644 src/pyflask/utils/_flask_errors.py diff --git a/src/pyflask/apis/__init__.py b/src/pyflask/apis/__init__.py index f492dc7df..cfb2085eb 100644 --- a/src/pyflask/apis/__init__.py +++ b/src/pyflask/apis/__init__.py @@ -1,3 +1,4 @@ +from ._neurosift import neurosift_api from .data import data_api from .neuroconv import neuroconv_api from .startup import startup_api diff --git a/src/pyflask/apis/_neurosift.py b/src/pyflask/apis/_neurosift.py new file mode 100644 index 000000000..f1544c31f --- /dev/null +++ b/src/pyflask/apis/_neurosift.py @@ -0,0 +1,68 @@ +"""An API for handling file system communication with the standalone Neurosift preview page.""" + +import collections +from typing import Union + +import flask +import flask.restx + +from ..utils import ( + abort_if_not_nwb_file, + catch_exception_and_abort, + server_error_responses, +) + +neurosift_api = flask_restx.Namespace( + name="neurosift", description="Handle file system communication with the " "standalone Neurosift preview page." +) + +# A global in-memory registry of NWB files - much faster than asking if file is in a global list +# Keys are NWB files; values are booleans indicating if the base URL has been exposed for that file +neurosift_file_registry = collections.defaultdict(bool) + + +@neurosift_api.route(rule="/files/", methods=["GET", "POST"]) +@neurosift_api.doc( + description="Handle adding and fetching NWB files from the global file registry.", +) +class AllInterfaces(flask_restx.Resource): + + @neurosift_api.doc( + description="If the file path has been added to the registry (and therefore sent its base " + "URL), return the absolute file path. This is implicitly called by Neurosift.", + responses=server_error_responses(codes=[200, 400, 500]), + ) + @catch_exception_and_abort(api=neurosift_api, code=500) + def get(self, file_path: str) -> Union[flask.Response, None]: + abort_if_not_nwb_file(file_path=file_path, api=neurosift_api) + if neurosift_file_registry[file_path]: + code = 404 + base_message = server_error_responses(codes=[code])[code] + message = f"{base_message}: The base URL has not been exposed for this NWB file." + api.abort(code=code, message=message) + + return + + # Decode any URL encoding applied to the file path + parsed_file_path = unquote(file_path) + + # Check if the file path is relative + is_file_relative = not isabs(parsed_file_path) + if is_file_relative: + parsed_file_path = f"/{parsed_file_path}" + + return flask.send_file(path_or_file=parsed_file_path) + + @neurosift_api.doc( + description="Add the file to a global in-memory registry (refreshes on App restart) and return " + "the base URL of the newly " + "added file", + responses=server_error_responses(codes=[200, 400, 500]), + ) + @catch_exception_and_abort(api=neurosift_api, code=500) + def post(self, file_path: str) -> Union[str, None]: + abort_if_not_nwb_file(file_path=file_path, api=neurosift_api) + + neurosift_file_registry[file_path] = True + + return request.base_url diff --git a/src/pyflask/apis/data.py b/src/pyflask/apis/data.py index de6dfdc6d..a3e05f985 100644 --- a/src/pyflask/apis/data.py +++ b/src/pyflask/apis/data.py @@ -2,17 +2,17 @@ import traceback -from errorHandlers import notBadRequestException from flask_restx import Namespace, Resource, reqparse from manageNeuroconv import generate_dataset, generate_test_data +from utils import catch_exception_and_abort, server_error_responses -data_api = Namespace("data", description="API route for dataset generation in the NWB GUIDE.") +data_api = Namespace(name="data", description="API route for dataset generation in the NWB GUIDE.") @data_api.errorhandler(Exception) def exception_handler(error): exceptiondata = traceback.format_exception(type(error), error, error.__traceback__) - return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)} + return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)}, 500 generate_test_data_parser = reqparse.RequestParser() @@ -22,15 +22,14 @@ def exception_handler(error): @data_api.route("/generate") @data_api.expect(generate_test_data_parser) class GeneratetestData(Resource): - @data_api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) + @data_api.doc( + description="Generate example ecephys data using SpikeInterface.", + responses=server_error_responses(codes=[200, 500]), + ) + @catch_exception_and_abort(api=data_api, code=500) def post(self): - try: - arguments = generate_test_data_parser.parse_args() - generate_test_data(output_path=arguments["output_path"]) - except Exception as exception: - if notBadRequestException(exception): - data_api.abort(500, str(exception)) - raise exception + arguments = generate_test_data_parser.parse_args() + generate_test_data(output_path=arguments["output_path"]) generate_test_dataset_parser = reqparse.RequestParser() @@ -48,5 +47,4 @@ def post(self): return generate_dataset(input_path=arguments["input_path"], output_path=arguments["output_path"]) except Exception as exception: - if notBadRequestException(exception): - data_api.abort(500, str(exception)) + data_api.abort(500, str(exception)) diff --git a/src/pyflask/apis/neuroconv.py b/src/pyflask/apis/neuroconv.py index 015cb61e8..98d85798b 100644 --- a/src/pyflask/apis/neuroconv.py +++ b/src/pyflask/apis/neuroconv.py @@ -1,8 +1,8 @@ """API endpoint definitions for interacting with NeuroConv.""" import traceback +from typing import Dict -from errorHandlers import notBadRequestException from flask import Response, request from flask_restx import Namespace, Resource, reqparse from manageNeuroconv import ( @@ -25,16 +25,18 @@ ) from manageNeuroconv.info import announcer -neuroconv_api = Namespace("neuroconv", description="Neuroconv neuroconv_api for the NWB GUIDE.") +neuroconv_api = Namespace(name="neuroconv", description="Neuroconv neuroconv_api for the NWB GUIDE.") parser = reqparse.RequestParser() -parser.add_argument("interfaces", type=str, action="split", help="Interfaces cannot be converted") +parser.add_argument("interfaces", type=str, action="split", help="Interfaces cannot be converted.") @neuroconv_api.errorhandler(Exception) -def exception_handler(error): - exceptiondata = traceback.format_exception(type(error), error, error.__traceback__) - return {"message": exceptiondata[-1], "traceback": "".join(exceptiondata)} +def exception_handler(error: Exception) -> Dict[str, str]: + full_traceback = traceback.format_exception(type(error), error, error.__traceback__) + message = full_traceback[-1] + remaining_traceback = "".join(full_traceback[:-1]) + return {"message": message, "traceback": remaining_traceback} @neuroconv_api.route("/") @@ -50,8 +52,7 @@ def get(self): **get_all_converter_info(), } except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) raise exception @@ -62,8 +63,7 @@ def post(self): try: return get_source_schema(neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/locate") @@ -73,8 +73,7 @@ def post(self): try: return locate_data(neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/locate/autocomplete") @@ -84,8 +83,7 @@ def post(self): try: return autocomplete_format_string(neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/metadata") @@ -97,8 +95,7 @@ def post(self): neuroconv_api.payload.get("source_data"), neuroconv_api.payload.get("interfaces") ) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/convert") @@ -109,8 +106,7 @@ def post(self): return convert_to_nwb(neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/alignment") @@ -121,8 +117,7 @@ def post(self): return get_interface_alignment(neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) validate_parser = neuroconv_api.parser() @@ -143,8 +138,7 @@ def post(self): return validate_metadata(args.get("parent"), args.get("function_name")) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/upload/project") @@ -163,8 +157,7 @@ def post(self): return upload_project_to_dandi(**upload_options) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/upload/folder") @@ -183,8 +176,7 @@ def post(self): return upload_folder_to_dandi(**upload_options) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/upload") @@ -206,8 +198,7 @@ def post(self): return upload_multiple_filesystem_objects_to_dandi(**neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/inspect_file") @@ -217,8 +208,7 @@ def post(self): try: return inspect_nwb_file(neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/inspect_folder") @@ -230,8 +220,7 @@ def post(self): return inspect_nwb_folder(url, neuroconv_api.payload) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/announce") @@ -243,8 +232,7 @@ def post(self): announcer.announce(data) return True except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/inspect") @@ -271,8 +259,7 @@ def post(self): return inspect_multiple_filesystem_objects(url, paths, **kwargs) except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) @neuroconv_api.route("/html") @@ -287,8 +274,7 @@ def post(self): return html except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) # Create an events endpoint @@ -301,5 +287,4 @@ def get(self): return Response(listen_to_neuroconv_events(), mimetype="text/event-stream") except Exception as exception: - if notBadRequestException(exception): - neuroconv_api.abort(500, str(exception)) + neuroconv_api.abort(500, str(exception)) diff --git a/src/pyflask/apis/startup.py b/src/pyflask/apis/startup.py index 63be4fd0a..c2ec8ab88 100644 --- a/src/pyflask/apis/startup.py +++ b/src/pyflask/apis/startup.py @@ -1,9 +1,8 @@ """API endpoint definitions for startup operations.""" -from errorHandlers import notBadRequestException from flask_restx import Namespace, Resource -startup_api = Namespace("startup", description="API for startup commands related to the NWB GUIDE.") +startup_api = Namespace(name="startup", description="API for startup commands related to the NWB GUIDE.") parser = startup_api.parser() parser.add_argument( @@ -40,6 +39,5 @@ def get(self): return True except Exception as exception: - if notBadRequestException(exception=exception): - startup_api.abort(500, str(exception)) + startup_api.abort(500, str(exception)) raise exception diff --git a/src/pyflask/app.py b/src/pyflask/app.py index 619ee9a0e..c254cc889 100644 --- a/src/pyflask/app.py +++ b/src/pyflask/app.py @@ -10,31 +10,30 @@ from os.path import isabs from pathlib import Path from signal import SIGINT +from typing import Union from urllib.parse import unquote -from errorHandlers import notBadRequestException - # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() from apis import data_api, neuroconv_api, startup_api -from flask import Flask, request, send_file, send_from_directory +from flask import Flask, Response, request, send_file, send_from_directory from flask_cors import CORS from flask_restx import Api, Resource from manageNeuroconv.info import ( CONVERSION_SAVE_FOLDER_PATH, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, - is_packaged, resource_path, ) +from utils import catch_exception_and_abort, server_error_responses -app = Flask(__name__) +flask_app = Flask(__name__) # Always enable CORS to allow distinct processes to handle frontend vs. backend -CORS(app) -app.config["CORS_HEADERS"] = "Content-Type" +CORS(flask_app) +flask_app.config["CORS_HEADERS"] = "Content-Type" # Create logger configuration LOG_FOLDER = Path(GUIDE_ROOT_FOLDER, "logs") @@ -43,7 +42,7 @@ LOG_FILE_PATH = Path(LOG_FOLDER, f"{timestamp}.log") # Initialize API -package_json_file_path = resource_path("package.json" if is_packaged() else "../package.json") +package_json_file_path = resource_path("package.json") with open(file=package_json_file_path) as fp: package_json = json.load(fp=fp) @@ -55,46 +54,72 @@ api.add_namespace(startup_api) api.add_namespace(neuroconv_api) api.add_namespace(data_api) -api.init_app(app) - -registered = {} +api.init_app(flask_app) +# 'nwbfile_registry' is a global list that keeps track of all NWB files that have been registered with the server +nwbfile_registry = [] -@app.route("/files") -def get_all_files(): - return list(registered.keys()) +# TODO: is there any advantage to using the api.route instead of app for resources added in this file? +@api.route(rule="/log") +@api.doc( + description="Any exception that occurs on the Flask server will save a full traceback to a log file on disk.", + responses=server_error_responses(codes=[200, 400, 500]), +) +class Log(Resource): + @api.doc( + description="Nicely format the exception and the payload that caused it.", + responses=server_error_responses(codes=[200, 400, 404, 500]), + ) + @catch_exception_and_abort(api=api, code=500) + def post(self): + payload = api.payload + type = payload["type"] + header = payload["header"] + inputs = payload["inputs"] + exception_traceback = payload["traceback"] + + message = f"{header}\n{'-'*len(header)}\n\n{json.dumps(inputs, indent=2)}\n\n{exception_traceback}\n" + selected_logger = getattr(api.logger, type) + selected_logger(message) + + +# Used for the standalone preview page +@flask_app.route(rule="/files/", methods=["GET", "POST"]) +@api.doc( + description="Handle adding and fetching NWB files from the global file registry.", + responses=server_error_responses(codes=[200, 400, 404, 500]), +) +def handle_file_request(file_path: str) -> Union[str, Response, None]: + """Used by the PreviewPage to serve the URL to Neurosift.""" + if ".nwb" not in file_path: + code = 400 + base_message = server_error_responses(codes=[code])[code] + message = f"{base_message}: Path does not point to an NWB file." + api.abort(code=code, message=message) + return + + if request.method == "GET" and file_path not in nwbfile_registry: + code = 404 + base_message = server_error_responses(codes=[code])[code] + message = f"{base_message}: Path does not point to an NWB file." + api.abort(code=code, message=message) + return -@app.route("/files/", methods=["GET", "POST"]) -def handle_file_request(path): if request.method == "GET": - if registered[path]: - path = unquote(path) - if not isabs(path): - path = f"/{path}" - return send_file(path) - else: - app.abort(404, "Resource is not accessible.") - - else: - if ".nwb" in path: - registered[path] = True - return request.base_url - else: - app.abort(400, str("Path does not point to an NWB file.")) - + parsed_file_path = unquote(file_path) # Decode any URL encoding applied to the file path + is_file_relative = not isabs(parsed_file_path) # Check if the file path is relative + if is_file_relative: + parsed_file_path = f"/{parsed_file_path}" + return send_file(path_or_file=parsed_file_path) -@app.route("/conversions/") -def send_conversions(path): - return send_from_directory(CONVERSION_SAVE_FOLDER_PATH, path) + # Register access to the provided file path + elif request.method == "POST": + nwbfile_registry.append(file_path) + return request.base_url # Return the URL of the newly added file -@app.route("/preview/") -def send_preview(path): - return send_from_directory(STUB_SAVE_FOLDER_PATH, path) - - -@app.route("/cpus") +@flask_app.route("/cpus") def get_cpu_count(): from psutil import cpu_count @@ -104,45 +129,24 @@ def get_cpu_count(): return dict(physical=physical, logical=logical) -@app.route("/get-recommended-species") +@flask_app.route("/get-recommended-species") def get_species(): from dandi.metadata.util import species_map return species_map -@api.route("/log") -class Log(Resource): - @api.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) - def post(self): - try: - - payload = api.payload - type = payload["type"] - header = payload["header"] - inputs = payload["inputs"] - traceback = payload["traceback"] - - message = f"{header}\n{'-'*len(header)}\n\n{json.dumps(inputs, indent=2)}\n\n{traceback}\n" - selected_logger = getattr(api.logger, type) - selected_logger(message) - - except Exception as exception: - if notBadRequestException(exception): - api.abort(500, str(exception)) - - @api.route("/server_shutdown", endpoint="shutdown") class Shutdown(Resource): - def get(self): - func = request.environ.get("werkzeug.server.shutdown") - api.logger.info("Shutting down server") + def get(self) -> None: + werkzeug_shutdown_function = request.environ.get("werkzeug.server.shutdown") + api.logger.info("Shutting down server...") - if func is None: + if werkzeug_shutdown_function is None: kill(getpid(), SIGINT) return - func() + werkzeug_shutdown_function() if __name__ == "__main__": @@ -157,13 +161,13 @@ def get(self): ) log_handler.setFormatter(log_formatter) - app.logger.addHandler(log_handler) - app.logger.setLevel(DEBUG) + flask_app.logger.addHandler(log_handler) + flask_app.logger.setLevel(DEBUG) - app.logger.info(f"Logging to {LOG_FILE_PATH}") + flask_app.logger.info(f"Logging to {LOG_FILE_PATH}") # Run the server api.logger.info(f"Starting server on port {port}") - app.run(host="127.0.0.1", port=port) + flask_app.run(host="127.0.0.1", port=port) else: raise Exception("No port provided for the NWB GUIDE backend.") diff --git a/src/pyflask/errorHandlers/__init__.py b/src/pyflask/errorHandlers/__init__.py deleted file mode 100644 index b88aad104..000000000 --- a/src/pyflask/errorHandlers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .notBadRequestException import notBadRequestException diff --git a/src/pyflask/errorHandlers/notBadRequestException.py b/src/pyflask/errorHandlers/notBadRequestException.py deleted file mode 100644 index c0f002952..000000000 --- a/src/pyflask/errorHandlers/notBadRequestException.py +++ /dev/null @@ -1,5 +0,0 @@ -def notBadRequestException(exception): - """ - Check if the exception is a generic exception. - """ - return type(exception).__name__ not in ["BadRequest", "Forbidden", "Unauthorized"] diff --git a/src/pyflask/manageNeuroconv/info/__init__.py b/src/pyflask/manageNeuroconv/info/__init__.py index 3a29439d2..edde04113 100644 --- a/src/pyflask/manageNeuroconv/info/__init__.py +++ b/src/pyflask/manageNeuroconv/info/__init__.py @@ -3,6 +3,5 @@ CONVERSION_SAVE_FOLDER_PATH, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, - is_packaged, resource_path, ) diff --git a/src/pyflask/manageNeuroconv/info/urls.py b/src/pyflask/manageNeuroconv/info/urls.py index d41355b1f..bf8a65116 100644 --- a/src/pyflask/manageNeuroconv/info/urls.py +++ b/src/pyflask/manageNeuroconv/info/urls.py @@ -4,16 +4,6 @@ from pathlib import Path -def is_packaged(): - deployed = True - try: - sys._MEIPASS # PyInstaller creates a temp folder and stores path in _MEIPASS - except Exception: - deployed = False - - return deployed - - def resource_path(relative_path): """Get absolute path to resource, works for dev and for PyInstaller""" try: @@ -30,7 +20,6 @@ def resource_path(relative_path): path_config = resource_path( "paths.config.json" ) # NOTE: Must have pyflask for running the GUIDE as a whole, but errors for just the server - f = path_config.open() data = json.load(f) GUIDE_ROOT_FOLDER = Path(Path.home(), data["root"]) diff --git a/src/pyflask/tests/conftest.py b/src/pyflask/tests/conftest.py index ec72c792b..0a19565fa 100644 --- a/src/pyflask/tests/conftest.py +++ b/src/pyflask/tests/conftest.py @@ -3,7 +3,7 @@ def pytest_addoption(parser): - parser.addoption("--target", action="store", help="Run the executable instead of the standard Flask app") + parser.addoption("--target", action="store", help="Run the executable instead of the standard Flask flask_app") @pytest.fixture(scope="session") @@ -12,7 +12,7 @@ def client(request): if target: return target else: - app = flask.app + app = flask.flask_app app.config.update( { "TESTING": True, diff --git a/src/pyflask/utils/__init__.py b/src/pyflask/utils/__init__.py new file mode 100644 index 000000000..408bd43da --- /dev/null +++ b/src/pyflask/utils/__init__.py @@ -0,0 +1,7 @@ +from ._flask_errors import ( + abort_if_not_nwb_file, + catch_exception_and_abort, + server_error_responses, +) + +__all__ = ["catch_exception_and_abort", "server_error_responses", "abort_if_not_nwb_file"] diff --git a/src/pyflask/utils/_flask_errors.py b/src/pyflask/utils/_flask_errors.py new file mode 100644 index 000000000..d530a048b --- /dev/null +++ b/src/pyflask/utils/_flask_errors.py @@ -0,0 +1,36 @@ +import contextlib +from typing import Dict, List, Union + +import flask_restx + + +def server_error_responses(*, codes: List[str]) -> Dict[int, str]: + all_server_error_responses = { + 200: "Success", + 400: "Bad request", + 404: "Resource is not accessible.", + 500: "Internal server error", + } + + selected_responses = {code: all_server_error_responses[code] for code in codes} + return selected_responses + + +@contextlib.contextmanager +def catch_exception_and_abort(*, api: Union[flask_restx.Api, flask_restx.Namespace], code: int) -> None: + try: + yield + except Exception as exception: + exception_type = type(exception) + exception_message = str(exception) + api.abort(code=code, message=f"{exception_type}: {exception_message}") + raise exception + + +def abort_if_not_nwb_file(file_path: str, api: flask_restx.Api) -> None: + """Check if the file path has a .nwb extension; otherwise, aport the API with code 400.""" + if ".nwb" not in file_path: + code = 400 + base_message = server_error_responses(codes=[code])[code] + message = f"{base_message}: Path does not point to an NWB file." + api.abort(code=code, message=message) From 43e16143827bdafa459d5b9c23a98fc5427da022 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Fri, 24 May 2024 13:01:54 -0400 Subject: [PATCH 02/10] split top-levle into distinct submodules --- src/pyflask/apis/__init__.py | 11 ++++ src/pyflask/apis/_dandi.py | 26 ++++++++ src/pyflask/apis/_neurosift.py | 4 +- src/pyflask/apis/_system.py | 26 ++++++++ src/pyflask/app.py | 105 +++++++++++---------------------- 5 files changed, 101 insertions(+), 71 deletions(-) create mode 100644 src/pyflask/apis/_dandi.py create mode 100644 src/pyflask/apis/_system.py diff --git a/src/pyflask/apis/__init__.py b/src/pyflask/apis/__init__.py index cfb2085eb..b52faa0a3 100644 --- a/src/pyflask/apis/__init__.py +++ b/src/pyflask/apis/__init__.py @@ -1,4 +1,15 @@ +from ._dandi import dandi_api from ._neurosift import neurosift_api +from ._system import system_api from .data import data_api from .neuroconv import neuroconv_api from .startup import startup_api + +__all__ = [ + "neurosift_api", + "dandi_api", + "system_api", + "data_api", + "neuroconv_api", + "startup_api", +] diff --git a/src/pyflask/apis/_dandi.py b/src/pyflask/apis/_dandi.py new file mode 100644 index 000000000..df8584acb --- /dev/null +++ b/src/pyflask/apis/_dandi.py @@ -0,0 +1,26 @@ +"""An API for handling requests to the DANDI Python API.""" + +from typing import List, Tuple, Union + +import flask.restx + +from ..utils import catch_exception_and_abort, server_error_responses + +dandi_api = flask_restx.Namespace( + name="dandi", description="Request various static listings from the DANDI Python API." +) + + +@dandi_api.route(rule="/get-recommended-species") +class SupportedSpecies(flask_restx.Resource): + + @neurosift_api.doc( + description="Request the list of currently supported species (by Latin Binomial name) for DANDI. Note that any " + "explicit NCBI taxonomy link is also supported.", + responses=server_error_responses(codes=[200, 500]), + ) + @catch_exception_and_abort(api=dandi_api, code=500) + def get(self) -> Union[List[Tuple[List[str], str, str, str]], None]: + from dandi.metadata.util import species_map + + return species_map diff --git a/src/pyflask/apis/_neurosift.py b/src/pyflask/apis/_neurosift.py index f1544c31f..e1e28e988 100644 --- a/src/pyflask/apis/_neurosift.py +++ b/src/pyflask/apis/_neurosift.py @@ -21,11 +21,11 @@ neurosift_file_registry = collections.defaultdict(bool) -@neurosift_api.route(rule="/files/", methods=["GET", "POST"]) +@neurosift_api.route(rule="/files/") @neurosift_api.doc( description="Handle adding and fetching NWB files from the global file registry.", ) -class AllInterfaces(flask_restx.Resource): +class NeurosiftFileManager(flask_restx.Resource): @neurosift_api.doc( description="If the file path has been added to the registry (and therefore sent its base " diff --git a/src/pyflask/apis/_system.py b/src/pyflask/apis/_system.py new file mode 100644 index 000000000..605852320 --- /dev/null +++ b/src/pyflask/apis/_system.py @@ -0,0 +1,26 @@ +"""An API for handling general system information.""" + +from typing import Dict, Union + +import flask.restx + +from ..utils import catch_exception_and_abort, server_error_responses + +system_api = flask_restx.Namespace(name="system", description="Request various system specific information.") + + +@system_api.route("/cpus") +class SupportedSpecies(flask_restx.Resource): + + @system_api.doc( + description="Request the number of physical and logical cores on the system.", + responses=server_error_responses(codes=[200, 500]), + ) + @catch_exception_and_abort(api=system_api, code=500) + def get(self) -> Union[Dict[str, int], None]: + from psutil import cpu_count + + physical = cpu_count(logical=False) + logical = cpu_count() + + return dict(physical=physical, logical=logical) diff --git a/src/pyflask/app.py b/src/pyflask/app.py index c254cc889..e8e2edb89 100644 --- a/src/pyflask/app.py +++ b/src/pyflask/app.py @@ -2,11 +2,11 @@ import json import multiprocessing +import os import sys from datetime import datetime from logging import DEBUG, Formatter from logging.handlers import RotatingFileHandler -from os import getpid, kill from os.path import isabs from pathlib import Path from signal import SIGINT @@ -16,9 +16,16 @@ # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() - -from apis import data_api, neuroconv_api, startup_api -from flask import Flask, Response, request, send_file, send_from_directory +import flask +from apis import ( + dandi_api, + data_api, + neuroconv_api, + neurosift_api, + startup_api, + system_api, +) +from flask import Flask, Response, send_file, send_from_directory from flask_cors import CORS from flask_restx import Api, Resource from manageNeuroconv.info import ( @@ -29,6 +36,8 @@ ) from utils import catch_exception_and_abort, server_error_responses +all_apis = [data_api, neuroconv_api, startup_api, neurosift_api, dandi_api, system_api] + flask_app = Flask(__name__) # Always enable CORS to allow distinct processes to handle frontend vs. backend @@ -36,89 +45,50 @@ flask_app.config["CORS_HEADERS"] = "Content-Type" # Create logger configuration -LOG_FOLDER = Path(GUIDE_ROOT_FOLDER, "logs") +LOG_FOLDER = Path(GUIDE_ROOT_FOLDER) / "logs" LOG_FOLDER.mkdir(exist_ok=True, parents=True) timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") -LOG_FILE_PATH = Path(LOG_FOLDER, f"{timestamp}.log") +LOG_FILE_PATH = Path(LOG_FOLDER) / f"{timestamp}.log" -# Initialize API +# Fetch version from package.json package_json_file_path = resource_path("package.json") with open(file=package_json_file_path) as fp: package_json = json.load(fp=fp) -api = Api( +# Initialize top-level API and set namespaces +flask_api = Api( version=package_json["version"], title="NWB GUIDE API", description="The REST API for the NWB GUIDE provided by the Python Flask Server.", ) -api.add_namespace(startup_api) -api.add_namespace(neuroconv_api) -api.add_namespace(data_api) -api.init_app(flask_app) - -# 'nwbfile_registry' is a global list that keeps track of all NWB files that have been registered with the server -nwbfile_registry = [] +for api in all_apis: + flask_api.add_namespace(api) +flask_api.init_app(flask_app) -# TODO: is there any advantage to using the api.route instead of app for resources added in this file? -@api.route(rule="/log") -@api.doc( +@flask_api.route(rule="/log") +@flask_api.doc( description="Any exception that occurs on the Flask server will save a full traceback to a log file on disk.", responses=server_error_responses(codes=[200, 400, 500]), ) class Log(Resource): - @api.doc( + @flask_api.doc( description="Nicely format the exception and the payload that caused it.", responses=server_error_responses(codes=[200, 400, 404, 500]), ) - @catch_exception_and_abort(api=api, code=500) + @catch_exception_and_abort(api=flask_api, code=500) def post(self): - payload = api.payload + payload = flask_api.payload type = payload["type"] header = payload["header"] inputs = payload["inputs"] exception_traceback = payload["traceback"] message = f"{header}\n{'-'*len(header)}\n\n{json.dumps(inputs, indent=2)}\n\n{exception_traceback}\n" - selected_logger = getattr(api.logger, type) + selected_logger = getattr(flask_api.logger, type) selected_logger(message) -# Used for the standalone preview page -@flask_app.route(rule="/files/", methods=["GET", "POST"]) -@api.doc( - description="Handle adding and fetching NWB files from the global file registry.", - responses=server_error_responses(codes=[200, 400, 404, 500]), -) -def handle_file_request(file_path: str) -> Union[str, Response, None]: - """Used by the PreviewPage to serve the URL to Neurosift.""" - if ".nwb" not in file_path: - code = 400 - base_message = server_error_responses(codes=[code])[code] - message = f"{base_message}: Path does not point to an NWB file." - api.abort(code=code, message=message) - return - - if request.method == "GET" and file_path not in nwbfile_registry: - code = 404 - base_message = server_error_responses(codes=[code])[code] - message = f"{base_message}: Path does not point to an NWB file." - api.abort(code=code, message=message) - return - - if request.method == "GET": - parsed_file_path = unquote(file_path) # Decode any URL encoding applied to the file path - is_file_relative = not isabs(parsed_file_path) # Check if the file path is relative - if is_file_relative: - parsed_file_path = f"/{parsed_file_path}" - return send_file(path_or_file=parsed_file_path) - - # Register access to the provided file path - elif request.method == "POST": - nwbfile_registry.append(file_path) - return request.base_url # Return the URL of the newly added file - - @flask_app.route("/cpus") def get_cpu_count(): from psutil import cpu_count @@ -129,21 +99,18 @@ def get_cpu_count(): return dict(physical=physical, logical=logical) -@flask_app.route("/get-recommended-species") -def get_species(): - from dandi.metadata.util import species_map - - return species_map - - -@api.route("/server_shutdown", endpoint="shutdown") +@flask_api.route("/server_shutdown", endpoint="shutdown") +@neurosift_api.doc( + description="Handle adding and fetching NWB files from the global file registry.", +) class Shutdown(Resource): def get(self) -> None: - werkzeug_shutdown_function = request.environ.get("werkzeug.server.shutdown") - api.logger.info("Shutting down server...") + werkzeug_shutdown_function = flask.request.environ.get("werkzeug.server.shutdown") + flask_api.logger.info("Shutting down server...") if werkzeug_shutdown_function is None: - kill(getpid(), SIGINT) + os.kill(os.getpid(), SIGINT) + return werkzeug_shutdown_function() @@ -167,7 +134,7 @@ def get(self) -> None: flask_app.logger.info(f"Logging to {LOG_FILE_PATH}") # Run the server - api.logger.info(f"Starting server on port {port}") + flask_api.logger.info(f"Starting server on port {port}") flask_app.run(host="127.0.0.1", port=port) else: raise Exception("No port provided for the NWB GUIDE backend.") From 518bc10306eb36b50c3f89adf511928aedb307ed Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Fri, 24 May 2024 13:06:23 -0400 Subject: [PATCH 03/10] restore frontend changes --- .../src/stories/pages/preview/PreviewPage.js | 96 ++--- .../src/stories/preview/NWBFilePreview.js | 358 ++++++++++-------- .../renderer/src/stories/preview/Neurosift.js | 158 ++++---- 3 files changed, 325 insertions(+), 287 deletions(-) diff --git a/src/electron/renderer/src/stories/pages/preview/PreviewPage.js b/src/electron/renderer/src/stories/pages/preview/PreviewPage.js index a8e6a320a..9c6c79fd0 100644 --- a/src/electron/renderer/src/stories/pages/preview/PreviewPage.js +++ b/src/electron/renderer/src/stories/pages/preview/PreviewPage.js @@ -6,51 +6,59 @@ import { Neurosift } from "../../preview/Neurosift.js"; import { baseUrl } from "../../../server/globals"; export class PreviewPage extends Page { - header = { - title: "NWB File Exploration", - subtitle: "Visualize your NWB file using Neurosift.", - }; - - constructor(...args) { - super(...args); - this.style.height = "100%"; // Fix main section - } + header = { + title: "NWB File Exploration", + subtitle: "Visualize your NWB file using Neurosift.", + }; + + constructor(...args) { + super(...args); + this.style.height = "100%"; // Fix main section + } + + updatePath = async (path) => { + if (path) { + // Enable access to the explicit file path + const result = await fetch(`${baseUrl}/files/${path}`, { + method: "POST", + }).then((res) => res.text()); + + // Set Neurosift to access the returned URL + if (result) this.neurosift.url = result; + } else this.neurosift.url = undefined; + }; - updatePath = async (path) => { - if (path) { - const result = await fetch(`${baseUrl}/files/${path}`, { method: "POST" }).then((res) => res.text()); - if (result) this.neurosift.url = result; - } else this.neurosift.url = undefined; - }; - - neurosift = new Neurosift(); - - input = new JSONSchemaInput({ - path: ["file_path"], - schema: { - type: "string", - format: "file", - description: - "Please provide a file path that you'd like to visualize using Neurosift. The GUIDE will serve this file and access the appropriate URL automatically.", - }, - onUpdate: this.updatePath, - onThrow, - }); - - render() { - const urlFilePath = new URL(document.location).searchParams.get("file"); - - if (urlFilePath) { - this.updatePath(urlFilePath); - this.input.value = urlFilePath; - } - - return html` -
- ${this.input} ${this.neurosift} -
- `; + neurosift = new Neurosift(); + + input = new JSONSchemaInput({ + path: ["file_path"], + schema: { + type: "string", + format: "file", + description: + "Please provide a file path that you'd like to visualize using Neurosift. The GUIDE will serve this file and access the appropriate URL automatically.", + }, + onUpdate: this.updatePath, + onThrow, + }); + + render() { + const urlFilePath = new URL(document.location).searchParams.get("file"); + + if (urlFilePath) { + this.updatePath(urlFilePath); + this.input.value = urlFilePath; } + + return html` +
+ ${this.input} ${this.neurosift} +
+ `; + } } -customElements.get("nwbguide-preview-page") || customElements.define("nwbguide-preview-page", PreviewPage); +customElements.get("nwbguide-preview-page") || + customElements.define("nwbguide-preview-page", PreviewPage); diff --git a/src/electron/renderer/src/stories/preview/NWBFilePreview.js b/src/electron/renderer/src/stories/preview/NWBFilePreview.js index 4bc8b7b71..67345f59b 100644 --- a/src/electron/renderer/src/stories/preview/NWBFilePreview.js +++ b/src/electron/renderer/src/stories/preview/NWBFilePreview.js @@ -1,191 +1,223 @@ import { LitElement, css, html } from "lit"; import { InspectorList } from "./inspector/InspectorList"; -import { Neurosift, getURLFromFilePath } from "./Neurosift"; +import { Neurosift } from "./Neurosift"; import { unsafeHTML } from "lit/directives/unsafe-html.js"; import { run } from "../pages/guided-mode/options/utils"; import { until } from "lit/directives/until.js"; import { InstanceManager } from "../InstanceManager"; import { path } from "../../electron"; import { FullScreenToggle } from "../FullScreenToggle"; +import { baseUrl } from "../../server/globals"; export function getSharedPath(array) { - array = array.map((str) => str.replace(/\\/g, "/")); // Convert to Mac-style path - const mapped = array.map((str) => str.split("/")); - let shared = mapped.shift(); - mapped.forEach((arr, i) => { - for (let j in arr) { - if (arr[j] !== shared[j]) { - shared = shared.slice(0, j); - break; - } - } - }); + array = array.map((str) => str.replace(/\\/g, "/")); // Convert to Mac-style path + const mapped = array.map((str) => str.split("/")); + let shared = mapped.shift(); + mapped.forEach((arr, i) => { + for (let j in arr) { + if (arr[j] !== shared[j]) { + shared = shared.slice(0, j); + break; + } + } + }); - return shared ? path.normalize(shared.join("/")) : ""; // Convert back to OS-specific path + return shared ? path.normalize(shared.join("/")) : ""; // Convert back to OS-specific path } export function truncateFilePaths(items, basepath) { - return items.map((item) => { - item = { ...item }; - item.file_path = item.file_path - .replace(`${basepath}/`, "") // Mac - .replace(`${basepath}\\`, ""); // Windows - return item; - }); + return items.map((item) => { + item = { ...item }; + item.file_path = item.file_path + .replace(`${basepath}/`, "") // Mac + .replace(`${basepath}\\`, ""); // Windows + return item; + }); } export const removeFilePaths = (items) => { - return items.map((item) => { - const copy = { ...item }; - delete copy.file_path; - return copy; - }); + return items.map((item) => { + const copy = { ...item }; + delete copy.file_path; + return copy; + }); }; class NWBPreviewInstance extends LitElement { - constructor({ file }, project) { - super(); - this.file = file; - this.project = project; - - window.addEventListener("online", () => this.requestUpdate()); - window.addEventListener("offline", () => this.requestUpdate()); - } - - render() { - const isOnline = navigator.onLine; - - return isOnline - ? new Neurosift({ url: getURLFromFilePath(this.file, this.project), fullscreen: false }) - : until( - (async () => { - const htmlRep = await run("html", { nwbfile_path: this.file }, { swal: false }); - return unsafeHTML(htmlRep); - })(), - html`Loading HTML representation...` - ); - } + constructor({ file }, project) { + super(); + this.file = file; + this.project = project; + + window.addEventListener("online", () => this.requestUpdate()); + window.addEventListener("offline", () => this.requestUpdate()); + } + + render() { + const isOnline = navigator.onLine; + + if (!isOnline) + return until( + (async () => { + const htmlRep = await run( + "html", + { nwbfile_path: this.file }, + { swal: false }, + ); + return unsafeHTML(htmlRep); + })(), + html`Loading HTML representation...`, + ); + + const neurosift = new Neurosift({ fullscreen: false }); + + // Enable access to the explicit file path + fetch(`${baseUrl}/files/${this.file}`, { method: "POST" }) + .then((res) => res.text()) + .then((result) => { + // Set Neurosift to access the returned URL + if (result) neurosift.url = result; + }); + + return neurosift; + } } -customElements.get("nwb-preview-instance") || customElements.define("nwb-preview-instance", NWBPreviewInstance); +customElements.get("nwb-preview-instance") || + customElements.define("nwb-preview-instance", NWBPreviewInstance); export class NWBFilePreview extends LitElement { - static get styles() { - return css` - :host { - display: block; - width: 100%; - height: 100%; - background: white; - position: relative; - } - - iframe { - width: 100%; - height: 100%; - border: 0; - } - - #inspect { - display: flex; - flex-direction: column; - border-left: 1px solid gray; - box-shadow: -5px 0 5px -5px rgba(0, 0, 0, 0.5); - z-index: 1; - } - `; - } - - constructor({ files = {}, project, inspect = false }) { - super(); - this.project = project; - this.files = files; - this.inspect = inspect; - } - - createInstance = ({ subject, session, info }) => { - return { - subject, - session, - display: () => new NWBPreviewInstance(info, this.project), - }; + static get styles() { + return css` + :host { + display: block; + width: 100%; + height: 100%; + background: white; + position: relative; + } + + iframe { + width: 100%; + height: 100%; + border: 0; + } + + #inspect { + display: flex; + flex-direction: column; + border-left: 1px solid gray; + box-shadow: -5px 0 5px -5px rgba(0, 0, 0, 0.5); + z-index: 1; + } + `; + } + + constructor({ files = {}, project, inspect = false }) { + super(); + this.project = project; + this.files = files; + this.inspect = inspect; + } + + createInstance = ({ subject, session, info }) => { + return { + subject, + session, + display: () => new NWBPreviewInstance(info, this.project), }; + }; + + render() { + const fileArr = Object.entries(this.files) + .map(([subject, v]) => + Object.entries(v).map(([session, info]) => { + return { subject, session, info }; + }), + ) + .flat(); + + const onlyFirstFile = fileArr.length <= 1; + + return html` ${new FullScreenToggle({ target: this })} +
+
+ ${(() => { + if (onlyFirstFile) + return new NWBPreviewInstance(fileArr[0].info, this.project); + else { + const _instances = fileArr.map(this.createInstance); + + const instances = _instances.reduce( + (acc, { subject, session, display }) => { + if (!acc[`sub-${subject}`]) acc[`sub-${subject}`] = {}; + acc[`sub-${subject}`][`ses-${session}`] = display; + return acc; + }, + {}, + ); - render() { - const fileArr = Object.entries(this.files) - .map(([subject, v]) => - Object.entries(v).map(([session, info]) => { - return { subject, session, info }; - }) - ) - .flat(); - - const onlyFirstFile = fileArr.length <= 1; - - return html` ${new FullScreenToggle({ target: this })} -
-
- ${(() => { - if (onlyFirstFile) return new NWBPreviewInstance(fileArr[0].info, this.project); - else { - const _instances = fileArr.map(this.createInstance); - - const instances = _instances.reduce((acc, { subject, session, display }) => { - if (!acc[`sub-${subject}`]) acc[`sub-${subject}`] = {}; - acc[`sub-${subject}`][`ses-${session}`] = display; - return acc; - }, {}); - - return new InstanceManager({ instances }); - } - })()} -
- ${this.inspect - ? html`
-

Inspector Report

- ${until( - (async () => { - const options = {}; // NOTE: Currently options are handled on the Python end until exposed to the user - - const title = "Inspecting your file"; - - const report = onlyFirstFile - ? await run( - "inspect_file", - { nwbfile_path: fileArr[0].info.file, ...options }, - { title } - ) // Inspect the first file - : await run("inspect_folder", { path, ...options }, { title: title + "s" }); // Inspect the folder - - const result = onlyFirstFile - ? { - ...report, - messages: removeFilePaths(report.messages), - } - : { - ...report, - messages: truncateFilePaths( - report.messages, - getSharedPath(fileArr.map(({ info }) => info.file)) - ), - }; - - const items = result.messages; - - const list = new InspectorList({ - items: items, - listStyles: { minWidth: "300px", maxWidth: "350px" }, - emptyMessage: "No issues found.", - }); - list.style.padding = "10px"; - return list; - })(), - html`Loading inspector report...` - )} -
` - : ""} -
`; - } + return new InstanceManager({ instances }); + } + })()} +
+ ${this.inspect + ? html`
+

+ Inspector Report +

+ ${until( + (async () => { + const options = {}; // NOTE: Currently options are handled on the Python end until exposed to the user + + const title = "Inspecting your file"; + + const report = onlyFirstFile + ? await run( + "inspect_file", + { nwbfile_path: fileArr[0].info.file, ...options }, + { title }, + ) // Inspect the first file + : await run( + "inspect_folder", + { path, ...options }, + { title: title + "s" }, + ); // Inspect the folder + + const result = onlyFirstFile + ? { + ...report, + messages: removeFilePaths(report.messages), + } + : { + ...report, + messages: truncateFilePaths( + report.messages, + getSharedPath(fileArr.map(({ info }) => info.file)), + ), + }; + + const items = result.messages; + + const list = new InspectorList({ + items: items, + listStyles: { minWidth: "300px", maxWidth: "350px" }, + emptyMessage: "No issues found.", + }); + list.style.padding = "10px"; + return list; + })(), + html`Loading inspector report...`, + )} +
` + : ""} +
`; + } } -customElements.get("nwb-file-preview") || customElements.define("nwb-file-preview", NWBFilePreview); +customElements.get("nwb-file-preview") || + customElements.define("nwb-file-preview", NWBFilePreview); diff --git a/src/electron/renderer/src/stories/preview/Neurosift.js b/src/electron/renderer/src/stories/preview/Neurosift.js index 8def8c8dd..1dbbc04d2 100644 --- a/src/electron/renderer/src/stories/preview/Neurosift.js +++ b/src/electron/renderer/src/stories/preview/Neurosift.js @@ -2,96 +2,94 @@ import { LitElement, css, html } from "lit"; import { Loader } from "../Loader"; import { FullScreenToggle } from "../FullScreenToggle"; -import { baseUrl } from "../../server/globals"; - -export function getURLFromFilePath(file, projectName) { - const regexp = new RegExp(`.+(${projectName}.+)`); - return `${baseUrl}/preview/${file.match(regexp)[1]}`; -} export class Neurosift extends LitElement { - static get styles() { - return css` - :host { - background: white; - width: 100%; - height: 100%; - display: grid; - grid-template-rows: 100%; - grid-template-columns: 100%; - position: relative; - --loader-color: hsl(200, 80%, 50%); - } + static get styles() { + return css` + :host { + background: white; + width: 100%; + height: 100%; + display: grid; + grid-template-rows: 100%; + grid-template-columns: 100%; + position: relative; + --loader-color: hsl(200, 80%, 50%); + } - iframe, - .loader-container { - width: 100%; - height: 100%; - } + iframe, + .loader-container { + width: 100%; + height: 100%; + } - .loader-container { - display: flex; - align-items: center; - justify-content: center; - position: absolute; - top: 0; - left: 0; - } + .loader-container { + display: flex; + align-items: center; + justify-content: center; + position: absolute; + top: 0; + left: 0; + } - .fullscreen-toggle { - display: flex; - position: absolute; - top: 10px; - right: 10px; - padding: 10px; - color: white; - background-color: gainsboro; - border: 1px solid gray; - border-radius: 10px; - cursor: pointer; - } + .fullscreen-toggle { + display: flex; + position: absolute; + top: 10px; + right: 10px; + padding: 10px; + color: white; + background-color: gainsboro; + border: 1px solid gray; + border-radius: 10px; + cursor: pointer; + } - span { - font-size: 14px; - } + span { + font-size: 14px; + } - small { - padding-left: 10px; - } + small { + padding-left: 10px; + } - iframe { - border: 0; - } - `; - } + iframe { + border: 0; + } + `; + } - static get properties() { - return { - url: { type: String, reflect: true }, - }; - } + static get properties() { + return { + url: { type: String, reflect: true }, + }; + } - constructor({ url, fullscreen = true } = {}) { - super(); - this.url = url; - this.fullscreen = fullscreen; - } + constructor({ url, fullscreen = true } = {}) { + super(); + this.url = url; + this.fullscreen = fullscreen; + } - render() { - return this.url - ? html`
- ${new Loader({ message: `Loading Neurosift view...
${this.url}` })} -
- ${this.fullscreen ? new FullScreenToggle({ target: this }) : ""} - ` - : ``; - } + render() { + return this.url + ? html`
+ ${new Loader({ + message: `Loading Neurosift view...
${this.url}`, + })} +
+ ${this.fullscreen ? new FullScreenToggle({ target: this }) : ""} + ` + : ``; + } } -customElements.get("neurosift-iframe") || customElements.define("neurosift-iframe", Neurosift); +customElements.get("neurosift-iframe") || + customElements.define("neurosift-iframe", Neurosift); From 3620d65e837c98dfcc9abe506b638130fdc88763 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Fri, 24 May 2024 13:06:58 -0400 Subject: [PATCH 04/10] alter dandi endpoint call --- src/electron/renderer/src/server/globals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/electron/renderer/src/server/globals.ts b/src/electron/renderer/src/server/globals.ts index 698243dc9..150362c59 100644 --- a/src/electron/renderer/src/server/globals.ts +++ b/src/electron/renderer/src/server/globals.ts @@ -56,7 +56,7 @@ export const activateServer = () => { export const serverGlobals = { species: new Promise((res, rej) => { onServerOpen(() => { - fetch(new URL("get-recommended-species", baseUrl)) + fetch(new URL("dandi/get-recommended-species", baseUrl)) .then((res) => res.json()) .then((species) => { res(species) From 8467a2772aa844fb96bf0f051813e4cb29f1768f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 17:08:18 +0000 Subject: [PATCH 05/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../src/stories/pages/preview/PreviewPage.js | 101 +++-- .../src/stories/preview/NWBFilePreview.js | 368 +++++++++--------- .../renderer/src/stories/preview/Neurosift.js | 154 ++++---- 3 files changed, 299 insertions(+), 324 deletions(-) diff --git a/src/electron/renderer/src/stories/pages/preview/PreviewPage.js b/src/electron/renderer/src/stories/pages/preview/PreviewPage.js index 9c6c79fd0..868b279ac 100644 --- a/src/electron/renderer/src/stories/pages/preview/PreviewPage.js +++ b/src/electron/renderer/src/stories/pages/preview/PreviewPage.js @@ -6,59 +6,56 @@ import { Neurosift } from "../../preview/Neurosift.js"; import { baseUrl } from "../../../server/globals"; export class PreviewPage extends Page { - header = { - title: "NWB File Exploration", - subtitle: "Visualize your NWB file using Neurosift.", - }; - - constructor(...args) { - super(...args); - this.style.height = "100%"; // Fix main section - } - - updatePath = async (path) => { - if (path) { - // Enable access to the explicit file path - const result = await fetch(`${baseUrl}/files/${path}`, { - method: "POST", - }).then((res) => res.text()); - - // Set Neurosift to access the returned URL - if (result) this.neurosift.url = result; - } else this.neurosift.url = undefined; - }; - - neurosift = new Neurosift(); - - input = new JSONSchemaInput({ - path: ["file_path"], - schema: { - type: "string", - format: "file", - description: - "Please provide a file path that you'd like to visualize using Neurosift. The GUIDE will serve this file and access the appropriate URL automatically.", - }, - onUpdate: this.updatePath, - onThrow, - }); - - render() { - const urlFilePath = new URL(document.location).searchParams.get("file"); - - if (urlFilePath) { - this.updatePath(urlFilePath); - this.input.value = urlFilePath; + header = { + title: "NWB File Exploration", + subtitle: "Visualize your NWB file using Neurosift.", + }; + + constructor(...args) { + super(...args); + this.style.height = "100%"; // Fix main section } - return html` -
- ${this.input} ${this.neurosift} -
- `; - } + updatePath = async (path) => { + if (path) { + // Enable access to the explicit file path + const result = await fetch(`${baseUrl}/files/${path}`, { + method: "POST", + }).then((res) => res.text()); + + // Set Neurosift to access the returned URL + if (result) this.neurosift.url = result; + } else this.neurosift.url = undefined; + }; + + neurosift = new Neurosift(); + + input = new JSONSchemaInput({ + path: ["file_path"], + schema: { + type: "string", + format: "file", + description: + "Please provide a file path that you'd like to visualize using Neurosift. The GUIDE will serve this file and access the appropriate URL automatically.", + }, + onUpdate: this.updatePath, + onThrow, + }); + + render() { + const urlFilePath = new URL(document.location).searchParams.get("file"); + + if (urlFilePath) { + this.updatePath(urlFilePath); + this.input.value = urlFilePath; + } + + return html` +
+ ${this.input} ${this.neurosift} +
+ `; + } } -customElements.get("nwbguide-preview-page") || - customElements.define("nwbguide-preview-page", PreviewPage); +customElements.get("nwbguide-preview-page") || customElements.define("nwbguide-preview-page", PreviewPage); diff --git a/src/electron/renderer/src/stories/preview/NWBFilePreview.js b/src/electron/renderer/src/stories/preview/NWBFilePreview.js index 67345f59b..9a82e8880 100644 --- a/src/electron/renderer/src/stories/preview/NWBFilePreview.js +++ b/src/electron/renderer/src/stories/preview/NWBFilePreview.js @@ -10,214 +10,194 @@ import { FullScreenToggle } from "../FullScreenToggle"; import { baseUrl } from "../../server/globals"; export function getSharedPath(array) { - array = array.map((str) => str.replace(/\\/g, "/")); // Convert to Mac-style path - const mapped = array.map((str) => str.split("/")); - let shared = mapped.shift(); - mapped.forEach((arr, i) => { - for (let j in arr) { - if (arr[j] !== shared[j]) { - shared = shared.slice(0, j); - break; - } - } - }); + array = array.map((str) => str.replace(/\\/g, "/")); // Convert to Mac-style path + const mapped = array.map((str) => str.split("/")); + let shared = mapped.shift(); + mapped.forEach((arr, i) => { + for (let j in arr) { + if (arr[j] !== shared[j]) { + shared = shared.slice(0, j); + break; + } + } + }); - return shared ? path.normalize(shared.join("/")) : ""; // Convert back to OS-specific path + return shared ? path.normalize(shared.join("/")) : ""; // Convert back to OS-specific path } export function truncateFilePaths(items, basepath) { - return items.map((item) => { - item = { ...item }; - item.file_path = item.file_path - .replace(`${basepath}/`, "") // Mac - .replace(`${basepath}\\`, ""); // Windows - return item; - }); + return items.map((item) => { + item = { ...item }; + item.file_path = item.file_path + .replace(`${basepath}/`, "") // Mac + .replace(`${basepath}\\`, ""); // Windows + return item; + }); } export const removeFilePaths = (items) => { - return items.map((item) => { - const copy = { ...item }; - delete copy.file_path; - return copy; - }); + return items.map((item) => { + const copy = { ...item }; + delete copy.file_path; + return copy; + }); }; class NWBPreviewInstance extends LitElement { - constructor({ file }, project) { - super(); - this.file = file; - this.project = project; - - window.addEventListener("online", () => this.requestUpdate()); - window.addEventListener("offline", () => this.requestUpdate()); - } - - render() { - const isOnline = navigator.onLine; - - if (!isOnline) - return until( - (async () => { - const htmlRep = await run( - "html", - { nwbfile_path: this.file }, - { swal: false }, - ); - return unsafeHTML(htmlRep); - })(), - html`Loading HTML representation...`, - ); - - const neurosift = new Neurosift({ fullscreen: false }); - - // Enable access to the explicit file path - fetch(`${baseUrl}/files/${this.file}`, { method: "POST" }) - .then((res) => res.text()) - .then((result) => { - // Set Neurosift to access the returned URL - if (result) neurosift.url = result; - }); - - return neurosift; - } + constructor({ file }, project) { + super(); + this.file = file; + this.project = project; + + window.addEventListener("online", () => this.requestUpdate()); + window.addEventListener("offline", () => this.requestUpdate()); + } + + render() { + const isOnline = navigator.onLine; + + if (!isOnline) + return until( + (async () => { + const htmlRep = await run("html", { nwbfile_path: this.file }, { swal: false }); + return unsafeHTML(htmlRep); + })(), + html`Loading HTML representation...` + ); + + const neurosift = new Neurosift({ fullscreen: false }); + + // Enable access to the explicit file path + fetch(`${baseUrl}/files/${this.file}`, { method: "POST" }) + .then((res) => res.text()) + .then((result) => { + // Set Neurosift to access the returned URL + if (result) neurosift.url = result; + }); + + return neurosift; + } } -customElements.get("nwb-preview-instance") || - customElements.define("nwb-preview-instance", NWBPreviewInstance); +customElements.get("nwb-preview-instance") || customElements.define("nwb-preview-instance", NWBPreviewInstance); export class NWBFilePreview extends LitElement { - static get styles() { - return css` - :host { - display: block; - width: 100%; - height: 100%; - background: white; - position: relative; - } - - iframe { - width: 100%; - height: 100%; - border: 0; - } - - #inspect { - display: flex; - flex-direction: column; - border-left: 1px solid gray; - box-shadow: -5px 0 5px -5px rgba(0, 0, 0, 0.5); - z-index: 1; - } - `; - } - - constructor({ files = {}, project, inspect = false }) { - super(); - this.project = project; - this.files = files; - this.inspect = inspect; - } - - createInstance = ({ subject, session, info }) => { - return { - subject, - session, - display: () => new NWBPreviewInstance(info, this.project), - }; - }; - - render() { - const fileArr = Object.entries(this.files) - .map(([subject, v]) => - Object.entries(v).map(([session, info]) => { - return { subject, session, info }; - }), - ) - .flat(); - - const onlyFirstFile = fileArr.length <= 1; - - return html` ${new FullScreenToggle({ target: this })} -
-
- ${(() => { - if (onlyFirstFile) - return new NWBPreviewInstance(fileArr[0].info, this.project); - else { - const _instances = fileArr.map(this.createInstance); - - const instances = _instances.reduce( - (acc, { subject, session, display }) => { - if (!acc[`sub-${subject}`]) acc[`sub-${subject}`] = {}; - acc[`sub-${subject}`][`ses-${session}`] = display; - return acc; - }, - {}, - ); - - return new InstanceManager({ instances }); + static get styles() { + return css` + :host { + display: block; + width: 100%; + height: 100%; + background: white; + position: relative; } - })()} -
- ${this.inspect - ? html`
-

- Inspector Report -

- ${until( - (async () => { - const options = {}; // NOTE: Currently options are handled on the Python end until exposed to the user - - const title = "Inspecting your file"; - - const report = onlyFirstFile - ? await run( - "inspect_file", - { nwbfile_path: fileArr[0].info.file, ...options }, - { title }, - ) // Inspect the first file - : await run( - "inspect_folder", - { path, ...options }, - { title: title + "s" }, - ); // Inspect the folder - - const result = onlyFirstFile - ? { - ...report, - messages: removeFilePaths(report.messages), - } - : { - ...report, - messages: truncateFilePaths( - report.messages, - getSharedPath(fileArr.map(({ info }) => info.file)), - ), - }; - - const items = result.messages; - - const list = new InspectorList({ - items: items, - listStyles: { minWidth: "300px", maxWidth: "350px" }, - emptyMessage: "No issues found.", - }); - list.style.padding = "10px"; - return list; - })(), - html`Loading inspector report...`, - )} -
` - : ""} -
`; - } + + iframe { + width: 100%; + height: 100%; + border: 0; + } + + #inspect { + display: flex; + flex-direction: column; + border-left: 1px solid gray; + box-shadow: -5px 0 5px -5px rgba(0, 0, 0, 0.5); + z-index: 1; + } + `; + } + + constructor({ files = {}, project, inspect = false }) { + super(); + this.project = project; + this.files = files; + this.inspect = inspect; + } + + createInstance = ({ subject, session, info }) => { + return { + subject, + session, + display: () => new NWBPreviewInstance(info, this.project), + }; + }; + + render() { + const fileArr = Object.entries(this.files) + .map(([subject, v]) => + Object.entries(v).map(([session, info]) => { + return { subject, session, info }; + }) + ) + .flat(); + + const onlyFirstFile = fileArr.length <= 1; + + return html` ${new FullScreenToggle({ target: this })} +
+
+ ${(() => { + if (onlyFirstFile) return new NWBPreviewInstance(fileArr[0].info, this.project); + else { + const _instances = fileArr.map(this.createInstance); + + const instances = _instances.reduce((acc, { subject, session, display }) => { + if (!acc[`sub-${subject}`]) acc[`sub-${subject}`] = {}; + acc[`sub-${subject}`][`ses-${session}`] = display; + return acc; + }, {}); + + return new InstanceManager({ instances }); + } + })()} +
+ ${this.inspect + ? html`
+

Inspector Report

+ ${until( + (async () => { + const options = {}; // NOTE: Currently options are handled on the Python end until exposed to the user + + const title = "Inspecting your file"; + + const report = onlyFirstFile + ? await run( + "inspect_file", + { nwbfile_path: fileArr[0].info.file, ...options }, + { title } + ) // Inspect the first file + : await run("inspect_folder", { path, ...options }, { title: title + "s" }); // Inspect the folder + + const result = onlyFirstFile + ? { + ...report, + messages: removeFilePaths(report.messages), + } + : { + ...report, + messages: truncateFilePaths( + report.messages, + getSharedPath(fileArr.map(({ info }) => info.file)) + ), + }; + + const items = result.messages; + + const list = new InspectorList({ + items: items, + listStyles: { minWidth: "300px", maxWidth: "350px" }, + emptyMessage: "No issues found.", + }); + list.style.padding = "10px"; + return list; + })(), + html`Loading inspector report...` + )} +
` + : ""} +
`; + } } -customElements.get("nwb-file-preview") || - customElements.define("nwb-file-preview", NWBFilePreview); +customElements.get("nwb-file-preview") || customElements.define("nwb-file-preview", NWBFilePreview); diff --git a/src/electron/renderer/src/stories/preview/Neurosift.js b/src/electron/renderer/src/stories/preview/Neurosift.js index 1dbbc04d2..ecce4cbb9 100644 --- a/src/electron/renderer/src/stories/preview/Neurosift.js +++ b/src/electron/renderer/src/stories/preview/Neurosift.js @@ -4,92 +4,90 @@ import { Loader } from "../Loader"; import { FullScreenToggle } from "../FullScreenToggle"; export class Neurosift extends LitElement { - static get styles() { - return css` - :host { - background: white; - width: 100%; - height: 100%; - display: grid; - grid-template-rows: 100%; - grid-template-columns: 100%; - position: relative; - --loader-color: hsl(200, 80%, 50%); - } + static get styles() { + return css` + :host { + background: white; + width: 100%; + height: 100%; + display: grid; + grid-template-rows: 100%; + grid-template-columns: 100%; + position: relative; + --loader-color: hsl(200, 80%, 50%); + } - iframe, - .loader-container { - width: 100%; - height: 100%; - } + iframe, + .loader-container { + width: 100%; + height: 100%; + } - .loader-container { - display: flex; - align-items: center; - justify-content: center; - position: absolute; - top: 0; - left: 0; - } + .loader-container { + display: flex; + align-items: center; + justify-content: center; + position: absolute; + top: 0; + left: 0; + } - .fullscreen-toggle { - display: flex; - position: absolute; - top: 10px; - right: 10px; - padding: 10px; - color: white; - background-color: gainsboro; - border: 1px solid gray; - border-radius: 10px; - cursor: pointer; - } + .fullscreen-toggle { + display: flex; + position: absolute; + top: 10px; + right: 10px; + padding: 10px; + color: white; + background-color: gainsboro; + border: 1px solid gray; + border-radius: 10px; + cursor: pointer; + } - span { - font-size: 14px; - } + span { + font-size: 14px; + } - small { - padding-left: 10px; - } + small { + padding-left: 10px; + } - iframe { - border: 0; - } - `; - } + iframe { + border: 0; + } + `; + } - static get properties() { - return { - url: { type: String, reflect: true }, - }; - } + static get properties() { + return { + url: { type: String, reflect: true }, + }; + } - constructor({ url, fullscreen = true } = {}) { - super(); - this.url = url; - this.fullscreen = fullscreen; - } + constructor({ url, fullscreen = true } = {}) { + super(); + this.url = url; + this.fullscreen = fullscreen; + } - render() { - return this.url - ? html`
- ${new Loader({ - message: `Loading Neurosift view...
${this.url}`, - })} -
- ${this.fullscreen ? new FullScreenToggle({ target: this }) : ""} - ` - : ``; - } + render() { + return this.url + ? html`
+ ${new Loader({ + message: `Loading Neurosift view...
${this.url}`, + })} +
+ ${this.fullscreen ? new FullScreenToggle({ target: this }) : ""} + ` + : ``; + } } -customElements.get("neurosift-iframe") || - customElements.define("neurosift-iframe", Neurosift); +customElements.get("neurosift-iframe") || customElements.define("neurosift-iframe", Neurosift); From 18b72295ef9f690343a7815948452717470434a2 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 24 May 2024 13:53:38 -0400 Subject: [PATCH 06/10] Update src/pyflask/tests/conftest.py --- src/pyflask/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyflask/tests/conftest.py b/src/pyflask/tests/conftest.py index 0a19565fa..c585a7124 100644 --- a/src/pyflask/tests/conftest.py +++ b/src/pyflask/tests/conftest.py @@ -3,7 +3,7 @@ def pytest_addoption(parser): - parser.addoption("--target", action="store", help="Run the executable instead of the standard Flask flask_app") + parser.addoption("--target", action="store", help="Run the executable instead of the standard Flask app.") @pytest.fixture(scope="session") From c70954e85a119dc5ef3e33e96012f62f9e8d893f Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Fri, 24 May 2024 14:50:17 -0400 Subject: [PATCH 07/10] debugging --- src/__init__.py | 0 src/pyflask/__init__.py | 0 src/pyflask/apis/_dandi.py | 6 +-- src/pyflask/apis/_neurosift.py | 4 +- src/pyflask/apis/_system.py | 4 +- src/pyflask/apis/data.py | 3 +- src/pyflask/{ => apis}/utils/__init__.py | 0 src/pyflask/{ => apis}/utils/_flask_errors.py | 0 src/pyflask/app.py | 37 ++++---------- src/pyflask/manageNeuroconv/info/__init__.py | 3 +- src/pyflask/manageNeuroconv/info/urls.py | 50 ++++++++++++------- 11 files changed, 54 insertions(+), 53 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/pyflask/__init__.py rename src/pyflask/{ => apis}/utils/__init__.py (100%) rename src/pyflask/{ => apis}/utils/_flask_errors.py (100%) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pyflask/__init__.py b/src/pyflask/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pyflask/apis/_dandi.py b/src/pyflask/apis/_dandi.py index df8584acb..003852c3a 100644 --- a/src/pyflask/apis/_dandi.py +++ b/src/pyflask/apis/_dandi.py @@ -2,9 +2,9 @@ from typing import List, Tuple, Union -import flask.restx +import flask_restx -from ..utils import catch_exception_and_abort, server_error_responses +from .utils import catch_exception_and_abort, server_error_responses dandi_api = flask_restx.Namespace( name="dandi", description="Request various static listings from the DANDI Python API." @@ -14,7 +14,7 @@ @dandi_api.route(rule="/get-recommended-species") class SupportedSpecies(flask_restx.Resource): - @neurosift_api.doc( + @dandi_api.doc( description="Request the list of currently supported species (by Latin Binomial name) for DANDI. Note that any " "explicit NCBI taxonomy link is also supported.", responses=server_error_responses(codes=[200, 500]), diff --git a/src/pyflask/apis/_neurosift.py b/src/pyflask/apis/_neurosift.py index e1e28e988..e43f89746 100644 --- a/src/pyflask/apis/_neurosift.py +++ b/src/pyflask/apis/_neurosift.py @@ -4,9 +4,9 @@ from typing import Union import flask -import flask.restx +import flask_restx -from ..utils import ( +from .utils import ( abort_if_not_nwb_file, catch_exception_and_abort, server_error_responses, diff --git a/src/pyflask/apis/_system.py b/src/pyflask/apis/_system.py index 605852320..d384362d9 100644 --- a/src/pyflask/apis/_system.py +++ b/src/pyflask/apis/_system.py @@ -2,9 +2,9 @@ from typing import Dict, Union -import flask.restx +import flask_restx -from ..utils import catch_exception_and_abort, server_error_responses +from .utils import catch_exception_and_abort, server_error_responses system_api = flask_restx.Namespace(name="system", description="Request various system specific information.") diff --git a/src/pyflask/apis/data.py b/src/pyflask/apis/data.py index a3e05f985..30de92ba1 100644 --- a/src/pyflask/apis/data.py +++ b/src/pyflask/apis/data.py @@ -4,7 +4,8 @@ from flask_restx import Namespace, Resource, reqparse from manageNeuroconv import generate_dataset, generate_test_data -from utils import catch_exception_and_abort, server_error_responses + +from .utils import catch_exception_and_abort, server_error_responses data_api = Namespace(name="data", description="API route for dataset generation in the NWB GUIDE.") diff --git a/src/pyflask/utils/__init__.py b/src/pyflask/apis/utils/__init__.py similarity index 100% rename from src/pyflask/utils/__init__.py rename to src/pyflask/apis/utils/__init__.py diff --git a/src/pyflask/utils/_flask_errors.py b/src/pyflask/apis/utils/_flask_errors.py similarity index 100% rename from src/pyflask/utils/_flask_errors.py rename to src/pyflask/apis/utils/_flask_errors.py diff --git a/src/pyflask/app.py b/src/pyflask/app.py index e8e2edb89..d1344ecf9 100644 --- a/src/pyflask/app.py +++ b/src/pyflask/app.py @@ -7,11 +7,8 @@ from datetime import datetime from logging import DEBUG, Formatter from logging.handlers import RotatingFileHandler -from os.path import isabs from pathlib import Path from signal import SIGINT -from typing import Union -from urllib.parse import unquote # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() @@ -25,16 +22,11 @@ startup_api, system_api, ) -from flask import Flask, Response, send_file, send_from_directory +from apis.utils import catch_exception_and_abort, server_error_responses +from flask import Flask from flask_cors import CORS from flask_restx import Api, Resource -from manageNeuroconv.info import ( - CONVERSION_SAVE_FOLDER_PATH, - GUIDE_ROOT_FOLDER, - STUB_SAVE_FOLDER_PATH, - resource_path, -) -from utils import catch_exception_and_abort, server_error_responses +from manageNeuroconv.info import GUIDE_ROOT_FOLDER, get_project_root_path all_apis = [data_api, neuroconv_api, startup_api, neurosift_api, dandi_api, system_api] @@ -51,8 +43,8 @@ LOG_FILE_PATH = Path(LOG_FOLDER) / f"{timestamp}.log" # Fetch version from package.json -package_json_file_path = resource_path("package.json") -with open(file=package_json_file_path) as fp: +package_json_file_path = get_project_root_path() / "package.json" +with open(file=package_json_file_path, mode="r") as fp: package_json = json.load(fp=fp) # Initialize top-level API and set namespaces @@ -89,21 +81,14 @@ def post(self): selected_logger(message) -@flask_app.route("/cpus") -def get_cpu_count(): - from psutil import cpu_count - - physical = cpu_count(logical=False) - logical = cpu_count() - - return dict(physical=physical, logical=logical) - - @flask_api.route("/server_shutdown", endpoint="shutdown") -@neurosift_api.doc( - description="Handle adding and fetching NWB files from the global file registry.", -) +@flask_api.doc(description="Close the Flask server.") class Shutdown(Resource): + + @flask_api.doc( + description="To trigger a shutdown, set a GET request to this endpoint. It will not return a response.", + responses=server_error_responses(codes=[200, 500]), + ) def get(self) -> None: werkzeug_shutdown_function = flask.request.environ.get("werkzeug.server.shutdown") flask_api.logger.info("Shutting down server...") diff --git a/src/pyflask/manageNeuroconv/info/__init__.py b/src/pyflask/manageNeuroconv/info/__init__.py index edde04113..fdf5a36ba 100644 --- a/src/pyflask/manageNeuroconv/info/__init__.py +++ b/src/pyflask/manageNeuroconv/info/__init__.py @@ -3,5 +3,6 @@ CONVERSION_SAVE_FOLDER_PATH, GUIDE_ROOT_FOLDER, STUB_SAVE_FOLDER_PATH, - resource_path, + get_project_root_path, + get_source_base_path, ) diff --git a/src/pyflask/manageNeuroconv/info/urls.py b/src/pyflask/manageNeuroconv/info/urls.py index bf8a65116..7bb586235 100644 --- a/src/pyflask/manageNeuroconv/info/urls.py +++ b/src/pyflask/manageNeuroconv/info/urls.py @@ -1,36 +1,50 @@ import json import os +import pathlib import sys -from pathlib import Path -def resource_path(relative_path): - """Get absolute path to resource, works for dev and for PyInstaller""" - try: +def get_source_base_path() -> pathlib.Path: + """Get absolute path of a relative resource to the app; works for both dev mode and for PyInstaller.""" + # Production: PyInstaller creates a temp folder and stores path in _MEIPASS + if hasattr(sys, "_MEIPASS"): # PyInstaller creates a temp folder and stores path in _MEIPASS - base_path = sys._MEIPASS - except Exception: - base_path = Path(__file__).parent.parent.parent.parent + base_path = pathlib.Path(sys._MEIPASS) - return Path(base_path) / relative_path + # Dev mode: base is the root of the `src` directory for the project + else: + base_path = pathlib.Path(__file__).parent.parent.parent.parent + + return base_path + + +def get_project_root_path() -> pathlib.Path: + """Get absolute path of a relative resource to the app; works for both dev mode and for PyInstaller.""" + # Production: PyInstaller creates a temp folder and stores path in _MEIPASS + if hasattr(sys, "_MEIPASS"): + # PyInstaller creates a temp folder and stores path in _MEIPASS + base_path = pathlib.Path(sys._MEIPASS).parent + + # Dev mode: base is the root of the `src` directory for the project + else: + base_path = pathlib.Path(__file__).parent.parent.parent.parent.parent + + return base_path is_test_environment = os.environ.get("VITEST") -path_config = resource_path( - "paths.config.json" -) # NOTE: Must have pyflask for running the GUIDE as a whole, but errors for just the server -f = path_config.open() -data = json.load(f) -GUIDE_ROOT_FOLDER = Path(Path.home(), data["root"]) + +path_config_file_path = get_source_base_path() / "paths.config.json" +with open(file=path_config_file_path, mode="r") as fp: + path_config = json.load(fp=fp) +GUIDE_ROOT_FOLDER = pathlib.Path.home() / path_config["root"] if is_test_environment: GUIDE_ROOT_FOLDER = GUIDE_ROOT_FOLDER / ".test" -STUB_SAVE_FOLDER_PATH = Path(GUIDE_ROOT_FOLDER, *data["subfolders"]["preview"]) -CONVERSION_SAVE_FOLDER_PATH = Path(GUIDE_ROOT_FOLDER, *data["subfolders"]["conversions"]) - -f.close() +STUB_SAVE_FOLDER_PATH = pathlib.Path(GUIDE_ROOT_FOLDER, *path_config["subfolders"]["preview"]) +CONVERSION_SAVE_FOLDER_PATH = pathlib.Path(GUIDE_ROOT_FOLDER, *path_config["subfolders"]["conversions"]) # Create all nested home folders STUB_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True) From 1ae8ede8fa753aa0351924d4f451c4555779f577 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sat, 25 May 2024 13:50:31 -0400 Subject: [PATCH 08/10] adjust frontend endpoints to new namespace; try removing private marker from APIs --- src/electron/renderer/src/server/globals.ts | 2 +- src/pyflask/apis/__init__.py | 6 +++--- src/pyflask/apis/{_dandi.py => dandi.py} | 0 src/pyflask/apis/{_neurosift.py => neurosift.py} | 0 src/pyflask/apis/{_system.py => system.py} | 0 src/schemas/interfaces.info.ts | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename src/pyflask/apis/{_dandi.py => dandi.py} (100%) rename src/pyflask/apis/{_neurosift.py => neurosift.py} (100%) rename src/pyflask/apis/{_system.py => system.py} (100%) diff --git a/src/electron/renderer/src/server/globals.ts b/src/electron/renderer/src/server/globals.ts index 150362c59..af69d9af9 100644 --- a/src/electron/renderer/src/server/globals.ts +++ b/src/electron/renderer/src/server/globals.ts @@ -67,7 +67,7 @@ export const serverGlobals = { }), cpus: new Promise((res, rej) => { onServerOpen(() => { - fetch(new URL("cpus", baseUrl)) + fetch(new URL("system/cpus", baseUrl)) .then((res) => res.json()) .then((cpus) => { res(cpus) diff --git a/src/pyflask/apis/__init__.py b/src/pyflask/apis/__init__.py index b52faa0a3..f8f83c054 100644 --- a/src/pyflask/apis/__init__.py +++ b/src/pyflask/apis/__init__.py @@ -1,9 +1,9 @@ -from ._dandi import dandi_api -from ._neurosift import neurosift_api -from ._system import system_api +from .dandi import dandi_api from .data import data_api from .neuroconv import neuroconv_api +from .neurosift import neurosift_api from .startup import startup_api +from .system import system_api __all__ = [ "neurosift_api", diff --git a/src/pyflask/apis/_dandi.py b/src/pyflask/apis/dandi.py similarity index 100% rename from src/pyflask/apis/_dandi.py rename to src/pyflask/apis/dandi.py diff --git a/src/pyflask/apis/_neurosift.py b/src/pyflask/apis/neurosift.py similarity index 100% rename from src/pyflask/apis/_neurosift.py rename to src/pyflask/apis/neurosift.py diff --git a/src/pyflask/apis/_system.py b/src/pyflask/apis/system.py similarity index 100% rename from src/pyflask/apis/_system.py rename to src/pyflask/apis/system.py diff --git a/src/schemas/interfaces.info.ts b/src/schemas/interfaces.info.ts index 7ecc13cc7..e52e671ad 100644 --- a/src/schemas/interfaces.info.ts +++ b/src/schemas/interfaces.info.ts @@ -13,7 +13,7 @@ export const ready = { interfaces: createPromise("interfaces"), } -// Get CPUs +// Fetch data formats onServerOpen(async () => { await fetch(`${baseUrl}/neuroconv`).then((res) => res.json()) .then((interfaces) => setReady.interfaces(interfaces)) From 2edfcf0203da7315333b610715b883f86c4a5d38 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Sat, 25 May 2024 16:34:36 -0400 Subject: [PATCH 09/10] cleanup some comments; still debugging --- src/pyflask/manageNeuroconv/info/urls.py | 8 +++----- src/pyflask/tests/conftest.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/pyflask/manageNeuroconv/info/urls.py b/src/pyflask/manageNeuroconv/info/urls.py index 7bb586235..aa296b73f 100644 --- a/src/pyflask/manageNeuroconv/info/urls.py +++ b/src/pyflask/manageNeuroconv/info/urls.py @@ -6,9 +6,8 @@ def get_source_base_path() -> pathlib.Path: """Get absolute path of a relative resource to the app; works for both dev mode and for PyInstaller.""" - # Production: PyInstaller creates a temp folder and stores path in _MEIPASS + # Production: PyInstaller creates a temp folder at runtime and stores path in _MEIPASS if hasattr(sys, "_MEIPASS"): - # PyInstaller creates a temp folder and stores path in _MEIPASS base_path = pathlib.Path(sys._MEIPASS) # Dev mode: base is the root of the `src` directory for the project @@ -20,12 +19,11 @@ def get_source_base_path() -> pathlib.Path: def get_project_root_path() -> pathlib.Path: """Get absolute path of a relative resource to the app; works for both dev mode and for PyInstaller.""" - # Production: PyInstaller creates a temp folder and stores path in _MEIPASS + # Production: PyInstaller creates a temp folder at runtime and stores path in _MEIPASS if hasattr(sys, "_MEIPASS"): - # PyInstaller creates a temp folder and stores path in _MEIPASS base_path = pathlib.Path(sys._MEIPASS).parent - # Dev mode: base is the root of the `src` directory for the project + # Dev mode: root of project is one level above root of `src` else: base_path = pathlib.Path(__file__).parent.parent.parent.parent.parent diff --git a/src/pyflask/tests/conftest.py b/src/pyflask/tests/conftest.py index c585a7124..c6f863924 100644 --- a/src/pyflask/tests/conftest.py +++ b/src/pyflask/tests/conftest.py @@ -12,7 +12,7 @@ def client(request): if target: return target else: - app = flask.flask_app + app = flask.app app.config.update( { "TESTING": True, From ae58fc70ff22fb3b270f634d4cd02dbb38fa8ae0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 25 May 2024 21:26:37 +0000 Subject: [PATCH 10/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyflask/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pyflask/app.py b/src/pyflask/app.py index 0d55a9bd5..25774027f 100644 --- a/src/pyflask/app.py +++ b/src/pyflask/app.py @@ -9,10 +9,8 @@ from logging.handlers import RotatingFileHandler from pathlib import Path from signal import SIGINT - from urllib.parse import unquote - # https://stackoverflow.com/questions/32672596/pyinstaller-loads-script-multiple-times#comment103216434_32677108 multiprocessing.freeze_support() @@ -84,7 +82,6 @@ def post(self): selected_logger(message) - @flask_api.route("/server_shutdown", endpoint="shutdown") @flask_api.doc(description="Close the Flask server.") class Shutdown(Resource):