diff --git a/README.md b/README.md index 31d4efd..e8e8d3e 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,59 @@ DIP Testbed Platform is an academic work which allows users to remotely program and experience physical, embedded devices through various virtual interfaces (uni-directional webcam stream, bi-directional serial connection stream). -## Installation -### Quick install +## Quick installation & usage +Download the CLI tool: ```bash curl https://github.com/kshaa/dip-testbed-dist/releases/latest/download/client_install.sh | bash ``` +Create a local authentication session: +```bash +dip_client session-auth -u -p +``` + +Upload software to the platform, forward it to a hardware board, run a serial connection against it: +```bash +dip_client quick-run -f firmware.bit -b ${BOARD_UUID} +``` + _Note: This assumes usage of bash, AMD64 architecture, testbed.veinbahs.lv as default server_ +_Note: Also the default buttonled interface is used_ +_Note: Quick run has all of the underlying mechanics configurable, see options with `quick-run --help`_ + +## Detailed platform usage -### Manual install +### Installation - Download `https://github.com/kshaa/dip-testbed-dist/releases/latest/download/dip_client_${TARGET_ARCH}` - Store in `${PATH}` - Set executable bit -- Set static URL using `dip_client session-static-server -s http://testbed.veinbahs.lv` -- Set control URL using `dip_client session-control-server -s ws://testbed.veinbahs.lv` ## Usage +Configure academig DIP Testbed platform server: +```bash +dip_client session-static-server -s http://testbed.veinbahs.lv +dip_client session-control-server -s ws://testbed.veinbahs.lv +``` + +Authenticate: ```bash dip_client session-auth -u -p ``` + +Upload software to platform: +```bash +dip_client software-upload -f firmware.bit +``` + +Forward software to a hardware board: +```bash +dip_client hardware-software-upload --hardware-id ${BOARD_UUID} --software-id ${SOFTWARE_UUID} +``` + +Create a serial connection to the board: +``` +dip_client hardware-serial-monitor --hardware-id ${BOARD_UUID} -t buttonleds +``` ## Documentation - See 🌼 🌻 [docs](./docs/README.md) 🌻 🌼 for user-centric documentation diff --git a/backend/web/app/diptestbed/web/controllers/SoftwareController.scala b/backend/web/app/diptestbed/web/controllers/SoftwareController.scala index 3ebdfe5..bd86eec 100644 --- a/backend/web/app/diptestbed/web/controllers/SoftwareController.scala +++ b/backend/web/app/diptestbed/web/controllers/SoftwareController.scala @@ -39,7 +39,8 @@ class SoftwareController( maybeUser .leftMap(databaseErrorResult) .flatMap(_.toRight(authorizationErrorResult)) - .flatMap(user => + .flatMap(user => { + println(request.body.file("software"), request.body.dataParts.get("name")) (request.body.file("software"), request.body.dataParts.get("name").flatMap(_.headOption)).tupled .toRight(Failure("Request must contain 'software' file and 'name' field").withHttpStatus(BAD_REQUEST)) .flatMap { @@ -50,8 +51,8 @@ class SoftwareController( Failure("Request too large").withHttpStatus(BAD_REQUEST), ) .map(_ => (data, name, user)) - }, - ), + } + }) ) val authWithSoftwareBytes = authWithSoftwareData.flatMap { diff --git a/backend/web/conf/application.conf b/backend/web/conf/application.conf index ab62309..8d2b169 100644 --- a/backend/web/conf/application.conf +++ b/backend/web/conf/application.conf @@ -15,6 +15,12 @@ play = { http.secret.key = "31imdCGyKb5IwITHhzlo" application.loader = diptestbed.web.DIPTestbedLoader akka.actor-system = "DIPTestbed" + # CSRF checks required only if cookies are present + # A lone Authorization header can bypass CSRF checks + filters.csrf.header.protectHeaders = { + "Cookie" = "*" + "Authorization" = "nocheck" + } } # Akka diff --git a/client/potato.txt b/client/potato.txt new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/client/potato.txt @@ -0,0 +1 @@ +1 diff --git a/client/src/engine/engine_serial_monitor.py b/client/src/engine/engine_serial_monitor.py index cc0e26c..7c27604 100644 --- a/client/src/engine/engine_serial_monitor.py +++ b/client/src/engine/engine_serial_monitor.py @@ -14,7 +14,7 @@ from src.domain.monitor_message import SerialMonitorMessageToAgent, SerialMonitorMessageToClient, MonitorUnavailable from src.engine.engine_events import COMMON_ENGINE_EVENT, StartSerialMonitor, SerialMonitorStartSuccess, \ SerialMonitorStartFailure, ReceivedSerialBytes, SendingBoardBytes, StoppingSerialMonitor, StoppedSerialMonitor, \ - SerialMonitorAboutToStart, SerialMonitorAlreadyConfigured, MonitorDied, LifecycleEnded + SerialMonitorAboutToStart, SerialMonitorAlreadyConfigured, MonitorDied, LifecycleEnded, UploadingBoardSoftware from src.engine.engine_state import EngineState, ManagedQueue, EngineBase from src.service.managed_serial import ManagedSerial from src.service.managed_serial_config import ManagedSerialConfig @@ -178,4 +178,7 @@ async def effect_project(self, previous_state: EngineSerialMonitorState, event: message = f"Agent engine stopped" if event.reason is not None: message = f"{message}, reason: {event.reason.text()}" - await previous_state.base.outgoing_message_queue.put(MonitorUnavailable(message)) \ No newline at end of file + await previous_state.base.outgoing_message_queue.put(MonitorUnavailable(message)) + elif isinstance(event, UploadingBoardSoftware): + await previous_state.base.outgoing_message_queue.put(MonitorUnavailable( + "Serial connection broken by new board software upload")) \ No newline at end of file diff --git a/client/src/service/backend.py b/client/src/service/backend.py index 2cecf4f..0c8cc0c 100644 --- a/client/src/service/backend.py +++ b/client/src/service/backend.py @@ -1,5 +1,5 @@ """Module for backend management service definitions""" - +import os from typing import List, TypeVar, Dict, Optional from dataclasses import dataclass import base64 @@ -87,8 +87,8 @@ def software_list(self) -> Result[List[Software], BackendManagementError]: def software_upload( self, - software_name: str, - file_path: str + file_path: str, + software_name: str ) -> Result[Software, BackendManagementError]: pass @@ -174,6 +174,7 @@ def static_get_json_result( if headers is None: headers = {} try: + LOGGER.debug(f"HTTP GET JSON: {url_text_result.value}, headers: {headers}") response = requests.get(url_text_result.value, headers=headers) LOGGER.debug(ManagedURL.response_log_text(response)) return BackendService.response_to_result(response, content_decoder) @@ -201,9 +202,11 @@ def static_post_json_result( if files is None: headers = {} try: if payload is None: + LOGGER.debug(f"HTTP POST JSON: {url_text_result.value}, headers: {headers}, files:{ files }") response = requests.post(url_text_result.value, headers=headers, files=files) else: encoded_payload = payload_encoder.encode(payload) if payload_encoder is not None else payload + LOGGER.debug(f"HTTP POST JSON: {url_text_result.value}, payload: {encoded_payload}, headers: {headers}, files:{files}") response = requests.post(url_text_result.value, encoded_payload, headers=headers, files=files) # Parse response LOGGER.debug(ManagedURL.response_log_text(response)) @@ -275,13 +278,15 @@ def software_list(self) -> Result[List[Software], str]: def software_upload( self, - software_name: str, - file_path: ExistingFilePath + file_path: ExistingFilePath, + software_name: Optional[str] ) -> Result[Software, BackendManagementError]: """Upload a new software""" path = f"{self.config.api_prefix}/software" decoder = s11n_json.SOFTWARE_DECODER_JSON files = {'software': open(file_path.value, 'rb')} + if software_name is None: + software_name = os.path.basename(file_path.value) payload = {'name': software_name} if self.config.auth is None: return BackendService.auth_error return self.static_post_json_result(path, decoder, payload, None, self.config.auth.auth_headers(), files) diff --git a/client/src/service/cli.py b/client/src/service/cli.py index 788edb2..0c3745f 100755 --- a/client/src/service/cli.py +++ b/client/src/service/cli.py @@ -216,10 +216,25 @@ def hardware_serial_monitor( control_server_str: Optional[str], hardware_id_str: str, monitor_type_str: str, - monitor_script_path_str: str + monitor_script_path_str: Optional[str] ): pass + @staticmethod + async def quick_run( + config_path_str: Optional[str], + control_server_str: Optional[str], + static_server_str: Optional[str], + username_str: Optional[str], + password_str: Optional[str], + file_path: Optional[str], + software_name: Optional[str], + hardware_id_str: str, + monitor_type_str: str, + monitor_script_path_str: Optional[str] + ) -> Result[DIPRunnable, DIPClientError]: + pass + @staticmethod def execute_runnable_result(agent_result: Result[DIPRunnable, DIPClientError]): pass @@ -633,14 +648,14 @@ def software_upload( static_server_str: Optional[str], username: Optional[str], password: Optional[str], - software_name: str, + software_name: Optional[str], file_path: str, ) -> Result[Software, DIPClientError]: backend_result = CLI.parsed_backend(config_path_str, None, static_server_str, username, password) if isinstance(backend_result, Err): return Err(backend_result.value) file_result = ExistingFilePath.build(file_path) if isinstance(file_result, Err): return Err(file_result.value.of_type("software")) - return backend_result.value.software_upload(software_name, file_result.value) + return backend_result.value.software_upload(file_result.value, software_name) @staticmethod def software_download( @@ -687,7 +702,7 @@ def hardware_serial_monitor( monitor_script_path_str: Optional[str] ) -> Result[MonitorSerial, DIPClientError]: # Build backend - backend_result = CLI.parsed_backend(control_server_str, None, None, None) + backend_result = CLI.parsed_backend(control_server_str, None, None, None, None) if isinstance(backend_result, Err): return Err(backend_result.value) # Hardware id @@ -709,6 +724,34 @@ def hardware_serial_monitor( websocket = WebSocket(url_result.value, decoder, encoder) return monitor_serial.resolve(websocket, monitor_script_path_str) + @staticmethod + async def quick_run( + config_path_str: Optional[str], + control_server_str: Optional[str], + static_server_str: Optional[str], + username_str: Optional[str], + password_str: Optional[str], + file_path: Optional[str], + software_name: Optional[str], + hardware_id_str: str, + monitor_type_str: str, + monitor_script_path_str: Optional[str] + ) -> Result[DIPRunnable, DIPClientError]: + # Upload software to platform + upload_result = await CLI.software_upload( + config_path_str, static_server_str, username_str, password_str, software_name, file_path) + if isinstance(upload_result, Err): return Err(upload_result.value) + software: Software = upload_result.value + # Forward software to board + forward_error = CLI.hardware_software_upload( + config_path_str, static_server_str, hardware_id_str, str(software.id)) + if forward_error is not None: return Err(forward_error) + # Create serial monitor connection to board + monitor_result = CLI.hardware_serial_monitor( + config_path_str, control_server_str, hardware_id_str, monitor_type_str, monitor_script_path_str) + if isinstance(monitor_result, Err): return Err(monitor_result.value) + return Ok(monitor_result.value) + @staticmethod async def execute_runnable_result( runnable_result: Result[DIPRunnable, DIPClientError], diff --git a/client/src/service/click.py b/client/src/service/click.py index 2509c08..efeba1e 100755 --- a/client/src/service/click.py +++ b/client/src/service/click.py @@ -48,12 +48,12 @@ help='Software id (e.g. \'16db6c30-3328-11ec-ae41-ff1d66202dcc\')' ) SOFTWARE_NAME_OPTION = click.option( - '--name', '-n', "software_name", show_envvar=True, - type=str, envvar=f"{ENV_PREFIX}_SOFTWARE_NAME", required=True, + '--software-name', '-n', "software_name", show_envvar=True, + type=str, envvar=f"{ENV_PREFIX}_SOFTWARE_NAME", required=False, help='Software name (e.g. \'adafruit-nrf52-hello-world.bin\' ' - 'or \'my-beautiful-program\' or whatever).') -SOFTWARE_FILE_PATH = click.option( - '--file', '-f', "file_path", show_envvar=True, + 'or \'my-beautiful-program\' or whatever), default: file name.') +SOFTWARE_FILE_PATH_OPTION = click.option( + '--software-file', '-f', "software_file_path", show_envvar=True, type=str, envvar="DIP_SOFTWARE_FILE_PATH", required=True, help='Software file path (e.g. \'./hello-world.bin\' ' 'or \'$HOME/code/project/hello-world.bin\' or whatever).') @@ -82,7 +82,7 @@ # Monitor options MONITOR_TYPE_OPTION = click.option( "--monitor-type", "-t", "monitor_type_str", type=click.Choice([t.name for t in MonitorType]), - show_envvar=True, envvar=f"{ENV_PREFIX}_MONITOR_TYPE", required=True, default=MonitorType.hexbytes.name, + show_envvar=True, envvar=f"{ENV_PREFIX}_MONITOR_TYPE", required=False, default=MonitorType.buttonleds.name, help="Sets the type of monitor implementation to be used") MONITOR_SCRIPT_PATH_OPTION = click.option( "--monitor-script-path", "-s", "monitor_script_path_str", type=str, default=None, @@ -123,7 +123,11 @@ def cli_client(): """ -@cli_client.command() +# Command +CLI_COMMAND = cli_client.command(context_settings=dict(max_content_width=300)) + + +@CLI_COMMAND @CONFIG_PATH_OPTION def session_debug(config_path_str: Optional[str]): """Print out all session data""" @@ -135,7 +139,7 @@ def session_debug(config_path_str: Optional[str]): ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @STATIC_SERVER_OPTION @USERNAME_OPTION @@ -154,7 +158,7 @@ def session_auth( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION def session_auth_remove( config_path_str: Optional[str], @@ -167,7 +171,7 @@ def session_auth_remove( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @STATIC_SERVER_OPTION def session_static_server( @@ -182,7 +186,7 @@ def session_static_server( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION def session_static_server_remove( config_path_str: Optional[str] @@ -195,7 +199,7 @@ def session_static_server_remove( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @CONTROL_SERVER_OPTION def session_control_server( @@ -210,7 +214,7 @@ def session_control_server( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION def session_control_server_remove( config_path_str: Optional[str] @@ -223,7 +227,7 @@ def session_control_server_remove( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @HARDWARE_ID_OPTION @CONTROL_SERVER_OPTION @@ -257,7 +261,7 @@ async def exec(): asyncio.run(exec()) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @HARDWARE_ID_OPTION @CONTROL_SERVER_OPTION @@ -297,7 +301,7 @@ async def exec(): asyncio.run(exec()) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @HARDWARE_ID_OPTION @CONTROL_SERVER_OPTION @@ -328,7 +332,7 @@ async def exec(): asyncio.run(exec()) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @JSON_OUTPUT_OPTION @STATIC_SERVER_OPTION @@ -346,7 +350,7 @@ def user_list( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @JSON_OUTPUT_OPTION @STATIC_SERVER_OPTION @@ -368,7 +372,7 @@ def user_create( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @JSON_OUTPUT_OPTION @STATIC_SERVER_OPTION @@ -386,7 +390,7 @@ def hardware_list( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @JSON_OUTPUT_OPTION @STATIC_SERVER_OPTION @@ -411,7 +415,7 @@ def hardware_create( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @JSON_OUTPUT_OPTION @STATIC_SERVER_OPTION @@ -429,53 +433,53 @@ def software_list( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @JSON_OUTPUT_OPTION @STATIC_SERVER_OPTION @USERNAME_OPTION @PASSWORD_OPTION @SOFTWARE_NAME_OPTION -@SOFTWARE_FILE_PATH +@SOFTWARE_FILE_PATH_OPTION def software_upload( config_path_str: Optional[str], json_output: bool, static_server_str: Optional[str], username_str: Optional[str], password_str: Optional[str], - software_name: str, - file_path: str, + software_name: Optional[str], + software_file_path: str, ): """Upload new software""" CLI.execute_table_result( json_output, CLI.software_upload( - config_path_str, static_server_str, username_str, password_str, software_name, file_path).map(lambda x: [x]), + config_path_str, static_server_str, username_str, password_str, software_name, software_file_path).map(lambda x: [x]), s11n_json.list_encoder_json(s11n_json.SOFTWARE_ENCODER_JSON), s11n_rich.RichSoftwareEncoder() ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @STATIC_SERVER_OPTION @SOFTWARE_ID_OPTION -@SOFTWARE_FILE_PATH +@SOFTWARE_FILE_PATH_OPTION def software_download( config_path_str: Optional[str], static_server_str: Optional[str], software_id_str: str, - file_path: str + software_file_path: str ): """Download existing software""" CLI.execute_optional_result( False, - CLI.software_download(config_path_str, static_server_str, software_id_str, file_path), - f"Downloaded software at '{file_path}'" + CLI.software_download(config_path_str, static_server_str, software_id_str, software_file_path), + f"Downloaded software at '{software_file_path}'" ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @STATIC_SERVER_OPTION @HARDWARE_ID_OPTION @@ -494,7 +498,7 @@ def hardware_software_upload( ) -@cli_client.command() +@CLI_COMMAND @CONFIG_PATH_OPTION @CONTROL_SERVER_OPTION @HARDWARE_ID_OPTION @@ -517,3 +521,41 @@ async def exec(): monitor_script_path_str), "Finished monitoring") asyncio.run(exec()) + +@CLI_COMMAND +@CONFIG_PATH_OPTION +@CONTROL_SERVER_OPTION +@STATIC_SERVER_OPTION +@USERNAME_OPTION +@PASSWORD_OPTION +@SOFTWARE_FILE_PATH_OPTION +@SOFTWARE_NAME_OPTION +@HARDWARE_ID_OPTION +@MONITOR_TYPE_OPTION +@MONITOR_SCRIPT_PATH_OPTION +def quick_run( + config_path_str: Optional[str], + control_server_str: Optional[str], + static_server_str: Optional[str], + username_str: Optional[str], + password_str: Optional[str], + software_file_path: Optional[str], + software_name: Optional[str], + hardware_id_str: str, + monitor_type_str: str, + monitor_script_path_str: str +): + """Upload, forward & monitor board software""" + async def exec(): + await CLI.execute_runnable_result(CLI.quick_run( + config_path_str, + control_server_str, + static_server_str, + username_str, + password_str, + software_file_path, + software_name, + hardware_id_str, + monitor_type_str, + monitor_script_path_str), "Finished quick run") + asyncio.run(exec()) diff --git a/client/src/service/managed_url.py b/client/src/service/managed_url.py index 269c347..9804d6b 100644 --- a/client/src/service/managed_url.py +++ b/client/src/service/managed_url.py @@ -11,6 +11,9 @@ from requests import Response from src.domain.dip_client_error import DIPClientError from src.domain.existing_file_path import ExistingFilePath +from src.util import log + +LOGGER = log.timed_named_logger("url") @dataclass @@ -72,6 +75,7 @@ def downloaded_file_in_path(self, path: str) -> Result[ExistingFilePath, str]: if isinstance(url_result, Err): return url_result url_text: str = url_result.value + LOGGER.debug(f"HTTP download. URL: {url_text}, file: {path}") urllib.request.urlretrieve(url_text, path) return Ok(ExistingFilePath(path)) except Exception as e: diff --git a/client/src/util/log.py b/client/src/util/log.py index bb4b92d..d13f83a 100644 --- a/client/src/util/log.py +++ b/client/src/util/log.py @@ -5,17 +5,21 @@ from logging import Logger import sys +def structure_logger(logger_name: str, logger: Logger): + log_level = os.environ.get('LOG_LEVEL', 'INFO').upper() + formatter = logging.Formatter( + fmt=f"[%(asctime)s] [%(levelname)s] [{logger_name}] %(message)s", + datefmt='%Y-%m-%d %H:%M:%S') + screen_handler = logging.StreamHandler(stream=sys.stdout) + screen_handler.setFormatter(formatter) + logger.setLevel(log_level) + logger.addHandler(screen_handler) + +# Custom logger def timed_named_logger(logger_name: str) -> Logger: """Create an opinionated, timestamped, named, formatted Logger instance""" logger = logging.getLogger(logger_name) if not logger.hasHandlers(): - log_level = os.environ.get('LOG_LEVEL', 'INFO').upper() - formatter = logging.Formatter( - fmt=f"[%(asctime)s] [%(levelname)s] [{logger_name}] %(message)s", - datefmt='%Y-%m-%d %H:%M:%S') - screen_handler = logging.StreamHandler(stream=sys.stdout) - screen_handler.setFormatter(formatter) - logger.setLevel(log_level) - logger.addHandler(screen_handler) + structure_logger(logger_name, logger) return logger diff --git a/docs/README.md b/docs/README.md index c642326..17304dc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,10 @@ # DIP Testbed Platform DIP Testbed Platform allows users to remotely program and experience physical, embedded devices through various virtual interfaces (uni-directional webcam stream, bi-directional serial connection stream). +## 🌸 🌼 DIP User Tutorial Guides 🌼 🌸 +_Hopefully if I have enough time I will create detailed tutorials for beginners_ +_Meanwhile you can check out the platform prototypes which are like "get your hands dirty" type of tutorials_ + ## 🌸 🌼 DIP Client 🌼 🌸 DIP Client is the main CLI tool to interact with the DIP Testbed platform. For more documentations see [CLIENT.md](./CLIENT.md) @@ -8,7 +12,7 @@ For more documentations see [CLIENT.md](./CLIENT.md) Latest release: https://github.com/kshaa/dip-testbed-dist/releases/latest/download/dip_client_amd64 Quick install: `curl https://github.com/kshaa/dip-testbed-dist/releases/latest/download/client_install.sh | bash` -## DIP Platform usage prototypes +## 🌸 🌼 DIP Platform Prototypes 🌼 🌸 The author of this academic work created various prototypes when manually emulating end-user usage of the platform. The prototypes, their source codes and descriptions can be seen in [prototypes](../prototypes/README.md)