diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index e2e228750..f6be66075 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.2.dev0' +__version__ = '5.2.dev1' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/donkeycar/config.py b/donkeycar/config.py index 607fd35f9..921dc7a07 100644 --- a/donkeycar/config.py +++ b/donkeycar/config.py @@ -6,9 +6,9 @@ """ import os import types -from logging import getLogger +import logging -logger = getLogger(__name__) +logger = logging.getLogger(__name__) class Config: @@ -29,7 +29,16 @@ def from_object(self, obj): for key in dir(obj): if key.isupper(): setattr(self, key, getattr(obj, key)) - + + def from_dict(self, d, keys=[]): + msg = 'Overwriting config with: ' + for k, v in d.items(): + if k.isupper(): + if k in keys or not keys: + setattr(self, k, v) + msg += f'{k}:{v}, ' + logger.info(msg) + def __str__(self): result = [] for key in dir(self): @@ -42,6 +51,17 @@ def show(self): if attr.isupper(): print(attr, ":", getattr(self, attr)) + def to_pyfile(self, path): + lines = [] + for attr in dir(self): + if attr.isupper(): + v = getattr(self, attr) + if isinstance(v, str): + v = f'"{v}"' + lines.append(f'{attr} = {v}{os.linesep}') + with open(path, 'w') as f: + f.writelines(lines) + def load_config(config_path=None, myconfig="myconfig.py"): diff --git a/donkeycar/management/__init__.py b/donkeycar/management/__init__.py index e69de29bb..eea436a37 100644 --- a/donkeycar/management/__init__.py +++ b/donkeycar/management/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/donkeycar/management/base.py b/donkeycar/management/base.py index 130353ac5..e906aa4a7 100644 --- a/donkeycar/management/base.py +++ b/donkeycar/management/base.py @@ -9,7 +9,6 @@ from progress.bar import IncrementalBar import donkeycar as dk from donkeycar.management.joystick_creator import CreateJoystick -from donkeycar.management.tub import TubManager from donkeycar.utils import normalize_image, load_image, math @@ -439,7 +438,7 @@ def run(self, args): class ShowPredictionPlots(BaseCommand): def plot_predictions(self, cfg, tub_paths, model_path, limit, model_type, - noshow): + noshow, dark=False): """ Plot model predictions for angle and throttle against data from tubs. """ @@ -472,9 +471,8 @@ def plot_predictions(self, cfg, tub_paths, model_path, limit, model_type, tub_record, lambda x: normalize_image(x)) pilot_angle, pilot_throttle = \ model.inference_from_dict(input_dict) - y_dict = model.y_transform(tub_record) - user_angle, user_throttle \ - = y_dict[output_names[0]], y_dict[output_names[1]] + user_angle = tub_record.underlying['user/angle'] + user_throttle = tub_record.underlying['user/throttle'] user_angles.append(user_angle) user_throttles.append(user_throttle) pilot_angles.append(pilot_angle) @@ -486,8 +484,10 @@ def plot_predictions(self, cfg, tub_paths, model_path, limit, model_type, 'pilot_angle': pilot_angles}) throttles_df = pd.DataFrame({'user_throttle': user_throttles, 'pilot_throttle': pilot_throttles}) - - fig = plt.figure() + if dark: + plt.style.use('dark_background') + fig = plt.figure('Tub Plot') + fig.set_layout_engine('tight') title = f"Model Predictions\nTubs: {tub_paths}\nModel: {model_path}\n" \ f"Type: {model_type}" fig.suptitle(title) @@ -594,7 +594,7 @@ def run(self, args): class Gui(BaseCommand): def run(self, args): - from donkeycar.management.kivy_ui import main + from donkeycar.management.ui.ui import main main() @@ -606,7 +606,6 @@ def execute_from_command_line(): 'createcar': CreateCar, 'findcar': FindCar, 'calibrate': CalibrateCar, - 'tubclean': TubManager, 'tubplot': ShowPredictionPlots, 'tubhist': ShowHistogram, 'makemovie': MakeMovieShell, diff --git a/donkeycar/management/kivy_ui.py b/donkeycar/management/kivy_ui.py deleted file mode 100644 index 8a42c2245..000000000 --- a/donkeycar/management/kivy_ui.py +++ /dev/null @@ -1,1334 +0,0 @@ -import json -import re -import time -from copy import copy, deepcopy -from datetime import datetime -from functools import partial -from subprocess import Popen, PIPE, STDOUT -from threading import Thread -from collections import namedtuple -from kivy.logger import Logger - -import io -import os -import atexit -import yaml -from PIL import Image as PilImage -import pandas as pd -import numpy as np -import plotly.express as px -from kivy.clock import Clock -from kivy.app import App -from kivy.uix.boxlayout import BoxLayout -from kivy.uix.image import Image -from kivy.core.image import Image as CoreImage -from kivy.properties import NumericProperty, ObjectProperty, StringProperty, \ - ListProperty, BooleanProperty -from kivy.uix.label import Label -from kivy.uix.button import Button -from kivy.uix.popup import Popup -from kivy.lang.builder import Builder -from kivy.core.window import Window -from kivy.uix.screenmanager import ScreenManager, Screen -from kivy.uix.scrollview import ScrollView -from kivy.uix.spinner import SpinnerOption, Spinner - -from donkeycar import load_config -from donkeycar.parts.image_transformations import ImageTransformations -from donkeycar.parts.tub_v2 import Tub -from donkeycar.pipeline.augmentations import ImageAugmentation -from donkeycar.pipeline.database import PilotDatabase -from donkeycar.pipeline.types import TubRecord -from donkeycar.utils import get_model_by_type -from donkeycar.pipeline.training import train - -Logger.propagate = False - -Builder.load_file(os.path.join(os.path.dirname(__file__), 'ui.kv')) -Window.clearcolor = (0.2, 0.2, 0.2, 1) -LABEL_SPINNER_TEXT = 'Add/remove' - -# Data struct to show tub field in the progress bar, containing the name, -# the name of the maximum value in the config file and if it is centered. -FieldProperty = namedtuple('FieldProperty', - ['field', 'max_value_id', 'centered']) - - -def get_norm_value(value, cfg, field_property, normalised=True): - max_val_key = field_property.max_value_id - max_value = getattr(cfg, max_val_key, 1.0) - out_val = value / max_value if normalised else value * max_value - return out_val - - -def tub_screen(): - return App.get_running_app().tub_screen if App.get_running_app() else None - - -def pilot_screen(): - return App.get_running_app().pilot_screen if App.get_running_app() else None - - -def train_screen(): - return App.get_running_app().train_screen if App.get_running_app() else None - - -def car_screen(): - return App.get_running_app().car_screen if App.get_running_app() else None - - -def recursive_update(target, source): - """ Recursively update dictionary """ - if isinstance(target, dict) and isinstance(source, dict): - for k, v in source.items(): - v_t = target.get(k) - if not recursive_update(v_t, v): - target[k] = v - return True - else: - return False - - -def decompose(field): - """ Function to decompose a string vector field like 'gyroscope_1' into a - tuple ('gyroscope', 1) """ - field_split = field.split('_') - if len(field_split) > 1 and field_split[-1].isdigit(): - return '_'.join(field_split[:-1]), int(field_split[-1]) - return field, None - - -class RcFileHandler: - """ This handles the config file which stores the data, like the field - mapping for displaying of bars and last opened car, tub directory. """ - - # These entries are expected in every tub, so they don't need to be in - # the file - known_entries = [ - FieldProperty('user/angle', '', centered=True), - FieldProperty('user/throttle', '', centered=False), - FieldProperty('pilot/angle', '', centered=True), - FieldProperty('pilot/throttle', '', centered=False), - ] - - def __init__(self, file_path='~/.donkeyrc'): - self.file_path = os.path.expanduser(file_path) - self.data = self.create_data() - recursive_update(self.data, self.read_file()) - self.field_properties = self.create_field_properties() - - def exit_hook(): - self.write_file() - # Automatically save config when program ends - atexit.register(exit_hook) - - def create_field_properties(self): - """ Merges known field properties with the ones from the file """ - field_properties = {entry.field: entry for entry in self.known_entries} - field_list = self.data.get('field_mapping') - if field_list is None: - field_list = {} - for entry in field_list: - assert isinstance(entry, dict), \ - 'Dictionary required in each entry in the field_mapping list' - field_property = FieldProperty(**entry) - field_properties[field_property.field] = field_property - return field_properties - - def create_data(self): - data = dict() - data['user_pilot_map'] = {'user/throttle': 'pilot/throttle', - 'user/angle': 'pilot/angle'} - return data - - def read_file(self): - if os.path.exists(self.file_path): - with open(self.file_path) as f: - data = yaml.load(f, Loader=yaml.FullLoader) - Logger.info(f'Donkeyrc: Donkey file {self.file_path} loaded.') - return data - else: - Logger.warning(f'Donkeyrc: Donkey file {self.file_path} does not ' - f'exist.') - return {} - - def write_file(self): - if os.path.exists(self.file_path): - Logger.info(f'Donkeyrc: Donkey file {self.file_path} updated.') - with open(self.file_path, mode='w') as f: - self.data['time_stamp'] = datetime.now() - data = yaml.dump(self.data, f) - return data - - -rc_handler = RcFileHandler() - - -class MySpinnerOption(SpinnerOption): - """ Customization for Spinner """ - pass - - -class MySpinner(Spinner): - """ Customization of Spinner drop down menu """ - def __init__(self, **kwargs): - super().__init__(option_cls=MySpinnerOption, **kwargs) - - -class FileChooserPopup(Popup): - """ File Chooser popup window""" - load = ObjectProperty() - root_path = StringProperty() - filters = ListProperty() - - -class FileChooserBase: - """ Base class for file chooser widgets""" - file_path = StringProperty("No file chosen") - popup = ObjectProperty(None) - root_path = os.path.expanduser('~') - title = StringProperty(None) - filters = ListProperty() - - def open_popup(self): - self.popup = FileChooserPopup(load=self.load, root_path=self.root_path, - title=self.title, filters=self.filters) - self.popup.open() - - def load(self, selection): - """ Method to load the chosen file into the path and call an action""" - self.file_path = str(selection[0]) - self.popup.dismiss() - self.load_action() - - def load_action(self): - """ Virtual method to run when file_path has been updated """ - pass - - -class ConfigManager(BoxLayout, FileChooserBase): - """ Class to mange loading of the config file from the car directory""" - config = ObjectProperty(None) - file_path = StringProperty(rc_handler.data.get('car_dir', '')) - - def load_action(self): - """ Load the config from the file path""" - if self.file_path: - try: - path = os.path.join(self.file_path, 'config.py') - self.config = load_config(path) - # If load successful, store into app config - rc_handler.data['car_dir'] = self.file_path - except FileNotFoundError: - Logger.error(f'Config: Directory {self.file_path} has no ' - f'config.py') - except Exception as e: - Logger.error(f'Config: {e}') - - -class TubLoader(BoxLayout, FileChooserBase): - """ Class to manage loading or reloading of the Tub from the tub directory. - Loading triggers many actions on other widgets of the app. """ - file_path = StringProperty(rc_handler.data.get('last_tub', '')) - tub = ObjectProperty(None) - len = NumericProperty(1) - records = None - - def load_action(self): - """ Update tub from the file path""" - if self.update_tub(): - # If update successful, store into app config - rc_handler.data['last_tub'] = self.file_path - - def update_tub(self, event=None): - if not self.file_path: - return False - # If config not yet loaded return - cfg = tub_screen().ids.config_manager.config - if not cfg: - return False - # At least check if there is a manifest file in the tub path - if not os.path.exists(os.path.join(self.file_path, 'manifest.json')): - tub_screen().status(f'Path {self.file_path} is not a valid tub.') - return False - try: - if self.tub: - self.tub.close() - self.tub = Tub(self.file_path) - except Exception as e: - tub_screen().status(f'Failed loading tub: {str(e)}') - return False - # Check if filter is set in tub screen - # expression = tub_screen().ids.tub_filter.filter_expression - train_filter = getattr(cfg, 'TRAIN_FILTER', None) - - # Use filter, this defines the function - def select(underlying): - if not train_filter: - return True - else: - try: - record = TubRecord(cfg, self.tub.base_path, underlying) - res = train_filter(record) - return res - except KeyError as err: - Logger.error(f'Filter: {err}') - return True - - self.records = [TubRecord(cfg, self.tub.base_path, record) - for record in self.tub if select(record)] - self.len = len(self.records) - if self.len > 0: - tub_screen().index = 0 - tub_screen().ids.data_plot.update_dataframe_from_tub() - msg = f'Loaded tub {self.file_path} with {self.len} records' - else: - msg = f'No records in tub {self.file_path}' - tub_screen().status(msg) - return True - - -class LabelBar(BoxLayout): - """ Widget that combines a label with a progress bar. This is used to - display the record fields in the data panel.""" - field = StringProperty() - field_property = ObjectProperty() - config = ObjectProperty() - msg = '' - - def update(self, record): - """ This function is called everytime the current record is updated""" - if not record: - return - field, index = decompose(self.field) - if field in record.underlying: - val = record.underlying[field] - if index is not None: - val = val[index] - # Update bar if a field property for this field is known - if self.field_property: - norm_value = get_norm_value(val, self.config, - self.field_property) - new_bar_val = (norm_value + 1) * 50 if \ - self.field_property.centered else norm_value * 100 - self.ids.bar.value = new_bar_val - self.ids.field_label.text = self.field - if isinstance(val, float) or isinstance(val, np.float32): - text = f'{val:+07.3f}' - elif isinstance(val, int): - text = f'{val:10}' - else: - text = str(val) - self.ids.value_label.text = text - else: - Logger.error(f'Record: Bad record {record.underlying["_index"]} - ' - f'missing field {self.field}') - - -class DataPanel(BoxLayout): - """ Data panel widget that contains the label/bar widgets and the drop - down menu to select/deselect fields.""" - record = ObjectProperty() - # dual mode is used in the pilot arena where we only show angle and - # throttle or speed - dual_mode = BooleanProperty(False) - auto_text = StringProperty(LABEL_SPINNER_TEXT) - throttle_field = StringProperty('user/throttle') - link = False - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.labels = {} - self.screen = ObjectProperty() - - def add_remove(self): - """ Method to add or remove a LabelBar. Depending on the value of the - drop down menu the LabelBar is added if it is not present otherwise - removed.""" - field = self.ids.data_spinner.text - if field is LABEL_SPINNER_TEXT: - return - if field in self.labels and not self.dual_mode: - self.remove_widget(self.labels[field]) - del(self.labels[field]) - self.screen.status(f'Removing {field}') - else: - # in dual mode replace the second entry with the new one - if self.dual_mode and len(self.labels) == 2: - k, v = list(self.labels.items())[-1] - self.remove_widget(v) - del(self.labels[k]) - field_property = rc_handler.field_properties.get(decompose(field)[0]) - cfg = tub_screen().ids.config_manager.config - lb = LabelBar(field=field, field_property=field_property, config=cfg) - self.labels[field] = lb - self.add_widget(lb) - lb.update(self.record) - if len(self.labels) == 2: - self.throttle_field = field - self.screen.status(f'Adding {field}') - if self.screen.name == 'tub': - self.screen.ids.data_plot.plot_from_current_bars() - self.ids.data_spinner.text = LABEL_SPINNER_TEXT - self.auto_text = field - - def on_record(self, obj, record): - """ Kivy function that is called every time self.record changes""" - for v in self.labels.values(): - v.update(record) - - def clear(self): - for v in self.labels.values(): - self.remove_widget(v) - self.labels.clear() - - -class FullImage(Image): - """ Widget to display an image that fills the space. """ - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.core_image = None - - def update(self, record): - """ This method is called ever time a record gets updated. """ - try: - img_arr = self.get_image(record) - pil_image = PilImage.fromarray(img_arr) - bytes_io = io.BytesIO() - pil_image.save(bytes_io, format='png') - bytes_io.seek(0) - self.core_image = CoreImage(bytes_io, ext='png') - self.texture = self.core_image.texture - except KeyError as e: - Logger.error(f'Record: Missing key: {e}') - except Exception as e: - Logger.error(f'Record: Bad record: {e}') - - def get_image(self, record): - return record.image() - - -class ControlPanel(BoxLayout): - """ Class for control panel navigation. """ - screen = ObjectProperty() - speed = NumericProperty(1.0) - record_display = StringProperty() - clock = None - fwd = None - - def start(self, fwd=True, continuous=False): - """ - Method to cycle through records if either single <,> or continuous - <<, >> buttons are pressed - :param fwd: If we go forward or backward - :param continuous: If we do <<, >> or <, > - :return: None - """ - # this widget it used in two screens, so reference the original location - # of the config which is the config manager in the tub screen - cfg = tub_screen().ids.config_manager.config - hz = cfg.DRIVE_LOOP_HZ if cfg else 20 - time.sleep(0.1) - call = partial(self.step, fwd, continuous) - if continuous: - self.fwd = fwd - s = float(self.speed) * hz - cycle_time = 1.0 / s - else: - cycle_time = 0.08 - self.clock = Clock.schedule_interval(call, cycle_time) - - def step(self, fwd=True, continuous=False, *largs): - """ - Updating a single step and cap/floor the index so we stay w/in the tub. - :param fwd: If we go forward or backward - :param continuous: If we are in continuous mode <<, >> - :param largs: dummy - :return: None - """ - if self.screen.index is None: - self.screen.status("No tub loaded") - return - new_index = self.screen.index + (1 if fwd else -1) - if new_index >= tub_screen().ids.tub_loader.len: - new_index = 0 - elif new_index < 0: - new_index = tub_screen().ids.tub_loader.len - 1 - self.screen.index = new_index - msg = f'Donkey {"run" if continuous else "step"} ' \ - f'{"forward" if fwd else "backward"}' - if not continuous: - msg += f' - you can also use {"" if fwd else ""} key' - else: - msg += ' - you can toggle run/stop with ' - self.screen.status(msg) - - def stop(self): - if self.clock: - self.clock.cancel() - self.clock = None - - def restart(self): - if self.clock: - self.stop() - self.start(self.fwd, True) - - def update_speed(self, up=True): - """ Method to update the speed on the controller""" - values = self.ids.control_spinner.values - idx = values.index(self.ids.control_spinner.text) - if up and idx < len(values) - 1: - self.ids.control_spinner.text = values[idx + 1] - elif not up and idx > 0: - self.ids.control_spinner.text = values[idx - 1] - - def set_button_status(self, disabled=True): - """ Method to disable(enable) all buttons. """ - self.ids.run_bwd.disabled = self.ids.run_fwd.disabled = \ - self.ids.step_fwd.disabled = self.ids.step_bwd.disabled = disabled - - def on_keyboard(self, key, scancode): - """ Method to chack with keystroke has ben sent. """ - if key == ' ': - if self.clock and self.clock.is_triggered: - self.stop() - self.set_button_status(disabled=False) - self.screen.status('Donkey stopped') - else: - self.start(continuous=True) - self.set_button_status(disabled=True) - elif scancode == 79: - self.step(fwd=True) - elif scancode == 80: - self.step(fwd=False) - elif scancode == 45: - self.update_speed(up=False) - elif scancode == 46: - self.update_speed(up=True) - - -class PaddedBoxLayout(BoxLayout): - pass - - -class TubEditor(PaddedBoxLayout): - """ Tub editor widget. Contains left/right index interval and the - manipulator buttons for deleting / restoring and reloading """ - lr = ListProperty([0, 0]) - - def set_lr(self, is_l=True): - """ Sets left or right range to the current tub record index """ - if not tub_screen().current_record: - return - self.lr[0 if is_l else 1] = tub_screen().current_record.underlying['_index'] - - def del_lr(self, is_del): - """ Deletes or restores records in chosen range """ - tub = tub_screen().ids.tub_loader.tub - if self.lr[1] >= self.lr[0]: - selected = list(range(*self.lr)) - else: - last_id = tub.manifest.current_index - selected = list(range(self.lr[0], last_id)) - selected += list(range(self.lr[1])) - tub.delete_records(selected) if is_del else tub.restore_records(selected) - - -class TubFilter(PaddedBoxLayout): - """ Tub filter widget. """ - filter_expression = StringProperty(None) - record_filter = StringProperty(rc_handler.data.get('record_filter', '')) - - def update_filter(self): - filter_text = self.ids.record_filter.text - config = tub_screen().ids.config_manager.config - # empty string resets the filter - if filter_text == '': - self.record_filter = '' - self.filter_expression = '' - rc_handler.data['record_filter'] = self.record_filter - if hasattr(config, 'TRAIN_FILTER'): - delattr(config, 'TRAIN_FILTER') - tub_screen().status(f'Filter cleared') - return - filter_expression = self.create_filter_string(filter_text) - try: - record = tub_screen().current_record - filter_func_text = f"""def filter_func(record): - return {filter_expression} - """ - # creates the function 'filter_func' - ldict = {} - exec(filter_func_text, globals(), ldict) - filter_func = ldict['filter_func'] - res = filter_func(record) - status = f'Filter result on current record: {res}' - if isinstance(res, bool): - self.record_filter = filter_text - self.filter_expression = filter_expression - rc_handler.data['record_filter'] = self.record_filter - setattr(config, 'TRAIN_FILTER', filter_func) - else: - status += ' - non bool expression can\'t be applied' - status += ' - press to see effect' - tub_screen().status(status) - except Exception as e: - tub_screen().status(f'Filter error on current record: {e}') - - @staticmethod - def create_filter_string(filter_text, record_name='record'): - """ Converts text like 'user/angle' into 'record.underlying['user/angle'] - so that it can be used in a filter. Will replace only expressions that - are found in the tub inputs list. - - :param filter_text: input text like 'user/throttle > 0.1' - :param record_name: name of the record in the expression - :return: updated string that has all input fields wrapped - """ - for field in tub_screen().current_record.underlying.keys(): - field_list = filter_text.split(field) - if len(field_list) > 1: - filter_text = f'{record_name}.underlying["{field}"]'\ - .join(field_list) - return filter_text - - -class DataPlot(PaddedBoxLayout): - """ Data plot panel which embeds matplotlib interactive graph""" - df = ObjectProperty(force_dispatch=True, allownone=True) - - def plot_from_current_bars(self, in_app=True): - """ Plotting from current selected bars. The DataFrame for plotting - should contain all bars except for strings fields and all data is - selected if bars are empty. """ - tub = tub_screen().ids.tub_loader.tub - field_map = dict(zip(tub.manifest.inputs, tub.manifest.types)) - # Use selected fields or all fields if nothing is slected - all_cols = tub_screen().ids.data_panel.labels.keys() or self.df.columns - cols = [c for c in all_cols if decompose(c)[0] in field_map - and field_map[decompose(c)[0]] not in ('image_array', 'str')] - - df = self.df[cols] - if df is None: - return - # Don't plot the milliseconds time stamp as this is a too big number - df = df.drop(labels=['_timestamp_ms'], axis=1, errors='ignore') - - if in_app: - tub_screen().ids.graph.df = df - else: - fig = px.line(df, x=df.index, y=df.columns, title=tub.base_path) - fig.update_xaxes(rangeslider=dict(visible=True)) - fig.show() - - def unravel_vectors(self): - """ Unravels vector and list entries in tub which are created - when the DataFrame is created from a list of records""" - manifest = tub_screen().ids.tub_loader.tub.manifest - for k, v in zip(manifest.inputs, manifest.types): - if v == 'vector' or v == 'list': - dim = len(tub_screen().current_record.underlying[k]) - df_keys = [k + f'_{i}' for i in range(dim)] - self.df[df_keys] = pd.DataFrame(self.df[k].tolist(), - index=self.df.index) - self.df.drop(k, axis=1, inplace=True) - - def update_dataframe_from_tub(self): - """ Called from TubManager when a tub is reloaded/recreated. Fills - the DataFrame from records, and updates the dropdown menu in the - data panel.""" - generator = (t.underlying for t in tub_screen().ids.tub_loader.records) - self.df = pd.DataFrame(generator).dropna() - to_drop = {'cam/image_array'} - self.df.drop(labels=to_drop, axis=1, errors='ignore', inplace=True) - self.df.set_index('_index', inplace=True) - self.unravel_vectors() - tub_screen().ids.data_panel.ids.data_spinner.values = self.df.columns - self.plot_from_current_bars() - - -class TabBar(BoxLayout): - manager = ObjectProperty(None) - - def disable_only(self, bar_name): - this_button_name = bar_name + '_btn' - for button_name, button in self.ids.items(): - button.disabled = button_name == this_button_name - - -class TubScreen(Screen): - """ First screen of the app managing the tub data. """ - index = NumericProperty(None, force_dispatch=True) - current_record = ObjectProperty(None) - keys_enabled = BooleanProperty(True) - - def initialise(self, e): - self.ids.config_manager.load_action() - self.ids.tub_loader.update_tub() - - def on_index(self, obj, index): - """ Kivy method that is called if self.index changes""" - if index >= 0: - self.current_record = self.ids.tub_loader.records[index] - self.ids.slider.value = index - - def on_current_record(self, obj, record): - """ Kivy method that is called if self.current_record changes.""" - self.ids.img.update(record) - i = record.underlying['_index'] - self.ids.control_panel.record_display = f"Record {i:06}" - - def status(self, msg): - self.ids.status.text = msg - - def on_keyboard(self, instance, keycode, scancode, key, modifiers): - if self.keys_enabled: - self.ids.control_panel.on_keyboard(key, scancode) - - -class PilotLoader(BoxLayout, FileChooserBase): - """ Class to mange loading of the config file from the car directory""" - num = StringProperty() - model_type = StringProperty() - pilot = ObjectProperty(None) - filters = ['*.h5', '*.tflite', '*.savedmodel', '*.trt'] - - def load_action(self): - if self.file_path and self.pilot: - try: - self.pilot.load(os.path.join(self.file_path)) - rc_handler.data['pilot_' + self.num] = self.file_path - rc_handler.data['model_type_' + self.num] = self.model_type - self.ids.pilot_spinner.text = self.model_type - Logger.info(f'Pilot: Successfully loaded {self.file_path}') - except FileNotFoundError: - Logger.error(f'Pilot: Model {self.file_path} not found') - except Exception as e: - Logger.error(f'Failed loading {self.file_path}: {e}') - - def on_model_type(self, obj, model_type): - """ Kivy method that is called if self.model_type changes. """ - if self.model_type and self.model_type != 'Model type': - cfg = tub_screen().ids.config_manager.config - if cfg: - self.pilot = get_model_by_type(self.model_type, cfg) - self.ids.pilot_button.disabled = False - if 'tflite' in self.model_type: - self.filters = ['*.tflite'] - elif 'tensorrt' in self.model_type: - self.filters = ['*.trt', '*.savedmodel'] - else: - self.filters = ['*.h5', '*.savedmodel'] - - def on_num(self, e, num): - """ Kivy method that is called if self.num changes. """ - self.file_path = rc_handler.data.get('pilot_' + self.num, '') - self.model_type = rc_handler.data.get('model_type_' + self.num, '') - - -class OverlayImage(FullImage): - """ Widget to display the image and the user/pilot data for the tub. """ - pilot = ObjectProperty() - pilot_record = ObjectProperty() - throttle_field = StringProperty('user/throttle') - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.is_left = True - - def augment(self, img_arr): - if pilot_screen().trans_list: - img_arr = pilot_screen().transformation.run(img_arr) - if pilot_screen().aug_list: - img_arr = pilot_screen().augmentation.run(img_arr) - if pilot_screen().post_trans_list: - img_arr = pilot_screen().post_transformation.run(img_arr) - return img_arr - - def get_image(self, record): - from donkeycar.management.makemovie import MakeMovie - config = tub_screen().ids.config_manager.config - orig_img_arr = super().get_image(record) - aug_img_arr = self.augment(orig_img_arr) - img_arr = copy(aug_img_arr) - angle = record.underlying['user/angle'] - throttle = get_norm_value( - record.underlying[self.throttle_field], config, - rc_handler.field_properties[self.throttle_field]) - rgb = (0, 255, 0) - MakeMovie.draw_line_into_image(angle, throttle, False, img_arr, rgb) - if not self.pilot: - return img_arr - - output = (0, 0) - try: - # Not each model is supported in each interpreter - output = self.pilot.run(aug_img_arr) - except Exception as e: - Logger.error(e) - - rgb = (0, 0, 255) - MakeMovie.draw_line_into_image(output[0], output[1], True, img_arr, rgb) - out_record = copy(record) - out_record.underlying['pilot/angle'] = output[0] - # rename and denormalise the throttle output - pilot_throttle_field \ - = rc_handler.data['user_pilot_map'][self.throttle_field] - out_record.underlying[pilot_throttle_field] \ - = get_norm_value(output[1], tub_screen().ids.config_manager.config, - rc_handler.field_properties[self.throttle_field], - normalised=False) - self.pilot_record = out_record - return img_arr - - -class TransformationPopup(Popup): - """ Transformation popup window""" - title = StringProperty() - transformations = \ - ["TRAPEZE", "CROP", "RGB2BGR", "BGR2RGB", "RGB2HSV", "HSV2RGB", - "BGR2HSV", "HSV2BGR", "RGB2GRAY", "RBGR2GRAY", "HSV2GRAY", "GRAY2RGB", - "GRAY2BGR", "CANNY", "BLUR", "RESIZE", "SCALE"] - transformations_obj = ObjectProperty() - selected = ListProperty() - - def __init__(self, selected, **kwargs): - super().__init__(**kwargs) - for t in self.transformations: - btn = Button(text=t) - btn.bind(on_release=self.toggle_transformation) - self.ids.trafo_list.add_widget(btn) - self.selected = selected - - def toggle_transformation(self, btn): - trafo = btn.text - if trafo in self.selected: - self.selected.remove(trafo) - else: - self.selected.append(trafo) - - def on_selected(self, obj, select): - self.ids.selected_list.clear_widgets() - for l in self.selected: - lab = Label(text=l) - self.ids.selected_list.add_widget(lab) - self.transformations_obj.selected = self.selected - - -class Transformations(Button): - """ Base class for transformation widgets""" - title = StringProperty(None) - pilot_screen = ObjectProperty() - is_post = False - selected = ListProperty() - - def open_popup(self): - popup = TransformationPopup(title=self.title, transformations_obj=self, - selected=self.selected) - popup.open() - - def on_selected(self, obj, select): - Logger.info(f"Selected {select}") - if self.is_post: - self.pilot_screen.post_trans_list = self.selected - else: - self.pilot_screen.trans_list = self.selected - - -class PilotScreen(Screen): - """ Screen to do the pilot vs pilot comparison .""" - index = NumericProperty(None, force_dispatch=True) - current_record = ObjectProperty(None) - keys_enabled = BooleanProperty(False) - aug_list = ListProperty(force_dispatch=True) - augmentation = ObjectProperty() - trans_list = ListProperty(force_dispatch=True) - transformation = ObjectProperty() - post_trans_list = ListProperty(force_dispatch=True) - post_transformation = ObjectProperty() - config = ObjectProperty() - - def on_index(self, obj, index): - """ Kivy method that is called if self.index changes. Here we update - self.current_record and the slider value. """ - if tub_screen().ids.tub_loader.records: - self.current_record = tub_screen().ids.tub_loader.records[index] - self.ids.slider.value = index - - def on_current_record(self, obj, record): - """ Kivy method that is called when self.current_index changes. Here - we update the images and the control panel entry.""" - if not record: - return - i = record.underlying['_index'] - self.ids.pilot_control.record_display = f"Record {i:06}" - self.ids.img_1.update(record) - self.ids.img_2.update(record) - - def initialise(self, e): - self.ids.pilot_loader_1.on_model_type(None, None) - self.ids.pilot_loader_1.load_action() - self.ids.pilot_loader_2.on_model_type(None, None) - self.ids.pilot_loader_2.load_action() - mapping = copy(rc_handler.data['user_pilot_map']) - del(mapping['user/angle']) - self.ids.data_in.ids.data_spinner.values = mapping.keys() - self.ids.data_in.ids.data_spinner.text = 'user/angle' - self.ids.data_panel_1.ids.data_spinner.disabled = True - self.ids.data_panel_2.ids.data_spinner.disabled = True - - def map_pilot_field(self, text): - """ Method to return user -> pilot mapped fields except for the - initial value called Add/remove. """ - if text == LABEL_SPINNER_TEXT: - return text - return rc_handler.data['user_pilot_map'][text] - - def set_brightness(self, val=None): - if not self.config: - return - if self.ids.button_bright.state == 'down': - self.config.AUG_BRIGHTNESS_RANGE = (val, val) - if 'BRIGHTNESS' not in self.aug_list: - self.aug_list.append('BRIGHTNESS') - else: - # Since we only changed the content of the config here, - # self.on_aug_list() would not be called, but we want to update - # the augmentation. Hence, update the dependency manually here. - self.on_aug_list(None, None) - elif 'BRIGHTNESS' in self.aug_list: - self.aug_list.remove('BRIGHTNESS') - - def set_blur(self, val=None): - if not self.config: - return - if self.ids.button_blur.state == 'down': - self.config.AUG_BLUR_RANGE = (val, val) - if 'BLUR' not in self.aug_list: - self.aug_list.append('BLUR') - elif 'BLUR' in self.aug_list: - self.aug_list.remove('BLUR') - # update dependency - self.on_aug_list(None, None) - - def on_aug_list(self, obj, aug_list): - if not self.config: - return - self.config.AUGMENTATIONS = self.aug_list - self.augmentation = ImageAugmentation( - self.config, 'AUGMENTATIONS', always_apply=True) - self.on_current_record(None, self.current_record) - - def on_trans_list(self, obj, trans_list): - if not self.config: - return - self.config.TRANSFORMATIONS = self.trans_list - self.transformation = ImageTransformations( - self.config, 'TRANSFORMATIONS') - self.on_current_record(None, self.current_record) - - def on_post_trans_list(self, obj, trans_list): - if not self.config: - return - self.config.POST_TRANSFORMATIONS = self.post_trans_list - self.post_transformation = ImageTransformations( - self.config, 'POST_TRANSFORMATIONS') - self.on_current_record(None, self.current_record) - - def set_mask(self, state): - if state == 'down': - self.ids.status.text = 'Trapezoidal mask on' - self.trans_list.append('TRAPEZE') - else: - self.ids.status.text = 'Trapezoidal mask off' - if 'TRAPEZE' in self.trans_list: - self.trans_list.remove('TRAPEZE') - - def set_crop(self, state): - if state == 'down': - self.ids.status.text = 'Crop on' - self.trans_list.append('CROP') - else: - self.ids.status.text = 'Crop off' - if 'CROP' in self.trans_list: - self.trans_list.remove('CROP') - - def status(self, msg): - self.ids.status.text = msg - - def on_keyboard(self, instance, keycode, scancode, key, modifiers): - if self.keys_enabled: - self.ids.pilot_control.on_keyboard(key, scancode) - - -class ScrollableLabel(ScrollView): - pass - - -class DataFrameLabel(Label): - pass - - -class TransferSelector(BoxLayout, FileChooserBase): - """ Class to select transfer model""" - filters = ['*.h5', '*.savedmodel'] - - -class TrainScreen(Screen): - """ Class showing the training screen. """ - config = ObjectProperty(force_dispatch=True, allownone=True) - database = ObjectProperty() - pilot_df = ObjectProperty(force_dispatch=True) - tub_df = ObjectProperty(force_dispatch=True) - train_checker = False - - def train_call(self, *args): - tub_path = tub_screen().ids.tub_loader.tub.base_path - transfer = self.ids.transfer_spinner.text - model_type = self.ids.train_spinner.text - if transfer != 'Choose transfer model': - h5 = os.path.join(self.config.MODELS_PATH, transfer + '.h5') - sm = os.path.join(self.config.MODELS_PATH, transfer + '.savedmodel') - if os.path.exists(sm): - transfer = sm - elif os.path.exists(h5): - transfer = h5 - else: - transfer = None - self.ids.status.text = \ - f'Could find neither {sm} nor {trans_h5} - training ' \ - f'without transfer' - else: - transfer = None - try: - history = train(self.config, tub_paths=tub_path, - model_type=model_type, - transfer=transfer, - comment=self.ids.comment.text) - except Exception as e: - Logger.error(e) - self.ids.status.text = f'Train failed see console' - - def train(self): - self.config.SHOW_PLOT = False - t = Thread(target=self.train_call) - self.ids.status.text = 'Training started.' - - def func(dt): - t.start() - - def check_training_done(dt): - if not t.is_alive(): - self.train_checker.cancel() - self.ids.comment.text = 'Comment' - self.ids.transfer_spinner.text = 'Choose transfer model' - self.ids.train_button.state = 'normal' - self.ids.status.text = 'Training completed.' - self.ids.train_button.disabled = False - self.reload_database() - - # schedules the call after the current frame - Clock.schedule_once(func, 0) - # checks if training finished and updates the window if - self.train_checker = Clock.schedule_interval(check_training_done, 0.5) - - def set_config_attribute(self, input): - try: - val = json.loads(input) - except ValueError: - val = input - - att = self.ids.cfg_spinner.text.split(':')[0] - setattr(self.config, att, val) - self.ids.cfg_spinner.values = self.value_list() - self.ids.status.text = f'Setting {att} to {val} of type ' \ - f'{type(val).__name__}' - - def value_list(self): - if self.config: - return [f'{k}: {v}' for k, v in self.config.__dict__.items()] - else: - return ['select'] - - def on_config(self, obj, config): - if self.config and self.ids: - self.ids.cfg_spinner.values = self.value_list() - self.reload_database() - - def reload_database(self): - if self.config: - self.database = PilotDatabase(self.config) - - def on_database(self, obj, database): - group_tubs = self.ids.check.state == 'down' - pilot_txt, tub_txt, pilot_names = self.database.pretty_print(group_tubs) - self.ids.scroll_tubs.text = tub_txt - self.ids.scroll_pilots.text = pilot_txt - self.ids.transfer_spinner.values \ - = ['Choose transfer model'] + pilot_names - self.ids.delete_spinner.values \ - = ['Pilot'] + pilot_names - - -class CarScreen(Screen): - """ Screen for interacting with the car. """ - config = ObjectProperty(force_dispatch=True, allownone=True) - files = ListProperty() - car_dir = StringProperty(rc_handler.data.get('robot_car_dir', '~/mycar')) - event = ObjectProperty(None, allownone=True) - connection = ObjectProperty(None, allownone=True) - pid = NumericProperty(None, allownone=True) - pilots = ListProperty() - is_connected = BooleanProperty(False) - - def initialise(self): - self.event = Clock.schedule_interval(self.connected, 3) - - def list_remote_dir(self, dir): - if self.is_connected: - cmd = f'ssh {self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}' + \ - f' "ls {dir}"' - listing = os.popen(cmd).read() - adjusted_listing = listing.split('\n')[1:-1] - return adjusted_listing - else: - return [] - - def list_car_dir(self, dir): - self.car_dir = dir - self.files = self.list_remote_dir(dir) - # non-empty director found - if self.files: - rc_handler.data['robot_car_dir'] = dir - - def update_pilots(self): - model_dir = os.path.join(self.car_dir, 'models') - self.pilots = self.list_remote_dir(model_dir) - - def pull(self, tub_dir): - target = f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}' + \ - f':{os.path.join(self.car_dir, tub_dir)}' - dest = self.config.DATA_PATH - if self.ids.create_dir.state == 'normal': - target += '/' - cmd = ['rsync', '-rv', '--progress', '--partial', target, dest] - Logger.info('car pull: ' + str(cmd)) - proc = Popen(cmd, shell=False, stdout=PIPE, text=True, - encoding='utf-8', universal_newlines=True) - repeats = 100 - call = partial(self.show_progress, proc, repeats, True) - event = Clock.schedule_interval(call, 0.0001) - - def send_pilot(self): - # add trailing '/' - src = os.path.join(self.config.MODELS_PATH,'') - # check if any sync buttons are pressed and update path accordingly - buttons = ['h5', 'savedmodel', 'tflite', 'trt'] - select = [btn for btn in buttons if self.ids[f'btn_{btn}'].state - == 'down'] - # build filter: for example this rsyncs all .tfilte and .trt models - # --include=*.trt/*** --include=*.tflite --exclude=* - filter = ['--include=database.json'] - for ext in select: - if ext in ['savedmodel', 'trt']: - ext += '/***' - filter.append(f'--include=*.{ext}') - # if nothing selected, sync all - if not select: - filter.append('--include=*') - else: - filter.append('--exclude=*') - dest = f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}:' + \ - f'{os.path.join(self.car_dir, "models")}' - cmd = ['rsync', '-rv', '--progress', '--partial', *filter, src, dest] - Logger.info('car push: ' + ' '.join(cmd)) - proc = Popen(cmd, shell=False, stdout=PIPE, - encoding='utf-8', universal_newlines=True) - repeats = 0 - call = partial(self.show_progress, proc, repeats, False) - event = Clock.schedule_interval(call, 0.0001) - - def show_progress(self, proc, repeats, is_pull, e): - # find 'to-check=33/4551)' in OSX or 'to-chk=33/4551)' in - # Linux which is end of line - pattern = 'to-(check|chk)=(.*)\)' - - def end(): - # call ended this stops the schedule - if is_pull: - button = self.ids.pull_tub - self.ids.pull_bar.value = 0 - # merge in previous deleted indexes which now might have been - # overwritten - old_tub = tub_screen().ids.tub_loader.tub - if old_tub: - deleted_indexes = old_tub.manifest.deleted_indexes - tub_screen().ids.tub_loader.update_tub() - if deleted_indexes: - new_tub = tub_screen().ids.tub_loader.tub - new_tub.manifest.add_deleted_indexes(deleted_indexes) - else: - button = self.ids.send_pilots - self.ids.push_bar.value = 0 - self.update_pilots() - button.disabled = False - - if proc.poll() is not None: - end() - return False - # find the next repeats lines with update info - count = 0 - while True: - stdout_data = proc.stdout.readline() - if stdout_data: - res = re.search(pattern, stdout_data) - if res: - if count < repeats: - count += 1 - else: - remain, total = tuple(res.group(2).split('/')) - bar = 100 * (1. - float(remain) / float(total)) - if is_pull: - self.ids.pull_bar.value = bar - else: - self.ids.push_bar.value = bar - return True - else: - # end of stream command completed - end() - return False - - def connected(self, event): - if not self.config: - return - if self.connection is None: - if not hasattr(self.config, 'PI_USERNAME') or \ - not hasattr(self.config, 'PI_HOSTNAME'): - self.ids.connected.text = 'Requires PI_USERNAME, PI_HOSTNAME' - return - # run new command to check connection status - cmd = ['ssh', - '-o ConnectTimeout=3', - f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}', - 'date'] - self.connection = Popen(cmd, shell=False, stdout=PIPE, - stderr=STDOUT, text=True, - encoding='utf-8', universal_newlines=True) - else: - # ssh is already running, check where we are - return_val = self.connection.poll() - self.is_connected = False - if return_val is None: - # command still running, do nothing and check next time again - status = 'Awaiting connection to...' - self.ids.connected.color = 0.8, 0.8, 0.0, 1 - else: - # command finished, check if successful and reset connection - if return_val == 0: - status = 'Connected to' - self.ids.connected.color = 0, 0.9, 0, 1 - self.is_connected = True - else: - status = 'Disconnected from' - self.ids.connected.color = 0.9, 0, 0, 1 - self.connection = None - self.ids.connected.text \ - = f'{status} {getattr(self.config, "PI_HOSTNAME")}' - - def drive(self): - model_args = '' - if self.ids.pilot_spinner.text != 'No pilot': - model_path = os.path.join(self.car_dir, "models", - self.ids.pilot_spinner.text) - model_args = f'--type {self.ids.type_spinner.text} ' + \ - f'--model {model_path}' - cmd = ['ssh', - f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}', - f'source env/bin/activate; cd {self.car_dir}; ./manage.py ' - f'drive {model_args} 2>&1'] - Logger.info(f'car connect: {cmd}') - proc = Popen(cmd, shell=False, stdout=PIPE, text=True, - encoding='utf-8', universal_newlines=True) - while True: - stdout_data = proc.stdout.readline() - if stdout_data: - # find 'PID: 12345' - pattern = 'PID: .*' - res = re.search(pattern, stdout_data) - if res: - try: - self.pid = int(res.group(0).split('PID: ')[1]) - Logger.info(f'car connect: manage.py drive PID: ' - f'{self.pid}') - except Exception as e: - Logger.error(f'car connect: {e}') - return - Logger.info(f'car connect: {stdout_data}') - else: - return - - def stop(self): - if self.pid: - cmd = f'ssh {self.config.PI_USERNAME}@{self.config.PI_HOSTNAME} '\ - + f'kill {self.pid}' - out = os.popen(cmd).read() - Logger.info(f"car connect: Kill PID {self.pid} + {out}") - self.pid = None - - -class StartScreen(Screen): - img_path = os.path.realpath(os.path.join( - os.path.dirname(__file__), - '../parts/web_controller/templates/static/donkeycar-logo-sideways.png')) - pass - - -class DonkeyApp(App): - start_screen = None - tub_screen = None - train_screen = None - pilot_screen = None - car_screen = None - title = 'Donkey Manager' - - def initialise(self, event): - self.tub_screen.ids.config_manager.load_action() - self.pilot_screen.initialise(event) - self.car_screen.initialise() - # This builds the graph which can only happen after everything else - # has run, therefore delay until the next round. - Clock.schedule_once(self.tub_screen.ids.tub_loader.update_tub) - - def build(self): - Window.bind(on_request_close=self.on_request_close) - self.start_screen = StartScreen(name='donkey') - self.tub_screen = TubScreen(name='tub') - self.train_screen = TrainScreen(name='train') - self.pilot_screen = PilotScreen(name='pilot') - self.car_screen = CarScreen(name='car') - Window.bind(on_keyboard=self.tub_screen.on_keyboard) - Window.bind(on_keyboard=self.pilot_screen.on_keyboard) - Clock.schedule_once(self.initialise) - sm = ScreenManager() - sm.add_widget(self.start_screen) - sm.add_widget(self.tub_screen) - sm.add_widget(self.train_screen) - sm.add_widget(self.pilot_screen) - sm.add_widget(self.car_screen) - return sm - - def on_request_close(self, *args): - tub = self.tub_screen.ids.tub_loader.tub - if tub: - tub.close() - Logger.info("Good bye Donkey") - return False - - -def main(): - tub_app = DonkeyApp() - tub_app.run() - - -if __name__ == '__main__': - main() diff --git a/donkeycar/management/tub.py b/donkeycar/management/tub.py deleted file mode 100644 index 7e60ab27f..000000000 --- a/donkeycar/management/tub.py +++ /dev/null @@ -1,116 +0,0 @@ -''' -tub.py - -Manage tubs -''' - -import json -import os -import sys -import time -from pathlib import Path -from stat import S_ISREG, ST_ATIME, ST_CTIME, ST_MODE, ST_MTIME - -import tornado.web - -from donkeycar.parts.tub_v2 import Tub - - -class TubManager: - - def run(self, args): - WebServer(args[0]).start() - - -class WebServer(tornado.web.Application): - - def __init__(self, data_path): - if not os.path.exists(data_path): - raise ValueError('The path {} does not exist.'.format(data_path)) - - this_dir = os.path.dirname(os.path.realpath(__file__)) - static_file_path = os.path.join(this_dir, 'tub_web', 'static') - - handlers = [ - (r"/", tornado.web.RedirectHandler, dict(url="/tubs")), - (r"/tubs", TubsView, dict(data_path=data_path)), - (r"/tubs/?(?P[^/]+)?", TubView), - (r"/api/tubs/?(?P[^/]+)?", TubApi, dict(data_path=data_path)), - (r"/static/(.*)", tornado.web.StaticFileHandler, {"path": static_file_path}), - (r"/tub_data/(.*)", tornado.web.StaticFileHandler, {"path": data_path}), - ] - - settings = {'debug': True} - - super().__init__(handlers, **settings) - - def start(self, port=8886): - self.port = int(port) - self.listen(self.port) - print('Listening on {}...'.format(port)) - tornado.ioloop.IOLoop.instance().start() - - -class TubsView(tornado.web.RequestHandler): - - def initialize(self, data_path): - self.data_path = data_path - - def get(self): - import fnmatch - dir_list = fnmatch.filter(os.listdir(self.data_path), '*') - dir_list.sort() - data = {"tubs": dir_list} - self.render("tub_web/tubs.html", **data) - - -class TubView(tornado.web.RequestHandler): - - def get(self, tub_id): - data = {} - self.render("tub_web/tub.html", **data) - - -class TubApi(tornado.web.RequestHandler): - - def initialize(self, data_path): - path = Path(os.path.expanduser(data_path)) - self.data_path = path.absolute() - - def clips_of_tub(self, tub_path): - tub = Tub(tub_path) - - clips = [] - for record in tub: - index = record['_index'] - images_relative_path = os.path.join(Tub.images(), record['cam/image_array']) - record['cam/image_array'] = images_relative_path - clips.append(record) - - return [clips] - - def get(self, tub_id): - base_path = os.path.join(self.data_path, tub_id) - clips = self.clips_of_tub(base_path) - self.set_header("Content-Type", "application/json; charset=UTF-8") - self.write(json.dumps({'clips': clips})) - - def post(self, tub_id): - tub_path = os.path.join(self.data_path, tub_id) - tub = Tub(tub_path) - old_clips = self.clips_of_tub(tub_path) - new_clips = tornado.escape.json_decode(self.request.body) - - import itertools - old_frames = list(itertools.chain(*old_clips)) - old_indexes = set() - for frame in old_frames: - old_indexes.add(frame['_index']) - - new_frames = list(itertools.chain(*new_clips['clips'])) - new_indexes = set() - for frame in new_frames: - new_indexes.add(frame['_index']) - - frames_to_delete = [index for index in old_indexes if index not in new_indexes] - tub.delete_records(frames_to_delete) diff --git a/donkeycar/management/ui.kv b/donkeycar/management/ui.kv deleted file mode 100644 index 34eeadd32..000000000 --- a/donkeycar/management/ui.kv +++ /dev/null @@ -1,780 +0,0 @@ -#: import TsPlot donkeycar.management.graph.TsPlot -#: import get_model_by_type donkeycar.utils.get_model_by_type -#: import platform sys.platform -#: import os os - -#:set common_height 30 if platform != 'darwin' else 60 -#:set layout_pad_x 10 if platform != 'darwin' else 20 -#:set layout_height common_height + layout_pad_x -#:set layout_height_double 2 * common_height + layout_pad_x -#:set layout_pad_xy [layout_pad_x, layout_pad_x // 2] -#:set supported_models ['linear', 'categorical', 'inferred', 'memory', 'behavior', 'localizer', 'rnn', '3d'] -#:set drive_models [pre + t for pre in['', 'tflite_', 'tensorrt_'] for t in supported_models ] - -: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - - -: - title: "Choose the directory" - size_hint: 1.0, 1.0 - auto_dismiss: False - pos_hint: {'center_x': .5, 'center_y': .5} - - BoxLayout: - orientation: "vertical" - FileChooser: - id: file_chooser - rootpath: root.root_path - dirselect: True - filter_dirs: True - filters: root.filters - FileChooserListLayout - - BoxLayout: - size_hint: (1, 0.1) - pos_hint: {'center_x': .5, 'center_y': .5} - spacing: 20 - Button: - text: "Cancel" - on_release: root.dismiss() - Button: - text: "Load" - on_release: root.load(file_chooser.selection) - disabled: file_chooser.selection==[] - - - text_size: self.size - halign: 'center' - valign: 'middle' - - - title: 'Choose the car directory' - orientation: 'horizontal' - on_config: - app.tub_screen.ids.tub_loader.ids.tub_button.disabled = False - app.tub_screen.ids.tub_loader.root_path = self.file_path - app.pilot_screen.ids.pilot_loader_1.root_path = self.config.MODELS_PATH - app.pilot_screen.ids.pilot_loader_2.root_path = self.config.MODELS_PATH - app.tub_screen.ids.status.text = 'Config loaded from' + self.file_path - Button: - text: 'Load car directory' - size_hint_x: 0.5 - on_press: root.open_popup() - AutoLabel: - id: car_dir - text: root.file_path - - - title: 'Choose the tub directory' - orientation: 'horizontal' - Button: - id: tub_button - text: 'Load tub' - size_hint_x: 0.5 - disabled: True - on_press: root.open_popup() - AutoLabel: - id: tub_dir - text: root.file_path - - -: - orientation: 'horizontal' - spacing: 4 - Label: - id: field_label - text_size: self.size - halign: 'left' - valign: 'middle' - font_size: '14sp' # 14 if platform == 'linux' else 28 - canvas.before: - Color: - rgba: 0.17, 0.18, 0.25, 1 - Rectangle: - pos: self.pos - size: self.size - Label: - id: value_label - text_size: self.size - halign: 'right' - valign: 'middle' - font_size: 14 if platform == 'linux' else 28 - size_hint_x: 0.8 - padding: 10, 0 - canvas.before: - Color: - rgba: 0.14, 0.15, 0.22, 1 - Rectangle: - pos: self.pos - size: self.size - ProgressBar: - id: bar - canvas.before: - Color: - rgba: 0.12, 0.13, 0.20, 1 - Rectangle: - pos: self.pos - size: self.size - -: - text_size: self.size - halign: 'center' - valign: 'middle' - height: common_height - - - - orientation: 'vertical' - spacing: 2 - GridLayout: - cols: 2 - Label: - text: 'Record field' - size_hint_y: None - height: common_height - MySpinner: - id: data_spinner - size_hint_y: None - height: common_height - text: root.auto_text if root.link else 'Add/remove' - on_text: root.add_remove() - on_values: root.clear() - - - orientation: 'vertical' - GridLayout: - cols: 2 - Label: - id: record_num - text: root.record_display - MySpinner: - id: control_spinner - pos_hint: {'center': (.5, .5)} - text: '1.00' - values: ['0.25', '0.50', '1.00', '1.50', '2.00', '3.00', '4.00'] - on_text: - root.speed = float(self.text) - app.tub_screen.ids.status.text = f'Setting speed to {self.text} - you can also use the +/- keys.' - root.restart() - Button: - id: step_bwd - text: '<' - on_press: - root.start(fwd=False) - on_release: - root.stop() - Button: - id: step_fwd - text: '>' - on_press: - root.start(fwd=True) - on_release: - root.stop() - Button: - id: run_bwd - text: '<<' - on_press: - root.start(fwd=False, continuous=True) - root.set_button_status(disabled=True) - Button: - id: run_fwd - text: '>>' - on_press: - root.start(fwd=True, continuous=True) - root.set_button_status(disabled=True) - Button: - size_hint_y: 0.3 - text: 'Stop' - on_press: - root.stop() - root.set_button_status(disabled=False) - - - orientation: 'horizontal' - on_lr: - msg = f'Setting range, ' - if root.lr[0] < root.lr[1]: msg += (f'affecting records inside [{root.lr[0]}, {root.lr[1]}) ' + \ - '- you can affect records outside by setting left > right') - else: msg += (f'affecting records outside ({root.lr[1]}, {root.lr[0]}] ' + \ - '- you can affect records inside by setting left < right') - app.tub_screen.ids.status.text = msg - Button: - text: 'Set left' - on_press: root.set_lr(True) - Button: - text: 'Set right' - on_press: root.set_lr(False) - Label: - text: '[' + str(root.lr[0]) + ', ' + str(root.lr[1]) + ')' - # text: f'Index range [{root.lr[0]}, {root.lr[1]})' - Button: - text: 'Delete' - on_press: - root.del_lr(True) - msg = f'Delete records {root.lr} - press to see the ' \ - + f'effect, but you can delete multiple ranges before doing so.' - app.tub_screen.ids.status.text = msg - Button: - text: 'Restore' - on_press: - root.del_lr(False) - msg = f'Restore records {root.lr} - press to see the ' \ - + f'effect, but you can restore multiple ranges before doing so.' - app.tub_screen.ids.status.text = msg - Button: - text: 'Reload Tub' - on_press: - app.tub_screen.ids.tub_loader.update_tub() - -: - id: tub_filter - orientation: 'horizontal' - Button: - text: 'Set filter' - size_hint_x: 0.2 - on_press: root.update_filter() - TextInput: - id: record_filter - text: root.record_filter - multiline: False - on_focus: app.tub_screen.keys_enabled = not app.tub_screen.keys_enabled - -<-FullImage>: - size_hint_x: 1.2 - size_hint_y: 1.2 - canvas: - Color: - rgb: (1, 1, 1) - Rectangle: - texture: self.texture - size: self.width, self.height / 1.2 - pos: self.x, self.y - -: - Button: - text: 'Reload Graph' - on_press: root.plot_from_current_bars() - Button: - text: 'Browser Graph' - on_press: root.plot_from_current_bars(in_app=False) - -: - size_hint_y: None - height: 30 - text_size: self.size - halign: 'left' - valign: 'bottom' - text: 'Donkey ready' - -: - size_hint_y: None - height: common_height + 40 - padding: [20, 20] - spacing: 20 - canvas.before: - Color: - rgba: 0.25, 0.138, 0.0, 1 - Rectangle: - pos: self.pos - size: self.size - Button: - id: tub_btn - text: 'Tub Manager' - on_press: - root.manager.current_screen.keys_enabled = False - root.manager.current = 'tub' - root.manager.current_screen.keys_enabled = True - root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) - Button: - id: train_btn - text: 'Trainer' - on_press: - root.manager.current_screen.keys_enabled = False - root.manager.current = 'train' - root.manager.current_screen.keys_enabled = True - root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) - Button: - id: pilot_btn - text: 'Pilot Arena' - on_press: - root.manager.current_screen.keys_enabled = False - root.manager.current = 'pilot' - root.manager.current_screen.keys_enabled = True - if not root.manager.current_screen.index: root.manager.current_screen.index = 0 - root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) - Button: - id: car_btn - text: 'Car Connector' - on_press: - root.manager.current_screen.keys_enabled = False - root.manager.current = 'car' - root.manager.current_screen.keys_enabled = True - root.manager.current_screen.ids.tab_bar.disable_only(root.manager.current_screen.name) - root.manager.current_screen.update_pilots() - -: - BoxLayout: - orientation: 'vertical' - TabBar: - id: tab_bar - manager: root.manager - PaddedBoxLayout: - orientation: 'horizontal' - ConfigManager: - id: config_manager - TubLoader: - id: tub_loader - PaddedBoxLayout: - size_hint_y: 1.0 - DataPanel: - id: data_panel - screen: root - record: root.current_record - FullImage: - id: img - ControlPanel - id: control_panel - screen: root - Slider: - id: slider - min: 0 - max: tub_loader.len - 1 - value: 0 - size_hint_y: None - height: common_height - on_value: root.index = int(self.value) - TubEditor - TubFilter: - id: tub_filter - TsPlot: - id: graph - #size_hint_y: 3.0 if platform == 'linux' else None - DataPlot: - id: data_plot - StatusLabel: - id: status - - -: - title: 'Choose the pilot' - orientation: 'horizontal' - model_type: pilot_spinner.text - Button: - id: pilot_button - text: 'Choose pilot' - size_hint_x: 0.5 - disabled: True - on_press: - root.open_popup() - MySpinner: - id: pilot_spinner - pos_hint: {'center': (.5, .5)} - size_hint_x: 0.5 - text: 'Model type' - values: drive_models - AutoLabel: - id: pilot_file - text: root.file_path - - -: - size_hint: 1.0, 1.0 - auto_dismiss: False - pos_hint: {'center_x': .5, 'center_y': .5} - - BoxLayout: - orientation: "horizontal" - BoxLayout: - orientation: 'vertical' - id: selected_list - spacing: 20 - - BoxLayout: - orientation: 'vertical' - spacing: 20 - BoxLayout: - orientation: 'vertical' - id: trafo_list - Button: - text: "Close" - size_hint_y: 0.1 - on_release: root.dismiss() - - -: - config: app.tub_screen.ids.config_manager.config - BoxLayout: - orientation: 'vertical' - TabBar: - id: tab_bar - manager: root.manager - PaddedBoxLayout: - PilotLoader: - id: pilot_loader_1 - num: '1' - data_panel: data_panel_1 - PilotLoader: - id: pilot_loader_2 - num: '2' - data_panel: data_panel_2 - BoxLayout: - padding: layout_pad_xy - spacing: layout_pad_x - OverlayImage: - id: img_1 - pilot: pilot_loader_1.pilot - throttle_field: data_in.throttle_field - is_left: True - OverlayImage: - id: img_2 - pilot: pilot_loader_2.pilot - throttle_field: data_in.throttle_field - is_left: False - PaddedBoxLayout: - size_hint_y: 0.5 - DataPanel: - id: data_panel_1 - screen: root - record: img_1.pilot_record - dual_mode: True - link: True - auto_text: root.map_pilot_field(data_in.auto_text) - DataPanel: - id: data_panel_2 - screen: root - record: img_2.pilot_record - dual_mode: True - link: True - auto_text: root.map_pilot_field(data_in.auto_text) - Slider: - id: slider - min: 0 - max: app.tub_screen.ids.tub_loader.len - 1 - value: 0 - size_hint_y: None - height: common_height - on_value: root.index = int(self.value) - - PaddedBoxLayout: - ToggleButton: - id: button_bright - size_hint_x: 0.5 - text: 'Brightness {:4.2f}'.format(slider_bright.value) - on_press: root.set_brightness(slider_bright.value) - Slider: - id: slider_bright - value: 0 - min: -0.5 - max: 0.5 - on_value: root.set_brightness(self.value) - ToggleButton: - id: button_blur - size_hint_x: 0.5 - text: 'Blur {:4.2f}'.format(slider_blur.value) - on_press: root.set_blur(slider_blur.value) - Slider: - id: slider_blur - value: 0 - min: 0.001 - max: 3 - on_value: root.set_blur(self.value) - PaddedBoxLayout: - Transformations: - id: pre_transformation - title: 'Pre-Augmentation Transformations' - text: 'Set pre transformation' - pilot_screen: root - is_post: False - on_press: self.open_popup() - Transformations: - id: post_transformation - title: 'Post-Augmentation Transformations' - text: 'Set post transformation' - pilot_screen: root - is_post: True - on_press: self.open_popup() - PaddedBoxLayout: - size_hint_y: 0.5 - ControlPanel: - id: pilot_control - screen: root - DataPanel: - id: data_in - screen: root - record: root.current_record - dual_mode: True - StatusLabel: - id: status - -: - font_name: 'data/fonts/RobotoMono-Regular.ttf' - font_size: 12 if platform == 'linux' else 24 - size_hint_y: None - size_hint_x: None - height: self.texture_size[1] - width: self.texture_size[0] - #text_size: self.width, None - -: - title: 'Choose transfer model' - orientation: 'horizontal' - Button: - id: transfer_button - text: 'Transfer model' - size_hint_x: 0.5 - on_press: root.open_popup() - AutoLabel: - text: root.file_path - -: - config: app.tub_screen.ids.config_manager.config - BoxLayout: - orientation: 'vertical' - TabBar: - id: tab_bar - manager: root.manager - Label: - size_hint_y: None - height: common_height - text: 'Overwrite config: use json syntax, i.e. "abc" for strings and true/false for bool' - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - MySpinner: - id: cfg_spinner - text: 'select' - values: root.value_list() - TextInput: - id: cfg_overwrite - multiline: False - text: 'New value' - on_text_validate: root.set_config_attribute(self.text) - Label: - size_hint_y: None - height: common_height - text: 'Train pilot' - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - Label: - size_hint_x: 0.5 - text: 'Model type' - MySpinner: - id: train_spinner - size_hint_x: 0.5 - text: 'linear' - values: supported_models - TextInput: - id: comment - multiline: False - text: 'Comment' - on_text: app.train_screen.ids.status.text = f'Adding comment: {self.text}' - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - MySpinner: - id: transfer_spinner - text: 'Choose transfer model' - ToggleButton: - id: train_button - text: 'Training running...' if self.state == 'down' else 'Train' - on_press: - root.train() - self.disabled = True - ScrollableLabel: - DataFrameLabel: - id: scroll_pilots - ScrollableLabel: - size_hint_y: 1.0 if check.state == 'down' else 0.1 - DataFrameLabel: - id: scroll_tubs - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - MySpinner: - id: delete_spinner - text_autoupdate: True - Button: - id: delete_btn - on_press: - root.database.delete_entry(delete_spinner.text) - root.reload_database() - text: 'Delete pilot' - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - Label: - text: 'Group multiple tubs' - ToggleButton: - id: check - on_press: root.on_database(None, None) - text: 'On' if self.state == 'down' else 'Off' - StatusLabel: - id: status - -: - config: app.tub_screen.ids.config_manager.config - BoxLayout: - orientation: 'vertical' - TabBar: - id: tab_bar - manager: root.manager - AutoLabel: - text: 'This screen is experimental and only works on Linux and OSX if ssh is configured to login without password. Please see the docs.' - color: (1, 0.35, 0, 1) - BoxLayout: - size_hint_y: None - height: common_height - Label: - text: 'Connection status' - Label: - id: connected - text: 'Checking...' - Label: - #size_hint_y: None - #height: common_height - text: 'Pull tub from car -> pc' - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - Label: - text: 'Car directory (hit return)' - Label: - text: 'Select tub' - - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - TextInput: - id: car_dir - multiline: False - text: root.car_dir - on_text_validate: - root.list_car_dir(self.text) - root.update_pilots() - MySpinner: - id: tub_dir_spinner - text: 'select' - values: root.files - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - ToggleButton: - id: create_dir - size_hint_x: 0.49 - text: 'Create new folder' - Button: - id: pull_tub - size_hint_x: 0.49 - text: 'Pull tub ' + tub_dir_spinner.text - on_press: - self.disabled = True - root.pull(tub_dir_spinner.text) - ProgressBar: - id: pull_bar - #value: root.pull_bar - Label: - #size_hint_y: None - #height: common_height - text: 'Push pilots from pc -> car' - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - ToggleButton: - id: btn_h5 - text: 'Sync h5' - ToggleButton: - id: btn_savedmodel - text: 'Sync savedmodel' - ToggleButton: - id: btn_tflite - text: 'Sync tflite' - ToggleButton: - id: btn_trt - text: 'Sync tensorrt' - BoxLayout: - size_hint_y: None - height: layout_height - padding: layout_pad_xy - spacing: layout_pad_x - Button: - id: send_pilots - multiline: False - text: 'Push pilots' - on_press: - self.disabled = True - root.send_pilot() - ProgressBar: - id: push_bar - # value: root.push_bar - - Label: - # size_hint_y: None - # height: common_height - text: 'Drive car' - BoxLayout: - size_hint_y: None - height: layout_height_double - padding: layout_pad_xy - spacing: layout_pad_x - MySpinner: - id: type_spinner - text: 'tflite_linear' - values: drive_models - MySpinner: - id: pilot_spinner - text: 'No pilot' - values: ['No pilot'] + root.pilots - BoxLayout: - size_hint_y: None - height: layout_height_double - padding: layout_pad_xy - spacing: layout_pad_x - Button: - id: drive_btn - text: 'Drive' - on_press: - root.drive() - self.disabled = True - stop_btn.disabled = False - Button: - id: stop_btn - text: 'Stop' - disabled: True - on_press: - root.stop() - self.disabled = True - drive_btn.disabled = False - - -: - BoxLayout: - orientation: 'vertical' - TabBar: - id: tab_bar - manager: root.manager - Image: - source: root.img_path - size: self.texture_size diff --git a/donkeycar/management/ui/__init__.py b/donkeycar/management/ui/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/donkeycar/management/ui/car_screen.kv b/donkeycar/management/ui/car_screen.kv new file mode 100644 index 000000000..358667014 --- /dev/null +++ b/donkeycar/management/ui/car_screen.kv @@ -0,0 +1,154 @@ + +: + name: 'car' + BoxLayout: + orientation: 'vertical' + padding: spacing + spacing: spacing + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: 0.6 + Header: + title: 'Car Connector' + description: + 'Connects to your car to pull tub data from the car or ' \ + 'send pilots to the car. The car must be connected to ' \ + 'your local network and the CONFIG settings PI_USERNAME ' \ + 'and PI_HOSTNAME have to be set correctly. SSH access ' \ + 'into the car must be unrestricted, i.e the command ' \ + '"ssh PI_USERNAME@PI_HOSTNAME" should open a shell on the car.' + BoxLayout: + size_hint_y: 0.25 + MyLabel: + text: 'Connection status' + MyLabel: + id: connected + text: 'Checking...' + + BackgroundBoxLayout: + BoxLayout: + orientation: 'vertical' + Header: + size_hint_y: 0.5 + title: 'Pull Tub' + description: + 'Type the Donkey Car directory on the host into the '\ + 'text field and hit . Then in the "select" '\ + 'dropdown, select the Tub directory within that '\ + 'directory, usually this is called "data"' + GridLayout: + spacing: spacing + cols: 2 + MyLabel: + text: 'Car directory (hit return)' + MyTextInput: + id: car_dir + multiline: False + text: root.car_dir + on_text_validate: + root.list_car_dir(self.text) + root.update_pilots() + MyLabel: + text: + 'Create a new tub within the data folder, '\ + 'instead of updating the data folder.' + CheckBox: + id: create_dir + MyLabel: + text: 'Select tub' + MySpinner: + id: tub_dir_spinner + text: 'select' + values: root.files + ProgressBar: + id: pull_bar + #value: root.pull_bar + RoundedButton: + id: pull_tub + text: 'Pull tub ' + tub_dir_spinner.text + on_release: + self.disabled = True + root.pull(tub_dir_spinner.text) + + BackgroundBoxLayout: + size_hint_y: 0.8 + BoxLayout: + orientation: 'vertical' + spacing: spacing + Header: + size_hint_y: 0.5 + title: 'Push Pilots' + description: + 'Copy trained pilots from the PC to the car. Select '\ + 'the format of the models. If nothing is selected, '\ + 'all formats will be copied. Usually you will only '\ + 'require the "tflite" models.' + BoxLayout: + size_hint_y: None + height: common_height + spacing: spacing + RoundedToggleButton: + id: btn_h5 + text: 'Sync h5' + RoundedToggleButton: + id: btn_savedmodel + text: 'Sync savedmodel' + RoundedToggleButton: + id: btn_tflite + text: 'Sync tflite' + RoundedToggleButton: + id: btn_trt + text: 'Sync tensorrt' + BoxLayout: + size_hint_y: None + height: common_height + spacing: spacing + ProgressBar: + id: push_bar + RoundedButton: + id: send_pilots + multiline: False + text: 'Push pilots' + on_release: + self.disabled = True + root.send_pilot() + + BackgroundBoxLayout: + size_hint_y: 0.75 + BoxLayout: + orientation: 'vertical' + Header: + size_hint_y: 0.25 + title: 'Drive Car' + description: + 'Highly experimental!!!' + GridLayout: + cols: 2 + spacing: spacing + MyLabel: + text: 'Set model type' + MySpinner: + id: type_spinner + text: 'tflite_linear' + values: drive_models + MyLabel: + text: 'Select pilot (or leave empty)' + MySpinner: + id: pilot_spinner + text: 'No pilot' + values: ['No pilot'] + root.pilots + RoundedButton: + id: drive_btn + text: 'Drive' + on_release: + root.drive() + self.disabled = True + stop_btn.disabled = False + RoundedButton: + id: stop_btn + text: 'Stop' + disabled: True + on_release: + root.stop() + self.disabled = True + drive_btn.disabled = False diff --git a/donkeycar/management/ui/car_screen.py b/donkeycar/management/ui/car_screen.py new file mode 100644 index 000000000..4676f6a56 --- /dev/null +++ b/donkeycar/management/ui/car_screen.py @@ -0,0 +1,219 @@ +import os +import re +from functools import partial +from subprocess import Popen, PIPE, STDOUT + +from kivy import Logger +from kivy.clock import Clock +from kivy.properties import NumericProperty, ObjectProperty, StringProperty, \ + ListProperty, BooleanProperty + +from donkeycar.management.ui.common import get_app_screen, AppScreen, status +from donkeycar.management.ui.rc_file_handler import rc_handler + + +class CarScreen(AppScreen): + """ Screen for interacting with the car. """ + config = ObjectProperty(force_dispatch=True, allownone=True) + files = ListProperty() + car_dir = StringProperty(rc_handler.data.get('robot_car_dir', '~/mycar')) + event = ObjectProperty(None, allownone=True) + connection = ObjectProperty(None, allownone=True) + pid = NumericProperty(None, allownone=True) + pilots = ListProperty() + is_connected = BooleanProperty(False) + + def initialise(self): + self.event = Clock.schedule_interval(self.connected, 3) + + def list_remote_dir(self, dir): + if self.is_connected: + cmd = f'ssh {self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}' + \ + f' "ls {dir}"' + listing = os.popen(cmd).read() + adjusted_listing = listing.split('\n')[1:-1] + return adjusted_listing + else: + return [] + + def list_car_dir(self, dir): + self.car_dir = dir + self.files = self.list_remote_dir(dir) + # non-empty, if directory found + if self.files: + rc_handler.data['robot_car_dir'] = dir + status(f'Successfully loaded directory: {dir}') + + def update_pilots(self): + model_dir = os.path.join(self.car_dir, 'models') + self.pilots = self.list_remote_dir(model_dir) + + def pull(self, tub_dir): + target = f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}' + \ + f':{os.path.join(self.car_dir, tub_dir)}' + dest = self.config.DATA_PATH + if not self.ids.create_dir.active: + target += '/' + cmd = ['rsync', '-rv', '--progress', '--partial', target, dest] + Logger.info('car pull: ' + str(cmd)) + proc = Popen(cmd, shell=False, stdout=PIPE, text=True, + encoding='utf-8', universal_newlines=True) + repeats = 100 + call = partial(self.show_progress, proc, repeats, True) + event = Clock.schedule_interval(call, 0.0001) + + def send_pilot(self): + # add trailing '/' + src = os.path.join(self.config.MODELS_PATH, '') + # check if any sync buttons are pressed and update path accordingly + buttons = ['h5', 'savedmodel', 'tflite', 'trt'] + select = [btn for btn in buttons if self.ids[f'btn_{btn}'].state + == 'down'] + # build filter: for example this rsyncs all .tfilte and .trt models + # --include=*.trt/*** --include=*.tflite --exclude=* + filter = ['--include=database.json'] + for ext in select: + if ext in ['savedmodel', 'trt']: + ext += '/***' + filter.append(f'--include=*.{ext}') + # if nothing selected, sync all + if not select: + filter.append('--include=*') + else: + filter.append('--exclude=*') + dest = f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}:' + \ + f'{os.path.join(self.car_dir, "models")}' + cmd = ['rsync', '-rv', '--progress', '--partial', *filter, src, dest] + Logger.info('car push: ' + ' '.join(cmd)) + proc = Popen(cmd, shell=False, stdout=PIPE, + encoding='utf-8', universal_newlines=True) + repeats = 0 + call = partial(self.show_progress, proc, repeats, False) + event = Clock.schedule_interval(call, 0.0001) + + def show_progress(self, proc, repeats, is_pull, e): + # find 'to-check=33/4551)' in OSX or 'to-chk=33/4551)' in + # Linux which is end of line + pattern = 'to-(check|chk)=(.*)\)' + + def end(): + # call ended this stops the schedule + if is_pull: + button = self.ids.pull_tub + self.ids.pull_bar.value = 0 + # merge in previous deleted indexes which now might have been + # overwritten + old_tub = get_app_screen('tub').ids.tub_loader.tub + if old_tub: + deleted_indexes = old_tub.manifest.deleted_indexes + get_app_screen('tub').ids.tub_loader.update_tub() + if deleted_indexes: + new_tub = get_app_screen('tub').ids.tub_loader.tub + new_tub.manifest.add_deleted_indexes(deleted_indexes) + else: + button = self.ids.send_pilots + self.ids.push_bar.value = 0 + self.update_pilots() + button.disabled = False + + if proc.poll() is not None: + end() + return False + # find the next repeats lines with update info + count = 0 + while True: + stdout_data = proc.stdout.readline() + if stdout_data: + res = re.search(pattern, stdout_data) + if res: + if count < repeats: + count += 1 + else: + remain, total = tuple(res.group(2).split('/')) + bar = 100 * (1. - float(remain) / float(total)) + if is_pull: + self.ids.pull_bar.value = bar + else: + self.ids.push_bar.value = bar + return True + else: + # end of stream command completed + end() + return False + + def connected(self, event): + if not self.config: + return + if self.connection is None: + if not hasattr(self.config, 'PI_USERNAME') or \ + not hasattr(self.config, 'PI_HOSTNAME'): + self.ids.connected.text = 'Requires PI_USERNAME, PI_HOSTNAME' + return + # run new command to check connection status + cmd = ['ssh', + '-o ConnectTimeout=3', + f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}', + 'date'] + self.connection = Popen(cmd, shell=False, stdout=PIPE, + stderr=STDOUT, text=True, + encoding='utf-8', universal_newlines=True) + else: + # ssh is already running, check where we are + return_val = self.connection.poll() + self.is_connected = False + if return_val is None: + # command still running, do nothing and check next time again + msg = 'Awaiting connection to...' + self.ids.connected.color = 0.8, 0.8, 0.0, 1 + else: + # command finished, check if successful and reset connection + if return_val == 0: + msg = 'Connected to' + self.ids.connected.color = 0, 0.9, 0, 1 + self.is_connected = True + else: + msg = 'Disconnected from' + self.ids.connected.color = 0.9, 0, 0, 1 + self.connection = None + self.ids.connected.text \ + = f'{msg} {getattr(self.config, "PI_HOSTNAME")}' + + def drive(self): + model_args = '' + if self.ids.pilot_spinner.text != 'No pilot': + model_path = os.path.join(self.car_dir, "models", + self.ids.pilot_spinner.text) + model_args = f'--type {self.ids.type_spinner.text} ' + \ + f'--model {model_path}' + cmd = ['ssh', + f'{self.config.PI_USERNAME}@{self.config.PI_HOSTNAME}', + f'source env/bin/activate; cd {self.car_dir}; ./manage.py ' + f'drive {model_args} 2>&1'] + Logger.info(f'car connect: {cmd}') + proc = Popen(cmd, shell=False, stdout=PIPE, text=True, + encoding='utf-8', universal_newlines=True) + while True: + stdout_data = proc.stdout.readline() + if stdout_data: + # find 'PID: 12345' + pattern = 'PID: .*' + res = re.search(pattern, stdout_data) + if res: + try: + self.pid = int(res.group(0).split('PID: ')[1]) + Logger.info(f'car connect: manage.py drive PID: ' + f'{self.pid}') + except Exception as e: + Logger.error(f'car connect: {e}') + return + Logger.info(f'car connect: {stdout_data}') + else: + return + + def stop(self): + if self.pid: + cmd = f'ssh {self.config.PI_USERNAME}@{self.config.PI_HOSTNAME} '\ + + f'kill -SIGINT {self.pid}' + out = os.popen(cmd).read() + Logger.info(f"car connect: Kill PID {self.pid} + {out}") + self.pid = None diff --git a/donkeycar/management/ui/common.kv b/donkeycar/management/ui/common.kv new file mode 100644 index 000000000..9499dc8c4 --- /dev/null +++ b/donkeycar/management/ui/common.kv @@ -0,0 +1,302 @@ +#:import platform sys.platform + +#:set common_height 30 +#:set spacing 10 +#:set layout_height common_height + spacing + + +#:set font_color 0.8, 0.9, 0.9, 1 +#:set action_text_color 0.6, 0.6, 0.4, 1 +#:set label_bar_back_color 0.15, 0.16, 0.18, 1 + + +# Define button background color template +: + background_color: 0,0,0,0 # the last zero is the critical on, make invisible + canvas.before: + Color: + rgba: (.3,.3,.3,1) if self.state=='normal' else (0.2, 0.6,.8,1) + RoundedRectangle: + pos: self.pos + size: self.size + radius: [10,] + color: font_color + text_size: self.width, None + valgin: 'middle' + halign: 'center' + size: self.texture_size + + +: + background_color: 0,0,0,0 # the last zero is the critical on, make invisible + canvas.before: + Color: + rgba: (.3,.3,.3,1) if self.state=='normal' else (0.2, 0.6,.8,1) + RoundedRectangle: + pos: self.pos + size: self.size + radius: [10,] + color: font_color + text_size: self.width, None + valgin: 'middle' + halign: 'center' + size: self.texture_size + + + + background_color: 1, 1, 1, 1 + canvas.before: + Color: + rgba: root.background_color + Rectangle: + size: self.size + pos: self.pos + + + + background_color: 0,0,0,0 # the last zero is the critical on, make invisible + padding: spacing + spacing: 5 + canvas.before: + Color: + rgba: 0.10, 0.11, 0.12, 1 + RoundedRectangle: + pos: self.pos + size: self.size + radius: [root.radius,] + + +: + halign: 'left' + valign: 'top' + color: font_color + text_size: self.size + + + + halign: 'center' + valign: 'middle' + color: font_color + text_size: self.size + + +
+ MyLabel: + font_size: '20sp' + text: root.title + MyLabel: + font_size: '13sp' + text: root.description + + +: + size_hint_y: None + height: layout_height + padding: [spacing, spacing // 2] + spacing: spacing + + +: + orientation: 'vertical' + padding: [10, 0, 10, 10] + size_hint_y: None + height: common_height + BackgroundBoxLayout: + radius: 6 + MyLabel: + valign: 'middle' + font_size: '12sp' + color: action_text_color + text: root.text + + +: + text_size: self.size + halign: 'center' + valign: 'middle' + height: common_height + background_color: 0,0,0,0 # the last zero is the critical on, make invisible + canvas.before: + Color: + rgba: (.3,.3,.3,1) if self.state=='normal' else (0.2, 0.6,.8,1) + RoundedRectangle: + pos: self.pos + size: self.size + radius: [10,] + color: font_color + + +: + text_size: self.size + halign: 'center' + valign: 'middle' + height: common_height + background_color: 0,0,0,0 # the last zero is the critical on, make invisible + canvas.before: + Color: + rgba: (.3,.3,.3,1) if self.state=='normal' else (0.2, 0.6,.8,1) + RoundedRectangle: + pos: self.pos + size: self.size + radius: [10,] + color: font_color + + +: + background_color: 0.15, 0.17, 0.19, 1 + foreground_color: action_text_color + + +: + title: "Choose the directory" + size_hint: 1.0, 1.0 + auto_dismiss: False + pos_hint: {'center_x': .5, 'center_y': .5} + BoxLayout: + orientation: "vertical" + FileChooser: + id: file_chooser + rootpath: root.root_path + dirselect: True + filter_dirs: True + filters: root.filters + FileChooserListLayout + + BoxLayout: + size_hint_y: None + height: common_height + pos_hint: {'center_x': .5, 'center_y': .5} + spacing: 20 + RoundedButton: + text: "Cancel" + on_release: root.dismiss() + RoundedButton: + text: "Load" + on_release: root.load(file_chooser.selection) + disabled: file_chooser.selection==[] + + +: + orientation: 'horizontal' + spacing: 4 + MyLabel: + id: field_label + text_size: self.size + halign: 'left' + valign: 'middle' + font_size: '12sp' + color: root.font_color + canvas.before: + Color: + rgba: label_bar_back_color + RoundedRectangle: + pos: self.pos + size: self.size + radius: [5,] + MyLabel: + id: value_label + text_size: self.size + halign: 'right' + valign: 'middle' + font_size: '12sp' + color: root.font_color + canvas.before: + Color: + rgba: label_bar_back_color + RoundedRectangle: + pos: self.pos + size: self.size + radius: [5,] + ProgressBar: + id: bar + +<-FullImage>: + canvas: + Color: + rgb: (1, 1, 1) + Rectangle: + texture: self.texture + size: self.width, self.height + pos: self.x, self.y + + + + cols: 1 + spacing: 2 + current_field: data_spinner.text + BoxLayout: + size_hint_y: None + height: 20 + spacing: 2 + MyLabel: + id: label + size_hint_x: 0.4 + text: 'Field' + RoundedToggleButton: + id: format_button + size_hint_x: 0.4 + font_size: '10sp' + text: 'Format timestamp' + disabled: root.is_linked + on_press: + root.format_timestamp = (self.state == 'down') + MySpinner: + id: data_spinner + text: 'Add/remove' + on_text: root.add_remove() + on_values: root.clear() + disabled: root.current_field != self.text or root.is_linked + + + + orientation: 'vertical' + spacing: 5 + GridLayout: + spacing: 5 + cols: 2 + MyLabel: + id: record_num + font_size: '14sp' + text: root.record_display + MySpinner: + id: control_spinner + pos_hint: {'center': (.5, .5)} + text: '1.00' + values: ['0.25', '0.50', '1.00', '1.50', '2.00', '3.00', '4.00'] + on_text: + root.speed = float(self.text) + app.root.ids.status.text = f'Setting speed to '\ + f'{self.text} - you can also use the / keys.' + root.restart() + RoundedButton: + id: step_bwd + text: '<' + on_press: + root.start(fwd=False) + on_release: + root.stop() + RoundedButton: + id: step_fwd + text: '>' + on_press: + root.start(fwd=True) + on_release: + root.stop() + RoundedButton: + id: run_bwd + text: '<<' + on_press: + root.start(fwd=False, continuous=True) + root.set_button_status(disabled=True) + RoundedButton: + id: run_fwd + text: '>>' + on_press: + root.start(fwd=True, continuous=True) + root.set_button_status(disabled=True) + RoundedButton: + size_hint_y: 0.3 + text: 'Stop' + on_press: + root.stop() + root.set_button_status(disabled=False) \ No newline at end of file diff --git a/donkeycar/management/ui/common.py b/donkeycar/management/ui/common.py new file mode 100644 index 000000000..3cd7c38e3 --- /dev/null +++ b/donkeycar/management/ui/common.py @@ -0,0 +1,376 @@ +import datetime +import os +import io +import time + +import numpy as np +from functools import partial +from PIL import Image as PilImage + +from kivy import Logger +from kivy.app import App +from kivy.clock import Clock +from kivy.core.window import Window +from kivy.properties import ObjectProperty, StringProperty, ListProperty, \ + BooleanProperty, NumericProperty +from kivy.uix.boxlayout import BoxLayout +from kivy.uix.button import Button +from kivy.uix.gridlayout import GridLayout +from kivy.uix.label import Label +from kivy.uix.popup import Popup +from kivy.uix.screenmanager import Screen +from kivy.uix.spinner import Spinner, SpinnerOption +from kivy.uix.image import Image +from kivy.core.image import Image as CoreImage +from kivy.uix.widget import Widget + +from donkeycar.management.ui.rc_file_handler import rc_handler + +LABEL_SPINNER_TEXT = 'Add/remove' + + +def get_app_screen(name): + root = App.get_running_app().root + if root is None: + return None + return root.ids.sm.get_screen(name) + + +def status(msg): + root = App.get_running_app().root + if root is None: + return + root.ids.status.text = msg + + +def decompose(field): + """ Function to decompose a string vector field like 'gyroscope_1' into a + tuple ('gyroscope', 1) """ + field_split = field.split('_') + if len(field_split) > 1 and field_split[-1].isdigit(): + return '_'.join(field_split[:-1]), int(field_split[-1]) + return field, None + + +def get_norm_value(value, cfg, field_property, normalised=True): + max_val_key = field_property.max_value_id + max_value = getattr(cfg, max_val_key, 1.0) + out_val = value / max_value if normalised else value * max_value + return out_val + + +class BackgroundColor(Widget): + pass + + +class BackgroundBoxLayout(BackgroundColor, BoxLayout): + radius = NumericProperty(10) + pass + + +class RoundedButton(Button): + pass + + +class MyLabel(Label): + pass + + +class Header(BoxLayout): + title = StringProperty() + description = StringProperty() + + +class MySpinnerOption(SpinnerOption): + """ Customization for Spinner """ + pass + + +class MySpinner(Spinner): + """ Customization of Spinner drop down menu """ + def __init__(self, **kwargs): + super().__init__(option_cls=MySpinnerOption, **kwargs) + + +class FileChooserPopup(Popup): + """ File Chooser popup window""" + load = ObjectProperty() + root_path = StringProperty() + filters = ListProperty() + + +class FileChooserBase: + """ Base class for file chooser widgets""" + file_path = StringProperty("No file chosen") + popup = ObjectProperty(None) + root_path = os.path.expanduser('~') + title = StringProperty(None) + filters = ListProperty() + + def open_popup(self): + self.popup = FileChooserPopup(load=self.load, root_path=self.root_path, + title=self.title, filters=self.filters) + self.popup.open() + + def load(self, selection): + """ Method to load the chosen file into the path and call an action""" + self.file_path = str(selection[0]) + self.popup.dismiss() + self.load_action() + + def load_action(self): + """ Virtual method to run when file_path has been updated """ + pass + + +class LabelBar(BoxLayout): + """ Widget that combines a label with a progress bar. This is used to + display the record fields in the data panel.""" + field = StringProperty() + field_property = ObjectProperty() + config = ObjectProperty() + # allowing the font color to change for using as user or pilot bar + font_color = ListProperty([0.8, 0.9, 0.9, 1]) + format_timestamp = BooleanProperty(False) + + def update(self, record): + """ This function is called everytime the current record is updated""" + if not record: + return + field, index = decompose(self.field) + if not field in record.underlying: + Logger.error(f'Record: Bad record {record.underlying["_index"]} - ' + f'missing field {self.field}') + return + val = record.underlying[field] + if index is not None: + val = val[index] + # Update bar if a field property for this field is known + if self.field_property: + norm_value = get_norm_value(val, self.config, + self.field_property) + new_bar_val = (norm_value + 1) * 50 if \ + self.field_property.centered else norm_value * 100 + self.ids.bar.value = new_bar_val + self.ids.field_label.text = self.field + if isinstance(val, float) or isinstance(val, np.float32): + text = f'{val:+07.3f}' + elif field == '_timestamp_ms' and self.format_timestamp: + ts = int(val) / 1000 + dt = datetime.datetime.fromtimestamp(ts) + text = dt.isoformat(sep=' ', timespec='milliseconds') + elif isinstance(val, int): + text = f'{val:10}' + else: + text = str(val) + self.ids.value_label.text = text + + +class DataPanel(GridLayout): + """ Data panel widget that contains the label/bar widgets and the drop + down menu to select/deselect fields.""" + record = ObjectProperty() + font_color = ListProperty([0.8, 0.9, 0.9, 1]) + current_field = StringProperty() + is_linked = BooleanProperty(False) + format_timestamp = BooleanProperty(False) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.labels = {} + self.screen = ObjectProperty() + + def add_remove(self): + """ Method to add or remove a LabelBar. Depending on the value of the + drop down menu the LabelBar is added if it is not present otherwise + removed.""" + field = self.ids.data_spinner.text + if field is LABEL_SPINNER_TEXT: + return + if field in self.labels: + self.remove_widget(self.labels[field]) + del self.labels[field] + status(f'Removing {field}') + else: + field_property = rc_handler.field_properties.get(decompose(field)[0]) + cfg = get_app_screen('tub').ids.config_manager.config + lb = LabelBar(field=field, field_property=field_property, + config=cfg, font_color=self.font_color, + format_timestamp=self.format_timestamp) + self.labels[field] = lb + self.add_widget(lb) + lb.update(self.record) + status(f'Adding {field}') + if self.screen.name == 'tub': + # update the history graphics on the tub screen + self.screen.ids.data_plot.plot_from_current_bars() + # Updating the spinner text again, forces to re-enter this method but + # then exiting early because of the text being 'add/remove' + self.ids.data_spinner.text = LABEL_SPINNER_TEXT + + def on_record(self, obj, record): + """ Kivy function that is called every time self.record changes""" + for v in self.labels.values(): + v.update(record) + + def clear(self): + for v in self.labels.values(): + self.remove_widget(v) + self.labels.clear() + + +class FullImage(Image): + """ Widget to display an image that fills the space. """ + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.core_image = None + + def update(self, record): + """ This method is called every time a record gets updated. """ + try: + img_arr = self.get_image(record) + pil_image = PilImage.fromarray(img_arr) + bytes_io = io.BytesIO() + pil_image.save(bytes_io, format='png') + bytes_io.seek(0) + self.core_image = CoreImage(bytes_io, ext='png') + self.texture = self.core_image.texture + except KeyError as e: + Logger.error(f'Record {record.underlying["_index"]}: ' + f'Missing key: {e}') + except Exception as e: + Logger.error(f'Record : {record.underlying["_index"]}' + f'Bad record: {e}') + + def get_image(self, record): + return record.image() + + +class ControlPanel(BoxLayout): + """ Class for control panel navigation. """ + screen = ObjectProperty() + speed = NumericProperty(1.0) + record_display = StringProperty() + clock = None + fwd = None + + def start(self, fwd=True, continuous=False): + """ + Method to cycle through records if either single <,> or continuous + <<, >> buttons are pressed + :param fwd: If we go forward or backward + :param continuous: If we do <<, >> or <, > + :return: None + """ + # this widget it used in two screens, so reference the original location + # of the config which is the config manager in the tub screen + cfg = get_app_screen('tub').ids.config_manager.config + hz = cfg.DRIVE_LOOP_HZ if cfg else 20 + time.sleep(0.1) + call = partial(self.step, fwd, continuous) + if continuous: + self.fwd = fwd + s = float(self.speed) * hz + cycle_time = 1.0 / s + else: + cycle_time = 0.08 + self.clock = Clock.schedule_interval(call, cycle_time) + + def step(self, fwd=True, continuous=False, *largs): + """ + Updating a single step and cap/floor the index so we stay w/in the tub. + :param fwd: If we go forward or backward + :param continuous: If we are in continuous mode <<, >> + :param largs: dummy + :return: None + """ + if self.screen.index is None: + status("No tub loaded") + return + new_index = self.screen.index + (1 if fwd else -1) + if new_index >= get_app_screen('tub').ids.tub_loader.len: + new_index = 0 + elif new_index < 0: + new_index = get_app_screen('tub').ids.tub_loader.len - 1 + self.screen.index = new_index + msg = f'Donkey {"run" if continuous else "step"} ' \ + f'{"forward" if fwd else "backward"}' + if not continuous: + msg += f' - you can also use {"" if fwd else ""} key' + else: + msg += (' - you can toggle run/stop with or + ' + ' for run backwards') + status(msg) + + def stop(self): + if self.clock: + self.clock.cancel() + self.clock = None + status('Donkey stopped') + + def restart(self): + if self.clock: + self.stop() + self.start(self.fwd, True) + + def update_speed(self, up=True): + """ Method to update the speed on the controller""" + values = self.ids.control_spinner.values + idx = values.index(self.ids.control_spinner.text) + if up and idx < len(values) - 1: + self.ids.control_spinner.text = values[idx + 1] + elif not up and idx > 0: + self.ids.control_spinner.text = values[idx - 1] + + def set_button_status(self, disabled=True): + """ Method to disable(enable) all buttons. """ + self.ids.run_bwd.disabled = self.ids.run_fwd.disabled = \ + self.ids.step_fwd.disabled = self.ids.step_bwd.disabled = disabled + + def on_keyboard(self, keyboard, scancode, text=None, modifier=None): + """ Method to check with keystroke has been sent. """ + if text == ' ': + if self.clock and self.clock.is_triggered: + self.stop() + self.set_button_status(disabled=False) + status('Donkey stopped') + else: + fwd = 'shift' not in modifier + self.start(fwd=fwd, continuous=True) + self.set_button_status(disabled=True) + elif scancode[1] == 'right': + self.step(fwd=True) + elif scancode[1] == 'left': + self.step(fwd=False) + elif scancode[1] == 'up': + self.update_speed(up=True) + elif scancode[1] == 'down': + self.update_speed(up=False) + + +class PaddedBoxLayout(BoxLayout): + pass + + +class StatusBar(BoxLayout): + text = StringProperty() + + +class AppScreen(Screen): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._keyboard = None + + def bind_keyboard(self): + self._keyboard = Window.request_keyboard( + self.keyboard_close, self, 'text') + self._keyboard.bind(on_key_down=self.on_keyboard) + + def keyboard_close(self): + self._keyboard.unbind(on_key_down=self.on_keyboard) + self._keyboard = None + + def on_keyboard(self, keyboard, scancode, text=None, modifier=None): + """ Method to check with keystroke has been sent. """ + pass diff --git a/donkeycar/management/ui/pilot_screen.kv b/donkeycar/management/ui/pilot_screen.kv new file mode 100644 index 000000000..511ae3743 --- /dev/null +++ b/donkeycar/management/ui/pilot_screen.kv @@ -0,0 +1,240 @@ + +: + spacing: spacing + title: 'Choose the pilot' + orientation: 'horizontal' + model_type: pilot_spinner.text + RoundedButton: + id: pilot_button + text: 'Choose pilot' + disabled: True + on_release: + root.open_popup() + MySpinner: + id: pilot_spinner + text: 'Model type' + values: drive_models + MyLabel: + id: pilot_file + size_hint_x: 3.0 + text: root.file_path + + +: + size_hint: 0.5, 1.0 + auto_dismiss: False + pos_hint: {'center_x': .25 if self.right else 0.75, 'center_y': .5} + + BoxLayout: + orientation: "horizontal" + BoxLayout: + orientation: 'vertical' + id: selected_list + spacing: 20 + + BoxLayout: + orientation: 'vertical' + spacing: 20 + BoxLayout: + spacing: spacing + orientation: 'vertical' + id: trafo_list + RoundedButton: + text: "Close" + size_hint_y: 0.1 + on_release: root.dismiss() + + +: + size_hint: 0.5, 0.5 + auto_dismiss: False + pos_hint: {'center_x': .5, 'center_y': .5} + BoxLayout: + orientation: "vertical" + padding: spacing + spacing: spacing + BackgroundBoxLayout: + orientation: "vertical" + spacing: spacing + BoxLayout: + MyLabel: + size_hint_x: 0.5 + text: 'Select pilot' + MySpinner: + id: tubplot_spinner + size_hint_y: 1 + text_autoupdate: True + values: root.screen.ids.pilot_board.get_pilot_names() + Slider: + id: slider + min: root.screen.ids.slider.min + max: root.screen.ids.slider.max + value: 1000 + MyLabel: + valign: 'center' + text: f'Selected records 0 to {int(slider.value)}' + RoundedButton: + text: 'Tub plot' + on_release: + root.screen.tub_plot(tubplot_spinner.text, int(slider.value)) + RoundedButton: + text: "Close" + on_release: root.dismiss() + + + + orientation: 'vertical' + current_user_field: root.screen.ids.data_in.current_field + on_current_user_field: + data_panel.ids.data_spinner.text = root.map_pilot_field(self.current_user_field) + BoxLayout: + orientation: 'horizontal' + size_hint_y: None + height: self.minimum_height + layout_height + spacing: spacing + PilotLoader: + id: pilot_loader + RoundedButton: + size_hint_x: 0.1 + text: '-' + on_release: + root.screen.ids.pilot_board.remove_viewer(root) + OverlayImage: + id: img + pilot_loader: pilot_loader + DataPanel: + size_hint_y: None + height: 20 + (self.minimum_height - 20) * 15 + id: data_panel + is_linked: True + font_color: [0.2, 0.2, 1, 1] + screen: root.screen + record: img.pilot_record + + + + size_hint_y: 1.5 + spacing: spacing + + +: + name: 'pilot' + BoxLayout: + orientation: 'vertical' + padding: spacing + spacing: spacing + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: None + height: self.minimum_height +60 + Header: + title: 'Multi Pilot Loader' + description: + 'Load pilots from the Trainer screen and compare the model '\ + 'inference with the recorded user data.' + BoxLayout: + orientation: 'horizontal' + size_hint_y: 0.6 + spacing: spacing + MyLabel: + text: 'Add pilot' + RoundedButton: + id: add_pilot_btn + text: '+' + on_release: pilot_board.add_viewer() + MyLabel: + text: 'Number of columns' + MySpinner: + id: col_spinner + values: ['1', '2', '3', '4'] + text: '2' + Tubplot: + text: 'Tub plot' + on_release: self.open_popup(root) + + PilotBoard: + id: pilot_board + cols: int(col_spinner.text or '2') + screen: root + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: None + height: self.minimum_height + common_height + Header: + title: 'Augmentations and Transformations' + description: + 'Set image augmentation and transformations on '\ + 'the fly. Transformations execute before augmentations'\ + 'and post transformations afterwards.' + + BoxLayout: + size_hint_y: None + height: common_height + orientation: 'horizontal' + spacing: spacing + RoundedToggleButton: + id: button_bright + size_hint_x: 0.5 + text: 'Brightness {:4.2f}'.format(slider_bright.value) + on_release: root.set_brightness(slider_bright.value) + Slider: + id: slider_bright + value: 0 + min: -0.5 + max: 0.5 + on_value: root.set_brightness(self.value) + RoundedToggleButton: + id: button_blur + size_hint_x: 0.5 + text: 'Blur {:4.2f}'.format(slider_blur.value) + on_release: root.set_blur(slider_blur.value) + Slider: + id: slider_blur + value: 0 + min: 0.001 + max: 3 + on_value: root.set_blur(self.value) + BoxLayout: + size_hint_y: None + height: common_height + orientation: 'horizontal' + spacing: spacing + Transformations: + id: pre_transformation + title: 'Pre-Augmentation Transformations' + text: 'Set pre transformation' + pilot_screen: root + is_post: False + on_release: self.open_popup() + Transformations: + id: post_transformation + title: 'Post-Augmentation Transformations' + text: 'Set post transformation' + pilot_screen: root + is_post: True + on_release: self.open_popup() + + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: None + height: self.minimum_height + 90 + Slider: + size_hint_y: None + height: common_height + id: slider + min: 0 + value: 0 + on_value: + root.index = int(self.value) + BoxLayout: + orientation: 'horizontal' + spacing: spacing + ControlPanel: + id: pilot_control + screen: root + DataPanel: + id: data_in + font_color: [0.3, 0.8, 0.3, 1] + screen: root + record: root.current_record + diff --git a/donkeycar/management/ui/pilot_screen.py b/donkeycar/management/ui/pilot_screen.py new file mode 100644 index 000000000..bd4bca64a --- /dev/null +++ b/donkeycar/management/ui/pilot_screen.py @@ -0,0 +1,386 @@ +from copy import copy #, deepcopy +import os + +from kivy import Logger +from kivy.properties import StringProperty, ObjectProperty, ListProperty, \ + NumericProperty, BooleanProperty +from kivy.uix.boxlayout import BoxLayout +from kivy.uix.gridlayout import GridLayout +from kivy.uix.popup import Popup + +from donkeycar.management.base import ShowPredictionPlots +from donkeycar.management.ui.common import FileChooserBase, \ + FullImage, get_app_screen, get_norm_value, LABEL_SPINNER_TEXT, AppScreen, \ + status, BackgroundBoxLayout, RoundedButton, MyLabel +from donkeycar.management.ui.rc_file_handler import rc_handler +from donkeycar.parts.image_transformations import ImageTransformations +from donkeycar.pipeline.augmentations import ImageAugmentation +from donkeycar.utils import get_model_by_type + + +ALL_FILTERS = ['*.h5', '*.tflite', '*.savedmodel', '*.trt'] + + +class PilotLoader(BoxLayout, FileChooserBase): + """ Class to manage loading of the config file from the car directory""" + model_type = StringProperty() + pilot = ObjectProperty(None) + is_loaded = BooleanProperty(False) + filters = copy(ALL_FILTERS) + + def load_action(self): + def remove_pilot_from_db(entry): + if entry in rc_handler.data['pilots']: + rc_handler.data['pilots'].remove(entry) + + if self.file_path and self.pilot: + entry = [self.file_path, self.model_type] + try: + self.pilot.load(os.path.join(self.file_path)) + self.is_loaded = True + self.ids.pilot_spinner.text = self.model_type + # if successfully loaded, add to rc file + if entry not in rc_handler.data['pilots']: + rc_handler.data['pilots'].append(entry) + Logger.info(f'Pilot: Successfully loaded {self.file_path}') + except FileNotFoundError: + Logger.error(f'Pilot: Model {self.file_path} not found') + remove_pilot_from_db(entry) + except Exception as e: + Logger.error(f'Failed loading {self.file_path}: {e}') + remove_pilot_from_db(entry) + + def on_model_type(self, obj, model_type): + """ Kivy method that is called if self.model_type changes. """ + if self.model_type and self.model_type != 'Model type': + # we cannot use get_app_screen() here as the app is not + # completely build when we are entering this the first time + tub_screen = get_app_screen('tub') + cfg = tub_screen.ids.config_manager.config if tub_screen else None + if not cfg: + return + try: + self.root_path = cfg.MODELS_PATH + self.pilot = get_model_by_type(self.model_type, cfg) + self.ids.pilot_button.disabled = False + if 'tflite' in self.model_type: + self.filters = ['*.tflite'] + elif 'tensorrt' in self.model_type: + self.filters = ['*.trt', '*.savedmodel'] + else: + self.filters = ['*.h5', '*.savedmodel'] + except Exception as e: + status(f'Error: {e}') + + def remove_from_rcfile(self): + if not self.is_loaded: + return + entry = [self.file_path, self.model_type] + if entry in rc_handler.data['pilots']: + rc_handler.data['pilots'].remove(entry) + + +class OverlayImage(FullImage): + """ Widget to display the image and the user/pilot data for the tub. """ + pilot_loader = ObjectProperty() + pilot_record = ObjectProperty() + throttle_field = StringProperty('user/throttle') + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.is_left = True + + def augment(self, img_arr): + pilot_screen = get_app_screen('pilot') + if pilot_screen.trans_list: + img_arr = pilot_screen.transformation.run(img_arr) + if pilot_screen.aug_list: + img_arr = pilot_screen.augmentation.run(img_arr) + if pilot_screen.post_trans_list: + img_arr = pilot_screen.post_transformation.run(img_arr) + return img_arr + + def get_image(self, record): + from donkeycar.management.makemovie import MakeMovie + config = get_app_screen('tub').ids.config_manager.config + orig_img_arr = super().get_image(record) + aug_img_arr = self.augment(orig_img_arr) + img_arr = copy(aug_img_arr) + angle = record.underlying['user/angle'] + throttle = get_norm_value( + record.underlying[self.throttle_field], config, + rc_handler.field_properties[self.throttle_field]) + rgb = (0, 255, 0) + MakeMovie.draw_line_into_image(angle, throttle, False, img_arr, rgb) + if not self.pilot_loader.is_loaded: + return img_arr + output = (0, 0) + try: + # Not each model is supported in each interpreter + output = self.pilot_loader.pilot.run(aug_img_arr) + except Exception as e: + Logger.error(e) + + rgb = (0, 0, 255) + MakeMovie.draw_line_into_image(output[0], output[1], True, img_arr, rgb) + out_record = copy(record) + out_record.underlying['pilot/angle'] = output[0] + # rename and denormalise the throttle output + pilot_throttle_field \ + = rc_handler.data['user_pilot_map'][self.throttle_field] + out_record.underlying[pilot_throttle_field] \ + = get_norm_value(output[1], + config, + rc_handler.field_properties[self.throttle_field], + normalised=False) + self.pilot_record = out_record + return img_arr + + +class TransformationPopup(Popup): + """ Transformation popup window""" + title = StringProperty() + transformations = \ + ["TRAPEZE", "CROP", "RGB2BGR", "BGR2RGB", "RGB2HSV", "HSV2RGB", + "BGR2HSV", "HSV2BGR", "RGB2GRAY", "RBGR2GRAY", "HSV2GRAY", "GRAY2RGB", + "GRAY2BGR", "CANNY", "BLUR", "RESIZE", "SCALE", "GAMMANORM"] + transformations_obj = ObjectProperty() + selected = ListProperty() + right = BooleanProperty() + + def __init__(self, selected, **kwargs): + super().__init__(**kwargs) + for t in self.transformations: + btn = RoundedButton(text=t) + btn.bind(on_release=self.toggle_transformation) + self.ids.trafo_list.add_widget(btn) + self.selected = selected + + def toggle_transformation(self, btn): + trafo = btn.text + if trafo in self.selected: + self.selected.remove(trafo) + else: + self.selected.append(trafo) + + def on_selected(self, obj, select): + self.ids.selected_list.clear_widgets() + for l in self.selected: + lab = MyLabel(text=l) + self.ids.selected_list.add_widget(lab) + self.transformations_obj.selected = self.selected + + +class Transformations(RoundedButton): + """ Base class for transformation widgets""" + title = StringProperty(None) + pilot_screen = ObjectProperty() + is_post = False + selected = ListProperty() + + def open_popup(self): + popup = TransformationPopup(title=self.title, transformations_obj=self, + selected=self.selected, right=self.is_post) + popup.open() + + def on_selected(self, obj, select): + Logger.info(f"Selected {select}") + if self.is_post: + self.pilot_screen.post_trans_list = self.selected + else: + self.pilot_screen.trans_list = self.selected + + +class PilotViewer(BackgroundBoxLayout): + screen = ObjectProperty() + current_user_field = StringProperty() + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.ids.data_panel.ids.data_spinner.disabled = True + # for any new create viewer, copy the user fields over + for field in self.screen.ids.data_in.labels.keys(): + self.current_user_field = field + + def update(self, record): + self.ids.img.update(record) + + def map_pilot_field(self, text): + """ Method to return user -> pilot mapped fields except for the + initial value called Add/remove. """ + if text == LABEL_SPINNER_TEXT: + return text + return rc_handler.data['user_pilot_map'][text] + + def remove_from_rcfile(self): + self.ids.pilot_loader.remove_from_rcfile() + + +class PilotBoard(GridLayout): + screen = ObjectProperty() + + def add_viewer(self): + viewer = PilotViewer(screen=self.screen) + self.add_widget(viewer) + return viewer + + def remove_viewer(self, viewer): + viewer.remove_from_rcfile() + self.remove_widget(viewer) + + def get_pilot_names(self): + return [c.ids.pilot_loader.file_path for c in self.children] + + +class TubplotPopup(Popup): + screen = ObjectProperty() + + +class Tubplot(RoundedButton): + + def open_popup(self, screen): + popup = TubplotPopup(title='Tub Plot', screen=screen) + popup.open() + + +class PilotScreen(AppScreen): + """ Screen to do the pilot vs pilot comparison .""" + index = NumericProperty(None, force_dispatch=True) + current_record = ObjectProperty(None) + aug_list = ListProperty(force_dispatch=True) + augmentation = ObjectProperty() + trans_list = ListProperty(force_dispatch=True) + transformation = ObjectProperty() + post_trans_list = ListProperty(force_dispatch=True) + post_transformation = ObjectProperty() + config = ObjectProperty(allownone=True) + + def on_index(self, obj, index): + """ Kivy method that is called if self.index changes. Here we update + self.current_record and the slider value. """ + if get_app_screen('tub').ids.tub_loader.records: + self.current_record \ + = get_app_screen('tub').ids.tub_loader.records[index] + self.ids.slider.value = index + + def on_current_record(self, obj, record): + """ Kivy method that is called when self.current_index changes. Here + we update the images and the control panel entry.""" + if not record: + return + i = record.underlying['_index'] + self.ids.pilot_control.record_display = f"Record {i:06}" + for c in self.ids.pilot_board.children: + c.update(record) + + def on_config(self, obj, cfg): + if not self.config: + return + try: + for c in self.ids.pilot_board.children: + c.ids.pilot_loader.root_path = self.config.MODELS_PATH + except Exception as e: + Logger.error(f'Error at config update in train screen: {e}') + + def initialise(self, e): + # self.ids.pilot_board.add_viewer() + for entry in rc_handler.data.get('pilots', []): + model_path = entry[0] + model_type = entry[1] + viewer = self.ids.pilot_board.add_viewer() + viewer.ids.pilot_loader.model_type = model_type + viewer.ids.pilot_loader.file_path = model_path + viewer.ids.pilot_loader.load_action() + + mapping = rc_handler.data['user_pilot_map'] + self.ids.data_in.ids.data_spinner.values = mapping.keys() + + def set_brightness(self, val=None): + if not self.config: + return + if self.ids.button_bright.state == 'down': + self.config.AUG_BRIGHTNESS_RANGE = (val, val) + if 'BRIGHTNESS' not in self.aug_list: + self.aug_list.append('BRIGHTNESS') + else: + # Since we only changed the content of the config here, + # self.on_aug_list() would not be called, but we want to update + # the augmentation. Hence, update the dependency manually here. + self.on_aug_list(None, None) + elif 'BRIGHTNESS' in self.aug_list: + self.aug_list.remove('BRIGHTNESS') + + def set_blur(self, val=None): + if not self.config: + return + if self.ids.button_blur.state == 'down': + self.config.AUG_BLUR_RANGE = (val, val) + if 'BLUR' not in self.aug_list: + self.aug_list.append('BLUR') + elif 'BLUR' in self.aug_list: + self.aug_list.remove('BLUR') + # update dependency + self.on_aug_list(None, None) + + def on_aug_list(self, obj, aug_list): + if not self.config: + return + # cast to python list, otherwise we have an ObservableList in the config + self.config.AUGMENTATIONS = list(self.aug_list) + self.augmentation = ImageAugmentation( + self.config, 'AUGMENTATIONS', always_apply=True) + self.on_current_record(None, self.current_record) + + def on_trans_list(self, obj, trans_list): + if not self.config: + return + # cast to python list, otherwise we have an ObservableList in the config + self.config.TRANSFORMATIONS = list(self.trans_list) + self.transformation = ImageTransformations( + self.config, 'TRANSFORMATIONS') + self.on_current_record(None, self.current_record) + + def on_post_trans_list(self, obj, trans_list): + if not self.config: + return + # cast to python list, otherwise we have an ObservableList in the config + self.config.POST_TRANSFORMATIONS = list(self.post_trans_list) + self.post_transformation = ImageTransformations( + self.config, 'POST_TRANSFORMATIONS') + self.on_current_record(None, self.current_record) + + def set_mask(self, state): + if state == 'down': + status('Trapezoidal mask on') + self.trans_list.append('TRAPEZE') + else: + status('Trapezoidal mask off') + if 'TRAPEZE' in self.trans_list: + self.trans_list.remove('TRAPEZE') + + def set_crop(self, state): + if state == 'down': + status('Crop on') + self.trans_list.append('CROP') + else: + status('Crop off') + if 'CROP' in self.trans_list: + self.trans_list.remove('CROP') + + def tub_plot(self, model_path, limit): + model_type = None + for c in self.ids.pilot_board.children: + if c.ids.pilot_loader.file_path == model_path: + model_type = c.ids.pilot_loader.model_type + assert model_type, f"Could not find{model_path} in pilot display" + ShowPredictionPlots().plot_predictions( + cfg=self.config, + tub_paths=get_app_screen('tub').ids.tub_loader.file_path, + model_path=model_path, + limit=limit, + model_type=model_type, + noshow=False, + dark=True) + + def on_keyboard(self, keyboard, scancode, text=None, modifier=None): + self.ids.pilot_control.on_keyboard(keyboard, scancode, text, modifier) diff --git a/donkeycar/management/ui/rc_file_handler.py b/donkeycar/management/ui/rc_file_handler.py new file mode 100644 index 000000000..f64254997 --- /dev/null +++ b/donkeycar/management/ui/rc_file_handler.py @@ -0,0 +1,92 @@ +import atexit +from datetime import datetime +import os +from collections import namedtuple + +import yaml +from kivy import Logger + +# Data struct to show tub field in the progress bar, containing the name, +# the name of the maximum value in the config file and if it is centered. +FieldProperty = namedtuple('FieldProperty', + ['field', 'max_value_id', 'centered']) + + +def recursive_update(target, source): + """ Recursively update dictionary """ + if isinstance(target, dict) and isinstance(source, dict): + for k, v in source.items(): + v_t = target.get(k) + if not recursive_update(v_t, v): + target[k] = v + return True + else: + return False + + +class RcFileHandler: + """ This handles the config file which stores the data, like the field + mapping for displaying of bars and last opened car, tub directory. """ + + # These entries are expected in every tub, so they don't need to be in + # the file + known_entries = [ + FieldProperty('user/angle', '', centered=True), + FieldProperty('user/throttle', '', centered=False), + FieldProperty('pilot/angle', '', centered=True), + FieldProperty('pilot/throttle', '', centered=False), + ] + + def __init__(self, file_path='~/.donkeyrc'): + self.file_path = os.path.expanduser(file_path) + self.data = self.create_data() + recursive_update(self.data, self.read_file()) + self.field_properties = self.create_field_properties() + + def exit_hook(): + self.write_file() + # Automatically save config when program ends + atexit.register(exit_hook) + + def create_field_properties(self): + """ Merges known field properties with the ones from the file """ + field_properties = {entry.field: entry for entry in self.known_entries} + field_list = self.data.get('field_mapping') + if field_list is None: + field_list = {} + for entry in field_list: + assert isinstance(entry, dict), \ + 'Dictionary required in each entry in the field_mapping list' + field_property = FieldProperty(**entry) + field_properties[field_property.field] = field_property + return field_properties + + def create_data(self): + data = dict() + data['user_pilot_map'] = {'user/throttle': 'pilot/throttle', + 'user/angle': 'pilot/angle'} + data['pilots'] = [] + data['config_params'] = [] + return data + + def read_file(self): + if os.path.exists(self.file_path): + with open(self.file_path) as f: + data = yaml.load(f, Loader=yaml.FullLoader) + Logger.info(f'Donkeyrc: Donkey file {self.file_path} loaded.') + return data + else: + Logger.warn(f'Donkeyrc: Donkey file {self.file_path} does not ' + f'exist.') + return {} + + def write_file(self): + if os.path.exists(self.file_path): + Logger.info(f'Donkeyrc: Donkey file {self.file_path} updated.') + with open(self.file_path, mode='w') as f: + self.data['time_stamp'] = datetime.now() + data = yaml.dump(self.data, f) + return data + + +rc_handler = RcFileHandler() diff --git a/donkeycar/management/ui/train_screen.kv b/donkeycar/management/ui/train_screen.kv new file mode 100644 index 000000000..e085d1937 --- /dev/null +++ b/donkeycar/management/ui/train_screen.kv @@ -0,0 +1,295 @@ +#:import ScrollEffect kivy.effects.scroll.ScrollEffect + +#:set supported_models ['linear', 'categorical', 'inferred', 'memory', 'behavior', 'localizer', 'rnn', '3d', 'sq', 'sq_imu', 'sq_mem', 'sq_mem_lap'] +#:set drive_models [pre + t for pre in['', 'tflite_', 'tensorrt_'] for t in supported_models ] + + + + size_hint_y: None + height: common_height + spacing: spacing + MySpinner: + id: cfg_spinner + # text_autoupdate: True + values: [] + on_text: + root.update_rc(self.text) + MyTextInput: + id: cfg_overwrite + multiline: False + text: + str(getattr(root.config, cfg_spinner.text, '')) if root.config else '' + on_text_validate: + root.set_config_attribute(self.text) + RoundedButton: + id: cfg_add_remove + size_hint_x: 0.15 + text: "-" + on_release: + root.screen.ids.config_panel.remove_widget(root) + + +: + spacing: spacing + cols: 1 + + +: + font_size: '12sp' + text_size: self.width, None + size: self.texture_size + valign: 'middle' + halign: 'left' + color: font_color + + +: + title: 'Choose transfer model' + orientation: 'horizontal' + RoundedButton: + id: transfer_button + text: 'Transfer model' + size_hint_x: 0.5 + on_release: root.open_popup() + AutoLabel: + text: root.file_path + + +: + orientation: 'horizontal' + size_hint_y: None + CheckBox: + active: True + on_active: + root.menu.selected.insert(root.i, root.text) if self.active else root.menu.selected.remove(root.text) + MyLabel: + valign: 'middle' + font_size: '12sp' + text: root.text + + +: + orientation: 'horizontal' + on_selected: self.screen.plot_dataframe(self.screen.dataframe, self.selected) + + +: + size_hint: 1.0, 1.0 + auto_dismiss: False + pos_hint: {'center_x': 0.5, 'center_y': .5} + BoxLayout: + orientation: 'vertical' + spacing: 20 + ScrollView: + effect_cls: ScrollEffect + GridLayout: + id: pilot_cfg_viewer_grid + cols: 2 + row_default_height: 26 + size_hint_y: None + height: self.minimum_height + RoundedButton: + text: "Close" + size_hint_y: 0.05 + on_release: root.dismiss() + + + auto_dismiss: False + pos_hint: {'center_x': .5, 'center_y': .5} + BoxLayout: + orientation: "vertical" + padding: spacing + spacing: spacing + BackgroundBoxLayout: + orientation: "vertical" + spacing: spacing + HistoryPlot: + id: history_plot + df: root.df + RoundedButton: + size_hint_y: None + height: common_height + text: "Close" + on_release: root.dismiss() + + +: + name: 'train' + BoxLayout: + orientation: 'vertical' + padding: spacing + spacing: spacing + BackgroundBoxLayout: + orientation: 'vertical' + spacing: spacing + size_hint_y: None + height: self.minimum_height + 80 + Header: + size_hint_y: 1.5 + id: cfg_header + title: 'Config Editor' + description: + 'Use dropdown menus to edit config parameters. Add more'\ + ' rows to keep track of more parameters without scrolling.'\ + ' Use json syntax, i.e. double-quoted "string" and true,'\ + ' false.' + BoxLayout: + id: lower_row + orientation: 'horizontal' + size_hint_y: 0.75 + spacing: spacing + MyLabel: + size_hint_x: 1.3 + text: 'Add config setter' + RoundedButton: + size_hint_x: 1.3 + text: '+' + on_release: config_panel.add() + MyLabel: + text: 'Number columns' + MySpinner: + id: col_setter + values: ['1', '2', '3', '4'] + text: '1' + on_text: + config_panel.cols = int(self.text) + RoundedToggleButton: + id: save_cfg + text: 'Save myconfig' + + ConfigParamPanel: + id: config_panel + size_hint_y: None + height: + (common_height + 10) * (int(len(self.children) // self.cols) + + (0 if len(self.children) % self.cols == 0 else 1)) - 10 + + + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: None + height: self.minimum_height + Header: + size_hint_y: None + height: 40 + title: 'Trainer' + description: + "Train pilot using config parameters from above. Choose a "\ + "model type and optionally a transfer model. Provide a "\ + "comment to help identify the training parameters." + + BoxLayout: + size_hint_y: None + height: common_height + spacing: spacing + MyLabel: + text: 'Select model type' + MySpinner: + id: train_spinner + text: 'linear' + values: supported_models + MyTextInput: + id: comment + size_hint_x: 2.05 + multiline: False + text: 'Comment' + on_text: app.root.ids.status.text = f'Adding comment: {self.text}' + BoxLayout: + size_hint_y: None + height: common_height + spacing: spacing + MySpinner: + id: transfer_spinner + text: 'Choose transfer model' + RoundedToggleButton: + id: train_button + text: 'Training running...' if self.state == 'down' else 'Train' + on_press: + root.train() + self.disabled = True + + BackgroundBoxLayout: + size_hint_y: 2 + orientation: 'vertical' + spacing: 0 + padding: 0 + CheckBoxRow: + size_hint_y: None + height: 90 + screen: root + id: column_chooser + ScrollView: + id: scroll + effect_cls: ScrollEffect + GridLayout: + size_hint_y: None + padding: 2 + spacing: 2 + cols: 2 + row_default_height: 28 + height: self.minimum_height + id: scroll_pilots + + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: None + height: self.minimum_height + Header: + size_hint_y: None + height: 50 + title: 'Pilot Viewer and Editor' + description: + "Select a pilot for viewing its config, updating the " \ + "comment or to deleting it. Deletion removes files from "\ + "disk and the entry from the database." + BoxLayout: + size_hint_y: None + height: common_height + spacing: spacing + MySpinner: + size_hint_x: 1.2 + id: select_spinner + text_autoupdate: True + on_text: + entry = root.database.get_entry(select_spinner.text) + txt = entry.get('Comment') or '' + comment.text = txt + RoundedToggleButton: + id: delete_switch + font_size: '11sp' + text: 'Enable delete' + size_hint_x: 0.3 + active: False + on_release: + delete_btn.disabled = False if self.state == 'down' else True + RoundedButton: + size_hint_x: 0.75 + id: delete_btn + disabled: True + color: (.4, .4, .4, 1) if self.disabled else (0.95, 0, 0, 1) + text: 'Delete pilot' + on_release: + root.database.delete_entry(select_spinner.text) + root.reload_database() + delete_switch.state = 'normal' + self.disabled = True + RoundedButton: + size_hint_x: 0.75 + id: update_comment + text: 'Update comment' + on_release: + root.database.get_entry(select_spinner.text)['Comment'] = comment.text + root.database.write() + root.on_database() + RoundedButton: + size_hint_x: 0.75 + id: show_config + text: 'Show Config' + on_release: + root.show_config() + RoundedButton: + size_hint_x: 0.75 + id: show_history() + text: 'Training history' + on_release: + root.show_history() \ No newline at end of file diff --git a/donkeycar/management/ui/train_screen.py b/donkeycar/management/ui/train_screen.py new file mode 100644 index 000000000..5bdce73e4 --- /dev/null +++ b/donkeycar/management/ui/train_screen.py @@ -0,0 +1,325 @@ +import datetime +import os +from threading import Thread +import json + +import pandas as pd +from kivy import Logger +from kivy.clock import Clock +from kivy.properties import ObjectProperty, NumericProperty, ListProperty, \ + StringProperty +from kivy.uix.boxlayout import BoxLayout +from kivy.uix.gridlayout import GridLayout +from kivy.uix.label import Label +from kivy.uix.popup import Popup +from kivy_garden.matplotlib import FigureCanvasKivyAgg +import matplotlib as mpl +import matplotlib.pyplot as plt + +from donkeycar.config import Config +from donkeycar.management.ui.common import FileChooserBase, get_app_screen, \ + AppScreen, status +from donkeycar.management.ui.rc_file_handler import rc_handler +from donkeycar.pipeline.database import PilotDatabase +from donkeycar.pipeline.training import train + + +mpl.rcParams.update({'font.size': 8}) +plt.style.use('dark_background') +fig1, ax1 = plt.subplots() +plt.tight_layout(pad=1.5) + + +class ConfigParamSetter(BoxLayout): + screen = ObjectProperty() + config = ObjectProperty(force_dispatch=True, allownone=True) + + def get_keys(self): + if self.config: + return [str(k) for k in self.config.__dict__.keys()] + else: + return [] + + def on_config(self, obj=None, config=None): + if self.ids: + self.ids.cfg_spinner.values = self.get_keys() + + def set_config_attribute(self, input): + try: + val = json.loads(input) + except ValueError: + val = input + att = self.ids.cfg_spinner.text + setattr(self.config, att, val) + msg = f'Setting {att} to {val} of type {type(val).__name__}' + if val in ('True', 'False', 'TRUE', 'FALSE'): + msg += f' - ATTENTION: {val} is not a Boolean but a String!' + status(msg) + if get_app_screen('train').ids.save_cfg.state == 'down': + car_path = get_app_screen('tub').ids.config_manager.file_path + my_cfg_path = os.path.join(car_path, 'myconfig.py') + my_cfg = Config() + my_cfg.from_pyfile(my_cfg_path) + my_cfg.from_dict({att: val}) + my_cfg.to_pyfile(my_cfg_path) + + @staticmethod + def update_rc(att): + cfg_params = rc_handler.data.get('config_params') + if cfg_params is None: + rc_handler.data['config_params'] = [att] + elif att not in cfg_params: + cfg_params.append(att) + + +class ConfigParamPanel(GridLayout): + + def row_count(self): + rows = int(len(self.children) // self.cols) \ + + 0 if len(self.children) % self.cols == 0 else 1 + return rows + + def add(self): + train_screen = get_app_screen('train') + cfg_setter = ConfigParamSetter(screen=train_screen, + config=train_screen.config) + # We need simulate a config change so the keys get populated + cfg_setter.on_config() + self.add_widget(cfg_setter) + return cfg_setter + + def remove_widget(self, cfg_setter, *args, **kwargs): + att = cfg_setter.ids.cfg_spinner.text + cfg_params = rc_handler.data.get('config_params', []) + if att in cfg_params: + cfg_params.remove(att) + super().remove_widget(cfg_setter) + + +class BackgroundLabel(Label): + pass + + +class MenuCheckbox(BoxLayout): + menu = ObjectProperty() + text = StringProperty() + i = NumericProperty() + + +class CheckBoxRow(BoxLayout): + selected = ListProperty() + screen = ObjectProperty() + + def build_widgets(self, labels): + self.clear_widgets() + self.selected.clear() + for i, label in enumerate(labels): + but = MenuCheckbox(i=i, text=label, menu=self) + self.add_widget(but) + self.selected = list(labels) + + +class TransferSelector(BoxLayout, FileChooserBase): + """ Class to select transfer model""" + filters = ['*.h5', '*.savedmodel'] + + +class ConfigViewerPopup(Popup): + """ Popup to view the config that was saved in the model database as part + of the training.""" + + config = ObjectProperty() + + def _config_to_dict(self): + # In old-style database format, the config was saved as list of (key, + # value) pairs. That string could not be de-jsonised and then + # self.config is a string. In new format is already a dict. + if isinstance(self.config, dict): + return self.config + cfg_list = [] + try: + s = self.config.replace("'", '"') + s = s.replace("(", '[').replace(")", "]") + s = s.replace("False", 'false').replace("True", "true") + s = s.replace("None", "null") + cfg_list = json.loads(s) + except Exception as e: + Logger.error(f'Failed json read of config: {e}') + assert isinstance(cfg_list, list), "De-jsonised config should be list" + return dict(cfg_list) + + def fill_grid(self): + d = self._config_to_dict() + for kv in d.items(): + for x in kv: + label = BackgroundLabel(text=str(x), font_size='13sp') + self.ids.pilot_cfg_viewer_grid.add_widget(label) + + +class HistoryViewerPopup(Popup): + df = ObjectProperty(force_dispatch=True, allownone=True) + + +class HistoryPlot(FigureCanvasKivyAgg): + df = ObjectProperty(force_dispatch=True, allownone=True) + + def __init__(self, **kwargs): + super().__init__(fig1, **kwargs) + + def on_df(self, e=None, z=None): + ax1.clear() + if self.df is None or self.df.empty: + return + # only look at loss graphs but not accuracy + drop = [c for c in self.df.columns if 'loss' not in c] + loss_df = self.df.drop(columns=drop) + n = len(loss_df.columns) + # arrange subplots + non_val_cols = [c for c in loss_df.columns if c[:4] != 'val_'] + if len(non_val_cols) != n / 2: + Logger.Error(f"Issue with history data, validation data history " + f"is not half of the loss data") + return + subplots = [(nv, f'val_{nv}') for nv in non_val_cols] + loss_df.plot(ax=ax1, linewidth=1.0, subplots=subplots) + self.draw() + + +class TrainScreen(AppScreen): + """ Class showing the training screen. """ + config = ObjectProperty(force_dispatch=True, allownone=True) + database = ObjectProperty() + dataframe = ObjectProperty(force_dispatch=True) + pilot_df = ObjectProperty(force_dispatch=True) + tub_df = ObjectProperty(force_dispatch=True) + train_checker = False + + def train_call(self, *args): + tub_path = get_app_screen('tub').ids.tub_loader.tub.base_path + transfer = self.ids.transfer_spinner.text + model_type = self.ids.train_spinner.text + h5 = os.path.join(self.config.MODELS_PATH, transfer + '.h5') + sm = os.path.join(self.config.MODELS_PATH, transfer + '.savedmodel') + + if transfer == 'Choose transfer model': + transfer_model = None + elif os.path.exists(sm): + transfer_model = str(sm) + elif os.path.exists(h5): + transfer_model = str(h5) + else: + transfer_model = None + status(f'Could find neither {sm} nor {h5} - training without ' + f'transfer') + try: + history = train(self.config, tub_paths=tub_path, + model_type=model_type, + transfer=transfer_model, + comment=self.ids.comment.text) + except Exception as e: + Logger.error(e) + status(f'Training failed see console') + + def train(self): + self.config.SHOW_PLOT = False + t = Thread(target=self.train_call) + status('Training started.') + + def func(dt): + t.start() + + def check_training_done(dt): + if t.is_alive(): + return + self.train_checker.cancel() + self.ids.comment.text = 'Comment' + self.ids.transfer_spinner.text = 'Choose transfer model' + self.ids.train_button.state = 'normal' + self.ids.train_button.disabled = False + self.reload_database() + status('Training completed.') + + # schedules the call after the current frame + Clock.schedule_once(func, 0) + # checks if training finished and updates the window if + self.train_checker = Clock.schedule_interval(check_training_done, 0.5) + + def on_config(self, obj, config): + if self.config and self.ids: + self.reload_database() + + def reload_database(self): + if self.config: + self.database = PilotDatabase(self.config) + + def on_database(self, obj=None, database=None): + df = self.database.to_df() + df.drop(columns=['History', 'Config'], errors='ignore', inplace=True) + self.dataframe = df + + def on_dataframe(self, obj, dataframe): + self.plot_dataframe(dataframe) + if self.dataframe.empty: + return + pilot_names = self.dataframe['Name'].values.tolist() + self.ids.transfer_spinner.values \ + = ['Choose transfer model'] + pilot_names + self.ids.select_spinner.values = pilot_names + self.ids.column_chooser.build_widgets(dataframe) + + def plot_dataframe(self, df, selected_cols=None): + grid = self.ids.scroll_pilots + grid.clear_widgets() + # only set column chooser labels on initialisation when selected_cols + # is not passed in. otherwise project df to selected columns + df1 = df[selected_cols] if selected_cols is not None else df + num_cols = len(df1.columns) + grid.cols = num_cols + + for i, col in enumerate(df1.columns): + lab = BackgroundLabel(text=f"[b]{col}[/b]", markup=True) + lab.size = lab.texture_size + grid.add_widget(lab) + # if col in ('Pilot', 'Comment'): + # grid.cols_minimum |= {i: 100} + + for idx in df1.index: + for col in df1.columns: + cell = df1[col][idx] + if col == 'Time': + cell = datetime.datetime.fromtimestamp(cell) + cell = cell.strftime("%Y-%m-%d %H:%M:%S") + cell = str(cell) + lab = BackgroundLabel(text=cell) + lab.size = lab.texture_size + grid.add_widget(lab) + + def show_config(self, obj=None): + pilot = self.ids.select_spinner.text + cfg = self.database.get_entry(pilot).get('Config') + if not cfg: + Logger.Error(f'Config for pilot {pilot} not found in database') + return + popup = ConfigViewerPopup(config=cfg, title=f'Config for {pilot}') + popup.fill_grid() + popup.open() + + def show_history(self, obj=None): + pilot = self.ids.select_spinner.text + history = self.database.get_entry(pilot).get('History') + if not history: + Logger.Error(f'History for pilot {pilot} not found in database') + return + df = pd.DataFrame(history) + popup = HistoryViewerPopup(df=df, title=f'Training history for {pilot}') + popup.open() + + def initialise(self): + cfg_params = rc_handler.data.get('config_params', []) + for param in cfg_params: + # only restore parameters that are in the config + if not hasattr(self.config, param): + continue + cfg_setter = self.ids.config_panel.add() + cfg_setter.ids.cfg_spinner.text = param + diff --git a/donkeycar/management/ui/tub_screen.kv b/donkeycar/management/ui/tub_screen.kv new file mode 100644 index 000000000..28122a46a --- /dev/null +++ b/donkeycar/management/ui/tub_screen.kv @@ -0,0 +1,174 @@ +# #:import TsPlot donkeycar.management.graph.TsPlot + + + title: 'Choose the car directory' + orientation: 'vertical' + + Header: + title: 'Config Loader' + description: + "Load config from car directory, typically ~/mycar" + BoxLayout: + orientation: 'horizontal' + spacing: spacing + RoundedButton: + text: 'Load config' + on_release: root.open_popup() + MyLabel: + id: car_dir + text: root.file_path + + + + title: 'Choose the tub directory' + orientation: 'vertical' + Header: + title: 'Tub Loader' + description: "Load tub from within the car directory, typically ./data" + BoxLayout: + spacing: spacing + RoundedButton: + id: tub_button + text: 'Load tub' + disabled: True + on_release: root.open_popup() + MyLabel: + id: tub_dir + text: root.file_path + + + + orientation: 'horizontal' + spacing: spacing + on_lr: + msg = f'Setting range, ' + if root.lr[0] < root.lr[1]: msg += (f'affecting records inside [{root.lr[0]}, {root.lr[1]}) ' + \ + '- you can affect records outside by setting left > right') + else: msg += (f'affecting records outside ({root.lr[1]}, {root.lr[0]}] ' + \ + '- you can affect records inside by setting left < right') + app.root.ids.status.text = msg + RoundedButton: + text: 'Set left' + on_release: root.set_lr(True) + RoundedButton: + text: 'Set right' + on_release: root.set_lr(False) + AutoLabel: + text: '[' + str(root.lr[0]) + ', ' + str(root.lr[1]) + ')' + RoundedButton: + text: 'Delete' + on_release: + root.del_lr(True) + msg = f'Delete records {root.lr} - press to see the ' \ + + f'effect, but you can delete multiple ranges before doing so.' + app.root.ids.status.text = msg + RoundedButton: + text: 'Restore' + on_release: + root.del_lr(False) + msg = f'Restore records {root.lr} - press to see the ' \ + + f'effect, but you can restore multiple ranges before doing so.' + app.root.ids.status.text = msg + RoundedButton: + text: 'Reload Tub' + on_release: + app.root.ids.tub_screen.ids.tub_loader.update_tub() + + +: + orientation: 'horizontal' + spacing: spacing + RoundedButton: + text: 'Set filter' + size_hint_x: 0.19 + on_release: root.update_filter() + MyTextInput: + id: record_filter + text: root.record_filter + multiline: False + on_focus: root.filter_focus() + + + +: + padding: 0 + spacing: spacing + RoundedButton: + text: 'Reload Graph' + on_release: root.plot_from_current_bars() + RoundedButton: + text: 'Browser Graph' + on_release: root.plot_from_current_bars(in_app=False) + + + + BoxLayout: + orientation: 'vertical' + padding: spacing + spacing: spacing + BoxLayout: + orientation: 'horizontal' + spacing: spacing + size_hint_y: 0.7 + ConfigManager: + id: config_manager + TubLoader: + id: tub_loader + + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: 1.8 + Header: + size_hint_y: 0.25 + title: 'Tub Navigator' + description: + "Use control panel or slider to navigate through tub and "\ + "select records. Add / remove record fields using the drop"\ + " down menu." + BoxLayout: + orientation: 'horizontal' + spacing: spacing + DataPanel: + id: data_panel + screen: root + record: root.current_record + FullImage: + id: img + ControlPanel + id: control_panel + size_hint_x: 0.75 + screen: root + Slider: + id: slider + size_hint_y: 0.25 + min: 0 + max: tub_loader.len - 1 + value: 0 + size_hint_y: None + height: common_height + on_value: root.index = int(self.value) + + BackgroundBoxLayout: + orientation: 'vertical' + Header: + size_hint_y: 1.2 + title: 'Tub Cleaner' + description: + "Select records using intervals to delete / restore or set "\ + "a filter. The filter is temporary in the app." + TubEditor: + id: tub_editor + TubFilter: + id: tub_filter + screen: root + + BackgroundBoxLayout: + orientation: 'vertical' + size_hint_y: 1.5 + Plot: + id: graph + DataPlot: + id: data_plot + + + diff --git a/donkeycar/management/ui/tub_screen.py b/donkeycar/management/ui/tub_screen.py new file mode 100644 index 000000000..7cf27879a --- /dev/null +++ b/donkeycar/management/ui/tub_screen.py @@ -0,0 +1,327 @@ +import os + +import numpy as np +import plotly.express as px +import pandas as pd + + +from kivy import Logger +from kivy.uix.boxlayout import BoxLayout +from kivy.properties import NumericProperty, ObjectProperty, StringProperty, \ + ListProperty, BooleanProperty +from kivy_garden.matplotlib.backend_kivyagg import FigureCanvasKivyAgg +import matplotlib as mpl +import matplotlib.pyplot as plt + + +from donkeycar import load_config +from donkeycar.management.ui.common import FileChooserBase, \ + PaddedBoxLayout, decompose, get_app_screen, BackgroundBoxLayout, AppScreen, \ + status +from donkeycar.management.ui.rc_file_handler import rc_handler +from donkeycar.parts.tub_v2 import Tub +from donkeycar.pipeline.types import TubRecord + + +mpl.rcParams.update({'font.size': 8}) +plt.style.use('dark_background') +fig, ax = plt.subplots() +plt.tight_layout(pad=1.5) +plt.subplots_adjust(bottom=0.16) +cmap = mpl.cm.get_cmap("plasma") + + +class ConfigManager(BackgroundBoxLayout, FileChooserBase): + """ Class to manage loading of the config file from the car directory""" + config = ObjectProperty(None, allownone=True) + file_path = StringProperty(rc_handler.data.get('car_dir', '')) + + def load_action(self): + """ Load the config from the file path""" + if not self.file_path: + return + try: + path = os.path.join(self.file_path, 'config.py') + new_conf = load_config(path) + self.config = new_conf + # If load successful, store into app config + rc_handler.data['car_dir'] = self.file_path + except FileNotFoundError: + Logger.error(f'Config: Directory {self.file_path} has no ' + f'config.py') + self.config = None + except Exception as e: + Logger.error(f'Config: {e}') + self.config = None + + def on_config(self, obj, cfg): + tub_screen = get_app_screen('tub') + if tub_screen: + tub_screen.ids.tub_loader.ids.tub_button.disabled = False + tub_screen.ids.tub_loader.root_path = self.file_path + pilot_screen = get_app_screen('pilot') + if pilot_screen: + pilot_screen.config = self.config + train_screen = get_app_screen('train') + if train_screen: + train_screen.config = self.config + car_screen = get_app_screen('car') + if car_screen: + car_screen.config = self.config + status('Config loaded from' + self.file_path) + + +class TubLoader(BackgroundBoxLayout, FileChooserBase): + """ Class to manage loading or reloading of the Tub from the tub directory. + Loading triggers many actions on other widgets of the app. """ + file_path = StringProperty(rc_handler.data.get('last_tub', '')) + tub = ObjectProperty(None) + len = NumericProperty(1) + records = None + + def load_action(self): + """ Update tub from the file path""" + if self.update_tub(): + # If update successful, store into app config + rc_handler.data['last_tub'] = self.file_path + + def update_tub(self, event=None): + if not self.file_path: + return False + # If config not yet loaded return + tub_screen = get_app_screen('tub') + cfg = tub_screen.ids.config_manager.config + if not cfg: + return False + # At least check if there is a manifest file in the tub path + if not os.path.exists(os.path.join(self.file_path, 'manifest.json')): + status(f'Path {self.file_path} is not a valid tub.') + return False + try: + if self.tub: + self.tub.close() + self.tub = Tub(self.file_path) + except Exception as e: + status(f'Failed loading tub: {str(e)}') + return False + # Check if filter is set in tub screen + # expression = tub_screen().ids.tub_filter.filter_expression + train_filter = getattr(cfg, 'TRAIN_FILTER', None) + + # Use filter, this defines the function + def select(underlying): + if not train_filter: + return True + else: + try: + record = TubRecord(cfg, self.tub.base_path, underlying) + res = train_filter(record) + return res + except KeyError as err: + Logger.error(f'Filter: {err}') + return True + + self.records = [TubRecord(cfg, self.tub.base_path, record) + for record in self.tub if select(record)] + self.len = len(self.records) + if self.len > 0: + tub_screen.index = 0 + tub_screen.ids.data_plot.update_dataframe_from_tub() + msg = f'Loaded tub {self.file_path} with {self.len} records' + get_app_screen('pilot').ids.slider.max = self.len - 1 + else: + msg = f'No records in tub {self.file_path}' + status(msg) + return True + + +class TubEditor(BoxLayout): + """ Tub editor widget. Contains left/right index interval and the + manipulator buttons for deleting / restoring and reloading """ + lr = ListProperty([0, 0]) + + def set_lr(self, is_l=True): + """ Sets left or right range to the current tub record index """ + if not get_app_screen('tub').current_record: + return + self.lr[0 if is_l else 1] \ + = get_app_screen('tub').current_record.underlying['_index'] + + def del_lr(self, is_del): + """ Deletes or restores records in chosen range """ + tub = get_app_screen('tub').ids.tub_loader.tub + if self.lr[1] >= self.lr[0]: + selected = list(range(*self.lr)) + else: + last_id = tub.manifest.current_index + selected = list(range(self.lr[0], last_id)) + selected += list(range(self.lr[1])) + tub.delete_records(selected) if is_del else tub.restore_records(selected) + + +class TubFilter(BoxLayout): + """ Tub filter widget. """ + screen = ObjectProperty() + filter_expression = StringProperty(None) + record_filter = StringProperty(rc_handler.data.get('record_filter', '')) + + def update_filter(self): + filter_text = self.ids.record_filter.text + config = get_app_screen('tub').ids.config_manager.config + # empty string resets the filter + if filter_text == '': + self.record_filter = '' + self.filter_expression = '' + rc_handler.data['record_filter'] = self.record_filter + if hasattr(config, 'TRAIN_FILTER'): + delattr(config, 'TRAIN_FILTER') + status(f'Filter cleared') + return + filter_expression = self.create_filter_string(filter_text) + try: + record = get_app_screen('tub').current_record + filter_func_text = f"""def filter_func(record): + return {filter_expression} + """ + # creates the function 'filter_func' + ldict = {} + exec(filter_func_text, globals(), ldict) + filter_func = ldict['filter_func'] + res = filter_func(record) + msg = f'Filter result on current record: {res}' + if isinstance(res, bool): + self.record_filter = filter_text + self.filter_expression = filter_expression + rc_handler.data['record_filter'] = self.record_filter + setattr(config, 'TRAIN_FILTER', filter_func) + else: + msg += ' - non bool expression can\'t be applied' + msg += ' - press to see effect' + status(msg) + except Exception as e: + status(f'Filter error on current record: {e}') + + def filter_focus(self): + focus = self.ids.record_filter.focus + self.screen.keys_enabled = not focus + if not focus: + self.screen.bind_keyboard() + + @staticmethod + def create_filter_string(filter_text, record_name='record'): + """ Converts text like 'user/angle' into 'record.underlying['user/angle'] + so that it can be used in a filter. Will replace only expressions that + are found in the tub inputs list. + + :param filter_text: input text like 'user/throttle > 0.1' + :param record_name: name of the record in the expression + :return: updated string that has all input fields wrapped + """ + for field in get_app_screen('tub').current_record.underlying.keys(): + field_list = filter_text.split(field) + if len(field_list) > 1: + filter_text = f'{record_name}.underlying["{field}"]'\ + .join(field_list) + return filter_text + + +class Plot(FigureCanvasKivyAgg): + df = ObjectProperty(force_dispatch=True, allownone=True) + + def __init__(self, **kwargs): + super().__init__(fig, **kwargs) + + def on_df(self, e=None, z=None): + ax.clear() + if not self.df.empty: + n = len(self.df.columns) + it = np.linspace(0, 1, n) + self.df.plot(ax=ax, linewidth=0.5, color=cmap(it)) + # Put a legend to the right of the current axis + ax.legend(loc='center left', bbox_to_anchor=(0.87, 0.5)) + self.draw() + + +class DataPlot(PaddedBoxLayout): + """ Data plot panel which embeds matplotlib interactive graph""" + df = ObjectProperty(force_dispatch=True, allownone=True) + + def plot_from_current_bars(self, in_app=True): + """ Plotting from current selected bars. The DataFrame for plotting + should contain all bars except for strings fields and all data is + selected if bars are empty. """ + tub = get_app_screen('tub').ids.tub_loader.tub + field_map = dict(zip(tub.manifest.inputs, tub.manifest.types)) + # Use selected fields or all fields if nothing is slected + all_cols = (get_app_screen('tub').ids.data_panel.labels.keys() + or self.df.columns) + cols = [c for c in all_cols if decompose(c)[0] in field_map + and field_map[decompose(c)[0]] not in ('image_array', 'str')] + + df = self.df[cols] + if df is None: + return + # Don't plot the milliseconds time stamp as this is a too big number + df = df.drop(labels=['_timestamp_ms'], axis=1, errors='ignore') + + if in_app: + get_app_screen('tub').ids.graph.df = df + else: + fig = px.line(df, x=df.index, y=df.columns, title=tub.base_path) + fig.update_xaxes(rangeslider=dict(visible=True)) + fig.show() + + def unravel_vectors(self): + """ Unravels vector and list entries in tub which are created + when the DataFrame is created from a list of records""" + manifest = get_app_screen('tub').ids.tub_loader.tub.manifest + for k, v in zip(manifest.inputs, manifest.types): + if v == 'vector' or v == 'list': + dim = len(get_app_screen('tub').current_record.underlying[k]) + df_keys = [k + f'_{i}' for i in range(dim)] + self.df[df_keys] = pd.DataFrame(self.df[k].tolist(), + index=self.df.index) + self.df.drop(k, axis=1, inplace=True) + + def update_dataframe_from_tub(self): + """ Called from TubManager when a tub is reloaded/recreated. Fills + the DataFrame from records, and updates the dropdown menu in the + data panel.""" + tub_screen = get_app_screen('tub') + generator = (t.underlying for t in tub_screen.ids.tub_loader.records) + self.df = pd.DataFrame(generator).dropna() + to_drop = ['cam/image_array'] + self.df.drop(labels=to_drop, axis=1, errors='ignore', inplace=True) + self.df.set_index('_index', inplace=True) + self.unravel_vectors() + tub_screen.ids.data_panel.ids.data_spinner.values = self.df.columns + self.plot_from_current_bars() + + +class TubScreen(AppScreen): + """ First screen of the app managing the tub data. """ + index = NumericProperty(None, force_dispatch=True) + current_record = ObjectProperty(None) + keys_enabled = BooleanProperty(True) + + def initialise(self, e): + self.ids.config_manager.load_action() + self.ids.tub_loader.update_tub() + + def on_index(self, obj, index): + """ Kivy method that is called if self.index changes""" + if index >= 0: + self.current_record = self.ids.tub_loader.records[index] + self.ids.slider.value = index + + def on_current_record(self, obj, record): + """ Kivy method that is called if self.current_record changes.""" + self.ids.img.update(record) + i = record.underlying['_index'] + self.ids.control_panel.record_display = f"Record {i:06}" + + def on_keyboard(self, keyboard, scancode, text=None, modifier=None): + if not self.keys_enabled: + return + self.ids.control_panel.on_keyboard(keyboard, scancode, text, modifier) + diff --git a/donkeycar/management/ui/ui.kv b/donkeycar/management/ui/ui.kv new file mode 100644 index 000000000..44675e874 --- /dev/null +++ b/donkeycar/management/ui/ui.kv @@ -0,0 +1,78 @@ + +: + BoxLayout: + padding: 10 + orientation: 'vertical' + Image: + source: root.img_path + size: self.texture_size + + +BoxLayout: + orientation: 'vertical' + ActionView: + id: av + size_hint_y: None + height: 20 + + ActionPrevious: + with_previous: False + app_icon: '' + size_hint_x: None + width: 0 + ActionButton: + id: tub_btn + color: font_color + text: 'Tub Manager' + on_release: + sm.current = 'tub' + sm.current_screen.bind_keyboard() + ActionButton: + id: train_btn + color: font_color + text: 'Trainer' + on_release: + sm.current = 'train' + sm.current_screen.bind_keyboard() + ActionButton: + id: pilot_btn + color: font_color + text: 'Pilot Arena' + on_release: + sm.current = 'pilot' + if not sm.current_screen.index: sm.current_screen.index = 0 + sm.current_screen.bind_keyboard() + ActionButton: + id: car_btn + color: font_color + text: 'Car Connector' + on_release: + sm.current = 'car' + sm.current_screen.update_pilots() + sm.current_screen.bind_keyboard() + ActionButton: + color: font_color + text: 'Quit' + on_release: app.get_running_app().stop() + + ScreenManager: + id: sm + StartScreen: + id: start_screen + name: 'start' + TubScreen: + id: tub_screen + name: 'tub' + TrainScreen: + id: train_screen + name: 'train' + PilotScreen: + id: pilot_screen + name: 'pilot' + CarScreen: + id: car_screen + name: 'car' + + StatusBar: + id: status + text: "Donkey ready" \ No newline at end of file diff --git a/donkeycar/management/ui/ui.py b/donkeycar/management/ui/ui.py new file mode 100644 index 000000000..7a1b8f20f --- /dev/null +++ b/donkeycar/management/ui/ui.py @@ -0,0 +1,68 @@ +import os +# need to do this before importing anything else +os.environ['KIVY_LOG_MODE'] = 'MIXED' + +from kivy.logger import Logger, LOG_LEVELS +from kivy.clock import Clock +from kivy.app import App +from kivy.properties import StringProperty +from kivy.uix.boxlayout import BoxLayout +from kivy.lang.builder import Builder +from kivy.core.window import Window + +from donkeycar.management.ui.car_screen import CarScreen +from donkeycar.management.ui.pilot_screen import PilotScreen +from donkeycar.management.ui.rc_file_handler import rc_handler +from donkeycar.management.ui.train_screen import TrainScreen +from donkeycar.management.ui.tub_screen import TubScreen +from donkeycar.management.ui.common import AppScreen + +Logger.setLevel(LOG_LEVELS["info"]) +Window.size = (800, 800) + + +class Header(BoxLayout): + title = StringProperty() + description = StringProperty() + + +class StartScreen(AppScreen): + img_path = os.path.realpath(os.path.join( + os.path.dirname(__file__), + '../../parts/web_controller/templates/' + 'static/donkeycar-logo-sideways.png')) + pass + + +class DonkeyApp(App): + title = 'Donkey Car' + + def initialise(self, event): + self.root.ids.tub_screen.ids.config_manager.load_action() + self.root.ids.train_screen.initialise() + self.root.ids.pilot_screen.initialise(event) + self.root.ids.car_screen.initialise() + self.root.ids.tub_screen.ids.tub_loader.update_tub() + self.root.ids.status.text = 'Donkey ready' + + def build(self): + # the builder returns the screen manager in ui.kv file + for kv in ['common.kv', 'tub_screen.kv', 'train_screen.kv', + 'pilot_screen.kv', 'car_screen.kv', 'ui.kv']: + dm = Builder.load_file(os.path.join(os.path.dirname(__file__), kv)) + Clock.schedule_once(self.initialise) + return dm + + def on_stop(self, *args): + tub = self.root.ids.tub_screen.ids.tub_loader.tub + if tub: + tub.close() + Logger.info("App: Good bye Donkey") + + +def main(): + DonkeyApp().run() + + +if __name__ == '__main__': + main() diff --git a/donkeycar/pipeline/augmentations.py b/donkeycar/pipeline/augmentations.py index b44e4e5fe..0842c9a75 100644 --- a/donkeycar/pipeline/augmentations.py +++ b/donkeycar/pipeline/augmentations.py @@ -20,8 +20,8 @@ def __init__(self, cfg, key, prob=0.5, always_apply=False): @classmethod def create(cls, aug_type: str, config: Config, prob, always) -> \ albumentations.core.transforms_interface.BasicTransform: - """ Augmenatition factory. Cropping and trapezoidal mask are - transfomations which should be applied in training, validation + """ Augmentation factory. Cropping and trapezoidal mask are + transformations which should be applied in training, validation and inference. Multiply, Blur and similar are augmentations which should be used only in training. """ diff --git a/donkeycar/pipeline/database.py b/donkeycar/pipeline/database.py index 4dc5a52fd..ef2221791 100644 --- a/donkeycar/pipeline/database.py +++ b/donkeycar/pipeline/database.py @@ -60,7 +60,7 @@ def to_df(self) -> pd.DataFrame: def write(self): try: with open(self.path, "w") as data_file: - json.dump(self.entries, data_file, + json.dump(self.entries, data_file, indent=4, default=lambda o: '') logger.info(f'Writing database file: {self.path}') except Exception as e: @@ -70,21 +70,27 @@ def add_entry(self, entry: Dict): self.entries.append(entry) def delete_entry(self, pilot_name): - to_delete_entry = None + to_delete_entry = self.get_entry(pilot_name) + if not to_delete_entry: + return + full_path = os.path.join(self.cfg.MODELS_PATH, pilot_name) + model_versions = glob.glob(f'{full_path}.*') + logger.info(f'Deleting {",".join(model_versions)}') + for model_version in model_versions: + if os.path.isdir(model_version): + shutil.rmtree(model_version, ignore_errors=True) + else: + os.remove(model_version) + self.entries.remove(to_delete_entry) + self.write() + + def get_entry(self, pilot_name): for entry in self.entries: if entry['Name'] == pilot_name: - to_delete_entry = entry - if to_delete_entry: - full_path = os.path.join(self.cfg.MODELS_PATH, pilot_name) - model_versions = glob.glob(f'{full_path}.*') - logger.info(f'Deleting {",".join(model_versions)}') - for model_version in model_versions: - if os.path.isdir(model_version): - shutil.rmtree(model_version, ignore_errors=True) - else: - os.remove(model_version) - self.entries.remove(to_delete_entry) - self.write() + return entry + logger.warning(f'Could not find pilot {pilot_name}, known pilots:' + f'{",".join(self.get_pilot_names())}') + return None def to_df_tubgrouped(self): def sorted_string(comma_separated_string): @@ -138,3 +144,6 @@ def pretty_print(self, group_tubs=False): pilot_text = pilot_df.to_string(formatters=self.formatter()) pilot_names = pilot_df['Name'].tolist() if not pilot_df.empty else [] return pilot_text, tub_text, pilot_names + + def get_pilot_names(self): + return [entry['Name'] for entry in self.entries] diff --git a/donkeycar/pipeline/training.py b/donkeycar/pipeline/training.py index 9f0a8d3dd..09d027cb6 100644 --- a/donkeycar/pipeline/training.py +++ b/donkeycar/pipeline/training.py @@ -187,7 +187,8 @@ def train(cfg: Config, tub_paths: str, model: str = None, database_entry = { 'Number': model_num, 'Name': os.path.basename(base_path), - 'Type': str(kl), + 'Pilot': str(kl), + 'Type': model_type, 'Tubs': tub_paths, 'Time': time(), 'History': history, @@ -198,4 +199,4 @@ def train(cfg: Config, tub_paths: str, model: str = None, database.add_entry(database_entry) database.write() - return history \ No newline at end of file + return history diff --git a/donkeycar/pipeline/types.py b/donkeycar/pipeline/types.py index 3cc61cfd7..27df5b1cb 100644 --- a/donkeycar/pipeline/types.py +++ b/donkeycar/pipeline/types.py @@ -1,17 +1,24 @@ -import copy +from copy import copy import os +from enum import Enum from typing import Any, List, Optional, TypeVar, Iterator, Iterable import logging import numpy as np from donkeycar.config import Config from donkeycar.parts.tub_v2 import Tub -from donkeycar.utils import load_image, load_pil_image +from donkeycar.utils import load_image, load_pil_image, binary_to_img, \ + img_to_arr, img_to_binary, arr_to_binary from typing_extensions import TypedDict logger = logging.getLogger(__name__) -X = TypeVar('X', covariant=True) + +class CachePolicy(Enum): + NOCACHE = 0 + BINARY = 1 + ARRAY = 2 + TubRecordDict = TypedDict( 'TubRecordDict', @@ -40,9 +47,23 @@ def __init__(self, config: Config, base_path: str, self.config = config self.base_path = base_path self.underlying = underlying + self._cache_policy = CachePolicy[ + getattr(self.config, 'CACHE_POLICY', 'ARRAY')] self._cache_images = getattr(self.config, 'CACHE_IMAGES', True) self._image: Optional[Any] = None + def __copy__(self): + """ Make shallow copies of config and image and full copies of the rest. + :return TubRecord: TubRecord copy + """ + tubrec = TubRecord(self.config, + copy(self.base_path), + copy(self.underlying)) + tubrec._cache_policy = copy(self._cache_policy) + tubrec._cache_images = copy(self._cache_images) + tubrec._image = self._image + return tubrec + def image(self, processor=None, as_nparray=True) -> np.ndarray: """ Loads the image. @@ -55,21 +76,77 @@ def image(self, processor=None, as_nparray=True) -> np.ndarray: :return: Image """ if self._image is None: - image_path = self.underlying['cam/image_array'] - full_path = os.path.join(self.base_path, 'images', image_path) - - if as_nparray: - _image = load_image(full_path, cfg=self.config) - else: - # If you just want the raw Image - _image = load_pil_image(full_path, cfg=self.config) + _image = self._extract_image(as_nparray, processor) + else: + _image = self._image_from_cache(as_nparray) if processor: _image = processor(_image) - # only cache images if config does not forbid it - if self._cache_images: + return _image + + def _image_from_cache(self, as_nparray): + """ + Cache policy only supports numpy array format + :return: Numpy array from cache + """ + if not as_nparray: + return self._image + + if self._cache_policy == CachePolicy.NOCACHE: + raise RuntimeError("Found cached image with policy NOCACHE") + elif self._cache_policy == CachePolicy.ARRAY: + return self._image + elif self._cache_policy == CachePolicy.BINARY: + return img_to_arr(binary_to_img(self._image)) + else: + raise RuntimeError(f"Unhandled cache policy {self._cache_policy}") + + def _load_image_and_cache(self, img_path): + # if no caching, just load but don't cache + if self._cache_policy == CachePolicy.NOCACHE: + _image = load_image(img_path, cfg=self.config) + # if caching full array, load and cache array + elif self._cache_policy == CachePolicy.ARRAY: + _image = load_image(img_path, cfg=self.config) + self._image = _image + # if caching is binary, only cache binary but return full array + elif self._cache_policy == CachePolicy.BINARY: + with open(img_path, 'rb') as f: + _image = f.read() self._image = _image + _image = img_to_arr(binary_to_img(_image)) + return _image + + def _load_pil_image_and_cache(self, img_path): + _image = load_pil_image(img_path, cfg=self.config) + if self._cache_policy != CachePolicy.NOCACHE: + self._image = _image + return _image + + def _cache_processed_image(self, image, as_nparray): + if not as_nparray: + if self._cache_policy != CachePolicy.NOCACHE: + self._image = image + return + # if numpy and array caching, cache the processed image + if self._cache_policy == CachePolicy.ARRAY: + self._image = image + # if numpy and binary caching, cache binary image, but return + # numpy + elif self._cache_policy == CachePolicy.BINARY: + self._image = arr_to_binary(image) + # in the case of no caching, nothing needs to be done here + + def _extract_image(self, as_nparray, processor): + image_path = self.underlying['cam/image_array'] + full_path = os.path.join(self.base_path, 'images', image_path) + if as_nparray: + _image = self._load_image_and_cache(full_path) else: - _image = self._image + _image = self._load_pil_image_and_cache(full_path) + if processor: + # _image is now either numpy or PIL, so processing applies always + _image = processor(_image) + self._cache_processed_image(_image, as_nparray) return _image def __repr__(self) -> str: @@ -110,7 +187,7 @@ def close(self): class Collator(Iterable[List[TubRecord]]): - """" Builds a sequence of continuous records for RNN and similar models. """ + """ Builds a sequence of continuous records for RNN and similar models. """ def __init__(self, seq_length: int, records: List[TubRecord]): """ :param seq_length: length of sequence @@ -123,6 +200,7 @@ def __init__(self, seq_length: int, records: List[TubRecord]): def is_continuous(rec_1: TubRecord, rec_2: TubRecord) -> bool: """ Checks if second record is next to first record + :param rec_1: first record :param rec_2: second record :return: if first record is followed by second record @@ -137,7 +215,7 @@ def __iter__(self) -> Iterator[List[TubRecord]]: it = iter(self.records) for this_record in it: seq = [this_record] - seq_it = copy.copy(it) + seq_it = copy(it) for next_record in seq_it: if self.is_continuous(this_record, next_record) and \ len(seq) < self.seq_length: diff --git a/donkeycar/templates/basic.py b/donkeycar/templates/basic.py index 7d2b71aeb..8881a1844 100755 --- a/donkeycar/templates/basic.py +++ b/donkeycar/templates/basic.py @@ -21,7 +21,7 @@ from donkeycar.parts.actuator import PCA9685, PWMSteering, PWMThrottle from donkeycar.pipeline.augmentations import ImageAugmentation -logger = logging.getLogger() +logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) diff --git a/donkeycar/templates/cfg_complete.py b/donkeycar/templates/cfg_complete.py index 464e8837f..d468f6b3c 100644 --- a/donkeycar/templates/cfg_complete.py +++ b/donkeycar/templates/cfg_complete.py @@ -384,7 +384,7 @@ CREATE_TF_LITE = True # automatically create tflite model in training CREATE_TENSOR_RT = False # automatically create tensorrt model in training SAVE_MODEL_AS_H5 = False # if old keras format should be used instead of savedmodel -CACHE_IMAGES = True # if images are cached in training for speed up +CACHE_POLICY = 'ARRAY' # if images are cached as array in training other options are 'NOCACHE' and 'BINARY' PRUNE_CNN = False #This will remove weights from your model. The primary goal is to increase performance. PRUNE_PERCENT_TARGET = 75 # The desired percentage of pruning. diff --git a/donkeycar/tests/test_scripts.py b/donkeycar/tests/test_scripts.py index 8d69f9261..3001feea7 100755 --- a/donkeycar/tests/test_scripts.py +++ b/donkeycar/tests/test_scripts.py @@ -1,7 +1,5 @@ import os -import platform import subprocess -import sys import tarfile from donkeycar import utils diff --git a/setup.cfg b/setup.cfg index 76ca5e58b..e08f42596 100644 --- a/setup.cfg +++ b/setup.cfg @@ -49,8 +49,15 @@ pi = adafruit-circuitpython-ssd1306 adafruit-circuitpython-rplidar RPi.GPIO + flatbuffers==24.3.* tensorflow-aarch64==2.15.* opencv-contrib-python + matplotlib + kivy + kivy-garden.matplotlib + pandas + plotly + albumentations nano = Adafruit_PCA9685 @@ -60,6 +67,7 @@ nano = numpy==1.23.* matplotlib==3.7.* kivy + kivy-garden.matplotlib plotly pandas==2.0.* @@ -67,6 +75,7 @@ pc = tensorflow==2.15.* matplotlib kivy + kivy-garden.matplotlib pandas plotly albumentations @@ -75,6 +84,7 @@ macos = tensorflow-macos==2.15.* matplotlib kivy + kivy-garden.matplotlib pandas plotly albumentations