1109 refactor augmentations/transformations (#1111)

* moved transformations to cv.py; added colorspace transforms
* add parts for crop and trapeze transformations

* Update training to use new transformations
- image_transformations.py use cv.py parts for transforms
- augmentations2.py remove old transformations
- training.py updated to allow for transformations
  both before and after training augmentation.
- cfg_complete.py added new tranformation config
  and a lot of documentation comments.

* Port albumentations augmentations from Dirk's branch
- replace imgaug with albumentations
- update install to include albumentations

* transforms from gray to color, cvcam resizes
- conversions from grey to BGR and RGB
- fixed CvCam so it will resize the image if
  the camera does not produce the desired size.

* Update deep learning to show inference image
- Now we can show the image used for inference
  when in autopilot mode.  set `SHOW_PILOT_IMAGE`
  to True.

* augmentations use albumentations library
- ported from Dirk's branch
- remove transformations from ImageAugmentation class in favor of new
  transformations in ImageTransformations class

* Can now add a custom image transformation to pipeline
- name the label starting with "CUSTOM", like "CUSTOM_CROP"
- include config to use to get the module and class names.
  these being with the custom label and end with
  "_MODULE" and "_CLASS" respectively, like
  "CUSTOM_CROP_MODULE" and "CUSTOM_CROP_CLASS"
- The custom transformer class must take a Config instance
  in the constructor and must have a run method that
  takes and image and returns an image.

* Change custom transformation loader to use file path
- the prior design did not work for both driving from the
  mycar folder and training (which uses the donkey module)
- the new design dynamically loads the module given it's
  file path and caches it.  As long as the file path
  is either an absolute path then this will work when
  driving or training without changing myconfig.py.
- this design allows more then on custom transformer
  class per file.

* Change imgaug -> albumentations in the UI. Update yaml files to remove opencv-headless which clashes w/ opencv. Unfreeze numpy version.

* Peg numpy to 1.19 in order to be aligned w/ tf 2.2.0 and replace remaining instances of MULTIPLY with BRIGHTNESS.

* Align interface of ImageTransformations with ImageAugmentations to take a config and a string, representing the key of transformations in the config file, and another optional string of post transformations. Update ui to support pre and post transformations.


* version='4.4.dev7'

---------

Co-authored-by: DocGarbanzo <[email protected]>

donkeycar/management/kivy_ui.py

@@ -25,6 +25,7 @@
 from kivy.properties import NumericProperty, ObjectProperty, StringProperty, \
     ListProperty, BooleanProperty
 from kivy.uix.label import Label
+from kivy.uix.button import Button
 from kivy.uix.popup import Popup
 from kivy.lang.builder import Builder
 from kivy.core.window import Window
@@ -33,6 +34,7 @@
 from kivy.uix.spinner import SpinnerOption, Spinner
 
 from donkeycar import load_config
+from donkeycar.parts.image_transformations import ImageTransformations
 from donkeycar.parts.tub_v2 import Tub
 from donkeycar.pipeline.augmentations import ImageAugmentation
 from donkeycar.pipeline.database import PilotDatabase
@@ -739,6 +741,8 @@ def augment(self, img_arr):
             img_arr = pilot_screen().transformation.run(img_arr)
         if pilot_screen().aug_list:
             img_arr = pilot_screen().augmentation.run(img_arr)
+        if pilot_screen().post_trans_list:
+            img_arr = pilot_screen().post_transformation.run(img_arr)
         return img_arr
 
     def get_image(self, record):
@@ -778,6 +782,59 @@ def get_image(self, record):
         return img_arr
 
 
+class TransformationPopup(Popup):
+    """ Transformation popup window"""
+    title = StringProperty()
+    transformations = \
+        ["TRAPEZE", "CROP", "RGB2BGR", "BGR2RGB", "RGB2HSV", "HSV2RGB",
+         "BGR2HSV", "HSV2BGR", "RGB2GRAY", "RBGR2GRAY", "HSV2GRAY", "GRAY2RGB",
+         "GRAY2BGR", "CANNY", "BLUR", "RESIZE", "SCALE"]
+    transformations_obj = ObjectProperty()
+    selected = ListProperty()
+
+    def __init__(self, selected, **kwargs):
+        super().__init__(**kwargs)
+        for t in self.transformations:
+            btn = Button(text=t)
+            btn.bind(on_release=self.toggle_transformation)
+            self.ids.trafo_list.add_widget(btn)
+        self.selected = selected
+
+    def toggle_transformation(self, btn):
+        trafo = btn.text
+        if trafo in self.selected:
+            self.selected.remove(trafo)
+        else:
+            self.selected.append(trafo)
+
+    def on_selected(self, obj, select):
+        self.ids.selected_list.clear_widgets()
+        for l in self.selected:
+            lab = Label(text=l)
+            self.ids.selected_list.add_widget(lab)
+        self.transformations_obj.selected = self.selected
+
+
+class Transformations(Button):
+    """ Base class for transformation widgets"""
+    title = StringProperty(None)
+    pilot_screen = ObjectProperty()
+    is_post = False
+    selected = ListProperty()
+
+    def open_popup(self):
+        popup = TransformationPopup(title=self.title, transformations_obj=self,
+                                    selected=self.selected)
+        popup.open()
+
+    def on_selected(self, obj, select):
+        Logger.info(f"Selected {select}")
+        if self.is_post:
+            self.pilot_screen.post_trans_list = self.selected
+        else:
+            self.pilot_screen.trans_list = self.selected
+
+
 class PilotScreen(Screen):
     """ Screen to do the pilot vs pilot comparison ."""
     index = NumericProperty(None, force_dispatch=True)
@@ -787,6 +844,8 @@ class PilotScreen(Screen):
     augmentation = ObjectProperty()
     trans_list = ListProperty(force_dispatch=True)
     transformation = ObjectProperty()
+    post_trans_list = ListProperty(force_dispatch=True)
+    post_transformation = ObjectProperty()
     config = ObjectProperty()
 
     def on_index(self, obj, index):
@@ -829,13 +888,16 @@ def set_brightness(self, val=None):
         if not self.config:
             return
         if self.ids.button_bright.state == 'down':
-            self.config.AUG_MULTIPLY_RANGE = (val, val)
-            if 'MULTIPLY' not in self.aug_list:
-                self.aug_list.append('MULTIPLY')
-        elif 'MULTIPLY' in self.aug_list:
-            self.aug_list.remove('MULTIPLY')
-        # update dependency
-        self.on_aug_list(None, None)
+            self.config.AUG_BRIGHTNESS_RANGE = (val, val)
+            if 'BRIGHTNESS' not in self.aug_list:
+                self.aug_list.append('BRIGHTNESS')
+            else:
+                # Since we only changed the content of the config here,
+                # self.on_aug_list() would not be called, but we want to update
+                # the augmentation. Hence, update the dependency manually here.
+                self.on_aug_list(None, None)
+        elif 'BRIGHTNESS' in self.aug_list:
+            self.aug_list.remove('BRIGHTNESS')
 
     def set_blur(self, val=None):
         if not self.config:
@@ -853,14 +915,24 @@ def on_aug_list(self, obj, aug_list):
         if not self.config:
             return
         self.config.AUGMENTATIONS = self.aug_list
-        self.augmentation = ImageAugmentation(self.config, 'AUGMENTATIONS')
+        self.augmentation = ImageAugmentation(
+            self.config, 'AUGMENTATIONS', always_apply=True)
         self.on_current_record(None, self.current_record)
 
     def on_trans_list(self, obj, trans_list):
         if not self.config:
             return
         self.config.TRANSFORMATIONS = self.trans_list
-        self.transformation = ImageAugmentation(self.config, 'TRANSFORMATIONS')
+        self.transformation = ImageTransformations(
+            self.config, 'TRANSFORMATIONS')
+        self.on_current_record(None, self.current_record)
+
+    def on_post_trans_list(self, obj, trans_list):
+        if not self.config:
+            return
+        self.config.POST_TRANSFORMATIONS = self.post_trans_list
+        self.post_transformation = ImageTransformations(
+            self.config, 'POST_TRANSFORMATIONS')
         self.on_current_record(None, self.current_record)
 
     def set_mask(self, state):

donkeycar/management/makemovie.py

@@ -84,7 +84,8 @@ def run(self, args, parser):
     def draw_line_into_image(angle, throttle, is_left, img, color):
         import cv2
 
-        height, width, _ = img.shape
+        height = img.shape[0]
+        width = img.shape[1]
         length = height
         a1 = angle * 45.0
         l1 = throttle * length

donkeycar/management/ui.kv

@@ -381,6 +381,30 @@
         text: root.file_path
 
 
+<TransformationPopup>:
+    size_hint: 1.0, 1.0
+    auto_dismiss: False
+    pos_hint: {'center_x': .5, 'center_y': .5}
+
+    BoxLayout:
+        orientation: "horizontal"
+        BoxLayout:
+            orientation: 'vertical'
+            id: selected_list
+            spacing: 20
+
+        BoxLayout:
+            orientation: 'vertical'
+            spacing: 20
+            BoxLayout:
+                orientation: 'vertical'
+                id: trafo_list
+            Button:
+                text: "Close"
+                size_hint_y: 0.1
+                on_release: root.dismiss()
+
+
 <PilotScreen>:
     config: app.tub_screen.ids.config_manager.config
     BoxLayout:
@@ -443,9 +467,9 @@
                 on_press: root.set_brightness(slider_bright.value)
             Slider:
                 id: slider_bright
-                value: 1
-                min: 0
-                max: 3
+                value: 0
+                min: -0.5
+                max: 0.5
                 on_value: root.set_brightness(self.value)
             ToggleButton:
                 id: button_blur
@@ -455,18 +479,24 @@
             Slider:
                 id: slider_blur
                 value: 0
-                min: 0
-                max: 4
+                min: 0.001
+                max: 3
                 on_value: root.set_blur(self.value)
         PaddedBoxLayout:
-            ToggleButton:
-                id: button_mask
-                text: 'Trapezoidal mask'
-                on_press: root.set_mask(self.state)
-            ToggleButton:
-                id: button_crop
-                text: 'Cropping mask'
-                on_press: root.set_crop(self.state)
+            Transformations:
+                id: pre_transformation
+                title: 'Pre-Augmentation Transformations'
+                text: 'Set pre transformation'
+                pilot_screen: root
+                is_post: False
+                on_press: self.open_popup()
+            Transformations:
+                id: post_transformation
+                title: 'Post-Augmentation Transformations'
+                text: 'Set post transformation'
+                pilot_screen: root
+                is_post: True
+                on_press: self.open_popup()
         PaddedBoxLayout:
             size_hint_y: 0.5
             ControlPanel: