Keras v3 Support

hls4ml/backends/catapult/catapult_backend.py

@@ -88,6 +88,7 @@ def _register_flows(self):
         init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
 
         streaming_passes = [
+            'catapult:inplace_stream_flatten',  # Inform downstream changed packsize in case of skipping flatten
             'catapult:reshape_stream',
             'catapult:clone_output',
             'catapult:insert_zero_padding_before_conv1d',

hls4ml/backends/fpga/passes/clone.py

@@ -54,41 +54,59 @@ def match(self, node):
         if isinstance(node, Clone):
             return False
 
-        return True
+        # Not needed for io_parallel
+        io_type = node.model.config.get_config_value('IOType')
+        if io_type != 'io_stream':
+            return False
+
+        # Check if the output is used more than once
+        output_map = node.get_output_use_map()
+        in_output = node.name in node.model.outputs
+        for output in node.outputs:
+            if len(output_map[output]) + in_output > 1:
+                # model output also need a stream
+                return True
+
+        return False
 
     def transform(self, model, node):
-        if model.config.get_config_value('IOType') != 'io_stream':
-            return False
 
         output_map = node.get_output_use_map()
+        in_output = node.name in node.model.outputs
 
         transformed = False
         for output in node.outputs:
-            if len(output_map[output]) > 1:
-                if len(output_map[output]) > 3:
-                    print(
-                        'WARNING: Cloning output {} of {} ({}) more than 3 times not currently supported'.format(
-                            output, node.__class__.__name__, node.name
-                        )
-                    )
-                    return False
-                out_var = node.get_output_variable(output)
-                for i, layer in enumerate(output_map[output], 1):
-                    attrs = {'size': np.prod(out_var.shape)}
-                    idx = layer.inputs.index(output)
-                    layer.inputs[idx] = output + '_cpy' + str(i)
-
-                clone_layer: Clone = model.make_node(
-                    Clone,
-                    'clone_' + node.name,
-                    attrs,
-                    [output],
-                    [output + '_cpy' + str(i + 1) for i in range(len(output_map[output]))],
-                )
-                for i in range(len(output_map[output])):
-                    key = output + '_cpy' + str(i + 1)
-                    clone_layer.attributes[key].type = node.attributes['result_t']
-                model.insert_node(clone_layer)
-                transformed = True
+            n_outputs = len(output_map[output]) + in_output
+            if n_outputs == 1:
+                continue
+            if n_outputs > 3:
+                msg = f'ERROR: Cloning output {output} of {node.__class__.__name__} ({node.name}) more than 3 times not currently supported'  # noqa: E501
+                raise ValueError(msg)
+
+            out_var = node.get_output_variable(output)
+            attrs = {'size': np.prod(out_var.shape)}
+
+            i0 = 1
+            if in_output:
+                # If the value is used as output, add one extra stream
+                idx = node.model.outputs.index(node.name)
+                node.model.outputs[idx] = node.name + '_cpy1'
+                i0 = 2
+            for i, layer in enumerate(output_map[output], i0):
+                idx = layer.inputs.index(output)
+                layer.inputs[idx] = output + f'_cpy{i}'
+
+            clone_layer: Clone = model.make_node(
+                Clone,
+                'clone_' + node.name,
+                attrs,
+                [output],
+                [output + '_cpy' + str(i + 1) for i in range(n_outputs)],
+            )
+            for i in range(n_outputs):
+                key = output + '_cpy' + str(i + 1)
+                clone_layer.attributes[key].type = node.attributes['result_t']
+            model.insert_node(clone_layer)
+            transformed = True
 
         return transformed

hls4ml/backends/fpga/passes/inplace_parallel_reshape.py

@@ -11,14 +11,19 @@ class InplaceParallelReshape(OptimizerPass):
     """
 
     def match(self, node):
-        return isinstance(node, Reshape)
+        if not isinstance(node, Reshape):
+            return
+        return node.model.config.get_config_value('IOType') == 'io_parallel'
 
     def transform(self, model, node):
-        if model.config.get_config_value('IOType') != 'io_parallel':
-            return False
-
         outvar = node.get_output_variable()
         invar = node.get_input_variable()
         newoutvar = InplaceTensorVariable(outvar, invar)
         node.set_attr(node.outputs[0], newoutvar)
+        if node.name in model.outputs:
+            prev_node = node.get_input_node()
+            assert (
+                prev_node.name not in model.outputs
+            ), f"Cannot output node {prev_node.name}: reshape is a no-op in io_parallel. As a result, the previous node {prev_node.name}'s output will be used as the output. However, this node is already an output."  # noqa: E501
+            model.outputs = [name if name != node.name else prev_node.name for name in model.outputs]
         return False

hls4ml/backends/fpga/passes/inplace_stream_flatten.py

@@ -12,12 +12,13 @@ class InplaceStreamFlatten(OptimizerPass):
 
     def match(self, node):
         # Reshape acts as a Flatten layer when the result has 1 dimension
-        return isinstance(node, Reshape) and len(node.get_output_variable().shape) == 1
-
-    def transform(self, model, node):
-        if model.config.get_config_value('IOType') != 'io_stream':
+        if not (isinstance(node, Reshape) and len(node.get_output_variable().shape)) == 1:
+            # Reshape with multiple outputs will be kept as is, or repack cannot handle different shapes
             return False
+        io_type = node.model.config.get_config_value('IOType')
+        return io_type == 'io_stream'
 
+    def transform(self, model, node):
         outvar = node.get_output_variable()
         invar = node.get_input_variable()
         newoutvar = InplaceTensorVariable(outvar, invar)

hls4ml/backends/quartus/quartus_backend.py

@@ -48,7 +48,12 @@ def _register_flows(self):
         initializers = self._get_layer_initializers()
         init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
 
-        streaming_passes = ['quartus:reshape_stream', 'quartus:clone_output']
+        streaming_passes = [
+            'quartus:inplace_stream_flatten',  # Inform downstream changed packsize in case of skipping flatten
+            'quartus:reshape_stream',
+            'quartus:clone_output',
+        ]
+
         streaming_flow = register_flow('streaming', streaming_passes, requires=[init_flow], backend=self.name)
 
         quartus_types = [

hls4ml/backends/vivado/vivado_backend.py

@@ -79,6 +79,7 @@ def _register_flows(self):
         init_flow = register_flow('init_layers', initializers, requires=['optimize'], backend=self.name)
 
         streaming_passes = [
+            'vivado:inplace_stream_flatten',  # Inform downstream changed packsize in case of skipping flatten
             'vivado:reshape_stream',
             'vivado:clone_output',
             'vivado:insert_zero_padding_before_conv1d',

hls4ml/converters/__init__.py

@@ -10,6 +10,7 @@
 from hls4ml.converters.keras_to_hls import get_supported_keras_layers  # noqa: F401
 from hls4ml.converters.keras_to_hls import parse_keras_model  # noqa: F401
 from hls4ml.converters.keras_to_hls import keras_to_hls, register_keras_layer_handler
+from hls4ml.converters.keras_v3_to_hls import parse_keras_v3_model  # noqa: F401
 
 # from hls4ml.converters.pytorch_to_hls import parse_pytorch_model  # noqa: F401
 from hls4ml.model import ModelGraph
@@ -94,10 +95,10 @@ def parse_yaml_config(config_file):
     """
 
     def construct_keras_model(loader, node):
-        from tensorflow.keras.models import load_model
-
         model_str = loader.construct_scalar(node)
-        return load_model(model_str)
+        import keras
+
+        return keras.models.load_model(model_str)
 
     yaml.add_constructor('!keras_model', construct_keras_model, Loader=yaml.SafeLoader)
 

hls4ml/converters/keras/qkeras.py

@@ -1,5 +1,3 @@
-from qkeras.quantizers import get_quantizer
-
 from hls4ml.converters.keras.convolution import parse_conv1d_layer, parse_conv2d_layer
 from hls4ml.converters.keras.core import parse_batchnorm_layer, parse_dense_layer
 from hls4ml.converters.keras.recurrent import parse_rnn_layer
@@ -88,6 +86,8 @@ def parse_qrnn_layer(keras_layer, input_names, input_shapes, data_reader):
 
 @keras_handler('QActivation')
 def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader):
+    from qkeras.quantizers import get_quantizer
+
     assert keras_layer['class_name'] == 'QActivation'
     supported_activations = [
         'quantized_relu',

hls4ml/converters/keras_to_hls.py

@@ -1,9 +1,12 @@
 import json
+from warnings import warn
 
 import h5py
 
 from hls4ml.model import ModelGraph
 
+from .keras_v3_to_hls import parse_keras_v3_model
+
 MAXMULT = 4096
 
 
@@ -160,9 +163,9 @@ def get_model_arch(config):
         # Model instance passed in config from API
         keras_model = config['KerasModel']
         if isinstance(keras_model, str):
-            from tensorflow.keras.models import load_model
+            import keras
 
-            keras_model = load_model(keras_model)
+            keras_model = keras.models.load_model(keras_model)
         model_arch = json.loads(keras_model.to_json())
         reader = KerasModelReader(keras_model)
     elif 'KerasJson' in config:
@@ -228,8 +231,8 @@ def parse_keras_model(model_arch, reader):
         layer_config = model_arch['config']
         if 'layers' in layer_config:  # Newer Keras versions have 'layers' in 'config' key
             layer_config = layer_config['layers']
-        # Sequential doesn't have InputLayer in TF < 2.3 (Keras 2.4.0)
         if layer_config[0]['class_name'] != 'InputLayer':
+            warn(DeprecationWarning('keras < 2.4.0 (tf 2.3) is deprecated. Please use a newer version.'))
             input_layer = {}
             input_layer['name'] = 'input1'
             input_layer['class_name'] = 'InputLayer'
@@ -241,25 +244,33 @@ def parse_keras_model(model_arch, reader):
         layer_config = model_arch['config']['layers']
         input_layers = [inp[0] for inp in model_arch['config']['input_layers']]
         output_layers = [out[0] for out in model_arch['config']['output_layers']]
+    else:
+        raise Exception(f'ERROR: Model class not supported: {model_arch["class_name"]}')
 
     # Get input shape and check for unsupported layer type
     for keras_layer in layer_config:
         if keras_layer['class_name'] not in supported_layers:
-            raise Exception('ERROR: Unsupported layer type: {}'.format(keras_layer['class_name']))
+            raise Exception(f'ERROR: Unsupported layer type: {keras_layer["class_name"]}')
 
     output_shapes = {}
     output_shape = None
 
     print('Topology:')
     for keras_layer in layer_config:
-        if 'batch_input_shape' in keras_layer['config']:
+        if 'batch_input_shape' in keras_layer['config'] or 'batch_shape' in keras_layer['config']:
             if 'inbound_nodes' in keras_layer and len(keras_layer['inbound_nodes']) > 0:
                 input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]]
             else:
-                input_shapes = [keras_layer['config']['batch_input_shape']]
+                _input_shapes = keras_layer['config'].get('batch_input_shape', None)
+                input_shapes = _input_shapes or keras_layer['config']['batch_shape']
         else:
             if 'inbound_nodes' in keras_layer:
-                input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]]
+                if 'args' in keras_layer['inbound_nodes'][0]:
+                    # keras v3
+                    input_shapes = [arg['config']['shape'] for arg in keras_layer['inbound_nodes'][0]['args']]
+                else:
+                    # keras v2
+                    input_shapes = [output_shapes[inbound_node[0]] for inbound_node in keras_layer['inbound_nodes'][0]]
             else:
                 # Sequential model, so output_shape from the previous layer is still valid
                 input_shapes = [output_shape]
@@ -323,6 +334,13 @@ def parse_keras_model(model_arch, reader):
 
 
 def keras_to_hls(config):
+    if 'KerasModel' in config:
+        import keras
+
+        if keras.__version__ >= '3.0':
+            layer_list, input_layers, output_layers, _ = parse_keras_v3_model(config['KerasModel'])
+            return ModelGraph(config, layer_list, input_layers, output_layers)
+
     model_arch, reader = get_model_arch(config)
     layer_list, input_layers, output_layers, _ = parse_keras_model(model_arch, reader)
     print('Creating HLS model')

hls4ml/converters/keras_v3/__init__.py

@@ -0,0 +1,5 @@
+from . import conv  # noqa: F401
+from . import core  # noqa: F401
+from ._base import registry as layer_handlers
+
+__all__ = ['layer_handlers']