diff --git a/EdgeImpulse.EI-SDK.pdsc b/EdgeImpulse.EI-SDK.pdsc index e9d9352..533d80c 100644 --- a/EdgeImpulse.EI-SDK.pdsc +++ b/EdgeImpulse.EI-SDK.pdsc @@ -5,13 +5,16 @@ EI-SDK LICENSE-apache-2.0.txt Edge Impulse SDK - https://github.com/edgeimpulse/edge-impulse-sdk-pack/releases/download/v1.48.1/ + https://github.com/edgeimpulse/edge-impulse-sdk-pack/releases/download/v1.48.2/ hello@edgeimpulse.com https://github.com/edgeimpulse/edge-impulse-sdk-pack.git - + EI-SDK + + EI-SDK + EI-SDK @@ -86,7 +89,7 @@ - + Edge Impulse SDK diff --git a/EdgeImpulse.pidx b/EdgeImpulse.pidx index fa585db..6c7a3fb 100644 --- a/EdgeImpulse.pidx +++ b/EdgeImpulse.pidx @@ -2,8 +2,8 @@ EdgeImpulse https://raw.githubusercontent.com/edgeimpulse/edge-impulse-sdk-pack/main/ - 2024-03-14 14:16:31 + 2024-03-18 16:38:05 - + diff --git a/edgeimpulse/edge-impulse-sdk/classifier/ei_fill_result_struct.h b/edgeimpulse/edge-impulse-sdk/classifier/ei_fill_result_struct.h index b3c2a41..087a80b 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/ei_fill_result_struct.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/ei_fill_result_struct.h @@ -197,6 +197,7 @@ __attribute__((unused)) static void fill_result_struct_from_cubes(ei_impulse_res #endif __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_fomo(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, float *data, int out_width, @@ -214,7 +215,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_fomo(cons for (size_t ix = 1; ix < impulse->label_count + 1; ix++) { float vf = data[loc+ix]; - ei_handle_cube(&cubes, x, y, vf, impulse->categories[ix - 1], impulse->object_detection_threshold); + ei_handle_cube(&cubes, x, y, vf, impulse->categories[ix - 1], block_config->threshold); } } } @@ -228,6 +229,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_fomo(cons } __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_i8_fomo(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, int8_t *data, float zero_point, @@ -248,7 +250,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_i8_fomo(const int8_t v = data[loc+ix]; float vf = static_cast(v - zero_point) * scale; - ei_handle_cube(&cubes, x, y, vf, impulse->categories[ix - 1], impulse->object_detection_threshold); + ei_handle_cube(&cubes, x, y, vf, impulse->categories[ix - 1], block_config->threshold); } } } @@ -266,6 +268,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_i8_fomo(const * (we don't support quantized here a.t.m.) */ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_object_detection(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, float *data, float *scores, @@ -280,7 +283,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_object_de float score = scores[ix]; float label = labels[ix]; - if (score >= impulse->object_detection_threshold) { + if (score >= block_config->threshold) { float ystart = data[(ix * 4) + 0]; float xstart = data[(ix * 4) + 1]; float yend = data[(ix * 4) + 2]; @@ -385,6 +388,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32(const ei_ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_visual_ad_struct_f32(const ei_impulse_t *impulse, ei_impulse_result_t *result, float *data, + float threshold, bool debug) { #if EI_CLASSIFIER_HAS_VISUAL_ANOMALY float max_val = 0; @@ -412,7 +416,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_visual_ad_struct_f32 for (uint32_t x = 0; x <= grid_size_x - 1; x++) { for (uint32_t y = 0; y <= grid_size_y - 1; y++) { - if (data[x * grid_size_x + y] >= impulse->object_detection_threshold) { + if (data[x * grid_size_x + y] >= threshold) { ei_impulse_result_bounding_box_t tmp = { .label = "anomaly", .x = static_cast(y * (static_cast(impulse->input_height) / grid_size_y)), @@ -446,6 +450,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_visual_ad_struct_f32 * Fill the result structure from an unquantized output tensor */ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolov5(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, int version, float *data, @@ -494,7 +499,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolov5(co } } - if (score >= impulse->object_detection_threshold && score <= 1.0f) { + if (score >= block_config->threshold && score <= 1.0f) { ei_impulse_result_bounding_box_t r; r.label = impulse->categories[label]; @@ -543,6 +548,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolov5(co */ template __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_yolov5(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, int version, T *data, @@ -593,7 +599,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_yol } } - if (score >= impulse->object_detection_threshold && score <= 1.0f) { + if (score >= block_config->threshold && score <= 1.0f) { ei_impulse_result_bounding_box_t r; r.label = ei_classifier_inferencing_categories[label]; @@ -641,7 +647,9 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_yol * Fill the result structure from an unquantized output tensor * (we don't support quantized here a.t.m.) */ -__attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox(const ei_impulse_t *impulse, ei_impulse_result_t *result, +__attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, + ei_impulse_result_t *result, float *data, size_t output_features_count, bool debug = false) { @@ -779,7 +787,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox(con for (int col = 0; col < (int)scores.cols; col++) { float confidence = scores.buffer[(row * scores.cols) + col]; - if (confidence >= impulse->object_detection_threshold && confidence <= 1.0f) { + if (confidence >= block_config->threshold && confidence <= 1.0f) { ei_impulse_result_bounding_box_t r; r.label = impulse->categories[col]; r.value = confidence; @@ -844,7 +852,9 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox(con * Fill the result structure from an unquantized output tensor * (we don't support quantized here a.t.m.) */ -__attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox_detect(const ei_impulse_t *impulse, ei_impulse_result_t *result, +__attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox_detect(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, + ei_impulse_result_t *result, float *data, size_t output_features_count) { #ifdef EI_HAS_YOLOX @@ -860,7 +870,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox_det float confidence = outputs.buffer[(row * outputs.cols) + 4]; int class_idx = (int)outputs.buffer[(row * outputs.cols) + 5]; - if (confidence >= impulse->object_detection_threshold && confidence <= 1.0f) { + if (confidence >= block_config->threshold && confidence <= 1.0f) { ei_impulse_result_bounding_box_t r; r.label = ei_classifier_inferencing_categories[class_idx]; r.value = confidence; @@ -912,7 +922,9 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolox_det * Fill the result structure from an unquantized output tensor * (we don't support quantized here a.t.m.) */ -__attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolov7(const ei_impulse_t *impulse, ei_impulse_result_t *result, +__attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolov7(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, + ei_impulse_result_t *result, float *data, size_t output_features_count) { #ifdef EI_HAS_YOLOV7 @@ -933,7 +945,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_yolov7(co uint32_t label = (uint32_t)data[base_ix + 5]; float score = data[base_ix + 6]; - if (score >= impulse->object_detection_threshold && score <= 1.0f) { + if (score >= block_config->threshold && score <= 1.0f) { ei_impulse_result_bounding_box_t r; r.label = ei_classifier_inferencing_categories[label]; @@ -1001,14 +1013,6 @@ __attribute__((unused)) static void prepare_tao_results_common(const ei_impulse_ #endif -#if (EI_HAS_TAO_DECODE_DETECTIONS == 1) || (EI_HAS_TAO_YOLO == 1) - -template -__attribute__((unused)) static T clip_val(T val, T min_val, T max_val) { - return std::min(std::max(val, min_val), max_val); -} -#endif - #ifdef EI_HAS_TAO_DECODE_DETECTIONS /** * Fill the result structure from an output tensor @@ -1020,24 +1024,28 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_decode_de float zero_point, float scale, size_t output_features_count, + float threshold, bool debug = false) { - static std::vector results; - results.clear(); - static std::vector class_results; - size_t col_size = 12 + impulse->label_count + 1; size_t row_count = output_features_count / col_size; + static std::vector results; + static std::vector class_results; + results.clear(); + for (size_t cls_idx = 1; cls_idx < (size_t)(impulse->label_count + 1); cls_idx++) { + std::vector boxes; + std::vector scores; + std::vector classes; class_results.clear(); + for (size_t ix = 0; ix < row_count; ix++) { float score = (static_cast(data[ix * col_size + cls_idx]) - zero_point) * scale; - score = clip_val(score, 0.0f, 1.0f); - if (score < impulse->object_detection_threshold) { + if ((score < threshold) || (score > 1.0f)) { continue; } @@ -1107,21 +1115,20 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_decode_de xmax *= impulse->input_width; ymax *= impulse->input_height; - float width = xmax - xmin; - float height = ymax - ymin; - - ei_impulse_result_bounding_box_t r; - r.label = impulse->categories[cls_idx - 1]; - r.x = static_cast(xmin); - r.y = static_cast(ymin); - r.width = static_cast(width); - r.height = static_cast(height); - r.value = score; - - class_results.push_back(r); + boxes.push_back(ymin); + boxes.push_back(xmin); + boxes.push_back(ymax); + boxes.push_back(xmax); + scores.push_back(score); + classes.push_back((int)(cls_idx-1)); } - EI_IMPULSE_ERROR nms_res = ei_run_nms(impulse, &class_results, false /*clip_boxes*/, debug); + size_t nr_boxes = scores.size(); + EI_IMPULSE_ERROR nms_res = ei_run_nms(impulse, &class_results, + boxes.data(), scores.data(), classes.data(), + nr_boxes, + false /*clip_boxes*/, + debug); if (nms_res != EI_IMPULSE_OK) { return nms_res; @@ -1143,6 +1150,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_decode_de */ template __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao_decode_detections(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, T *data, float zero_point, @@ -1150,7 +1158,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao size_t output_features_count, bool debug = false) { #ifdef EI_HAS_TAO_DECODE_DETECTIONS - return fill_result_struct_tao_decode_detections_common(impulse, result, data, zero_point, scale, output_features_count, debug); + return fill_result_struct_tao_decode_detections_common(impulse, result, data, zero_point, scale, output_features_count, block_config->threshold, debug); #else return EI_IMPULSE_LAST_LAYER_NOT_AVAILABLE; #endif // #ifdef EI_HAS_TAO_DETECT_DETECTIONS @@ -1161,12 +1169,13 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao * Fill the result structure from an unquantized output tensor */ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_tao_decode_detections(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, float *data, size_t output_features_count, bool debug = false) { #ifdef EI_HAS_TAO_DECODE_DETECTIONS - return fill_result_struct_tao_decode_detections_common(impulse, result, data, 0.0f, 1.0f, output_features_count, debug); + return fill_result_struct_tao_decode_detections_common(impulse, result, data, 0.0f, 1.0f, output_features_count, block_config->threshold, debug); #else return EI_IMPULSE_LAST_LAYER_NOT_AVAILABLE; #endif // #ifdef EI_HAS_TAO_DETECT_DETECTIONS @@ -1189,6 +1198,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov3_c float zero_point, float scale, size_t output_features_count, + float threshold, bool debug) { // # x: 3-D tensor. Last dimension is // (cy, cx, ph, pw, step_y, step_x, pred_y, pred_x, pred_h, pred_w, object, cls...) @@ -1196,10 +1206,14 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov3_c size_t row_count = output_features_count / col_size; static std::vector results; - results.clear(); static std::vector class_results; + results.clear(); for (size_t cls_idx = 0; cls_idx < (size_t)impulse->label_count; cls_idx++) { + + std::vector boxes; + std::vector scores; + std::vector classes; class_results.clear(); for (size_t ix = 0; ix < row_count; ix++) { @@ -1219,7 +1233,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov3_c float cls = (static_cast(data[data_ix + 11 + cls_idx]) - zero_point) * scale; float score = sigmoid(cls) * sigmoid(r_10); - if (score < impulse->object_detection_threshold) { + if ((score < threshold) || (score > 1.0f)) { continue; } @@ -1239,21 +1253,20 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov3_c ymax *= impulse->input_height; xmax *= impulse->input_width; - float width = xmax - xmin; - float height = ymax - ymin; - - ei_impulse_result_bounding_box_t r; - r.label = impulse->categories[cls_idx]; - r.x = static_cast(xmin); - r.y = static_cast(ymin); - r.width = static_cast(width); - r.height = static_cast(height); - r.value = score; - - class_results.push_back(r); + boxes.push_back(ymin); + boxes.push_back(xmin); + boxes.push_back(ymax); + boxes.push_back(xmax); + scores.push_back(score); + classes.push_back((int)cls_idx); } - EI_IMPULSE_ERROR nms_res = ei_run_nms(impulse, &class_results, true /*clip_boxes*/, debug); + size_t nr_boxes = scores.size(); + EI_IMPULSE_ERROR nms_res = ei_run_nms(impulse, &class_results, + boxes.data(), scores.data(), classes.data(), + nr_boxes, + true /*clip_boxes*/, + debug); if (nms_res != EI_IMPULSE_OK) { return nms_res; } @@ -1279,17 +1292,24 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov4_co float zero_point, float scale, size_t output_features_count, + float threshold, bool debug) { // # x: 3-D tensor. Last dimension is // (cy, cx, ph, pw, step_y, step_x, pred_y, pred_x, pred_h, pred_w, object, cls...) size_t col_size = 11 + impulse->label_count; size_t row_count = output_features_count / col_size; - const float grid_scale_xy = 1.0f; + static std::vector results; - results.clear(); static std::vector class_results; + results.clear(); + + const float grid_scale_xy = 1.0f; for (size_t cls_idx = 0; cls_idx < (size_t)impulse->label_count; cls_idx++) { + + std::vector boxes; + std::vector scores; + std::vector classes; class_results.clear(); for (size_t ix = 0; ix < row_count; ix++) { @@ -1309,7 +1329,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov4_co float cls = (static_cast(data[ix * col_size + 11 + cls_idx]) - zero_point) * scale; float score = sigmoid(cls) * sigmoid(r_10); - if (score < impulse->object_detection_threshold) { + if ((score < threshold) || (score > 1.0f)) { continue; } @@ -1339,21 +1359,20 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov4_co ymax *= impulse->input_height; xmax *= impulse->input_width; - float width = xmax - xmin; - float height = ymax - ymin; - - ei_impulse_result_bounding_box_t r; - r.label = impulse->categories[cls_idx]; - r.x = static_cast(xmin); - r.y = static_cast(ymin); - r.width = static_cast(width); - r.height = static_cast(height); - r.value = score; - - class_results.push_back(r); + boxes.push_back(ymin); + boxes.push_back(xmin); + boxes.push_back(ymax); + boxes.push_back(xmax); + scores.push_back(score); + classes.push_back((int)cls_idx); } - EI_IMPULSE_ERROR nms_res = ei_run_nms(impulse, &class_results, true /*clip_boxes*/, debug); + size_t nr_boxes = scores.size(); + EI_IMPULSE_ERROR nms_res = ei_run_nms(impulse, &class_results, + boxes.data(), scores.data(), classes.data(), + nr_boxes, + true /*clip_boxes*/, + debug); if (nms_res != EI_IMPULSE_OK) { return nms_res; } @@ -1372,12 +1391,13 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_tao_yolov4_co * Fill the result structure from an unquantized output tensor */ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_tao_yolov3(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, float *data, size_t output_features_count, bool debug = false) { #ifdef EI_HAS_TAO_YOLOV3 - return fill_result_struct_tao_yolov3_common(impulse, result, data, 0.0f, 1.0f, output_features_count, debug); + return fill_result_struct_tao_yolov3_common(impulse, result, data, 0.0f, 1.0f, output_features_count, block_config->threshold, debug); #else return EI_IMPULSE_LAST_LAYER_NOT_AVAILABLE; #endif // #ifdef EI_HAS_TAO_YOLOV3 @@ -1388,6 +1408,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_tao_yolov */ template __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao_yolov3(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, T *data, float zero_point, @@ -1395,7 +1416,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao size_t output_features_count, bool debug = false) { #ifdef EI_HAS_TAO_YOLOV3 - return fill_result_struct_tao_yolov3_common(impulse, result, data, zero_point, scale, output_features_count, debug); + return fill_result_struct_tao_yolov3_common(impulse, result, data, zero_point, scale, output_features_count, block_config->threshold, debug); #else return EI_IMPULSE_LAST_LAYER_NOT_AVAILABLE; #endif // #ifdef EI_HAS_TAO_YOLOV3 @@ -1405,12 +1426,13 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao * Fill the result structure from an unquantized output tensor */ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_tao_yolov4(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, float *data, size_t output_features_count, bool debug = false) { #ifdef EI_HAS_TAO_YOLOV4 - return fill_result_struct_tao_yolov4_common(impulse, result, data, 0.0f, 1.0f, output_features_count, debug); + return fill_result_struct_tao_yolov4_common(impulse, result, data, 0.0f, 1.0f, output_features_count, block_config->threshold, debug); #else return EI_IMPULSE_LAST_LAYER_NOT_AVAILABLE; #endif // #ifdef EI_HAS_TAO_YOLOV4 @@ -1421,6 +1443,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_f32_tao_yolov */ template __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao_yolov4(const ei_impulse_t *impulse, + const ei_learning_block_config_tflite_graph_t *block_config, ei_impulse_result_t *result, T *data, float zero_point, @@ -1428,7 +1451,7 @@ __attribute__((unused)) static EI_IMPULSE_ERROR fill_result_struct_quantized_tao size_t output_features_count, bool debug = false) { #ifdef EI_HAS_TAO_YOLOV4 - return fill_result_struct_tao_yolov4_common(impulse, result, data, zero_point, scale, output_features_count, debug); + return fill_result_struct_tao_yolov4_common(impulse, result, data, zero_point, scale, output_features_count, block_config->threshold, debug); #else return EI_IMPULSE_LAST_LAYER_NOT_AVAILABLE; #endif // #ifdef EI_HAS_TAO_YOLOV4 diff --git a/edgeimpulse/edge-impulse-sdk/classifier/ei_model_types.h b/edgeimpulse/edge-impulse-sdk/classifier/ei_model_types.h index deda9a3..0a8d147 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/ei_model_types.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/ei_model_types.h @@ -164,6 +164,8 @@ typedef struct { uint8_t output_data_tensor; uint8_t output_labels_tensor; uint8_t output_score_tensor; + /* object detection and visual AD */ + float threshold; /* tflite graph params */ bool quantized; bool compiled; @@ -218,10 +220,7 @@ typedef struct ei_impulse { ei_model_dsp_t *dsp_blocks; /* object detection */ - bool object_detection; uint16_t object_detection_count; - float object_detection_threshold; - int8_t object_detection_last_layer; uint32_t fomo_output_size; uint32_t tflite_output_features_count; diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/akida.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/akida.h index b86e9a7..f15677b 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/akida.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/akida.h @@ -353,7 +353,7 @@ EI_IMPULSE_ERROR run_nn_inference( std::vector potentials_v;// = potentials.cast>(); // TODO: output conversion depending on output shape? - if (impulse->object_detection == false) { + if (block_config->object_detection == false) { potentials_v = potentials.squeeze().cast>(); } else { @@ -396,11 +396,12 @@ EI_IMPULSE_ERROR run_nn_inference( engine_info << "Power consumption: " << std::fixed << std::setprecision(2) << active_power << " mW\n"; engine_info << "Inferences per second: " << (1000000 / result->timing.classification_us); - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_FOMO: { fill_res = fill_result_struct_f32_fomo( impulse, + block_config, result, potentials_v.data(), impulse->fomo_output_size, @@ -409,17 +410,17 @@ EI_IMPULSE_ERROR run_nn_inference( } case EI_CLASSIFIER_LAST_LAYER_SSD: { ei_printf("ERR: MobileNet SSD models are not implemented for Akida (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } case EI_CLASSIFIER_LAST_LAYER_YOLOV5: { ei_printf("ERR: YOLO v5 models are not implemented for Akida (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } default: { ei_printf("ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } } @@ -547,6 +548,9 @@ __attribute__((unused)) int extract_tflite_features(signal_t *signal, matrix_t * .output_data_tensor = 0, .output_labels_tensor = 255, .output_score_tensor = 255, + .threshold = 0, + .quantized = 0, + .compiled = 1, .graph_config = &ei_config_tflite_graph_0 }; diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/anomaly.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/anomaly.h index 0d24bac..5a800eb 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/anomaly.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/anomaly.h @@ -214,6 +214,7 @@ EI_IMPULSE_ERROR run_gmm_anomaly( .output_data_tensor = 0, .output_labels_tensor = 0, .output_score_tensor = 0, + .threshold = block_config->anomaly_threshold, .quantized = 0, .compiled = 0, .graph_config = block_config->graph_config diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/drpai.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/drpai.h index 7d6f6d3..01d4053 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/drpai.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/drpai.h @@ -455,10 +455,12 @@ EI_IMPULSE_ERROR drpai_close(uint32_t input_frame_size) { #if ((EI_CLASSIFIER_OBJECT_DETECTION == 1) && (EI_CLASSIFIER_OBJECT_DETECTION_LAST_LAYER == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI)) EI_IMPULSE_ERROR drpai_run_yolov5_postprocessing( const ei_impulse_t *impulse, + ei_learning_block_config_tflite_graph_t *block_config, signal_t *signal, ei_impulse_result_t *result, bool debug = false) { + static std::unique_ptr model = nullptr; static std::unique_ptr interpreter = nullptr; @@ -564,7 +566,7 @@ EI_IMPULSE_ERROR drpai_run_yolov5_postprocessing( // } // printf("\n"); - return fill_result_struct_f32_yolov5(impulse, result, 5, out_data, out_size); + return fill_result_struct_f32_yolov5(impulse, block_config, result, 5, out_data, out_size); } #endif @@ -602,6 +604,8 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( void *config_ptr, bool debug = false) { + ei_learning_block_config_tflite_graph_t *block_config = (ei_learning_block_config_tflite_graph_t*)config_ptr; + // this needs to be changed for multi-model, multi-impulse static bool first_run = true; uint64_t ctx_start_us; @@ -678,8 +682,8 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( EI_IMPULSE_ERROR fill_res = EI_IMPULSE_OK; - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_FOMO: { if (debug) { ei_printf("DEBUG: raw drpai output"); @@ -693,6 +697,7 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( fill_res = fill_result_struct_f32_fomo( impulse, + block_config, result, drpai_output_buf, impulse->fomo_output_size, @@ -701,7 +706,7 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( } case EI_CLASSIFIER_LAST_LAYER_SSD: { ei_printf("ERR: MobileNet SSD models are not implemented for DRP-AI (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } case EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI: { @@ -723,7 +728,7 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( #if ((EI_CLASSIFIER_OBJECT_DETECTION == 1) && (EI_CLASSIFIER_OBJECT_DETECTION_LAST_LAYER == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI)) // do post processing - fill_res = drpai_run_yolov5_postprocessing(impulse, signal, result, debug); + fill_res = drpai_run_yolov5_postprocessing(impulse, block_config, signal, result, debug); #endif #endif @@ -732,7 +737,7 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( } default: { ei_printf("ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } } diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/memryx.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/memryx.h index fd227a4..b8693ab 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/memryx.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/memryx.h @@ -208,6 +208,8 @@ EI_IMPULSE_ERROR run_nn_inference( void *config_ptr, bool debug = false) { + ei_learning_block_config_tflite_graph_t *block_config = (ei_learning_block_config_tflite_graph_t*)config_ptr; + memx_status status = MEMX_STATUS_OK; int32_t ifmap_height, ifmap_width, ifmap_channel_number, ifmap_format; int32_t ofmap_height, ofmap_width, ofmap_channel_number, ofmap_format; @@ -311,12 +313,13 @@ EI_IMPULSE_ERROR run_nn_inference( tflite::reference_ops::Softmax(dummy_params, softmax_shape, ofmap, softmax_shape, ofmap); // handle inference outputs - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_FOMO: { ei_printf("FOMO executed on Memryx\n"); fill_result_struct_f32_fomo( impulse, + block_config, result, ofmap, impulse->fomo_output_size, @@ -329,7 +332,7 @@ EI_IMPULSE_ERROR run_nn_inference( } default: { ei_printf("ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } } @@ -353,6 +356,8 @@ EI_IMPULSE_ERROR run_nn_inference( void *config_ptr, bool debug = false) { + ei_learning_block_config_tflite_graph_t *block_config = (ei_learning_block_config_tflite_graph_t*)config_ptr; + // init Python embedded interpreter (should be called once!) static py::scoped_interpreter guard{}; @@ -420,7 +425,7 @@ EI_IMPULSE_ERROR run_nn_inference( potentials = outputs.squeeze().cast>(); - if (impulse->object_detection == false) { + if (block_config->object_detection == false) { potentials_v = outputs.squeeze().cast>(); } else { @@ -439,12 +444,13 @@ EI_IMPULSE_ERROR run_nn_inference( ei_printf("Memryx raw output:\n%s\n", ret_str.c_str()); } - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_FOMO: { ei_printf("FOMO executed on Memryx\n"); fill_result_struct_f32_fomo( impulse, + block_config, result, potentials_v.data(), impulse->fomo_output_size, diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/onnx_tidl.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/onnx_tidl.h index 8e806d6..0c957b1 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/onnx_tidl.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/onnx_tidl.h @@ -347,6 +347,7 @@ static EI_IMPULSE_ERROR inference_onnx_setup( * @return EI_IMPULSE_OK if successful */ static EI_IMPULSE_ERROR inference_onnx_run(const ei_impulse_t *impulse, + void *config_ptr, uint64_t ctx_start_us, std::vector* input_tensors, std::vector* output_tensors, @@ -356,6 +357,8 @@ static EI_IMPULSE_ERROR inference_onnx_run(const ei_impulse_t *impulse, ei_impulse_result_t *result, bool debug) { + ei_learning_block_config_tflite_graph_t *block_config = (ei_learning_block_config_tflite_graph_t*)config_ptr; + session->Run(*run_options, *binding); uint64_t ctx_end_us = ei_read_timer_us(); @@ -381,8 +384,8 @@ static EI_IMPULSE_ERROR inference_onnx_run(const ei_impulse_t *impulse, EI_IMPULSE_ERROR fill_res = EI_IMPULSE_OK; // NOTE: for now only yolox object detection supported - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_YOLOX: { #if EI_CLASSIFIER_TFLITE_OUTPUT_QUANTIZED == 1 ei_printf("ERR: YOLOX does not support quantized inference\n"); @@ -399,6 +402,7 @@ static EI_IMPULSE_ERROR inference_onnx_run(const ei_impulse_t *impulse, } fill_res = fill_result_struct_f32_yolox_detect( impulse, + block_config, result, (float*)out_data, output_tensor_features_count); @@ -407,7 +411,7 @@ static EI_IMPULSE_ERROR inference_onnx_run(const ei_impulse_t *impulse, } default: { ei_printf("ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); break; } } @@ -549,6 +553,7 @@ EI_IMPULSE_ERROR run_nn_inference( ctx_start_us = ei_read_timer_us(); EI_IMPULSE_ERROR run_res = inference_onnx_run(impulse, + config_ptr, ctx_start_us, &input_tensors, &output_tensors, @@ -678,6 +683,7 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( ctx_start_us = ei_read_timer_us(); EI_IMPULSE_ERROR run_res = inference_onnx_run(impulse, + config_ptr, ctx_start_us, &input_tensors, &output_tensors, diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensaiflow.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensaiflow.h index 5fec55b..7ce1591 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensaiflow.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensaiflow.h @@ -81,7 +81,7 @@ EI_IMPULSE_ERROR run_nn_inference( ei_learning_block_config_tflite_graph_t *block_config = (ei_learning_block_config_tflite_graph_t*)config_ptr; ei_config_tensaiflow_graph_t *graph_config = (ei_config_tensaiflow_graph_t*)block_config->graph_config; - if (impulse->object_detection) { + if (block_config->object_detection) { ei_printf("ERR: Object detection models are not supported with TensaiFlow\n"); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } @@ -138,8 +138,15 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( processed_features = (int8_t *) features_matrix.buffer; // run DSP process and quantize automatically - int ret = extract_image_features_quantized(signal, &features_matrix, impulse->dsp_blocks[0].config, graph_config->input_scale, graph_config->input_zeropoint, - impulse->frequency, impulse->learning_blocks[0].image_scaling); + int ret = extract_image_features_quantized( + signal, + &features_matrix, + impulse->dsp_blocks[0].config, + graph_config->input_scale, + graph_config->input_zeropoint, + impulse->frequency, + impulse->learning_blocks[0].image_scaling); + if (ret != EIDSP_OK) { ei_printf("ERR: Failed to run DSP process (%d)\n", ret); return EI_IMPULSE_DSP_ERROR; @@ -171,13 +178,19 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( EI_IMPULSE_ERROR fill_res = EI_IMPULSE_OK; - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_FOMO: { #if EI_CLASSIFIER_TFLITE_OUTPUT_QUANTIZED == 1 - fill_res = fill_result_struct_i8_fomo(impulse, result, infer_result, - graph_config->output_zeropoint, graph_config->output_scale, - impulse->fomo_output_size, impulse->fomo_output_size); + fill_res = fill_result_struct_i8_fomo( + impulse, + block_config, + result, + infer_result, + graph_config->output_zeropoint, + graph_config->output_scale, + impulse->fomo_output_size, + impulse->fomo_output_size); #else ei_printf("ERR: TensaiFlow does not support float32 inference\n"); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; @@ -186,15 +199,20 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( } default: { ei_printf("ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } } } else { #if EI_CLASSIFIER_TFLITE_OUTPUT_QUANTIZED == 1 - fill_res = fill_result_struct_i8(impulse, result, infer_result, - graph_config->output_zeropoint, graph_config->output_scale, debug); + fill_res = fill_result_struct_i8( + impulse, + result, + infer_result, + graph_config->output_zeropoint, + graph_config->output_scale, + debug); #else ei_printf("ERR: TensaiFlow does not support float32 inference\n"); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; @@ -205,7 +223,6 @@ EI_IMPULSE_ERROR run_nn_inference_image_quantized( return fill_res; } - result->timing.classification_us = ei_read_timer_us() - ctx_start_us; result->timing.classification = (int)(result->timing.classification_us / 1000); return EI_IMPULSE_OK; diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensorrt.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensorrt.h index ac92ebf..d9fbb29 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensorrt.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tensorrt.h @@ -155,8 +155,8 @@ EI_IMPULSE_ERROR run_nn_inference( uint32_t out_data_size = 0; - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_TAO_SSD: case EI_CLASSIFIER_LAST_LAYER_TAO_RETINANET: case EI_CLASSIFIER_LAST_LAYER_TAO_YOLOV3: @@ -170,7 +170,7 @@ EI_IMPULSE_ERROR run_nn_inference( default: { ei_printf( "ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } } @@ -224,11 +224,12 @@ EI_IMPULSE_ERROR run_nn_inference( EI_IMPULSE_ERROR fill_res = EI_IMPULSE_OK; - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_FOMO: { fill_res = fill_result_struct_f32_fomo( impulse, + block_config, result, out_data, impulse->fomo_output_size, @@ -237,10 +238,11 @@ EI_IMPULSE_ERROR run_nn_inference( } case EI_CLASSIFIER_LAST_LAYER_YOLOV5: case EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI: { - int version = impulse->object_detection_last_layer == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI ? + int version = block_config->object_detection_last_layer == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI ? 5 : 6; fill_res = fill_result_struct_f32_yolov5( impulse, + block_config, result, version, out_data, @@ -251,6 +253,7 @@ EI_IMPULSE_ERROR run_nn_inference( case EI_CLASSIFIER_LAST_LAYER_TAO_RETINANET: { fill_res = fill_result_struct_f32_tao_decode_detections( impulse, + block_config, result, out_data, impulse->tflite_output_features_count, @@ -260,6 +263,7 @@ EI_IMPULSE_ERROR run_nn_inference( case EI_CLASSIFIER_LAST_LAYER_TAO_YOLOV3: fill_res = fill_result_struct_f32_tao_yolov3( impulse, + block_config, result, out_data, impulse->tflite_output_features_count, @@ -268,6 +272,7 @@ EI_IMPULSE_ERROR run_nn_inference( case EI_CLASSIFIER_LAST_LAYER_TAO_YOLOV4: { fill_res = fill_result_struct_f32_tao_yolov4( impulse, + block_config, result, out_data, impulse->tflite_output_features_count, @@ -277,7 +282,7 @@ EI_IMPULSE_ERROR run_nn_inference( default: { ei_printf( "ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } } diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_eon.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_eon.h index 167d121..c1053e0 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_eon.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_eon.h @@ -388,6 +388,7 @@ __attribute__((unused)) int extract_tflite_eon_features(signal_t *signal, matrix .output_data_tensor = 0, .output_labels_tensor = 255, .output_score_tensor = 255, + .threshold = 0, .quantized = 0, .compiled = 1, .graph_config = &ei_config_tflite_graph_0 diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_full.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_full.h index 65ddb24..e1f0d42 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_full.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_full.h @@ -229,6 +229,9 @@ __attribute__((unused)) int extract_tflite_features(signal_t *signal, matrix_t * .output_data_tensor = 0, .output_labels_tensor = 255, .output_score_tensor = 255, + .threshold = 0, + .quantized = 0, + .compiled = 0, .graph_config = &ei_config_tflite_graph_0 }; diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_helper.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_helper.h index 5c2f266..85cee68 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_helper.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_helper.h @@ -291,12 +291,24 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( case EI_CLASSIFIER_LAST_LAYER_FOMO: { bool int8_output = output->type == TfLiteType::kTfLiteInt8; if (int8_output) { - fill_res = fill_result_struct_i8_fomo(impulse, result, output->data.int8, output->params.zero_point, output->params.scale, - impulse->fomo_output_size, impulse->fomo_output_size); + fill_res = fill_result_struct_i8_fomo( + impulse, + block_config, + result, + output->data.int8, + output->params.zero_point, + output->params.scale, + impulse->fomo_output_size, + impulse->fomo_output_size); } else { - fill_res = fill_result_struct_f32_fomo(impulse, result, output->data.f, - impulse->fomo_output_size, impulse->fomo_output_size); + fill_res = fill_result_struct_f32_fomo( + impulse, + block_config, + result, + output->data.f, + impulse->fomo_output_size, + impulse->fomo_output_size); } break; } @@ -309,7 +321,14 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( return EI_IMPULSE_LABEL_TENSOR_WAS_NULL; } if (output->type == kTfLiteFloat32) { - fill_res = fill_result_struct_f32_object_detection(impulse, result, output->data.f, scores_tensor->data.f, labels_tensor->data.f, debug); + fill_res = fill_result_struct_f32_object_detection( + impulse, + block_config, + result, + output->data.f, + scores_tensor->data.f, + labels_tensor->data.f, + debug); } else { ei_printf("ERR: MobileNet SSD does not support quantized inference\n"); @@ -325,12 +344,13 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( #endif // EI_CLASSIFIER_INFERENCING_ENGINE == EI_CLASSIFIER_TFLITE_FULL case EI_CLASSIFIER_LAST_LAYER_YOLOV5: case EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI: { - int version = impulse->object_detection_last_layer == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI ? + int version = block_config->object_detection_last_layer == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI ? 5 : 6; if (output->type == kTfLiteInt8) { fill_res = fill_result_struct_quantized_yolov5( impulse, + block_config, result, version, output->data.int8, @@ -342,6 +362,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteUInt8) { fill_res = fill_result_struct_quantized_yolov5( impulse, + block_config, result, version, output->data.uint8, @@ -353,6 +374,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteFloat32) { fill_res = fill_result_struct_f32_yolov5( impulse, + block_config, result, version, output->data.f, @@ -372,6 +394,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( #else fill_res = fill_result_struct_f32_yolox( impulse, + block_config, result, output->data.f, impulse->tflite_output_features_count, @@ -390,6 +413,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( } fill_res = fill_result_struct_f32_yolov7( impulse, + block_config, result, output->data.f, output_feature_count); @@ -402,6 +426,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( if (output->type == kTfLiteInt8) { fill_res = fill_result_struct_quantized_tao_decode_detections( impulse, + block_config, result, output->data.int8, output->params.zero_point, @@ -412,6 +437,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteUInt8) { fill_res = fill_result_struct_quantized_tao_decode_detections( impulse, + block_config, result, output->data.uint8, output->params.zero_point, @@ -422,6 +448,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteFloat32) { fill_res = fill_result_struct_f32_tao_decode_detections( impulse, + block_config, result, output->data.f, impulse->tflite_output_features_count, @@ -438,6 +465,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( if (output->type == kTfLiteInt8) { fill_res = fill_result_struct_quantized_tao_yolov3( impulse, + block_config, result, output->data.int8, output->params.zero_point, @@ -448,6 +476,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteUInt8) { fill_res = fill_result_struct_quantized_tao_yolov3( impulse, + block_config, result, output->data.uint8, output->params.zero_point, @@ -458,6 +487,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteFloat32) { fill_res = fill_result_struct_f32_tao_yolov3( impulse, + block_config, result, output->data.f, impulse->tflite_output_features_count, @@ -474,6 +504,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( if (output->type == kTfLiteInt8) { fill_res = fill_result_struct_quantized_tao_yolov4( impulse, + block_config, result, output->data.int8, output->params.zero_point, @@ -484,6 +515,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteUInt8) { fill_res = fill_result_struct_quantized_tao_yolov4( impulse, + block_config, result, output->data.uint8, output->params.zero_point, @@ -494,6 +526,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (output->type == kTfLiteFloat32) { fill_res = fill_result_struct_f32_tao_yolov4( impulse, + block_config, result, output->data.f, impulse->tflite_output_features_count, @@ -507,7 +540,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( } default: { ei_printf("ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; } } @@ -515,7 +548,7 @@ EI_IMPULSE_ERROR fill_result_struct_from_output_tensor_tflite( else if (block_config->classification_mode == EI_CLASSIFIER_CLASSIFICATION_MODE_VISUAL_ANOMALY) { if (!result->copy_output) { - fill_res = fill_result_visual_ad_struct_f32(impulse, result, output->data.f, debug); + fill_res = fill_result_visual_ad_struct_f32(impulse, result, output->data.f, block_config->threshold, debug); } } // if we copy the output, we don't need to process it as classification diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_micro.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_micro.h index a0ba8e7..fd3cb26 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_micro.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_micro.h @@ -452,6 +452,7 @@ __attribute__((unused)) int extract_tflite_features(signal_t *signal, matrix_t * .output_data_tensor = 0, .output_labels_tensor = 255, .output_score_tensor = 255, + .threshold = 0, .quantized = 0, .compiled = 0, .graph_config = &ei_config_tflite_graph_0 diff --git a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_tidl.h b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_tidl.h index 07019bb..c9642f1 100644 --- a/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_tidl.h +++ b/edgeimpulse/edge-impulse-sdk/classifier/inferencing_engines/tflite_tidl.h @@ -60,7 +60,7 @@ EI_IMPULSE_ERROR run_nn_inference( void *config_ptr, bool debug) { - ei_learning_block_config_tflite_graph_t *config = (ei_learning_block_config_tflite_graph_t*)config_ptr; + ei_learning_block_config_tflite_graph_t *block_config = (ei_learning_block_config_tflite_graph_t*)config_ptr; static std::unique_ptr model = nullptr; static std::unique_ptr interpreter = nullptr; @@ -160,8 +160,8 @@ EI_IMPULSE_ERROR run_nn_inference( size_t mtx_size = impulse->dsp_blocks_size + impulse->learning_blocks_size; for (size_t i = 0; i < input_block_ids_size; i++) { - uint16_t cur_mtx = input_block_ids[i]; #if EI_CLASSIFIER_SINGLE_FEATURE_INPUT == 0 + uint16_t cur_mtx = input_block_ids[i]; ei::matrix_t* matrix = NULL; if (!find_mtx_by_idx(fmatrix, &matrix, cur_mtx, mtx_size)) { @@ -173,7 +173,7 @@ EI_IMPULSE_ERROR run_nn_inference( #endif for (uint32_t ix = 0; ix < matrix->rows * matrix->cols; ix++) { - if (impulse->object_detection) { + if (block_config->object_detection) { #if EI_CLASSIFIER_QUANTIZATION_ENABLED == 1 float pixel = (float)matrix->buffer[ix]; input[ix] = static_cast((pixel / input->tflite_input_scale) + input->tflite_input_zeropoint); @@ -201,9 +201,9 @@ EI_IMPULSE_ERROR run_nn_inference( result->timing.classification = (int)(result->timing.classification_us / 1000); #if EI_CLASSIFIER_TFLITE_OUTPUT_QUANTIZED == 1 - int8_t* out_data = interpreter->typed_output_tensor(config->output_data_tensor); + int8_t* out_data = interpreter->typed_output_tensor(block_config->output_data_tensor); #else - float* out_data = interpreter->typed_output_tensor(config->output_data_tensor); + float* out_data = interpreter->typed_output_tensor(block_config->output_data_tensor); #endif if (debug) { @@ -246,21 +246,33 @@ EI_IMPULSE_ERROR run_nn_inference( EI_IMPULSE_ERROR fill_res = EI_IMPULSE_OK; - if (impulse->object_detection) { - switch (impulse->object_detection_last_layer) { + if (block_config->object_detection) { + switch (block_config->object_detection_last_layer) { case EI_CLASSIFIER_LAST_LAYER_FOMO: { #if EI_CLASSIFIER_TFLITE_OUTPUT_QUANTIZED == 1 - fill_res = fill_result_struct_i8_fomo(impulse, result, out_data, out_data->tflite_output_zeropoint, out_data->tflite_output_scale, - impulse->fomo_output_size, impulse->fomo_output_size); + fill_res = fill_result_struct_i8_fomo( + impulse, + block_config, + result, + out_data, + out_data->tflite_output_zeropoint, + out_data->tflite_output_scale, + impulse->fomo_output_size, + impulse->fomo_output_size); #else - fill_res = fill_result_struct_f32_fomo(impulse, result, out_data, - impulse->fomo_output_size, impulse->fomo_output_size); + fill_res = fill_result_struct_f32_fomo( + impulse, + block_config, + result, + out_data, + impulse->fomo_output_size, + impulse->fomo_output_size); #endif break; } case EI_CLASSIFIER_LAST_LAYER_SSD: { - float *scores_tensor = interpreter->typed_output_tensor(config->output_score_tensor); - float *label_tensor = interpreter->typed_output_tensor(config->output_labels_tensor); + float *scores_tensor = interpreter->typed_output_tensor(block_config->output_score_tensor); + float *label_tensor = interpreter->typed_output_tensor(block_config->output_labels_tensor); if (!scores_tensor) { return EI_IMPULSE_SCORE_TENSOR_WAS_NULL; } @@ -271,7 +283,14 @@ EI_IMPULSE_ERROR run_nn_inference( ei_printf("ERR: MobileNet SSD does not support quantized inference\n"); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; #else - fill_res = fill_result_struct_f32_object_detection(impulse, result, out_data, scores_tensor, label_tensor, debug); + fill_res = fill_result_struct_f32_object_detection( + impulse, + block_config, + result, + out_data, + scores_tensor, + label_tensor, + debug); #endif break; } @@ -281,10 +300,11 @@ EI_IMPULSE_ERROR run_nn_inference( ei_printf("ERR: YOLOv5 does not support quantized inference\n"); return EI_IMPULSE_UNSUPPORTED_INFERENCING_ENGINE; #else - int version = impulse->object_detection_last_layer == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI ? + int version = block_config->object_detection_last_layer == EI_CLASSIFIER_LAST_LAYER_YOLOV5_V5_DRPAI ? 5 : 6; fill_res = fill_result_struct_f32_yolov5( impulse, + block_config, result, version, out_data, @@ -300,6 +320,7 @@ EI_IMPULSE_ERROR run_nn_inference( #else fill_res = fill_result_struct_f32_yolox( impulse, + block_config, result, out_data, impulse->tflite_output_features_count, @@ -319,6 +340,7 @@ EI_IMPULSE_ERROR run_nn_inference( } fill_res = fill_result_struct_f32_yolov7( impulse, + block_config, result, output->data.f, output_feature_count); @@ -327,7 +349,7 @@ EI_IMPULSE_ERROR run_nn_inference( } default: { ei_printf("ERR: Unsupported object detection last layer (%d)\n", - impulse->object_detection_last_layer); + block_config->object_detection_last_layer); break; } }