first

README.MD

@@ -0,0 +1,39 @@
+# Depth Anything - python infer by QNN and TFLite
+
+## Model
+QNN model and TFLite model are converted from small scale Depth-Anything ONNX model depth_anything_vits14.onnx.
+
+## Infer by QNN 
+Prapare qnn-net-run env as in Qualcomm doc. This part is tested under ubuntu20.4, python==3.8, qnn==2.20.0.240223.
+
+Run:
+```bash
+cd qnn_python
+python infer.py
+```
+
+## Infer by TFLite
+Prapare tensorflow, onnx2tf pip package. This part is tested under python==3.10.14, onnx==1.16.1, tensorflow==2.17.0 and onnx2tf==1.25.8. ref:
+```bash
+pip install -U onnx==1.16.1 && pip install -U nvidia-pyindex && pip install -U onnx-graphsurgeon && pip install -U onnxruntime==1.18.1 && pip install -U onnxsim==0.4.33 && pip install -U simple_onnx_processing_tools && pip install -U sne4onnx>=1.0.13 && pip install -U sng4onnx>=1.0.4 && pip install -U tensorflow==2.17.0 && pip install -U protobuf==3.20.3 && pip install -U onnx2tf && pip install -U h5py==3.11.0 && pip install -U psutil==5.9.5 && pip install -U ml_dtypes==0.3.2 && pip install -U tf-keras~=2.16 && pip install flatbuffers>=23.5.26
+```
+
+Run:
+```bash
+cd tflite
+python infer.py
+```
+
+## Result
+![input](image.jpg)
+![depth result - QNN](qnn_python/res_qnn.jpg)
+![depth result - TFLite](tflite/res_tflite.jpg)
+
+
+## Reference
+
+https://github.com/fabio-sim/Depth-Anything-ONNX 
+
+https://github.com/LiheYoung/Depth-Anything 
+
+https://github.com/PINTO0309/onnx2tf

qnn_python/infer.py

@@ -0,0 +1,65 @@
+import cv2, os
+import numpy as np
+
+img= '../image.jpg'
+image = cv2.imread(img)
+origin_h, origin_w, _ = image.shape
+print('ori ', origin_h, origin_w)
+length = 518
+if origin_h > origin_w:
+    new_w = round(origin_w * float(length) / origin_h)
+    new_h = length
+else:
+    new_h = round(origin_h * float(length) / origin_w)
+    new_w = length
+scale_w = new_w / origin_w
+sclae_h = new_h / origin_h
+
+input_var = cv2.resize(image, (new_w, new_h))
+mean_raw= [123.675, 116.28, 103.53]
+std= [1/58.395, 1/57.12, 1/57.375]
+
+snpe_raw = input_var - mean_raw
+snpe_raw= snpe_raw* std
+snpe_raw = snpe_raw.astype(np.float32)
+
+print('snpe_raw after norm', snpe_raw.shape)
+
+# 计算需要添加的填充
+top, left= 0,0
+bottom = length - new_h  # 计算底部填充行数
+right = length - new_w  # 右侧填充列数
+snpe_raw = cv2.copyMakeBorder(snpe_raw, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(0, 0, 0))
+
+print('snpe_raw after pad', snpe_raw.shape)
+# scalar data divide
+# snpe_raw /= 255
+snpe_raw = snpe_raw[..., ::-1] # bgr变rgb
+
+print('snpe_raw after trans', snpe_raw.shape)
+
+snpe_raw_filename = 'building.raw'
+snpe_raw.tofile(snpe_raw_filename)
+os.system('echo "building.raw" > input_list.txt')
+
+cmd= 'qnn-net-run --backend /opt/qcom/aistack/qnn/2.20.0.240223/lib/x86_64-linux-clang/libQnnCpu.so  --input_list input_list.txt  --output_dir ./out/ --model x86_64-linux-clang/libdam.so '
+os.system(cmd)
+
+file_path= 'building.raw'
+data = np.fromfile(file_path, dtype= np.float32)
+print(data.shape)
+
+file_path= 'out/Result_0/depth.raw'
+depth = np.fromfile(file_path, dtype= np.float32)
+print(data.shape, origin_w, origin_h)
+depth= depth.reshape((length, length))
+depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
+depth= depth[:new_h, :new_w]
+depth = cv2.resize(depth, (origin_w, origin_h))
+depth = depth.astype(np.uint8)
+print('after depth ', depth.shape)
+depth_color = cv2.applyColorMap(depth, cv2.COLORMAP_INFERNO)
+print('after depth ', depth.shape)
+
+image = cv2.addWeighted(image, 0.2, depth_color.astype(np.uint8), 0.8, 0)
+cv2.imwrite('res_qnn.jpg', image)

tflite/infer.py

@@ -0,0 +1,98 @@
+import tensorflow as tf
+import cv2
+import numpy as np
+
+# 调用tflite
+mp= './saved_model/depth_anything_vits14_float32.tflite'
+interpreter = tf.lite.Interpreter(model_path= mp)
+interpreter.allocate_tensors()
+
+img= '../image.jpg'
+image = cv2.imread(img)
+origin_h, origin_w, _ = image.shape
+print('ori ', origin_h, origin_w)
+length = 518
+if origin_h > origin_w:
+    new_w = round(origin_w * float(length) / origin_h)
+    new_h = length
+else:
+    new_h = round(origin_h * float(length) / origin_w)
+    new_w = length
+scale_w = new_w / origin_w
+sclae_h = new_h / origin_h
+input_var = cv2.resize(image, (new_w, new_h))
+mean_raw= [123.675, 116.28, 103.53]
+std= [1/58.395, 1/57.12, 1/57.375]
+snpe_raw = input_var - mean_raw
+snpe_raw= snpe_raw* std
+snpe_raw = snpe_raw.astype(np.float32)
+print('snpe_raw after norm', snpe_raw.shape)
+# 计算需要添加的填充
+top, left= 0,0
+bottom = length - new_h  # 计算底部填充行数
+right = length - new_w  # 右侧填充列数
+snpe_raw = cv2.copyMakeBorder(snpe_raw, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(0, 0, 0))
+print('snpe_raw after pad', snpe_raw.shape)
+input_image = snpe_raw[..., ::-1] # bgr变rgb
+input_image = np.expand_dims(input_image, axis=0)
+input_image = tf.cast(input_image, dtype=tf.float32)
+
+# Inference
+input_details = interpreter.get_input_details()
+interpreter.set_tensor(input_details[0]['index'], input_image.numpy())
+interpreter.invoke()
+
+output_details = interpreter.get_output_details()
+print('input_details ', input_details)
+print('output_details ', output_details)
+
+depth_map = interpreter.get_tensor(output_details[0]['index'])
+
+depth= depth_map.reshape((length, length))
+depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
+depth= depth[:new_h, :new_w]
+depth = cv2.resize(depth, (origin_w, origin_h))
+depth = depth.astype(np.uint8)
+print('after depth ', depth.shape)
+depth_color = cv2.applyColorMap(depth, cv2.COLORMAP_INFERNO)
+print('after depth ', depth.shape)
+
+image = cv2.addWeighted(image, 0.2, depth_color.astype(np.uint8), 0.8, 0)
+cv2.imwrite('res_tflite.jpg', image)
+
+
+
+
+# 按照知乎， 配置onnx-tf-tflite的环境，tensorflow的环境非常麻烦，各种不好用
+# pip install tensorflow
+# pip install onnx_tf 
+# pip install tensorflow
+# pip install tensorflow-probability
+# pip install tf_keras
+# onnx-->tf
+if False:
+    onnxpath= '/home/local/sota/depthanything/Depth-Anything-ONNX/weights/depth_anything_vits14.onnx'
+    onnx_model = onnx.load(onnxpath)
+    tf_rep = prepare(onnx_model)
+    tf_rep.export_graph('dam.tf')
+
+    # tf-->tflite
+    converter = tf.lite.TFLiteConverter.from_saved_model('dam.tf')
+    converter.target_spec.supported_ops = [
+        tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops.
+        tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.
+        ]
+    tflite_model = converter.convert()
+    with open('dam.tflite', 'wb') as f:
+        f.write(tflite_model)
+# /home/local/miniconda3/envs/dinov2/lib/python3.9/site-packages/tensorflow_addons/utils/ensure_tf_install.py:53: UserWarning: Tensorflow Addons supports using Python ops for all Tensorflow versions above or equal to 2.13.0 and strictly below 2.16.0 (nightly versions are not supported). 
+
+
+# 使用 onnx2tf库 取代 onnx-tf-tflite，取代tinynn
+# conda create -n py310 python=3.10
+# 按照官方安装方法 https://github.com/PINTO0309/onnx2tf
+# pip install -U onnx==1.16.1 && pip install -U nvidia-pyindex && pip install -U onnx-graphsurgeon && pip install -U onnxruntime==1.18.1 && pip install -U onnxsim==0.4.33 && pip install -U simple_onnx_processing_tools && pip install -U sne4onnx>=1.0.13 && pip install -U sng4onnx>=1.0.4 && pip install -U tensorflow==2.17.0 && pip install -U protobuf==3.20.3 && pip install -U onnx2tf && pip install -U h5py==3.11.0 && pip install -U psutil==5.9.5 && pip install -U ml_dtypes==0.3.2 && pip install -U tf-keras~=2.16 && pip install flatbuffers>=23.5.26
+
+# 直接执行失败，固定了onnx的输入后成功
+# onnx2tf -i depth_anything_vits14.onnx -ois image:1,3,518,518
+# 代码如上，已经跑通