Skip to content

Commit

Permalink
Merge pull request atenpas#47 from sharronliu/master
Browse files Browse the repository at this point in the history
Enable OpenVINO option for GPD
  • Loading branch information
atenpas authored Sep 6, 2018
2 parents 344a770 + 494d0b7 commit 11ebd16
Show file tree
Hide file tree
Showing 20 changed files with 919 additions and 35 deletions.
34 changes: 26 additions & 8 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,28 @@ find_package(Caffe)
include_directories(${Caffe_INCLUDE_DIRS})
add_definitions(${Caffe_DEFINITIONS})

# OpenVINO
option(USE_OPENVINO "use OpenVINO toolkit" OFF)
if(USE_OPENVINO STREQUAL "ON")
find_package(InferenceEngine 1.0)
if (NOT InferenceEngine_FOUND)
message(FATAL_ERROR "Please install OpenVINO https://software.intel.com/en-us/articles/OpenVINO-Install-Linux")
endif()
include_directories(${InferenceEngine_INCLUDE_DIRS})
link_directories(${InferenceEngine_LIBRARY_DIRS})
add_definitions(-DUSE_OPENVINO)
get_filename_component(MODELS_DIR "models/openvino" ABSOLUTE)
configure_file(include/gpd/openvino_classifier.h.in gpd/openvino_classifier.h)
include_directories(${CMAKE_CURRENT_BINARY_DIR})
set(classifier_src src/${PROJECT_NAME}/classifier.cpp src/${PROJECT_NAME}/openvino_classifier.cpp)
set(classifier_dep ${InferenceEngine_LIBRARIES})
else()
set(classifier_src src/${PROJECT_NAME}/classifier.cpp src/${PROJECT_NAME}/caffe_classifier.cpp)
set(classifier_dep ${Caffe_LIBRARIES} ${OpenCV_LIBRARIES})
endif()

## Set compiler optimization flags
set(CMAKE_CXX_FLAGS "-DNDEBUG -O3 -fopenmp -flto -Wno-deprecated -Wenum-compare")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNDEBUG -O3 -fopenmp -flto -Wno-deprecated -Wenum-compare -std=c++11")
# set(CMAKE_CXX_FLAGS "-DNDEBUG -O3 -fopenmp -flto -mavx -mfma -Wno-deprecated -Wenum-compare")

## Uncomment this if the package has a setup.py. This macro ensures
Expand Down Expand Up @@ -130,7 +150,7 @@ DEPENDS Eigen OpenCV PCL
include_directories(include ${catkin_INCLUDE_DIRS} ${PCL_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS})

## Declare a C++ library
add_library(${PROJECT_NAME}_caffe_classifier src/${PROJECT_NAME}/caffe_classifier.cpp)
add_library(${PROJECT_NAME}_classifier ${classifier_src})
add_library(${PROJECT_NAME}_clustering src/${PROJECT_NAME}/clustering.cpp)
add_library(${PROJECT_NAME}_data_generator src/${PROJECT_NAME}/data_generator.cpp)
add_library(${PROJECT_NAME}_grasp_detector src/${PROJECT_NAME}/grasp_detector.cpp)
Expand Down Expand Up @@ -158,9 +178,8 @@ add_executable(${PROJECT_NAME}_test_occlusion src/tests/test_occlusion.cpp)
# add_dependencies(grasp_candidates_classifier_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})

## Specify libraries to link a library or executable target against
target_link_libraries(${PROJECT_NAME}_caffe_classifier
${Caffe_LIBRARIES}
${OpenCV_LIBRARIES})
target_link_libraries(${PROJECT_NAME}_classifier
${classifier_dep})

target_link_libraries(${PROJECT_NAME}_clustering
${GENERATOR_LIB})
Expand Down Expand Up @@ -194,12 +213,12 @@ target_link_libraries(${PROJECT_NAME}_classify_candidates
${PCL_LIBRARIES})

target_link_libraries(${PROJECT_NAME}_grasp_detector
${PROJECT_NAME}_caffe_classifier
${PROJECT_NAME}_classifier
${PROJECT_NAME}_clustering
${PROJECT_NAME}_learning
${GENERATOR_LIB}
${catkin_LIBRARIES}
${Caffe_LIBRARIES})
${classifier_dep})

target_link_libraries(${PROJECT_NAME}_grasp_image
${OpenCV_LIBRARIES})
Expand Down Expand Up @@ -232,7 +251,6 @@ target_link_libraries(${PROJECT_NAME}_learning
${catkin_LIBRARIES}
${Caffe_LIBRARIES})


# Rename targets to simplify their names.
set_target_properties(${PROJECT_NAME}_detect_grasps
PROPERTIES OUTPUT_NAME detect_grasps
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ The output should look similar to the screenshot shown below.

1. [Detect Grasps With an RGBD camera](tutorials/tutorial_1_grasps_camera.md)
2. [Detect Grasps on a Specific Object](tutorials/tutorial_2_grasp_select.md)
3. [Detect Grasps with OpenVINO](tutorials/tutorial_openvino.md)


## 7) Parameters
Expand Down
6 changes: 3 additions & 3 deletions include/gpd/caffe_classifier.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
#ifndef CAFFE_CLASSIFIER_H_
#define CAFFE_CLASSIFIER_H_


// System
#include <string>
#include <vector>
Expand All @@ -46,6 +45,7 @@
// OpenCV
#include <opencv2/core/core.hpp>

#include "gpd/classifier.h"

/** CaffeClassifier class
*
Expand All @@ -54,7 +54,7 @@
* This class classifies grasps as viable or not using a convolutional neural network (CNN) in the Caffe framework.
*
*/
class CaffeClassifier
class CaffeClassifier : public Classifier
{
public:

Expand All @@ -63,7 +63,7 @@ class CaffeClassifier
* \param model_file the location of the file that describes the network model
* \param weights_file the location of the file that contains the network weights
*/
CaffeClassifier(const std::string& model_file, const std::string& weights_file, bool use_gpu = true);
CaffeClassifier(const std::string& model_file, const std::string& weights_file, Classifier::Device device);

/**
* \brief Classify grasp candidates as viable grasps or not.
Expand Down
80 changes: 80 additions & 0 deletions include/gpd/classifier.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Software License Agreement (BSD License)
*
* Copyright (c) 2018 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#ifndef CLASSIFIER_H_
#define CLASSIFIER_H_

// System
#include <memory>
#include <string>
#include <vector>

// OpenCV
#include <opencv2/core/core.hpp>

/** Classifier class
*
* \brief Abstract base class for classifier that Classify grasp candidates as viable grasps or not
*
*/
class Classifier
{
public:

enum class Device : uint8_t {
eCPU = 0,
eGPU = 1,
eVPU = 2,
eFPGA = 3
};

/**
* \brief Create Classifier per build option.
* - If "USE_OPENVINO", an OpenVINO classifier will be created
* - Otherwise a Caffe classifier will be created
* \param model_file The location of the file that describes the network model
* \param weights_file The location of the file that contains the network weights
* \param device The target device where the network computation executes
*/
static std::shared_ptr<Classifier> create(const std::string& model_file, const std::string& weights_file, Device device = Device::eCPU);

/**
* \brief Classify grasp candidates as viable grasps or not.
* \param image_list the list of grasp images
* \return the classified grasp candidates
*/
virtual std::vector<float> classifyImages(const std::vector<cv::Mat>& image_list) = 0;

virtual int getBatchSize() const = 0;
};


#endif /* CLASSIFIER_H_ */
10 changes: 2 additions & 8 deletions include/gpd/grasp_detector.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,6 @@
#include <algorithm>
#include <vector>

// Caffe
#include "caffe/caffe.hpp"
#include "caffe/layers/memory_data_layer.hpp"
#include "caffe/util/io.hpp"

// PCL
#include <pcl/common/common.h>
#include <pcl/filters/statistical_outlier_removal.h>
Expand All @@ -60,7 +55,7 @@
#include <gpg/plot.h>

// Custom
#include "../gpd/caffe_classifier.h"
#include "gpd/classifier.h"
#include "../gpd/clustering.h"
#include "../gpd/learning.h"

Expand Down Expand Up @@ -91,7 +86,6 @@ class GraspDetector
delete candidates_generator_;
delete learning_;
delete clustering_;
delete classifier_;
}

/**
Expand Down Expand Up @@ -184,7 +178,7 @@ class GraspDetector
CandidatesGenerator* candidates_generator_; ///< pointer to object for grasp candidate generation
Learning* learning_; ///< pointer to object for grasp image creation
Clustering* clustering_; ///< pointer to object for clustering geometrically aligned grasps
CaffeClassifier* classifier_; ///< pointer to object for classification of candidates
std::shared_ptr<Classifier> classifier_; ///< pointer to object for classification of candidates

Learning::ImageParameters image_params_; // grasp image parameters

Expand Down
80 changes: 80 additions & 0 deletions include/gpd/openvino_classifier.h.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Software License Agreement (BSD License)
*
* Copyright (c) 2018 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#ifndef OPENVINO_CLASSIFIER_H_
#define OPENVINO_CLASSIFIER_H_

// System
#include <string>
#include <vector>

// OpenVINO
#include <inference_engine.hpp>

// OpenCV
#include <opencv2/opencv.hpp>

#include <gpd/classifier.h>

#define MODELS_DIR "@MODELS_DIR@"

/** CaffeClassifier class
*
* \brief Classify grasp candidates as viable grasps or not
*
* This class classifies grasps as viable or not using a convolutional neural network (CNN) in the Caffe framework.
*
*/
class OpenVINOClassifier : public Classifier
{
public:

/**
* \brief Constructor.
* \param model_dir The directory that contains the openvino network model
* \param device The target device where the computation executes
*/
OpenVINOClassifier(Classifier::Device device);

std::vector<float> classifyImages(const std::vector<cv::Mat>& image_list);

int getBatchSize() const;

private:
static std::map<Classifier::Device, InferenceEngine::TargetDevice> device_map_;
std::vector<cv::Mat> batch_image_list_;
InferenceEngine::Blob::Ptr input_blob_, output_blob_;
InferenceEngine::CNNNetwork network_;
InferenceEngine::InferRequest infer_request_;
InferenceEngine::InferencePlugin plugin_;
};

#endif /* OPENVINO_CLASSIFIER_H_ */
2 changes: 0 additions & 2 deletions launch/caffe/classifier_15channels.launch
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@

<arg name="caffe_path" value="$(find gpd)/caffe/15channels" />

<param name="$(arg node)/use_gpu" value="true" /> <!-- if Caffe uses the GPU or CPU -->

<param name="$(arg node)/model_file" value="$(arg caffe_path)/lenet_15_channels.prototxt" />
<param name="$(arg node)/trained_file" value="$(arg caffe_path)/two_views_15_channels_53_deg.caffemodel" />
<!-- <param name="$(arg node)/trained_file" value="$(arg caffe_path)/single_view_15_channels.caffemodel" /> -->
Expand Down
2 changes: 0 additions & 2 deletions launch/caffe/ur5_classifier_15channels.launch
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@

<arg name="caffe_path" value="$(find gpd)/caffe/15channels" />

<param name="$(arg node)/use_gpu" value="true" /> <!-- if Caffe uses the GPU or CPU -->

<param name="$(arg node)/model_file" value="$(arg caffe_path)/lenet_15_channels.prototxt" />
<param name="$(arg node)/trained_file" value="$(arg caffe_path)/two_views_15_channels_90_deg_no_flipping.caffemodel" />

Expand Down
13 changes: 9 additions & 4 deletions launch/tutorial1.launch
Original file line number Diff line number Diff line change
@@ -1,17 +1,22 @@
<launch>


<arg name="device" default="0"
doc="target device to execute inference. [0:CPU]|1:GPU|2:VPU|3:FPGA" />

<!-- Load hand geometry parameters -->
<include file="$(find gpd)/launch/hand_geometry.launch">
<arg name="node" value="detect_grasps" />
</include>
<!-- Load classifier parameters -->

<!-- Load classifier parameters -->
<include file="$(find gpd)/launch/caffe/classifier_15channels.launch">
<arg name="node" value="detect_grasps" />
</include>

<node name="detect_grasps" pkg="gpd" type="detect_grasps" output="screen">


<param name="device" type="int" value="$(arg device)" />

<!-- If sequential importance sampling is used (default: false) -->
<param name="use_importance_sampling" value="false" />

Expand Down
Binary file added models/openvino/fp16/single_view_15_channels.bin
Binary file not shown.
Loading

0 comments on commit 11ebd16

Please sign in to comment.