From 89f5c80394330f783ab2e7e438a1a847fa1bea2b Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 14:36:01 +0800 Subject: [PATCH 01/41] refactoring extraction --- .gitignore | 2 + .gitmodules | 3 + CMakeLists.txt | 23 ++++- denseFlow.cpp | 33 +----- denseFlow_gpu.cpp | 225 +++++++++++++++++++---------------------- include/common.h | 37 +++++++ include/dense_flow.h | 19 ++++ include/easylogging++ | 1 + src/common.cpp | 44 ++++++++ src/dense_flow.cpp | 9 ++ src/dense_flow_gpu.cpp | 83 +++++++++++++++ 11 files changed, 321 insertions(+), 158 deletions(-) create mode 100644 .gitmodules create mode 100644 include/common.h create mode 100644 include/dense_flow.h create mode 160000 include/easylogging++ create mode 100644 src/common.cpp create mode 100644 src/dense_flow.cpp create mode 100644 src/dense_flow_gpu.cpp diff --git a/.gitignore b/.gitignore index 5761abc..50e7488 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.o +.idea/ +build/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..208c0b4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "include/easylogging++"] + path = include/easylogging++ + url = https://github.com/easylogging/easyloggingpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 92217d2..47b3fc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,23 @@ cmake_minimum_required(VERSION 2.8) project( denseFlow ) + +include(CheckCXXCompilerFlag) +CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) +CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) +if(COMPILER_SUPPORTS_CXX11) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +elseif(COMPILER_SUPPORTS_CXX0X) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") +else() + message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") +endif() + find_package( OpenCV REQUIRED ) -include_directories( ${OpenCV_INCLUDE_DIRS} ) -add_executable( denseFlow_gpu denseFlow_gpu.cpp ) -target_link_libraries( denseFlow_gpu ${OpenCV_LIBS} ) \ No newline at end of file + +include_directories( ${OpenCV_INCLUDE_DIRS} include/ include/easylogging++/src) + +add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp) +target_link_libraries( denseflow ${OpenCV_LIBS}) + +add_executable( extract_gpu denseFlow_gpu.cpp) +target_link_libraries( extract_gpu ${OpenCV_LIBS} denseflow) \ No newline at end of file diff --git a/denseFlow.cpp b/denseFlow.cpp index e15a46e..b723464 100644 --- a/denseFlow.cpp +++ b/denseFlow.cpp @@ -1,35 +1,4 @@ -#include "opencv2/video/tracking.hpp" -#include "opencv2/imgproc/imgproc.hpp" -#include "opencv2/highgui/highgui.hpp" - -#include -#include -using namespace cv; - -static void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y, double lowerBound, double higherBound) { - #define CAST(v, L, H) ((v) > (H) ? 255 : (v) < (L) ? 0 : cvRound(255*((v) - (L))/((H)-(L)))) - for (int i = 0; i < flow_x.rows; ++i) { - for (int j = 0; j < flow_y.cols; ++j) { - float x = flow_x.at(i,j); - float y = flow_y.at(i,j); - img_x.at(i,j) = CAST(x, lowerBound, higherBound); - img_y.at(i,j) = CAST(y, lowerBound, higherBound); - } - } - #undef CAST -} - -static void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scalar& color) -{ - for(int y = 0; y < cflowmap.rows; y += step) - for(int x = 0; x < cflowmap.cols; x += step) - { - const Point2f& fxy = flow.at(y, x); - line(cflowmap, Point(x,y), Point(cvRound(x+fxy.x), cvRound(y+fxy.y)), - color); - circle(cflowmap, Point(x,y), 2, color, -1); - } -} +#include "common.h" int main(int argc, char** argv) { diff --git a/denseFlow_gpu.cpp b/denseFlow_gpu.cpp index 3e2983f..3867643 100644 --- a/denseFlow_gpu.cpp +++ b/denseFlow_gpu.cpp @@ -1,37 +1,9 @@ -#include "opencv2/video/tracking.hpp" -#include "opencv2/imgproc/imgproc.hpp" -#include "opencv2/highgui/highgui.hpp" -#include "opencv2/gpu/gpu.hpp" +#include "common.h" +#include "dense_flow.h" -#include -#include -using namespace cv; -using namespace cv::gpu; - -static void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y, - double lowerBound, double higherBound) { - #define CAST(v, L, H) ((v) > (H) ? 255 : (v) < (L) ? 0 : cvRound(255*((v) - (L))/((H)-(L)))) - for (int i = 0; i < flow_x.rows; ++i) { - for (int j = 0; j < flow_y.cols; ++j) { - float x = flow_x.at(i,j); - float y = flow_y.at(i,j); - img_x.at(i,j) = CAST(x, lowerBound, higherBound); - img_y.at(i,j) = CAST(y, lowerBound, higherBound); - } - } - #undef CAST -} +INITIALIZE_EASYLOGGINGPP -static void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scalar& color){ - for(int y = 0; y < cflowmap.rows; y += step) - for(int x = 0; x < cflowmap.cols; x += step) - { - const Point2f& fxy = flow.at(y, x); - line(cflowmap, Point(x,y), Point(cvRound(x+fxy.x), cvRound(y+fxy.y)), - color); - circle(cflowmap, Point(x,y), 2, color, -1); - } -} +using namespace cv::gpu; int main(int argc, char** argv){ // IO operation @@ -57,96 +29,103 @@ int main(int argc, char** argv){ int device_id = cmd.get("device_id"); int step = cmd.get("step"); - VideoCapture capture(vidFile); - if(!capture.isOpened()) { - printf("Could not initialize capturing..\n"); - return -1; - } - - int frame_num = 0; - Mat image, prev_image, prev_grey, grey, frame, flow_x, flow_y; - GpuMat frame_0, frame_1, flow_u, flow_v; - - setDevice(device_id); - FarnebackOpticalFlow alg_farn; - OpticalFlowDual_TVL1_GPU alg_tvl1; - BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); - - while(true) { - capture >> frame; - if(frame.empty()) - break; - if(frame_num == 0) { - image.create(frame.size(), CV_8UC3); - grey.create(frame.size(), CV_8UC1); - prev_image.create(frame.size(), CV_8UC3); - prev_grey.create(frame.size(), CV_8UC1); - - frame.copyTo(prev_image); - cvtColor(prev_image, prev_grey, CV_BGR2GRAY); - - frame_num++; - - int step_t = step; - while (step_t > 1){ - capture >> frame; - step_t--; - } - continue; - } - - frame.copyTo(image); - cvtColor(image, grey, CV_BGR2GRAY); - - - frame_0.upload(prev_grey); - frame_1.upload(grey); - - - // GPU optical flow - switch(type){ - case 0: - alg_farn(frame_0,frame_1,flow_u,flow_v); - break; - case 1: - alg_tvl1(frame_0,frame_1,flow_u,flow_v); - break; - case 2: - GpuMat d_frame0f, d_frame1f; - frame_0.convertTo(d_frame0f, CV_32F, 1.0 / 255.0); - frame_1.convertTo(d_frame1f, CV_32F, 1.0 / 255.0); - alg_brox(d_frame0f, d_frame1f, flow_u,flow_v); - break; - } - - flow_u.download(flow_x); - flow_v.download(flow_y); - - // Output optical flow - Mat imgX(flow_x.size(),CV_8UC1); - Mat imgY(flow_y.size(),CV_8UC1); - convertFlowToImage(flow_x,flow_y, imgX, imgY, -bound, bound); - char tmp[20]; - sprintf(tmp,"_%04d.jpg",int(frame_num)); - - Mat imgX_, imgY_, image_; - resize(imgX,imgX_,cv::Size(340,256)); - resize(imgY,imgY_,cv::Size(340,256)); - resize(image,image_,cv::Size(340,256)); - - imwrite(xFlowFile + tmp,imgX_); - imwrite(yFlowFile + tmp,imgY_); - imwrite(imgFile + tmp, image_); - - std::swap(prev_grey, grey); - std::swap(prev_image, image); - frame_num = frame_num + 1; - - int step_t = step; - while (step_t > 1){ - capture >> frame; - step_t--; - } - } + vector > out_vec_x, out_vec_y, out_vec_img; + + calcDenseFlowGPU(vidFile, bound, type, step, device_id, + out_vec_x, out_vec_y, out_vec_img); + + + +// VideoCapture capture(vidFile); +// if(!capture.isOpened()) { +// printf("Could not initialize capturing..\n"); +// return -1; +// } +// +// int frame_num = 0; +// Mat image, prev_image, prev_grey, grey, frame, flow_x, flow_y; +// GpuMat frame_0, frame_1, flow_u, flow_v; +// +// setDevice(device_id); +// FarnebackOpticalFlow alg_farn; +// OpticalFlowDual_TVL1_GPU alg_tvl1; +// BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); +// +// while(true) { +// capture >> frame; +// if(frame.empty()) +// break; +// if(frame_num == 0) { +// image.create(frame.size(), CV_8UC3); +// grey.create(frame.size(), CV_8UC1); +// prev_image.create(frame.size(), CV_8UC3); +// prev_grey.create(frame.size(), CV_8UC1); +// +// frame.copyTo(prev_image); +// cvtColor(prev_image, prev_grey, CV_BGR2GRAY); +// +// frame_num++; +// +// int step_t = step; +// while (step_t > 1){ +// capture >> frame; +// step_t--; +// } +// continue; +// } +// +// frame.copyTo(image); +// cvtColor(image, grey, CV_BGR2GRAY); +// +// +// frame_0.upload(prev_grey); +// frame_1.upload(grey); +// +// +// // GPU optical flow +// switch(type){ +// case 0: +// alg_farn(frame_0,frame_1,flow_u,flow_v); +// break; +// case 1: +// alg_tvl1(frame_0,frame_1,flow_u,flow_v); +// break; +// case 2: +// GpuMat d_frame0f, d_frame1f; +// frame_0.convertTo(d_frame0f, CV_32F, 1.0 / 255.0); +// frame_1.convertTo(d_frame1f, CV_32F, 1.0 / 255.0); +// alg_brox(d_frame0f, d_frame1f, flow_u,flow_v); +// break; +// } +// +// flow_u.download(flow_x); +// flow_v.download(flow_y); +// +// // Output optical flow +// Mat imgX(flow_x.size(),CV_8UC1); +// Mat imgY(flow_y.size(),CV_8UC1); +// convertFlowToImage(flow_x,flow_y, imgX, imgY, -bound, bound); +// char tmp[20]; +// sprintf(tmp,"_%04d.jpg",int(frame_num)); +// +// Mat imgX_, imgY_, image_; +// resize(imgX,imgX_,cv::Size(340,256)); +// resize(imgY,imgY_,cv::Size(340,256)); +// resize(image,image_,cv::Size(340,256)); +// +// imwrite(xFlowFile + tmp,imgX_); +// imwrite(yFlowFile + tmp,imgY_); +// imwrite(imgFile + tmp, image_); +// +// std::swap(prev_grey, grey); +// std::swap(prev_image, image); +// frame_num = frame_num + 1; +// +// int step_t = step; +// while (step_t > 1){ +// capture >> frame; +// step_t--; +// } +// } return 0; } diff --git a/include/common.h b/include/common.h new file mode 100644 index 0000000..16e24d8 --- /dev/null +++ b/include/common.h @@ -0,0 +1,37 @@ +// +// Created by yjxiong on 11/18/15. +// + +#ifndef DENSEFLOW_COMMON_H_H +#define DENSEFLOW_COMMON_H_H + +#include "easylogging++.h" + +#include "opencv2/video/tracking.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/highgui/highgui.hpp" + +#include +#include +using namespace cv; + +void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y, + double lowerBound, double higherBound); +void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scalar& color); + +void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, + vector& encoded_x, vector& encoded_y, + int bound); + +inline void initializeMats(const Mat& frame, + Mat& capture_image, Mat& capture_gray, + Mat& prev_image, Mat& prev_gray){ + capture_image.create(frame.size(), CV_8UC3); + capture_gray.create(frame.size(), CV_8UC1); + + prev_image.create(frame.size(), CV_8UC3); + prev_gray.create(frame.size(), CV_8UC1); +} + +#endif //DENSEFLOW_COMMON_H_H + diff --git a/include/dense_flow.h b/include/dense_flow.h new file mode 100644 index 0000000..b67851c --- /dev/null +++ b/include/dense_flow.h @@ -0,0 +1,19 @@ +// +// Created by yjxiong on 11/18/15. +// + +#ifndef DENSEFLOW_DENSE_FLOW_H +#define DENSEFLOW_DENSE_FLOW_H + +#include "common.h" + +void calcDenseFlow(string file_name, int bound, int type, int step, + vector >& output_x, + vector >& output_y, + vector >& output_img); +void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_id, + vector >& output_x, + vector >& output_y, + vector >& output_img); + +#endif //DENSEFLOW_DENSE_FLOW_H diff --git a/include/easylogging++ b/include/easylogging++ new file mode 160000 index 0000000..f926802 --- /dev/null +++ b/include/easylogging++ @@ -0,0 +1 @@ +Subproject commit f926802dfbde716d82b64b8ef3c25b7f0fcfec65 diff --git a/src/common.cpp b/src/common.cpp new file mode 100644 index 0000000..5dedae2 --- /dev/null +++ b/src/common.cpp @@ -0,0 +1,44 @@ +// +// Created by yjxiong on 11/18/15. +// + +#include "common.h" + +void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y, + double lowerBound, double higherBound) { +#define CAST(v, L, H) ((v) > (H) ? 255 : (v) < (L) ? 0 : cvRound(255*((v) - (L))/((H)-(L)))) + for (int i = 0; i < flow_x.rows; ++i) { + for (int j = 0; j < flow_y.cols; ++j) { + float x = flow_x.at(i,j); + float y = flow_y.at(i,j); + img_x.at(i,j) = CAST(x, lowerBound, higherBound); + img_y.at(i,j) = CAST(y, lowerBound, higherBound); + } + } +#undef CAST +} + +void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scalar& color){ + for(int y = 0; y < cflowmap.rows; y += step) + for(int x = 0; x < cflowmap.cols; x += step) + { + const Point2f& fxy = flow.at(y, x); + line(cflowmap, Point(x,y), Point(cvRound(x+fxy.x), cvRound(y+fxy.y)), + color); + circle(cflowmap, Point(x,y), 2, color, -1); + } +} + +void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, + vector& encoded_x, vector& encoded_y, + int bound){ + Mat flow_img_x(flow_map_x.size(), CV_8UC1); + Mat flow_img_y(flow_map_y.size(), CV_8UC1); + + convertFlowToImage(flow_map_x, flow_map_y, flow_img_x, flow_img_y, + -bound, bound); + + imencode(".jpg", flow_img_x, encoded_x); + imencode(".jpg", flow_img_y, encoded_y); +} + diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp new file mode 100644 index 0000000..bc08a40 --- /dev/null +++ b/src/dense_flow.cpp @@ -0,0 +1,9 @@ +// +// Created by yjxiong on 11/18/15. +// +#include "common.h" + +void calcDenseFlow(string file_name, int bound, int type, int step, + vector >& output){ + +} \ No newline at end of file diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp new file mode 100644 index 0000000..850ac17 --- /dev/null +++ b/src/dense_flow_gpu.cpp @@ -0,0 +1,83 @@ +// +// Created by yjxiong on 11/18/15. +// +#include "common.h" +#include "opencv2/gpu/gpu.hpp" +using namespace cv::gpu; + + +void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_id, + vector >& output_x, + vector >& output_y, + vector >& output_img){ + VideoCapture video_stream(file_name); + CHECK(video_stream.isOpened())<<"Cannot open video stream" + <> capture_frame; + if (capture_frame.empty()) break; // read frames until end + + //build mats for the first frame + if (~initialized){ + initializeMats(capture_frame, capture_image, capture_gray, + prev_image, prev_gray); + capture_frame.copyTo(prev_image); + cvtColor(prev_image, prev_gray, CV_BGR2GRAY); + initialized = true; + }else if(iter % step == 0){ + capture_frame.copyTo(capture_image); + cvtColor(capture_image, capture_gray, CV_BGR2GRAY); + d_frame_0.upload(prev_gray); + d_frame_1.upload(capture_gray); + + switch(type){ + case 0: { + alg_farn(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + break; + } + case 1: { + alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + break; + } + case 2: { + GpuMat d_buf_0, d_buf_1; + d_frame_0.convertTo(d_buf_0, CV_32F, 1.0 / 255.0); + d_frame_1.convertTo(d_buf_1, CV_32F, 1.0 / 255.0); + alg_brox(d_buf_0, d_buf_1, d_flow_x, d_flow_y); + break; + } + default: + LOG(ERROR)<<"Unknown optical method: "< str_x, str_y, str_img; + encodeFlowMap(flow_x, flow_y, str_x, str_y, bound); + imencode(".jpg", capture_image, str_img); + + output_x.push_back(str_x); + output_y.push_back(str_y); + output_img.push_back(str_img); + } + + + } + +} \ No newline at end of file From 00d13d0716c1deaf49c5fdb1b0c3a16e297867d6 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 14:43:43 +0800 Subject: [PATCH 02/41] change log format --- src/dense_flow_gpu.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index 850ac17..3ea0fd2 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -11,9 +11,9 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i vector >& output_y, vector >& output_img){ VideoCapture video_stream(file_name); - CHECK(video_stream.isOpened())<<"Cannot open video stream" + CHECK(video_stream.isOpened())<<"Cannot open video stream \"" < Date: Wed, 18 Nov 2015 15:28:16 +0800 Subject: [PATCH 03/41] change project structure --- CMakeLists.txt | 5 +- .../extracOpticalFlow.m | 0 .../extractOpticalFlow.m | 0 .../extractOpticalFlow_gpu.m | 0 denseFlow.cpp | 76 ---------- denseFlow_gpu.cpp | 131 ------------------ src/dense_flow.cpp | 45 +++++- src/dense_flow_gpu.cpp | 2 +- tools/extract_flow.cpp | 30 ++++ tools/extract_flow_gpu.cpp | 38 +++++ 10 files changed, 117 insertions(+), 210 deletions(-) rename extracOpticalFlow.m => _deprecated/extracOpticalFlow.m (100%) rename extractOpticalFlow.m => _deprecated/extractOpticalFlow.m (100%) rename extractOpticalFlow_gpu.m => _deprecated/extractOpticalFlow_gpu.m (100%) delete mode 100644 denseFlow.cpp delete mode 100644 denseFlow_gpu.cpp create mode 100644 tools/extract_flow.cpp create mode 100644 tools/extract_flow_gpu.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 47b3fc6..e7fb3e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,5 +19,8 @@ include_directories( ${OpenCV_INCLUDE_DIRS} include/ include/easylogging++/src) add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp) target_link_libraries( denseflow ${OpenCV_LIBS}) -add_executable( extract_gpu denseFlow_gpu.cpp) +add_executable( extract_cpu tools/extract_flow.cpp) +target_link_libraries( extract_cpu ${OpenCV_LIBS} denseflow) + +add_executable( extract_gpu tools/extract_flow_gpu.cpp) target_link_libraries( extract_gpu ${OpenCV_LIBS} denseflow) \ No newline at end of file diff --git a/extracOpticalFlow.m b/_deprecated/extracOpticalFlow.m similarity index 100% rename from extracOpticalFlow.m rename to _deprecated/extracOpticalFlow.m diff --git a/extractOpticalFlow.m b/_deprecated/extractOpticalFlow.m similarity index 100% rename from extractOpticalFlow.m rename to _deprecated/extractOpticalFlow.m diff --git a/extractOpticalFlow_gpu.m b/_deprecated/extractOpticalFlow_gpu.m similarity index 100% rename from extractOpticalFlow_gpu.m rename to _deprecated/extractOpticalFlow_gpu.m diff --git a/denseFlow.cpp b/denseFlow.cpp deleted file mode 100644 index b723464..0000000 --- a/denseFlow.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "common.h" - -int main(int argc, char** argv) -{ - // IO operation - - const char* keys = - { - "{ f | vidFile | ex2.avi | filename of video }" - "{ x | xFlowFile | flow_x | filename of flow x component }" - "{ y | yFlowFile | flow_y | filename of flow x component }" - "{ i | imgFile | flow_i | filename of flow image}" - "{ b | bound | 15 | specify the maximum of optical flow}" - }; - - CommandLineParser cmd(argc, argv, keys); - string vidFile = cmd.get("vidFile"); - string xFlowFile = cmd.get("xFlowFile"); - string yFlowFile = cmd.get("yFlowFile"); - string imgFile = cmd.get("imgFile"); - int bound = cmd.get("bound"); - - - VideoCapture capture(vidFile); - if(!capture.isOpened()) { - printf("Could not initialize capturing..\n"); - return -1; - } - - int frame_num = 0; - Mat image, prev_image, prev_grey, grey, frame, flow, cflow; - - while(true) { - capture >> frame; - if(frame.empty()) - break; - - if(frame_num == 0) { - image.create(frame.size(), CV_8UC3); - grey.create(frame.size(), CV_8UC1); - prev_image.create(frame.size(), CV_8UC3); - prev_grey.create(frame.size(), CV_8UC1); - - frame.copyTo(prev_image); - cvtColor(prev_image, prev_grey, CV_BGR2GRAY); - - frame_num++; - continue; - } - - frame.copyTo(image); - cvtColor(image, grey, CV_BGR2GRAY); - - // calcOpticalFlowFarneback(prev_grey,grey,flow,0.5, 3, 15, 3, 5, 1.2, 0 ); - calcOpticalFlowFarneback(prev_grey, grey, flow, 0.702, 5, 10, 2, 7, 1.5, cv::OPTFLOW_FARNEBACK_GAUSSIAN ); - - // prev_image.copyTo(cflow); - // drawOptFlowMap(flow, cflow, 12, 1.5, Scalar(0, 255, 0)); - - Mat flows[2]; - split(flow,flows); - Mat imgX(flows[0].size(),CV_8UC1); - Mat imgY(flows[0].size(),CV_8UC1); - convertFlowToImage(flows[0],flows[1], imgX, imgY, -bound, bound); - char tmp[20]; - sprintf(tmp,"_%04d.jpg",int(frame_num)); - imwrite(xFlowFile + tmp,imgX); - imwrite(yFlowFile + tmp,imgY); - imwrite(imgFile + tmp, image); - - std::swap(prev_grey, grey); - std::swap(prev_image, image); - frame_num = frame_num + 1; - } - return 0; -} diff --git a/denseFlow_gpu.cpp b/denseFlow_gpu.cpp deleted file mode 100644 index 3867643..0000000 --- a/denseFlow_gpu.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "common.h" -#include "dense_flow.h" - -INITIALIZE_EASYLOGGINGPP - -using namespace cv::gpu; - -int main(int argc, char** argv){ - // IO operation - const char* keys = - { - "{ f | vidFile | ex2.avi | filename of video }" - "{ x | xFlowFile | flow_x | filename of flow x component }" - "{ y | yFlowFile | flow_y | filename of flow x component }" - "{ i | imgFile | flow_i | filename of flow image}" - "{ b | bound | 15 | specify the maximum of optical flow}" - "{ t | type | 0 | specify the optical flow algorithm }" - "{ d | device_id | 0 | set gpu id}" - "{ s | step | 1 | specify the step for frame sampling}" - }; - - CommandLineParser cmd(argc, argv, keys); - string vidFile = cmd.get("vidFile"); - string xFlowFile = cmd.get("xFlowFile"); - string yFlowFile = cmd.get("yFlowFile"); - string imgFile = cmd.get("imgFile"); - int bound = cmd.get("bound"); - int type = cmd.get("type"); - int device_id = cmd.get("device_id"); - int step = cmd.get("step"); - - vector > out_vec_x, out_vec_y, out_vec_img; - - calcDenseFlowGPU(vidFile, bound, type, step, device_id, - out_vec_x, out_vec_y, out_vec_img); - - - -// VideoCapture capture(vidFile); -// if(!capture.isOpened()) { -// printf("Could not initialize capturing..\n"); -// return -1; -// } -// -// int frame_num = 0; -// Mat image, prev_image, prev_grey, grey, frame, flow_x, flow_y; -// GpuMat frame_0, frame_1, flow_u, flow_v; -// -// setDevice(device_id); -// FarnebackOpticalFlow alg_farn; -// OpticalFlowDual_TVL1_GPU alg_tvl1; -// BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); -// -// while(true) { -// capture >> frame; -// if(frame.empty()) -// break; -// if(frame_num == 0) { -// image.create(frame.size(), CV_8UC3); -// grey.create(frame.size(), CV_8UC1); -// prev_image.create(frame.size(), CV_8UC3); -// prev_grey.create(frame.size(), CV_8UC1); -// -// frame.copyTo(prev_image); -// cvtColor(prev_image, prev_grey, CV_BGR2GRAY); -// -// frame_num++; -// -// int step_t = step; -// while (step_t > 1){ -// capture >> frame; -// step_t--; -// } -// continue; -// } -// -// frame.copyTo(image); -// cvtColor(image, grey, CV_BGR2GRAY); -// -// -// frame_0.upload(prev_grey); -// frame_1.upload(grey); -// -// -// // GPU optical flow -// switch(type){ -// case 0: -// alg_farn(frame_0,frame_1,flow_u,flow_v); -// break; -// case 1: -// alg_tvl1(frame_0,frame_1,flow_u,flow_v); -// break; -// case 2: -// GpuMat d_frame0f, d_frame1f; -// frame_0.convertTo(d_frame0f, CV_32F, 1.0 / 255.0); -// frame_1.convertTo(d_frame1f, CV_32F, 1.0 / 255.0); -// alg_brox(d_frame0f, d_frame1f, flow_u,flow_v); -// break; -// } -// -// flow_u.download(flow_x); -// flow_v.download(flow_y); -// -// // Output optical flow -// Mat imgX(flow_x.size(),CV_8UC1); -// Mat imgY(flow_y.size(),CV_8UC1); -// convertFlowToImage(flow_x,flow_y, imgX, imgY, -bound, bound); -// char tmp[20]; -// sprintf(tmp,"_%04d.jpg",int(frame_num)); -// -// Mat imgX_, imgY_, image_; -// resize(imgX,imgX_,cv::Size(340,256)); -// resize(imgY,imgY_,cv::Size(340,256)); -// resize(image,image_,cv::Size(340,256)); -// -// imwrite(xFlowFile + tmp,imgX_); -// imwrite(yFlowFile + tmp,imgY_); -// imwrite(imgFile + tmp, image_); -// -// std::swap(prev_grey, grey); -// std::swap(prev_image, image); -// frame_num = frame_num + 1; -// -// int step_t = step; -// while (step_t > 1){ -// capture >> frame; -// step_t--; -// } -// } - return 0; -} diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp index bc08a40..c9a2812 100644 --- a/src/dense_flow.cpp +++ b/src/dense_flow.cpp @@ -4,6 +4,49 @@ #include "common.h" void calcDenseFlow(string file_name, int bound, int type, int step, - vector >& output){ + vector >& output_x, + vector >& output_y, + vector >& output_img){ + + VideoCapture video_stream(file_name); + CHECK(video_stream.isOpened())<<"Cannot open video stream \"" + <> capture_frame; + if (capture_frame.empty()) break; // read frames until end + + //build mats for the first frame + if (!initialized){ + initializeMats(capture_frame, capture_image, capture_gray, + prev_image, prev_gray); + capture_frame.copyTo(prev_image); + cvtColor(prev_image, prev_gray, CV_BGR2GRAY); + initialized = true; +// LOG(INFO)<<"Initialized"; + }else if(iter % step == 0){ + capture_frame.copyTo(capture_image); + cvtColor(capture_image, capture_gray, CV_BGR2GRAY); + calcOpticalFlowFarneback(prev_gray, capture_gray, flow, + 0.702, 5, 10, 2, 7, 1.5, + cv::OPTFLOW_FARNEBACK_GAUSSIAN ); + + vector str_x, str_y, str_img; + split(flow, flow_split); + encodeFlowMap(flow_split[0], flow_split[0], str_x, str_y, bound); + imencode(".jpg", capture_image, str_img); + + output_x.push_back(str_x); + output_y.push_back(str_y); + output_img.push_back(str_img); +// LOG(INFO)<("vidFile"); + string xFlowFile = cmd.get("xFlowFile"); + string yFlowFile = cmd.get("yFlowFile"); + string imgFile = cmd.get("imgFile"); + int bound = cmd.get("bound"); + + vector > out_vec_x, out_vec_y, out_vec_img; + + calcDenseFlow(vidFile, bound, 0, 1, + out_vec_x, out_vec_y, out_vec_img); + + return 0; +} diff --git a/tools/extract_flow_gpu.cpp b/tools/extract_flow_gpu.cpp new file mode 100644 index 0000000..76f9892 --- /dev/null +++ b/tools/extract_flow_gpu.cpp @@ -0,0 +1,38 @@ +#include "common.h" +#include "dense_flow.h" + +INITIALIZE_EASYLOGGINGPP + +using namespace cv::gpu; + +int main(int argc, char** argv){ + // IO operation + const char* keys = + { + "{ f | vidFile | ex2.avi | filename of video }" + "{ x | xFlowFile | flow_x | filename of flow x component }" + "{ y | yFlowFile | flow_y | filename of flow x component }" + "{ i | imgFile | flow_i | filename of flow image}" + "{ b | bound | 15 | specify the maximum of optical flow}" + "{ t | type | 0 | specify the optical flow algorithm }" + "{ d | device_id | 0 | set gpu id}" + "{ s | step | 1 | specify the step for frame sampling}" + }; + + CommandLineParser cmd(argc, argv, keys); + string vidFile = cmd.get("vidFile"); + string xFlowFile = cmd.get("xFlowFile"); + string yFlowFile = cmd.get("yFlowFile"); + string imgFile = cmd.get("imgFile"); + int bound = cmd.get("bound"); + int type = cmd.get("type"); + int device_id = cmd.get("device_id"); + int step = cmd.get("step"); + + vector > out_vec_x, out_vec_y, out_vec_img; + + calcDenseFlowGPU(vidFile, bound, type, step, device_id, + out_vec_x, out_vec_y, out_vec_img); + + return 0; +} From d721d8ce4772761c068ea09a972a7682a6002d12 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 15:46:07 +0800 Subject: [PATCH 04/41] save previous frame --- src/dense_flow.cpp | 5 ++++- src/dense_flow_gpu.cpp | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp index c9a2812..096367e 100644 --- a/src/dense_flow.cpp +++ b/src/dense_flow.cpp @@ -45,7 +45,10 @@ void calcDenseFlow(string file_name, int bound, int type, int step, output_x.push_back(str_x); output_y.push_back(str_y); output_img.push_back(str_img); -// LOG(INFO)< Date: Wed, 18 Nov 2015 15:55:05 +0800 Subject: [PATCH 05/41] add gpu video reader --- include/dense_flow.h | 5 ++ src/dense_flow_gpu.cpp | 86 +++++++++++++++++++++++++++++++++ tools/extract_flow_pure_gpu.cpp | 38 +++++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 tools/extract_flow_pure_gpu.cpp diff --git a/include/dense_flow.h b/include/dense_flow.h index b67851c..4ba1da9 100644 --- a/include/dense_flow.h +++ b/include/dense_flow.h @@ -16,4 +16,9 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i vector >& output_y, vector >& output_img); +void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int dev_id, + vector >& output_x, + vector >& output_y, + vector >& output_img); + #endif //DENSEFLOW_DENSE_FLOW_H diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index b0bdf13..5d052da 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -83,4 +83,90 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i } +} + +/** + * This function use pure GPU backend for video loading and optical flow calculation + */ +void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int dev_id, + vector >& output_x, + vector >& output_y, + vector >& output_img){ + + setDevice(dev_id); + VideoReader_GPU video_stream(file_name); +// VideoCapture video_stream(file_name); + CHECK(video_stream.isOpened())<<"Cannot open video stream \"" + < str_x, str_y, str_img; + encodeFlowMap(flow_x, flow_y, str_x, str_y, bound); + imencode(".jpg", img, str_img); + + output_x.push_back(str_x); + output_y.push_back(str_y); + output_img.push_back(str_img); + + std::swap(prev_gray, capture_gray); + std::swap(prev_image, capture_image); + } + + + } + } \ No newline at end of file diff --git a/tools/extract_flow_pure_gpu.cpp b/tools/extract_flow_pure_gpu.cpp new file mode 100644 index 0000000..7b643d8 --- /dev/null +++ b/tools/extract_flow_pure_gpu.cpp @@ -0,0 +1,38 @@ +#include "common.h" +#include "dense_flow.h" + +INITIALIZE_EASYLOGGINGPP + +using namespace cv::gpu; + +int main(int argc, char** argv){ + // IO operation + const char* keys = + { + "{ f | vidFile | ex2.avi | filename of video }" + "{ x | xFlowFile | flow_x | filename of flow x component }" + "{ y | yFlowFile | flow_y | filename of flow x component }" + "{ i | imgFile | flow_i | filename of flow image}" + "{ b | bound | 15 | specify the maximum of optical flow}" + "{ t | type | 0 | specify the optical flow algorithm }" + "{ d | device_id | 0 | set gpu id}" + "{ s | step | 1 | specify the step for frame sampling}" + }; + + CommandLineParser cmd(argc, argv, keys); + string vidFile = cmd.get("vidFile"); + string xFlowFile = cmd.get("xFlowFile"); + string yFlowFile = cmd.get("yFlowFile"); + string imgFile = cmd.get("imgFile"); + int bound = cmd.get("bound"); + int type = cmd.get("type"); + int device_id = cmd.get("device_id"); + int step = cmd.get("step"); + + vector > out_vec_x, out_vec_y, out_vec_img; + + calcDenseFlowPureGPU(vidFile, bound, type, step, device_id, + out_vec_x, out_vec_y, out_vec_img); + + return 0; +} From 071809fd8fb15d5285416edb515175256ebfc201 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 15:55:50 +0800 Subject: [PATCH 06/41] add gpu video reader --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e7fb3e2..0d9fe29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,4 +23,7 @@ add_executable( extract_cpu tools/extract_flow.cpp) target_link_libraries( extract_cpu ${OpenCV_LIBS} denseflow) add_executable( extract_gpu tools/extract_flow_gpu.cpp) -target_link_libraries( extract_gpu ${OpenCV_LIBS} denseflow) \ No newline at end of file +target_link_libraries( extract_gpu ${OpenCV_LIBS} denseflow) + +add_executable( extract_pure_gpu tools/extract_flow_gpu.cpp) +target_link_libraries( extract_pure_gpu ${OpenCV_LIBS} denseflow) \ No newline at end of file From c708f9edb8e2ba283bcb9aa5ec7fe9673eba28d7 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 16:04:34 +0800 Subject: [PATCH 07/41] add gpu video reader --- src/dense_flow_gpu.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index 5d052da..5e63cc6 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -110,12 +110,12 @@ void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int d BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); bool initialized = false; - for(int iter = 0;; iter++){ - bool success = video_stream.read(capture_frame); - if (!success) break; // read frames until end + while(true){ //build mats for the first frame if (!initialized){ + bool success = video_stream.read(capture_frame); + if (!success) break; // read frames until end capture_image.create(capture_frame.size(), CV_8UC3); capture_gray.create(capture_frame.size(), CV_8UC1); @@ -125,7 +125,11 @@ void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int d capture_frame.copyTo(prev_image); cvtColor(prev_image, prev_gray, CV_BGR2GRAY); initialized = true; - }else if(iter % step == 0){ + + for (int s = 0; s < step; ++s){ + video_stream.read(capture_frame); + } + }else { capture_frame.copyTo(capture_image); cvtColor(capture_image, capture_gray, CV_BGR2GRAY); @@ -149,6 +153,10 @@ void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int d LOG(ERROR)<<"Unknown optical method: "< Date: Wed, 18 Nov 2015 16:13:58 +0800 Subject: [PATCH 08/41] add gpu video reader --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d9fe29..0bb4836 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,5 +25,5 @@ target_link_libraries( extract_cpu ${OpenCV_LIBS} denseflow) add_executable( extract_gpu tools/extract_flow_gpu.cpp) target_link_libraries( extract_gpu ${OpenCV_LIBS} denseflow) -add_executable( extract_pure_gpu tools/extract_flow_gpu.cpp) +add_executable( extract_pure_gpu tools/extract_flow_pure_gpu.cpp) target_link_libraries( extract_pure_gpu ${OpenCV_LIBS} denseflow) \ No newline at end of file From 721647d439ef26d03209481bed7170145512c418 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 16:30:42 +0800 Subject: [PATCH 09/41] try prefetch frames --- CMakeLists.txt | 3 --- src/dense_flow_gpu.cpp | 22 ++++++++++++++++--- tools/extract_flow_pure_gpu.cpp | 38 --------------------------------- 3 files changed, 19 insertions(+), 44 deletions(-) delete mode 100644 tools/extract_flow_pure_gpu.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bb4836..35a8072 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,3 @@ target_link_libraries( extract_cpu ${OpenCV_LIBS} denseflow) add_executable( extract_gpu tools/extract_flow_gpu.cpp) target_link_libraries( extract_gpu ${OpenCV_LIBS} denseflow) - -add_executable( extract_pure_gpu tools/extract_flow_pure_gpu.cpp) -target_link_libraries( extract_pure_gpu ${OpenCV_LIBS} denseflow) \ No newline at end of file diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index 5e63cc6..751ebf6 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -27,9 +27,9 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); bool initialized = false; - for(int iter = 0;; iter++){ + while(true){ video_stream >> capture_frame; - if (capture_frame.empty()) break; // read frames until end + if (capture_frame.empty()) return; // read frames until end //build mats for the first frame if (!initialized){ @@ -38,7 +38,11 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i capture_frame.copyTo(prev_image); cvtColor(prev_image, prev_gray, CV_BGR2GRAY); initialized = true; - }else if(iter % step == 0){ + for(int s = 0; s < step; ++s){ + video_stream >> capture_frame; + if (capture_frame.empty()) return; // read frames until end + } + }else { capture_frame.copyTo(capture_image); cvtColor(capture_image, capture_gray, CV_BGR2GRAY); d_frame_0.upload(prev_gray); @@ -64,6 +68,14 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i LOG(ERROR)<<"Unknown optical method: "<> capture_frame; + hasnext = !capture_frame.empty(); + // read frames until end + } + //get back flow map d_flow_x.download(flow_x); d_flow_y.download(flow_y); @@ -78,6 +90,10 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i std::swap(prev_gray, capture_gray); std::swap(prev_image, capture_image); + + if (!hasnext){ + return; + } } diff --git a/tools/extract_flow_pure_gpu.cpp b/tools/extract_flow_pure_gpu.cpp deleted file mode 100644 index 7b643d8..0000000 --- a/tools/extract_flow_pure_gpu.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "common.h" -#include "dense_flow.h" - -INITIALIZE_EASYLOGGINGPP - -using namespace cv::gpu; - -int main(int argc, char** argv){ - // IO operation - const char* keys = - { - "{ f | vidFile | ex2.avi | filename of video }" - "{ x | xFlowFile | flow_x | filename of flow x component }" - "{ y | yFlowFile | flow_y | filename of flow x component }" - "{ i | imgFile | flow_i | filename of flow image}" - "{ b | bound | 15 | specify the maximum of optical flow}" - "{ t | type | 0 | specify the optical flow algorithm }" - "{ d | device_id | 0 | set gpu id}" - "{ s | step | 1 | specify the step for frame sampling}" - }; - - CommandLineParser cmd(argc, argv, keys); - string vidFile = cmd.get("vidFile"); - string xFlowFile = cmd.get("xFlowFile"); - string yFlowFile = cmd.get("yFlowFile"); - string imgFile = cmd.get("imgFile"); - int bound = cmd.get("bound"); - int type = cmd.get("type"); - int device_id = cmd.get("device_id"); - int step = cmd.get("step"); - - vector > out_vec_x, out_vec_y, out_vec_img; - - calcDenseFlowPureGPU(vidFile, bound, type, step, device_id, - out_vec_x, out_vec_y, out_vec_img); - - return 0; -} From ceb9350be9ab821523901974d9df169ceb6a221b Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 16:56:24 +0800 Subject: [PATCH 10/41] add image writeout --- include/common.h | 2 ++ src/common.cpp | 11 +++++++++++ src/dense_flow.cpp | 2 +- tools/extract_flow.cpp | 3 +++ tools/extract_flow_gpu.cpp | 4 ++++ 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/include/common.h b/include/common.h index 16e24d8..7aaa628 100644 --- a/include/common.h +++ b/include/common.h @@ -33,5 +33,7 @@ inline void initializeMats(const Mat& frame, prev_gray.create(frame.size(), CV_8UC1); } +void writeImages(vector> images, string name_temp); + #endif //DENSEFLOW_COMMON_H_H diff --git a/src/common.cpp b/src/common.cpp index 5dedae2..27354be 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -42,3 +42,14 @@ void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, imencode(".jpg", flow_img_y, encoded_y); } +void writeImages(vector> images, string name_temp){ + for (int i = 0; i < images.size(); ++i){ + char tmp[256]; + sprintf(tmp, "_%05d.jpg", i); + FILE* fp; + fp = fopen((name_temp + tmp).c_str(), "wb"); + fwrite( images[i].data(), 1, images[i].size(), fp); + fclose(fp); + } +} + diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp index 096367e..818c4c7 100644 --- a/src/dense_flow.cpp +++ b/src/dense_flow.cpp @@ -45,7 +45,7 @@ void calcDenseFlow(string file_name, int bound, int type, int step, output_x.push_back(str_x); output_y.push_back(str_y); output_img.push_back(str_img); - LOG(INFO)< Date: Wed, 18 Nov 2015 17:00:06 +0800 Subject: [PATCH 11/41] update usage info --- README | 2 +- {_deprecated => matlab}/extracOpticalFlow.m | 2 +- {_deprecated => matlab}/extractOpticalFlow.m | 2 +- {_deprecated => matlab}/extractOpticalFlow_gpu.m | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename {_deprecated => matlab}/extracOpticalFlow.m (90%) rename {_deprecated => matlab}/extractOpticalFlow.m (90%) rename {_deprecated => matlab}/extractOpticalFlow_gpu.m (94%) diff --git a/README b/README index 8372cfd..2afa176 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ This is a sample code for extrating dense flow field given a video. Usage: -./denseFlow_gpu -f test.avi -x tmp/flow_x -y tmp/flow_x -i tmp/image -b 20 -t 1 -d 0 -s 1 +./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_x -i tmp/image -b 20 -t 1 -d 0 -s 1 test.avi: input video tmp: folder containing RGB images and optical flow images diff --git a/_deprecated/extracOpticalFlow.m b/matlab/extracOpticalFlow.m similarity index 90% rename from _deprecated/extracOpticalFlow.m rename to matlab/extracOpticalFlow.m index 69f2675..6a7fd82 100644 --- a/_deprecated/extracOpticalFlow.m +++ b/matlab/extracOpticalFlow.m @@ -21,7 +21,7 @@ file2 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_x']; file3 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_y']; file4 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_i']; - cmd = sprintf('./denseFlow -f %s -x %s -y %s -i %s -b 20',file1,file2,file3,file4); + cmd = sprintf('./extract_gpu -f %s -x %s -y %s -i %s -b 20',file1,file2,file3,file4); system(cmd); end i diff --git a/_deprecated/extractOpticalFlow.m b/matlab/extractOpticalFlow.m similarity index 90% rename from _deprecated/extractOpticalFlow.m rename to matlab/extractOpticalFlow.m index eb51bf7..1dd9df0 100644 --- a/_deprecated/extractOpticalFlow.m +++ b/matlab/extractOpticalFlow.m @@ -21,7 +21,7 @@ file2 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_x']; file3 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_y']; file4 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_i']; - cmd = sprintf('./denseFlow -f %s -x %s -y %s -i %s -b 20',file1,file2,file3,file4); + cmd = sprintf('./extract_gpu -f %s -x %s -y %s -i %s -b 20',file1,file2,file3,file4); system(cmd); end i diff --git a/_deprecated/extractOpticalFlow_gpu.m b/matlab/extractOpticalFlow_gpu.m similarity index 94% rename from _deprecated/extractOpticalFlow_gpu.m rename to matlab/extractOpticalFlow_gpu.m index e7602d9..853d05d 100644 --- a/_deprecated/extractOpticalFlow_gpu.m +++ b/matlab/extractOpticalFlow_gpu.m @@ -36,7 +36,7 @@ file2 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_x']; file3 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_y']; file4 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_i']; - cmd = sprintf('./denseFlow_gpu -f %s -x %s -y %s -i %s -b 20 -t %d -d %d -s %d',... + cmd = sprintf('./extract_gpu -f %s -x %s -y %s -i %s -b 20 -t %d -d %d -s %d',... file1,file2,file3,file4,type,device_id,1); system(cmd); end From d0df2e5149c0875d64b742d13b06563514700996 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 18 Nov 2015 17:01:26 +0800 Subject: [PATCH 12/41] update usage info --- README | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README b/README index 2afa176..f71f7fb 100644 --- a/README +++ b/README @@ -1,5 +1,10 @@ This is a sample code for extrating dense flow field given a video. +Install: +git clone --recursive http://github.com/yjxiong/dense_flow +mkdir build && cd build +cmake .. && make -j + Usage: ./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_x -i tmp/image -b 20 -t 1 -d 0 -s 1 test.avi: input video From de4bc61b261ab1ad37457e421914c38ffe9a860e Mon Sep 17 00:00:00 2001 From: Yuanjun Xiong Date: Wed, 18 Nov 2015 23:08:00 +0800 Subject: [PATCH 13/41] now support zip file output --- CMakeLists.txt | 13 +++++++----- cmake/Modules/FindLibZip.cmake | 37 ++++++++++++++++++++++++++++++++++ include/utils.h | 12 +++++++++++ src/zip_utils.cpp | 27 +++++++++++++++++++++++++ tools/extract_flow.cpp | 17 +++++++++++++--- tools/extract_flow_gpu.cpp | 17 ++++++++++++---- 6 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 cmake/Modules/FindLibZip.cmake create mode 100644 include/utils.h create mode 100644 src/zip_utils.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 35a8072..18041d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 2.8) project( denseFlow ) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") + include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) @@ -13,14 +15,15 @@ else() endif() find_package( OpenCV REQUIRED ) +find_package( LibZip REQUIRED ) -include_directories( ${OpenCV_INCLUDE_DIRS} include/ include/easylogging++/src) +include_directories( ${OpenCV_INCLUDE_DIRS} ${LIBZIP_INCLUDE_DIR_ZIP} ${LIBZIP_INCLUDE_DIR_ZIPCONF} include/ include/easylogging++/src) -add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp) -target_link_libraries( denseflow ${OpenCV_LIBS}) +add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp src/zip_utils.cpp) +target_link_libraries( denseflow ${OpenCV_LIBS} ${LIBZIP_LIBRARY}) add_executable( extract_cpu tools/extract_flow.cpp) -target_link_libraries( extract_cpu ${OpenCV_LIBS} denseflow) +target_link_libraries( extract_cpu ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) add_executable( extract_gpu tools/extract_flow_gpu.cpp) -target_link_libraries( extract_gpu ${OpenCV_LIBS} denseflow) +target_link_libraries( extract_gpu ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) diff --git a/cmake/Modules/FindLibZip.cmake b/cmake/Modules/FindLibZip.cmake new file mode 100644 index 0000000..76ab224 --- /dev/null +++ b/cmake/Modules/FindLibZip.cmake @@ -0,0 +1,37 @@ +# Finds libzip. +# +# This module defines: +# LIBZIP_INCLUDE_DIR_ZIP +# LIBZIP_INCLUDE_DIR_ZIPCONF +# LIBZIP_LIBRARY +# + +find_package(PkgConfig) +pkg_check_modules(PC_LIBZIP QUIET libzip) + +find_path(LIBZIP_INCLUDE_DIR_ZIP + NAMES zip.h + HINTS ${PC_LIBZIP_INCLUDE_DIRS}) + +find_path(LIBZIP_INCLUDE_DIR_ZIPCONF + NAMES zipconf.h + HINTS ${PC_LIBZIP_INCLUDE_DIRS}) + +find_library(LIBZIP_LIBRARY + NAMES zip) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS( + LIBZIP DEFAULT_MSG + LIBZIP_LIBRARY LIBZIP_INCLUDE_DIR_ZIP LIBZIP_INCLUDE_DIR_ZIPCONF) + +set(LIBZIP_VERSION 0) + +if (LIBZIP_INCLUDE_DIR_ZIPCONF) + FILE(READ "${LIBZIP_INCLUDE_DIR_ZIPCONF}/zipconf.h" _LIBZIP_VERSION_CONTENTS) + if (_LIBZIP_VERSION_CONTENTS) + STRING(REGEX REPLACE ".*#define LIBZIP_VERSION \"([0-9.]+)\".*" "\\1" LIBZIP_VERSION "${_LIBZIP_VERSION_CONTENTS}") + endif () +endif () + +set(LIBZIP_VERSION ${LIBZIP_VERSION} CACHE STRING "Version number of libzip") diff --git a/include/utils.h b/include/utils.h new file mode 100644 index 0000000..d69e22c --- /dev/null +++ b/include/utils.h @@ -0,0 +1,12 @@ +// +// Created by Yuanjun Xiong on 18/11/2015. +// + +#ifndef DENSEFLOW_UTILS_H +#define DENSEFLOW_UTILS_H + +#include "common.h" + +void writeZipFile(vector >& data, string name_temp, string archive_name); + +#endif //DENSEFLOW_UTILS_H diff --git a/src/zip_utils.cpp b/src/zip_utils.cpp new file mode 100644 index 0000000..926c529 --- /dev/null +++ b/src/zip_utils.cpp @@ -0,0 +1,27 @@ +// +// Created by Yuanjun Xiong on 18/11/2015. +// + +#include "utils.h" +#include "zip.h" + +void writeZipFile(vector >& data, string name_temp, string archive_name){ + int err=0; + struct zip* archive = zip_open(archive_name.c_str(), ZIP_CREATE|ZIP_TRUNCATE, &err); + + CHECK_EQ(err, 0)<<"Failed to open Zip file with error code: "<("xFlowFile"); string yFlowFile = cmd.get("yFlowFile"); string imgFile = cmd.get("imgFile"); + string output_style = cmd.get("out"); int bound = cmd.get("bound"); +// LOG(INFO)<<"Starting extraction"; vector > out_vec_x, out_vec_y, out_vec_img; calcDenseFlow(vidFile, bound, 0, 1, out_vec_x, out_vec_y, out_vec_img); - writeImages(out_vec_x, xFlowFile); - writeImages(out_vec_y, yFlowFile); - writeImages(out_vec_img, imgFile); + if (output_style == "dir") { + writeImages(out_vec_x, xFlowFile); + writeImages(out_vec_y, yFlowFile); + writeImages(out_vec_img, imgFile); + }else{ +// LOG(INFO)<<"Writing results to Zip archives"; + writeZipFile(out_vec_x, "x_%05d.jpg", xFlowFile+".zip"); + writeZipFile(out_vec_y, "y_%05d.jpg", yFlowFile+".zip"); + writeZipFile(out_vec_img, "img_%05d.jpg", imgFile+".zip"); + } return 0; } diff --git a/tools/extract_flow_gpu.cpp b/tools/extract_flow_gpu.cpp index 097adbd..89c7686 100644 --- a/tools/extract_flow_gpu.cpp +++ b/tools/extract_flow_gpu.cpp @@ -1,5 +1,5 @@ -#include "common.h" #include "dense_flow.h" +#include "utils.h" INITIALIZE_EASYLOGGINGPP @@ -17,6 +17,7 @@ int main(int argc, char** argv){ "{ t | type | 0 | specify the optical flow algorithm }" "{ d | device_id | 0 | set gpu id}" "{ s | step | 1 | specify the step for frame sampling}" + "{ o | out | zip | output style}" }; CommandLineParser cmd(argc, argv, keys); @@ -24,6 +25,7 @@ int main(int argc, char** argv){ string xFlowFile = cmd.get("xFlowFile"); string yFlowFile = cmd.get("yFlowFile"); string imgFile = cmd.get("imgFile"); + string output_style = cmd.get("out"); int bound = cmd.get("bound"); int type = cmd.get("type"); int device_id = cmd.get("device_id"); @@ -34,9 +36,16 @@ int main(int argc, char** argv){ calcDenseFlowGPU(vidFile, bound, type, step, device_id, out_vec_x, out_vec_y, out_vec_img); - writeImages(out_vec_x, xFlowFile); - writeImages(out_vec_y, yFlowFile); - writeImages(out_vec_img, imgFile); + if (output_style == "dir") { + writeImages(out_vec_x, xFlowFile); + writeImages(out_vec_y, yFlowFile); + writeImages(out_vec_img, imgFile); + }else{ +// LOG(INFO)<<"Writing results to Zip archives"; + writeZipFile(out_vec_x, "x_%05d.jpg", xFlowFile+".zip"); + writeZipFile(out_vec_y, "y_%05d.jpg", yFlowFile+".zip"); + writeZipFile(out_vec_img, "img_%05d.jpg", imgFile+".zip"); + } return 0; } From 791e37e925d891609e44b6f61c22ed4e4f297319 Mon Sep 17 00:00:00 2001 From: Yuanjun Xiong Date: Wed, 18 Nov 2015 23:21:25 +0800 Subject: [PATCH 14/41] remove zip truncate to fix linux libzip --- src/zip_utils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zip_utils.cpp b/src/zip_utils.cpp index 926c529..bff1f2b 100644 --- a/src/zip_utils.cpp +++ b/src/zip_utils.cpp @@ -7,7 +7,7 @@ void writeZipFile(vector >& data, string name_temp, string archive_name){ int err=0; - struct zip* archive = zip_open(archive_name.c_str(), ZIP_CREATE|ZIP_TRUNCATE, &err); + struct zip* archive = zip_open(archive_name.c_str(), ZIP_CREATE, &err); CHECK_EQ(err, 0)<<"Failed to open Zip file with error code: "< >& data, string name_temp, string archive sprintf(name, name_temp.c_str(), i); if ((src_ptr = zip_source_buffer(archive, data[i].data(), data[i].size(), 0)) == NULL || - zip_file_add(archive, name, src_ptr, ZIP_FL_ENC_UTF_8) < 0) { + zip_file_add(archive, name, src_ptr, ZIP_FL_OVERWRITE|ZIP_FL_ENC_UTF_8) < 0) { zip_source_free(src_ptr); LOG(FATAL)<<"error adding file "< Date: Wed, 18 Nov 2015 23:52:57 +0800 Subject: [PATCH 15/41] fix old version libzip --- CMakeLists.txt | 5 +++++ src/zip_utils.cpp | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 18041d4..3e35752 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,11 @@ endif() find_package( OpenCV REQUIRED ) find_package( LibZip REQUIRED ) +if(LIBZIP_VERSION VERSION_LESS 0.11) + #old version LibZip + add_definitions(-DUSE_OBSEL_LIBZIP) +endif() + include_directories( ${OpenCV_INCLUDE_DIRS} ${LIBZIP_INCLUDE_DIR_ZIP} ${LIBZIP_INCLUDE_DIR_ZIPCONF} include/ include/easylogging++/src) add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp src/zip_utils.cpp) diff --git a/src/zip_utils.cpp b/src/zip_utils.cpp index bff1f2b..6f5842e 100644 --- a/src/zip_utils.cpp +++ b/src/zip_utils.cpp @@ -7,7 +7,11 @@ void writeZipFile(vector >& data, string name_temp, string archive_name){ int err=0; +#ifdef USE_OBSEL_LIBZIP struct zip* archive = zip_open(archive_name.c_str(), ZIP_CREATE, &err); +#else + struct zip* archive = zip_open(archive_name.c_str(), ZIP_CREATE|ZIP_TRUNCATE, &err); +#endif CHECK_EQ(err, 0)<<"Failed to open Zip file with error code: "< >& data, string name_temp, string archive sprintf(name, name_temp.c_str(), i); if ((src_ptr = zip_source_buffer(archive, data[i].data(), data[i].size(), 0)) == NULL || - zip_file_add(archive, name, src_ptr, ZIP_FL_OVERWRITE|ZIP_FL_ENC_UTF_8) < 0) { +#ifdef USE_OBSEL_LIBZIP + zip_add(archive, name, src_ptr) < 0) { +#else + zip_file_add(archive, name, src_ptr, ZIP_FL_ENC_UTF_8) < 0) { +#endif zip_source_free(src_ptr); LOG(FATAL)<<"error adding file "< Date: Thu, 19 Nov 2015 00:07:22 +0800 Subject: [PATCH 16/41] Update and rename README to README.md --- README | 11 ----------- README.md | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 11 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index f71f7fb..0000000 --- a/README +++ /dev/null @@ -1,11 +0,0 @@ -This is a sample code for extrating dense flow field given a video. - -Install: -git clone --recursive http://github.com/yjxiong/dense_flow -mkdir build && cd build -cmake .. && make -j - -Usage: -./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_x -i tmp/image -b 20 -t 1 -d 0 -s 1 -test.avi: input video -tmp: folder containing RGB images and optical flow images diff --git a/README.md b/README.md new file mode 100644 index 0000000..05709ce --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +Extracting dense flow field given a video. + +####Depencies: +- LibZip: +to install on ubuntu "apt-get install libzip-dev" on mac "brew intall libzip" + +###Install +``` +git clone --recursive http://github.com/yjxiong/dense_flow +mkdir build && cd build +cmake .. && make -j +``` + +###Usage +``` +./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_x -i tmp/image -b 20 -t 1 -d 0 -s 1 -o dir +``` +- `test.avi`: input video +- `tmp`: folder containing RGB images and optical flow images From 1822a88a7d1c2c2e86a1792344806b9d3e023885 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Thu, 19 Nov 2015 00:08:28 +0800 Subject: [PATCH 17/41] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05709ce..ea63a1e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Extracting dense flow field given a video. ####Depencies: - LibZip: -to install on ubuntu "apt-get install libzip-dev" on mac "brew intall libzip" +to install on ubuntu ```apt-get install libzip-dev``` on mac ```brew install libzip``` ###Install ``` From 27d3d6631c69a9aca520da489b201636d8c740ef Mon Sep 17 00:00:00 2001 From: yjxiong Date: Sat, 21 Nov 2015 00:07:50 +0800 Subject: [PATCH 18/41] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ea63a1e..6e4711f 100644 --- a/README.md +++ b/README.md @@ -17,3 +17,4 @@ cmake .. && make -j ``` - `test.avi`: input video - `tmp`: folder containing RGB images and optical flow images +- `dir`: output generated images to folder. if set to `zip`, will write images to zip files instead. From d99a2140a235c4d715d65733649cfd0a41ccca37 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Thu, 17 Dec 2015 13:11:41 +0800 Subject: [PATCH 19/41] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e4711f..2a0f1ce 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ cmake .. && make -j ###Usage ``` -./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_x -i tmp/image -b 20 -t 1 -d 0 -s 1 -o dir +./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_y -i tmp/image -b 20 -t 1 -d 0 -s 1 -o dir ``` - `test.avi`: input video - `tmp`: folder containing RGB images and optical flow images From 794b003bacd479c4ce7a48d5ada0fdfbaeeb10a1 Mon Sep 17 00:00:00 2001 From: Yuanjun Xiong Date: Thu, 17 Dec 2015 13:55:42 +0800 Subject: [PATCH 20/41] fix flow extraction bug --- src/common.cpp | 2 +- src/dense_flow.cpp | 2 +- src/dense_flow_gpu.cpp | 11 +++++++---- src/zip_utils.cpp | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/common.cpp b/src/common.cpp index 27354be..a85c907 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -45,7 +45,7 @@ void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, void writeImages(vector> images, string name_temp){ for (int i = 0; i < images.size(); ++i){ char tmp[256]; - sprintf(tmp, "_%05d.jpg", i); + sprintf(tmp, "_%05d.jpg", i+1); FILE* fp; fp = fopen((name_temp + tmp).c_str(), "wb"); fwrite( images[i].data(), 1, images[i].size(), fp); diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp index 818c4c7..8129c70 100644 --- a/src/dense_flow.cpp +++ b/src/dense_flow.cpp @@ -52,4 +52,4 @@ void calcDenseFlow(string file_name, int bound, int type, int step, } } -} \ No newline at end of file +} diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index 751ebf6..93fdfe5 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -27,19 +27,21 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); bool initialized = false; + int cnt = 0; while(true){ - video_stream >> capture_frame; - if (capture_frame.empty()) return; // read frames until end //build mats for the first frame if (!initialized){ - initializeMats(capture_frame, capture_image, capture_gray, + video_stream >> capture_frame; + if (capture_frame.empty()) return; // read frames until end + initializeMats(capture_frame, capture_image, capture_gray, prev_image, prev_gray); capture_frame.copyTo(prev_image); cvtColor(prev_image, prev_gray, CV_BGR2GRAY); initialized = true; for(int s = 0; s < step; ++s){ video_stream >> capture_frame; + cnt ++; if (capture_frame.empty()) return; // read frames until end } }else { @@ -72,6 +74,7 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i bool hasnext = true; for(int s = 0; s < step; ++s){ video_stream >> capture_frame; + cnt ++; hasnext = !capture_frame.empty(); // read frames until end } @@ -193,4 +196,4 @@ void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int d } -} \ No newline at end of file +} diff --git a/src/zip_utils.cpp b/src/zip_utils.cpp index 6f5842e..ea3d0a1 100644 --- a/src/zip_utils.cpp +++ b/src/zip_utils.cpp @@ -18,7 +18,7 @@ void writeZipFile(vector >& data, string name_temp, string archive struct zip_source *src_ptr; for (int i = 0; i < data.size(); ++i) { - sprintf(name, name_temp.c_str(), i); + sprintf(name, name_temp.c_str(), i+1); if ((src_ptr = zip_source_buffer(archive, data[i].data(), data[i].size(), 0)) == NULL || #ifdef USE_OBSEL_LIBZIP zip_add(archive, name, src_ptr) < 0) { From 891f5b1f7694fa7978a9fa797b4ea5cae2d9c7f9 Mon Sep 17 00:00:00 2001 From: Yuanjun Xiong Date: Thu, 24 Dec 2015 14:45:04 +0800 Subject: [PATCH 21/41] add anet_builder --- build_of.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 build_of.py diff --git a/build_of.py b/build_of.py new file mode 100644 index 0000000..5116e8b --- /dev/null +++ b/build_of.py @@ -0,0 +1,66 @@ +__author__ = 'yjxiong' + +import cv2 +import os +from multiprocessing import Pool, current_process + +out_path = '' + + +def dump_frames(vid_path): + video = cv2.VideoCapture(vid_path) + vid_name = vid_path.split('/')[-1].split('.')[0] + out_full_path = os.path.join(out_path, vid_name) + + fcount = int(video.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)) + try: + os.mkdir(out_full_path) + except OSError: + pass + file_list = [] + for i in xrange(fcount): + ret, frame = video.read() + assert ret + cv2.imwrite('{}/{:06d}.jpg'.format(out_full_path, i), frame) + access_path = '{}/{:06d}.jpg'.format(vid_name, i) + file_list.append(access_path) + print '{} done'.format(vid_name) + return file_list + + +def run_optical_flow(vid_item, dev_id=0): + vid_path = vid_item[0] + vid_id = vid_item[1] + vid_name = vid_path.split('/')[-1].split('.')[0] + out_full_path = os.path.join(out_path, vid_name) + try: + os.mkdir(out_full_path) + except OSError: + pass + + current = current_process() + dev_id = int(current._identity[0]) - 1 + image_path = '{}/img'.format(out_full_path) + flow_x_path = '{}/flow_x'.format(out_full_path) + flow_y_path = '{}/flow_y'.format(out_full_path) + + cmd = './build/extract_gpu -f {} -x {} -y {} -i {} -b 20 -t 1 -d {} -s 1 -o zip'.format(vid_path, flow_x_path, flow_y_path, image_path, dev_id) + + os.system(cmd) + print '{} {} done'.format(vid_id, vid_name) + return True + + +if __name__ == '__main__': + out_path = '/data1/alex/anet/flow_tvl1' + import glob + vid_list = glob.glob('/home/alex/clips/*avi') + print len(vid_list) + pool = Pool(4) + pool.map(run_optical_flow, zip(vid_list, xrange(len(vid_list)))) + #file_list = pool.map(dump_frames, vid_list) + #all_file_list = [f for x in file_list for f in x] + #open('anet_image_list_nov_17.txt','w').writelines('\n'.join(all_file_list)) + #for i,v in enumerate(vid_list): + # run_optical_flow(v, 0) + # print i From 99bc8348b646d51f5da35c9b7ba7c089842d4ce9 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Sun, 3 Apr 2016 18:58:29 +0800 Subject: [PATCH 22/41] add experimental python wrapper --- .gitmodules | 3 + CMakeLists.txt | 21 ++++- build_of.py | 10 ++- include/CLUE | 1 + include/common.h | 2 +- include/dense_flow.h | 1 + src/dense_flow.cpp | 1 + src/dense_flow_gpu.cpp | 2 +- src/py_denseflow.cpp | 85 +++++++++++++++++++ src/zip_utils.cpp | 1 + tools/benchmark_video_io.cpp | 158 +++++++++++++++++++++++++++++++++++ 11 files changed, 278 insertions(+), 7 deletions(-) create mode 160000 include/CLUE create mode 100644 src/py_denseflow.cpp create mode 100644 tools/benchmark_video_io.cpp diff --git a/.gitmodules b/.gitmodules index 208c0b4..8d989cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "include/easylogging++"] path = include/easylogging++ url = https://github.com/easylogging/easyloggingpp +[submodule "include/CLUE"] + path = include/CLUE + url = https://github.com/lindahua/CLUE diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e35752..d9c9e08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,28 +7,45 @@ include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) if(COMPILER_SUPPORTS_CXX11) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC") elseif(COMPILER_SUPPORTS_CXX0X) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") else() message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") endif() + find_package( OpenCV REQUIRED ) find_package( LibZip REQUIRED ) +# BOOST +FIND_PACKAGE(Boost REQUIRED python) +FIND_PACKAGE(PythonLibs REQUIRED) + if(LIBZIP_VERSION VERSION_LESS 0.11) #old version LibZip add_definitions(-DUSE_OBSEL_LIBZIP) endif() -include_directories( ${OpenCV_INCLUDE_DIRS} ${LIBZIP_INCLUDE_DIR_ZIP} ${LIBZIP_INCLUDE_DIR_ZIPCONF} include/ include/easylogging++/src) +include_directories( ${OpenCV_INCLUDE_DIRS} ${LIBZIP_INCLUDE_DIR_ZIP} ${LIBZIP_INCLUDE_DIR_ZIPCONF} include/ include/easylogging++/src include/CLUE/include) +include_directories(SYSTEM ${Boost_INCLUDE_DIR} ${PYTHON_INCLUDE_DIR}) add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp src/zip_utils.cpp) target_link_libraries( denseflow ${OpenCV_LIBS} ${LIBZIP_LIBRARY}) +add_library(pydenseflow SHARED src/py_denseflow.cpp) +target_link_libraries(pydenseflow + denseflow + ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} ${OpenCV_LIBS} + ) + add_executable( extract_cpu tools/extract_flow.cpp) target_link_libraries( extract_cpu ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) add_executable( extract_gpu tools/extract_flow_gpu.cpp) target_link_libraries( extract_gpu ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) + + +add_executable( benchmark_io tools/benchmark_video_io.cpp) +target_link_libraries( benchmark_io ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) + diff --git a/build_of.py b/build_of.py index 5116e8b..0f51d20 100644 --- a/build_of.py +++ b/build_of.py @@ -45,6 +45,7 @@ def run_optical_flow(vid_item, dev_id=0): flow_y_path = '{}/flow_y'.format(out_full_path) cmd = './build/extract_gpu -f {} -x {} -y {} -i {} -b 20 -t 1 -d {} -s 1 -o zip'.format(vid_path, flow_x_path, flow_y_path, image_path, dev_id) + #cmd = './build/extract_gpu -f {} -x {} -y {} -b 20 -t 1 -d {} -s 1 -o zip'.format(vid_path, flow_x_path, flow_y_path, dev_id) os.system(cmd) print '{} {} done'.format(vid_id, vid_name) @@ -52,12 +53,15 @@ def run_optical_flow(vid_item, dev_id=0): if __name__ == '__main__': - out_path = '/data1/alex/anet/flow_tvl1' + out_path = '/data2/alex/anet_v1.2_testing_flow_tvl1' import glob - vid_list = glob.glob('/home/alex/clips/*avi') + #vid_list = glob.glob('/data1/alex/anet_clips_v1.2/*avi') + vid_list = glob.glob('/data1/alex/ActivityNet/testing_videos_v1.2/*.mp4') + #vid_list = ['/data1/alex/anet_clips_v1.2/{}.avi'.format(x.strip()) for x in open('/data2/alex/prob_clip_list.txt')] print len(vid_list) - pool = Pool(4) + pool = Pool(16) pool.map(run_optical_flow, zip(vid_list, xrange(len(vid_list)))) + #map(run_optical_flow, zip(vid_list, xrange(len(vid_list)))) #file_list = pool.map(dump_frames, vid_list) #all_file_list = [f for x in file_list for f in x] #open('anet_image_list_nov_17.txt','w').writelines('\n'.join(all_file_list)) diff --git a/include/CLUE b/include/CLUE new file mode 160000 index 0000000..1d81996 --- /dev/null +++ b/include/CLUE @@ -0,0 +1 @@ +Subproject commit 1d819960b815dd69d4adf606e09bc8bc69269c30 diff --git a/include/common.h b/include/common.h index 7aaa628..67b9cda 100644 --- a/include/common.h +++ b/include/common.h @@ -5,7 +5,7 @@ #ifndef DENSEFLOW_COMMON_H_H #define DENSEFLOW_COMMON_H_H -#include "easylogging++.h" + #include "opencv2/video/tracking.hpp" #include "opencv2/imgproc/imgproc.hpp" diff --git a/include/dense_flow.h b/include/dense_flow.h index 4ba1da9..611d31b 100644 --- a/include/dense_flow.h +++ b/include/dense_flow.h @@ -6,6 +6,7 @@ #define DENSEFLOW_DENSE_FLOW_H #include "common.h" +#include "easylogging++.h" void calcDenseFlow(string file_name, int bound, int type, int step, vector >& output_x, diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp index 8129c70..fa2ee75 100644 --- a/src/dense_flow.cpp +++ b/src/dense_flow.cpp @@ -2,6 +2,7 @@ // Created by yjxiong on 11/18/15. // #include "common.h" +#include "dense_flow.h" void calcDenseFlow(string file_name, int bound, int type, int step, vector >& output_x, diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index 93fdfe5..e4b52e7 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -1,7 +1,7 @@ // // Created by yjxiong on 11/18/15. // -#include "common.h" +#include "dense_flow.h" #include "opencv2/gpu/gpu.hpp" using namespace cv::gpu; diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp new file mode 100644 index 0000000..e201379 --- /dev/null +++ b/src/py_denseflow.cpp @@ -0,0 +1,85 @@ +#include +#include +#include +#include + + +#include "common.h" +#include "opencv2/gpu/gpu.hpp" +using namespace cv::gpu; +using namespace cv; + +namespace bp = boost::python; + +class TVL1FlowExtractor{ +public: + + TVL1FlowExtractor(int bound){ + bound_ = bound; + } + + static void set_device(int dev_id){ + setDevice(dev_id); + } + + bp::list extract_flow(bp::list frames, int img_width, int img_height){ + bp::list output; + Mat input_frame, prev_frame, next_frame, prev_gray, next_gray; + Mat flow_x, flow_y; + + GpuMat d_frame_0, d_frame_1; + GpuMat d_flow_x, d_flow_y; + OpticalFlowDual_TVL1_GPU alg_tvl1; + + + printf("Initialized Mats\n"); + // initialize the first frame + const char* first_data = ((const char*)bp::extract(frames[0])); + input_frame = Mat(img_height, img_width, CV_8UC3); + initializeMats(input_frame, prev_frame, prev_gray, next_frame, next_gray); + + printf("Copy to buffer, size %d %d\n", prev_frame.size[0], prev_frame.size[1]); + memcpy(prev_frame.data, first_data, bp::len(frames[0])); + printf("Convert to grayscale\n"); + cvtColor(prev_frame, prev_gray, CV_BGR2GRAY); + for (int idx = 1; idx < bp::len(frames); idx++){ + printf("Running frame %d\n", idx); + const char* this_data = ((const char*)bp::extract(frames[idx])); + memcpy(next_frame.data, this_data, bp::len(frames[0])); + cvtColor(next_frame, next_gray, CV_BGR2GRAY); + + printf("Uploading to GPU\n"); + d_frame_0.upload(prev_gray); + d_frame_1.upload(next_gray); + + alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + + d_flow_x.download(flow_x); + d_flow_y.download(flow_y); + + vector str_x, str_y; + encodeFlowMap(flow_x, flow_y, str_x, str_y, bound_); + output.append( + bp::make_tuple( + bp::str((const char*) str_x.data(), str_x.size()), + bp::str((const char*) str_y.data(), str_y.size()) + ) + ); + + std::swap(prev_gray, next_gray); + } + printf("extraction done\n"); + return output; + }; +private: + int bound_; +}; + + +//// Boost Python Related Decl +BOOST_PYTHON_MODULE(libpydenseflow){ + bp::class_("TVL1FlowExtractor", bp::init()) + .def("extract_flow", &TVL1FlowExtractor::extract_flow) + .def("set_device", &TVL1FlowExtractor::set_device) + .staticmethod("set_device"); +} \ No newline at end of file diff --git a/src/zip_utils.cpp b/src/zip_utils.cpp index ea3d0a1..6f2b626 100644 --- a/src/zip_utils.cpp +++ b/src/zip_utils.cpp @@ -4,6 +4,7 @@ #include "utils.h" #include "zip.h" +#include "easylogging++.h" void writeZipFile(vector >& data, string name_temp, string archive_name){ int err=0; diff --git a/tools/benchmark_video_io.cpp b/tools/benchmark_video_io.cpp new file mode 100644 index 0000000..0b99c37 --- /dev/null +++ b/tools/benchmark_video_io.cpp @@ -0,0 +1,158 @@ +// +// Created by alex on 16-3-27. +// + +#include "dense_flow.h" +#include "utils.h" +#include +#include +#include +#include "easylogging++.h" + +INITIALIZE_EASYLOGGINGPP + +using namespace cv::gpu; + +void OpenVideoOnly(string video_path){ + VideoCapture video_stream(video_path); + CHECK(video_stream.isOpened())<<"Cannot open video stream \"" + < SeekByFrame(string video_path, vector ratio, int read_num, double& first_seek_time){ + VideoCapture video_stream(video_path); + + double seeking_time = 0, reading_time = 0; + int cnt = 0; + for (float r : ratio) { + CHECK(video_stream.isOpened())<<"Cannot open video stream \"" + <> capture_frame; + } + reading_time += sw.elapsed().msecs(); + + if (cnt == 0){ + first_seek_time = seeking_time; + LOG(INFO)<<"first seek "<>capture_frame; + } + video_stream.release(); +} + +void SeekByRATIO(string video_path, float ratio, int read_num){ + VideoCapture video_stream(video_path); + CHECK(video_stream.isOpened())<<"Cannot open video stream \"" + <>capture_frame; + } + video_stream.release(); +} + +int main(int argc, char** argv){ + + CHECK(argc==6)<<"Need a input video list to test"; + + //read video list + vector video_list; + std::ifstream list_file(argv[1]); + CHECK(list_file.is_open()); + + int read_num; + clue::try_parse(argv[2], read_num); + + string vid_path = string(argv[3]); + + while(list_file.good()){ + string name; + list_file>>name; + video_list.push_back(name); + } + list_file.close(); + + int test_limit; + clue::try_parse(argv[4], test_limit); + + int seek_num; + clue::try_parse(argv[5], seek_num); + + LOG(INFO)<<"Video list loaded, "< dist(0.01, 0.99); + + std::shuffle(video_list.begin(), video_list.end(), gen); + + // time seeking by frame number + LOG(INFO)<<"Timing started. Reading "< ratios; + for (int i = 0; i < seek_num; i++){ + ratios.push_back(dist(gen)); + } + double fs; + auto pair = SeekByFrame(video_full_path, ratios, read_num, fs); + accum_seeking_time += pair.first; + accum_reading_time += pair.second; + accum_first_seeking_time += fs; + cnt ++; + if (cnt >= test_limit){ + break; + } + } + + double elapsed = aw.elapsed().msecs(); + double avg_time = elapsed / cnt; + + LOG(INFO)<<"Seeking by Frame Finished. Total time "< Date: Sun, 3 Apr 2016 18:59:31 +0800 Subject: [PATCH 23/41] remove debug print --- src/py_denseflow.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp index e201379..27e5d32 100644 --- a/src/py_denseflow.cpp +++ b/src/py_denseflow.cpp @@ -32,23 +32,18 @@ class TVL1FlowExtractor{ OpticalFlowDual_TVL1_GPU alg_tvl1; - printf("Initialized Mats\n"); // initialize the first frame const char* first_data = ((const char*)bp::extract(frames[0])); input_frame = Mat(img_height, img_width, CV_8UC3); initializeMats(input_frame, prev_frame, prev_gray, next_frame, next_gray); - printf("Copy to buffer, size %d %d\n", prev_frame.size[0], prev_frame.size[1]); memcpy(prev_frame.data, first_data, bp::len(frames[0])); - printf("Convert to grayscale\n"); cvtColor(prev_frame, prev_gray, CV_BGR2GRAY); for (int idx = 1; idx < bp::len(frames); idx++){ - printf("Running frame %d\n", idx); const char* this_data = ((const char*)bp::extract(frames[idx])); memcpy(next_frame.data, this_data, bp::len(frames[0])); cvtColor(next_frame, next_gray, CV_BGR2GRAY); - printf("Uploading to GPU\n"); d_frame_0.upload(prev_gray); d_frame_1.upload(next_gray); @@ -68,7 +63,6 @@ class TVL1FlowExtractor{ std::swap(prev_gray, next_gray); } - printf("extraction done\n"); return output; }; private: From 0563a3f0b665e49e3a4558259a1a2ea4b79b1e7a Mon Sep 17 00:00:00 2001 From: yjxiong Date: Mon, 4 Apr 2016 20:59:37 +0800 Subject: [PATCH 24/41] improve python wrapper --- include/common.h | 2 +- src/common.cpp | 13 ++++++++++--- src/py_denseflow.cpp | 10 ++++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/include/common.h b/include/common.h index 67b9cda..b8717e5 100644 --- a/include/common.h +++ b/include/common.h @@ -21,7 +21,7 @@ void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scala void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, vector& encoded_x, vector& encoded_y, - int bound); + int bound, bool to_jpg=true); inline void initializeMats(const Mat& frame, Mat& capture_image, Mat& capture_gray, diff --git a/src/common.cpp b/src/common.cpp index a85c907..35fca2e 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -31,15 +31,22 @@ void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scala void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, vector& encoded_x, vector& encoded_y, - int bound){ + int bound, bool to_jpg){ Mat flow_img_x(flow_map_x.size(), CV_8UC1); Mat flow_img_y(flow_map_y.size(), CV_8UC1); convertFlowToImage(flow_map_x, flow_map_y, flow_img_x, flow_img_y, -bound, bound); - imencode(".jpg", flow_img_x, encoded_x); - imencode(".jpg", flow_img_y, encoded_y); + if (to_jpg) { + imencode(".jpg", flow_img_x, encoded_x); + imencode(".jpg", flow_img_y, encoded_y); + }else { + encoded_x.resize(flow_img_x.total()); + encoded_y.resize(flow_img_y.total()); + memcpy(encoded_x.data(), flow_img_x.data, flow_img_x.total()); + memcpy(encoded_y.data(), flow_img_y.data, flow_img_y.total()); + } } void writeImages(vector> images, string name_temp){ diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp index 27e5d32..250a4b0 100644 --- a/src/py_denseflow.cpp +++ b/src/py_denseflow.cpp @@ -27,9 +27,7 @@ class TVL1FlowExtractor{ Mat input_frame, prev_frame, next_frame, prev_gray, next_gray; Mat flow_x, flow_y; - GpuMat d_frame_0, d_frame_1; - GpuMat d_flow_x, d_flow_y; - OpticalFlowDual_TVL1_GPU alg_tvl1; + // initialize the first frame @@ -53,7 +51,8 @@ class TVL1FlowExtractor{ d_flow_y.download(flow_y); vector str_x, str_y; - encodeFlowMap(flow_x, flow_y, str_x, str_y, bound_); + + encodeFlowMap(flow_x, flow_y, str_x, str_y, bound_, false); output.append( bp::make_tuple( bp::str((const char*) str_x.data(), str_x.size()), @@ -67,6 +66,9 @@ class TVL1FlowExtractor{ }; private: int bound_; + GpuMat d_frame_0, d_frame_1; + GpuMat d_flow_x, d_flow_y; + OpticalFlowDual_TVL1_GPU alg_tvl1; }; From a1288819610022eeb96a74fde4d9c9610186d4c4 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 27 Apr 2016 22:41:36 +0800 Subject: [PATCH 25/41] init warp flow extraction --- CMakeLists.txt | 2 +- include/dense_flow.h | 4 + src/dense_warp_flow_gpu.cpp | 359 ++++++++++++++++++++++++++++++++++++ 3 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 src/dense_warp_flow_gpu.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e35752..cb05977 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ endif() include_directories( ${OpenCV_INCLUDE_DIRS} ${LIBZIP_INCLUDE_DIR_ZIP} ${LIBZIP_INCLUDE_DIR_ZIPCONF} include/ include/easylogging++/src) -add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp src/zip_utils.cpp) +add_library(denseflow src/common.cpp src/dense_flow.cpp src/dense_flow_gpu.cpp src/dense_warp_flow_gpu.cpp src/zip_utils.cpp) target_link_libraries( denseflow ${OpenCV_LIBS} ${LIBZIP_LIBRARY}) add_executable( extract_cpu tools/extract_flow.cpp) diff --git a/include/dense_flow.h b/include/dense_flow.h index 4ba1da9..97c609c 100644 --- a/include/dense_flow.h +++ b/include/dense_flow.h @@ -21,4 +21,8 @@ void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int d vector >& output_y, vector >& output_img); +void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int dev_id, + vector >& output_x, + vector >& output_y); + #endif //DENSEFLOW_DENSE_FLOW_H diff --git a/src/dense_warp_flow_gpu.cpp b/src/dense_warp_flow_gpu.cpp new file mode 100644 index 0000000..0697091 --- /dev/null +++ b/src/dense_warp_flow_gpu.cpp @@ -0,0 +1,359 @@ +#include "common.h" +#include "dense_flow.h" + +#include "opencv2/video/tracking.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/calib3d/calib3d.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/features2d/features2d.hpp" +#include "opencv2/core/core.hpp" +#include "opencv2/nonfree/nonfree.hpp" +#include "opencv2/gpu/gpu.hpp" +#include "opencv2/gpu/gpu.hpp" + +#include +#include +using namespace cv; +using namespace cv::gpu; + + +static void MyWarpPerspective(Mat& prev_src, Mat& src, Mat& dst, Mat& M0, int flags = INTER_LINEAR, + int borderType = BORDER_CONSTANT, const Scalar& borderValue = Scalar()) +{ + int width = src.cols; + int height = src.rows; + dst.create( height, width, CV_8UC1 ); + + Mat mask = Mat::zeros(height, width, CV_8UC1); + const int margin = 5; + + const int BLOCK_SZ = 32; + short XY[BLOCK_SZ*BLOCK_SZ*2], A[BLOCK_SZ*BLOCK_SZ]; + + int interpolation = flags & INTER_MAX; + if( interpolation == INTER_AREA ) + interpolation = INTER_LINEAR; + + double M[9]; + Mat matM(3, 3, CV_64F, M); + M0.convertTo(matM, matM.type()); + if( !(flags & WARP_INVERSE_MAP) ) + invert(matM, matM); + + int x, y, x1, y1; + + int bh0 = min(BLOCK_SZ/2, height); + int bw0 = min(BLOCK_SZ*BLOCK_SZ/bh0, width); + bh0 = min(BLOCK_SZ*BLOCK_SZ/bw0, height); + + for( y = 0; y < height; y += bh0 ) { + for( x = 0; x < width; x += bw0 ) { + int bw = min( bw0, width - x); + int bh = min( bh0, height - y); + + Mat _XY(bh, bw, CV_16SC2, XY); + Mat matA; + Mat dpart(dst, Rect(x, y, bw, bh)); + + for( y1 = 0; y1 < bh; y1++ ) { + + short* xy = XY + y1*bw*2; + double X0 = M[0]*x + M[1]*(y + y1) + M[2]; + double Y0 = M[3]*x + M[4]*(y + y1) + M[5]; + double W0 = M[6]*x + M[7]*(y + y1) + M[8]; + short* alpha = A + y1*bw; + + for( x1 = 0; x1 < bw; x1++ ) { + + double W = W0 + M[6]*x1; + W = W ? INTER_TAB_SIZE/W : 0; + double fX = max((double)INT_MIN, min((double)INT_MAX, (X0 + M[0]*x1)*W)); + double fY = max((double)INT_MIN, min((double)INT_MAX, (Y0 + M[3]*x1)*W)); + + double _X = fX/double(INTER_TAB_SIZE); + double _Y = fY/double(INTER_TAB_SIZE); + + if( _X > margin && _X < width-1-margin && _Y > margin && _Y < height-1-margin ) + mask.at(y+y1, x+x1) = 1; + + int X = saturate_cast(fX); + int Y = saturate_cast(fY); + + xy[x1*2] = saturate_cast(X >> INTER_BITS); + xy[x1*2+1] = saturate_cast(Y >> INTER_BITS); + alpha[x1] = (short)((Y & (INTER_TAB_SIZE-1))*INTER_TAB_SIZE + (X & (INTER_TAB_SIZE-1))); + } + } + + Mat _matA(bh, bw, CV_16U, A); + remap( src, dpart, _XY, _matA, interpolation, borderType, borderValue ); + } + } + + for( y = 0; y < height; y++ ) { + const uchar* m = mask.ptr(y); + const uchar* s = prev_src.ptr(y); + uchar* d = dst.ptr(y); + for( x = 0; x < width; x++ ) { + if(m[x] == 0) + d[x] = s[x]; + } + } +} + +void ComputeMatch(const std::vector& prev_kpts, const std::vector& kpts, + const Mat& prev_desc, const Mat& desc, std::vector& prev_pts, std::vector& pts) +{ + prev_pts.clear(); + pts.clear(); + + if(prev_kpts.size() == 0 || kpts.size() == 0) + return; + + Mat mask = windowedMatchingMask(kpts, prev_kpts, 25, 25); + + BFMatcher desc_matcher(NORM_L2); + std::vector matches; + + desc_matcher.match(desc, prev_desc, matches, mask); + + prev_pts.reserve(matches.size()); + pts.reserve(matches.size()); + + for(size_t i = 0; i < matches.size(); i++) { + const DMatch& dmatch = matches[i]; + // get the point pairs that are successfully matched + prev_pts.push_back(prev_kpts[dmatch.trainIdx].pt); + pts.push_back(kpts[dmatch.queryIdx].pt); + } + + return; +} + +void MergeMatch(const std::vector& prev_pts1, const std::vector& pts1, + const std::vector& prev_pts2, const std::vector& pts2, + std::vector& prev_pts_all, std::vector& pts_all) +{ + prev_pts_all.clear(); + prev_pts_all.reserve(prev_pts1.size() + prev_pts2.size()); + + pts_all.clear(); + pts_all.reserve(pts1.size() + pts2.size()); + + for(size_t i = 0; i < prev_pts1.size(); i++) { + prev_pts_all.push_back(prev_pts1[i]); + pts_all.push_back(pts1[i]); + } + + for(size_t i = 0; i < prev_pts2.size(); i++) { + prev_pts_all.push_back(prev_pts2[i]); + pts_all.push_back(pts2[i]); + } + + return; +} + +void MatchFromFlow(const Mat& prev_grey, const Mat& flow, std::vector& prev_pts, std::vector& pts, const Mat& mask) +{ + int width = prev_grey.cols; + int height = prev_grey.rows; + prev_pts.clear(); + pts.clear(); + + const int MAX_COUNT = 1000; + goodFeaturesToTrack(prev_grey, prev_pts, MAX_COUNT, 0.001, 3, mask); + + if(prev_pts.size() == 0) + return; + + for(int i = 0; i < prev_pts.size(); i++) { + int x = std::min(std::max(cvRound(prev_pts[i].x), 0), width-1); + int y = std::min(std::max(cvRound(prev_pts[i].y), 0), height-1); + + const float* f = flow.ptr(y); + pts.push_back(Point2f(x+f[2*x], y+f[2*x+1])); + } +} + +void MatchFromFlow_copy(const Mat& prev_grey, const Mat& flow_x, const Mat& flow_y, std::vector& prev_pts, std::vector& pts, const Mat& mask) +{ + int width = prev_grey.cols; + int height = prev_grey.rows; + prev_pts.clear(); + pts.clear(); + + const int MAX_COUNT = 1000; + goodFeaturesToTrack(prev_grey, prev_pts, MAX_COUNT, 0.001, 3, mask); + + if(prev_pts.size() == 0) + return; + + for(int i = 0; i < prev_pts.size(); i++) { + int x = std::min(std::max(cvRound(prev_pts[i].x), 0), width-1); + int y = std::min(std::max(cvRound(prev_pts[i].y), 0), height-1); + + const float* f_x = flow_x.ptr(y); + const float* f_y = flow_y.ptr(y); + pts.push_back(Point2f(x+f_x[x], y+f_y[y])); + } +} + +void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int dev_id, + vector >& output_x, + vector >& output_y){ + VideoCapture video_stream(file_name); + CHECK(video_stream.isOpened())<<"Cannot open video stream \"" + < prev_pts_flow, pts_flow; + std::vector prev_pts_surf, pts_surf; + std::vector prev_pts_all, pts_all; + std::vector prev_kpts_surf, kpts_surf; + Mat prev_desc_surf, desc_surf; + + setDevice(dev_id); + Mat capture_frame, capture_image, prev_image, capture_gray, prev_gray, human_mask; + Mat flow_x, flow_y; + + GpuMat d_frame_0, d_frame_1; + GpuMat d_flow_x, d_flow_y; + + FarnebackOpticalFlow alg_farn; + OpticalFlowDual_TVL1_GPU alg_tvl1; + BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); + + bool initialized = false; + int cnt = 0; + while(true){ + + //build mats for the first frame + if (!initialized){ + video_stream >> capture_frame; + if (capture_frame.empty()) return; // read frames until end + initializeMats(capture_frame, capture_image, capture_gray, + prev_image, prev_gray); + capture_frame.copyTo(prev_image); + cvtColor(prev_image, prev_gray, CV_BGR2GRAY); + + //detect key points + human_mask = Mat::ones(capture_frame.size(), CV_8UC1); + detector_surf.detect(prev_gray, prev_kpts_surf, human_mask); + extractor_surf.compute(prev_gray, prev_kpts_surf, prev_desc_surf); + + initialized = true; + for(int s = 0; s < step; ++s){ + video_stream >> capture_frame; + cnt ++; + if (capture_frame.empty()) return; // read frames until end + } + }else { + capture_frame.copyTo(capture_image); + cvtColor(capture_image, capture_gray, CV_BGR2GRAY); + d_frame_0.upload(prev_gray); + d_frame_1.upload(capture_gray); + + switch(type){ + case 0: { + alg_farn(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + break; + } + case 1: { + alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + break; + } + case 2: { + GpuMat d_buf_0, d_buf_1; + d_frame_0.convertTo(d_buf_0, CV_32F, 1.0 / 255.0); + d_frame_1.convertTo(d_buf_1, CV_32F, 1.0 / 255.0); + alg_brox(d_buf_0, d_buf_1, d_flow_x, d_flow_y); + break; + } + default: + LOG(ERROR)<<"Unknown optical method: "< 50) { + std::vector match_mask; + Mat temp = findHomography(prev_pts_all, pts_all, RANSAC, 1, match_mask); + if(countNonZero(Mat(match_mask)) > 25) + H = temp; + } + + Mat H_inv = H.inv(); + Mat gray_warp = Mat::zeros(capture_gray.size(), CV_8UC1); + MyWarpPerspective(prev_gray, capture_gray, gray_warp, H_inv); + + // re-extract flow on warped images + d_frame_0.upload(prev_gray); + d_frame_1.upload(gray_warp); + + switch(type){ + case 0: { + alg_farn(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + break; + } + case 1: { + alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + break; + } + case 2: { + GpuMat d_buf_0, d_buf_1; + d_frame_0.convertTo(d_buf_0, CV_32F, 1.0 / 255.0); + d_frame_1.convertTo(d_buf_1, CV_32F, 1.0 / 255.0); + alg_brox(d_buf_0, d_buf_1, d_flow_x, d_flow_y); + break; + } + default: + LOG(ERROR)<<"Unknown optical method: "< str_x, str_y; + encodeFlowMap(flow_x, flow_y, str_x, str_y, bound); + + output_x.push_back(str_x); + output_y.push_back(str_y); + + std::swap(prev_gray, capture_gray); + std::swap(prev_image, capture_image); + + + //get next frame + bool hasnext = true; + for(int s = 0; s < step; ++s){ + video_stream >> capture_frame; + cnt ++; + hasnext = !capture_frame.empty(); + // read frames until end + } + if (!hasnext){ + return; + } + } + + + } +} \ No newline at end of file From f64d0d0154d6862f995c66416136b9477a54e530 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Thu, 28 Apr 2016 21:41:14 +0800 Subject: [PATCH 26/41] add tool to extract warp --- CMakeLists.txt | 3 +++ tools/extract_warp_flow_gpu.cpp | 47 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tools/extract_warp_flow_gpu.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cb05977..ad794ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,3 +32,6 @@ target_link_libraries( extract_cpu ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) add_executable( extract_gpu tools/extract_flow_gpu.cpp) target_link_libraries( extract_gpu ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) + +add_executable( extract_warp_gpu tools/extract_warp_flow_gpu.cpp) +target_link_libraries( extract_warp_gpu ${OpenCV_LIBS} ${LIBZIP_LIBRARY} denseflow) diff --git a/tools/extract_warp_flow_gpu.cpp b/tools/extract_warp_flow_gpu.cpp new file mode 100644 index 0000000..d0bb92f --- /dev/null +++ b/tools/extract_warp_flow_gpu.cpp @@ -0,0 +1,47 @@ +#include "dense_flow.h" +#include "utils.h" + +INITIALIZE_EASYLOGGINGPP + +using namespace cv::gpu; + +int main(int argc, char** argv){ + // IO operation + const char* keys = + { + "{ f | vidFile | ex2.avi | filename of video }" + "{ x | xFlowFile | flow_x | filename of flow x component }" + "{ y | yFlowFile | flow_y | filename of flow x component }" + "{ b | bound | 15 | specify the maximum of optical flow}" + "{ t | type | 0 | specify the optical flow algorithm }" + "{ d | device_id | 0 | set gpu id}" + "{ s | step | 1 | specify the step for frame sampling}" + "{ o | out | zip | output style}" + }; + + CommandLineParser cmd(argc, argv, keys); + string vidFile = cmd.get("vidFile"); + string xFlowFile = cmd.get("xFlowFile"); + string yFlowFile = cmd.get("yFlowFile"); + string output_style = cmd.get("out"); + int bound = cmd.get("bound"); + int type = cmd.get("type"); + int device_id = cmd.get("device_id"); + int step = cmd.get("step"); + + vector > out_vec_x, out_vec_y; + + calcDenseWarpFlowGPU(vidFile, bound, type, step, device_id, + out_vec_x, out_vec_y); + + if (output_style == "dir") { + writeImages(out_vec_x, xFlowFile); + writeImages(out_vec_y, yFlowFile); + }else{ +// LOG(INFO)<<"Writing results to Zip archives"; + writeZipFile(out_vec_x, "x_%05d.jpg", xFlowFile+".zip"); + writeZipFile(out_vec_y, "y_%05d.jpg", yFlowFile+".zip"); + } + + return 0; +} From 6dd37bca97bafbebbd82783dfba05f4f6c58ec07 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Thu, 28 Apr 2016 22:12:54 +0800 Subject: [PATCH 27/41] updates --- include/dense_flow.h | 2 + src/py_denseflow.cpp | 121 +++++++++++++++++++++++++++ tools/benchmark_video_io.cpp | 158 ----------------------------------- 3 files changed, 123 insertions(+), 158 deletions(-) delete mode 100644 tools/benchmark_video_io.cpp diff --git a/include/dense_flow.h b/include/dense_flow.h index 6a81774..3203b3c 100644 --- a/include/dense_flow.h +++ b/include/dense_flow.h @@ -26,4 +26,6 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d vector >& output_x, vector >& output_y); +void MatchFromFlow_copy(const Mat& prev_grey, const Mat& flow_x, const Mat& flow_y, std::vector& prev_pts, std::vector& pts, const Mat& mask); + #endif //DENSEFLOW_DENSE_FLOW_H diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp index 250a4b0..32bc40c 100644 --- a/src/py_denseflow.cpp +++ b/src/py_denseflow.cpp @@ -6,6 +6,20 @@ #include "common.h" #include "opencv2/gpu/gpu.hpp" + +#include "opencv2/video/tracking.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/calib3d/calib3d.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/features2d/features2d.hpp" +#include "opencv2/core/core.hpp" +#include "opencv2/nonfree/nonfree.hpp" +#include "opencv2/gpu/gpu.hpp" +#include "opencv2/gpu/gpu.hpp" + + using namespace cv::gpu; using namespace cv; @@ -71,6 +85,109 @@ class TVL1FlowExtractor{ OpticalFlowDual_TVL1_GPU alg_tvl1; }; +extern void MyWarpPerspective(Mat& prev_src, Mat& src, Mat& dst, Mat& M0, int flags = INTER_LINEAR, + int borderType = BORDER_CONSTANT, const Scalar& borderValue = Scalar()); +extern void ComputeMatch(const std::vector& prev_kpts, const std::vector& kpts, + const Mat& prev_desc, const Mat& desc, std::vector& prev_pts, std::vector& pts); +extern void MergeMatch(const std::vector& prev_pts1, const std::vector& pts1, + const std::vector& prev_pts2, const std::vector& pts2, + std::vector& prev_pts_all, std::vector& pts_all); +extern void MatchFromFlow_copy(const Mat& prev_grey, const Mat& flow_x, const Mat& flow_y, std::vector& prev_pts, std::vector& pts, const Mat& mask); + +class TVL1WarpFlowExtractor { +public: + + TVL1WarpFlowExtractor(int bound) + :detector_surf(200), extractor_surf(true, true){ + bound_ = bound; + } + + static void set_device(int dev_id){ + setDevice(dev_id); + } + + bp::list extract_warp_flow(bp::list frames, int img_width, int img_height){ + bp::list output; + Mat input_frame, prev_frame, next_frame, prev_gray, next_gray, human_mask; + Mat flow_x, flow_y; + + // initialize the first frame + const char* first_data = ((const char*)bp::extract(frames[0])); + input_frame = Mat(img_height, img_width, CV_8UC3); + initializeMats(input_frame, prev_frame, prev_gray, next_frame, next_gray); + human_mask = Mat::ones(input_frame.size(), CV_8UC1); + + memcpy(prev_frame.data, first_data, bp::len(frames[0])); + cvtColor(prev_frame, prev_gray, CV_BGR2GRAY); + for (int idx = 1; idx < bp::len(frames); idx++){ + const char* this_data = ((const char*)bp::extract(frames[idx])); + memcpy(next_frame.data, this_data, bp::len(frames[0])); + cvtColor(next_frame, next_gray, CV_BGR2GRAY); + + d_frame_0.upload(prev_gray); + d_frame_1.upload(next_gray); + + alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + + d_flow_x.download(flow_x); + d_flow_y.download(flow_y); + + // warp to reduce holistic motion + detector_surf.detect(next_gray, kpts_surf, human_mask); + extractor_surf.compute(next_gray, kpts_surf, desc_surf); + ComputeMatch(prev_kpts_surf, kpts_surf, prev_desc_surf, desc_surf, prev_pts_surf, pts_surf); + MatchFromFlow_copy(next_gray, flow_x, flow_y, prev_pts_flow, pts_flow, human_mask); + MergeMatch(prev_pts_flow, pts_flow, prev_pts_surf, pts_surf, prev_pts_all, pts_all); + Mat H = Mat::eye(3, 3, CV_64FC1); + if(pts_all.size() > 50) { + std::vector match_mask; + Mat temp = findHomography(prev_pts_all, pts_all, RANSAC, 1, match_mask); + if(countNonZero(Mat(match_mask)) > 25) + H = temp; + } + + Mat H_inv = H.inv(); + Mat gray_warp = Mat::zeros(next_gray.size(), CV_8UC1); + MyWarpPerspective(prev_gray, next_gray, gray_warp, H_inv); + + d_frame_0.upload(prev_gray); + d_frame_1.upload(gray_warp); + + alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + + d_flow_x.download(flow_x); + d_flow_y.download(flow_y); + + vector str_x, str_y; + + encodeFlowMap(flow_x, flow_y, str_x, str_y, bound_, false); + output.append( + bp::make_tuple( + bp::str((const char*) str_x.data(), str_x.size()), + bp::str((const char*) str_y.data(), str_y.size()) + ) + ); + + std::swap(prev_gray, next_gray); + } + return output; + } +private: + SurfFeatureDetector detector_surf; + SurfDescriptorExtractor extractor_surf; + std::vector prev_pts_flow, pts_flow; + std::vector prev_pts_surf, pts_surf; + std::vector prev_pts_all, pts_all; + std::vector prev_kpts_surf, kpts_surf; + Mat prev_desc_surf, desc_surf; + + GpuMat d_frame_0, d_frame_1; + GpuMat d_flow_x, d_flow_y; + + OpticalFlowDual_TVL1_GPU alg_tvl1; + int bound_; +}; + //// Boost Python Related Decl BOOST_PYTHON_MODULE(libpydenseflow){ @@ -78,4 +195,8 @@ BOOST_PYTHON_MODULE(libpydenseflow){ .def("extract_flow", &TVL1FlowExtractor::extract_flow) .def("set_device", &TVL1FlowExtractor::set_device) .staticmethod("set_device"); + bp::class_("TVL1WarpFlowExtractor", bp::init()) + .def("extract_warp_flow", &TVL1WarpFlowExtractor::extract_warp_flow) + .def("set_device", &TVL1WarpFlowExtractor::set_device) + .staticmethod("set_device"); } \ No newline at end of file diff --git a/tools/benchmark_video_io.cpp b/tools/benchmark_video_io.cpp deleted file mode 100644 index 0b99c37..0000000 --- a/tools/benchmark_video_io.cpp +++ /dev/null @@ -1,158 +0,0 @@ -// -// Created by alex on 16-3-27. -// - -#include "dense_flow.h" -#include "utils.h" -#include -#include -#include -#include "easylogging++.h" - -INITIALIZE_EASYLOGGINGPP - -using namespace cv::gpu; - -void OpenVideoOnly(string video_path){ - VideoCapture video_stream(video_path); - CHECK(video_stream.isOpened())<<"Cannot open video stream \"" - < SeekByFrame(string video_path, vector ratio, int read_num, double& first_seek_time){ - VideoCapture video_stream(video_path); - - double seeking_time = 0, reading_time = 0; - int cnt = 0; - for (float r : ratio) { - CHECK(video_stream.isOpened())<<"Cannot open video stream \"" - <> capture_frame; - } - reading_time += sw.elapsed().msecs(); - - if (cnt == 0){ - first_seek_time = seeking_time; - LOG(INFO)<<"first seek "<>capture_frame; - } - video_stream.release(); -} - -void SeekByRATIO(string video_path, float ratio, int read_num){ - VideoCapture video_stream(video_path); - CHECK(video_stream.isOpened())<<"Cannot open video stream \"" - <>capture_frame; - } - video_stream.release(); -} - -int main(int argc, char** argv){ - - CHECK(argc==6)<<"Need a input video list to test"; - - //read video list - vector video_list; - std::ifstream list_file(argv[1]); - CHECK(list_file.is_open()); - - int read_num; - clue::try_parse(argv[2], read_num); - - string vid_path = string(argv[3]); - - while(list_file.good()){ - string name; - list_file>>name; - video_list.push_back(name); - } - list_file.close(); - - int test_limit; - clue::try_parse(argv[4], test_limit); - - int seek_num; - clue::try_parse(argv[5], seek_num); - - LOG(INFO)<<"Video list loaded, "< dist(0.01, 0.99); - - std::shuffle(video_list.begin(), video_list.end(), gen); - - // time seeking by frame number - LOG(INFO)<<"Timing started. Reading "< ratios; - for (int i = 0; i < seek_num; i++){ - ratios.push_back(dist(gen)); - } - double fs; - auto pair = SeekByFrame(video_full_path, ratios, read_num, fs); - accum_seeking_time += pair.first; - accum_reading_time += pair.second; - accum_first_seeking_time += fs; - cnt ++; - if (cnt >= test_limit){ - break; - } - } - - double elapsed = aw.elapsed().msecs(); - double avg_time = elapsed / cnt; - - LOG(INFO)<<"Seeking by Frame Finished. Total time "< Date: Sun, 1 May 2016 13:41:53 +0800 Subject: [PATCH 28/41] add python scripts --- build_of.py | 57 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/build_of.py b/build_of.py index 0f51d20..83f6b13 100644 --- a/build_of.py +++ b/build_of.py @@ -4,6 +4,7 @@ import os from multiprocessing import Pool, current_process +import argparse out_path = '' @@ -45,26 +46,52 @@ def run_optical_flow(vid_item, dev_id=0): flow_y_path = '{}/flow_y'.format(out_full_path) cmd = './build/extract_gpu -f {} -x {} -y {} -i {} -b 20 -t 1 -d {} -s 1 -o zip'.format(vid_path, flow_x_path, flow_y_path, image_path, dev_id) - #cmd = './build/extract_gpu -f {} -x {} -y {} -b 20 -t 1 -d {} -s 1 -o zip'.format(vid_path, flow_x_path, flow_y_path, dev_id) os.system(cmd) print '{} {} done'.format(vid_id, vid_name) return True +def run_warp_optical_flow(vid_item, dev_id=0): + vid_path = vid_item[0] + vid_id = vid_item[1] + vid_name = vid_path.split('/')[-1].split('.')[0] + out_full_path = os.path.join(out_path, vid_name) + try: + os.mkdir(out_full_path) + except OSError: + pass + + current = current_process() + dev_id = int(current._identity[0]) - 1 + flow_x_path = '{}/flow_x'.format(out_full_path) + flow_y_path = '{}/flow_y'.format(out_full_path) + + cmd = './build/extract_warp_gpu -f {} -x {} -y {} -b 20 -t 1 -d {} -s 1 -o zip'.format(vid_path, flow_x_path, flow_y_path, dev_id) + + os.system(cmd) + print 'warp on {} {} done'.format(vid_id, vid_name) + return True + if __name__ == '__main__': - out_path = '/data2/alex/anet_v1.2_testing_flow_tvl1' - import glob - #vid_list = glob.glob('/data1/alex/anet_clips_v1.2/*avi') - vid_list = glob.glob('/data1/alex/ActivityNet/testing_videos_v1.2/*.mp4') - #vid_list = ['/data1/alex/anet_clips_v1.2/{}.avi'.format(x.strip()) for x in open('/data2/alex/prob_clip_list.txt')] + parser = argparse.ArgumentParser(description="extract optical flows") + parser.add_argument("src_dir") + parser.add_argument("out_dir") + parser.add_argument("--num_worker", type=int, default=8) + parser.add_argument("--flow_type", type=str, default='tvl1', choices=['tvl1', 'warp_tvl1']) + + args = parser.parse_args() + + out_path = args.out_dir + src_path = args.src_dir + num_worker = args.num_worker + flow_type = args.flow_type + + + vid_list = glob.glob(src_path+'/*.mp4') print len(vid_list) - pool = Pool(16) - pool.map(run_optical_flow, zip(vid_list, xrange(len(vid_list)))) - #map(run_optical_flow, zip(vid_list, xrange(len(vid_list)))) - #file_list = pool.map(dump_frames, vid_list) - #all_file_list = [f for x in file_list for f in x] - #open('anet_image_list_nov_17.txt','w').writelines('\n'.join(all_file_list)) - #for i,v in enumerate(vid_list): - # run_optical_flow(v, 0) - # print i + pool = Pool(num_worker) + if flow_type == 'tvl1': + pool.map(run_optical_flow, zip(vid_list, xrange(len(vid_list)))) + elif flow_type == 'warp_tvl1': + pool.map(run_warp_optical_flow, zip(vid_list, xrange(len(vid_list)))) From 86450274abcaa544f7a7e2d171d0a086dcf923a1 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 25 May 2016 15:52:07 +0800 Subject: [PATCH 29/41] fix linking problem of warp flow --- include/warp_flow.h | 189 ++++++++++++++++++++++++++++++++++++ src/dense_warp_flow_gpu.cpp | 185 +---------------------------------- src/py_denseflow.cpp | 10 +- 3 files changed, 194 insertions(+), 190 deletions(-) create mode 100644 include/warp_flow.h diff --git a/include/warp_flow.h b/include/warp_flow.h new file mode 100644 index 0000000..f0d560e --- /dev/null +++ b/include/warp_flow.h @@ -0,0 +1,189 @@ +// +// Created by alex on 16-5-25. +// + +#ifndef DENSEFLOW_WARP_FLOW_H_H +#define DENSEFLOW_WARP_FLOW_H_H + +void MyWarpPerspective(Mat& prev_src, Mat& src, Mat& dst, Mat& M0, int flags=INTER_LINEAR, + int borderType=BORDER_CONSTANT, const Scalar& borderValue=Scalar()) +{ + int width = src.cols; + int height = src.rows; + dst.create( height, width, CV_8UC1 ); + + Mat mask = Mat::zeros(height, width, CV_8UC1); + const int margin = 5; + + const int BLOCK_SZ = 32; + short XY[BLOCK_SZ*BLOCK_SZ*2], A[BLOCK_SZ*BLOCK_SZ]; + + int interpolation = flags & INTER_MAX; + if( interpolation == INTER_AREA ) + interpolation = INTER_LINEAR; + + double M[9]; + Mat matM(3, 3, CV_64F, M); + M0.convertTo(matM, matM.type()); + if( !(flags & WARP_INVERSE_MAP) ) + invert(matM, matM); + + int x, y, x1, y1; + + int bh0 = min(BLOCK_SZ/2, height); + int bw0 = min(BLOCK_SZ*BLOCK_SZ/bh0, width); + bh0 = min(BLOCK_SZ*BLOCK_SZ/bw0, height); + + for( y = 0; y < height; y += bh0 ) { + for( x = 0; x < width; x += bw0 ) { + int bw = min( bw0, width - x); + int bh = min( bh0, height - y); + + Mat _XY(bh, bw, CV_16SC2, XY); + Mat matA; + Mat dpart(dst, Rect(x, y, bw, bh)); + + for( y1 = 0; y1 < bh; y1++ ) { + + short* xy = XY + y1*bw*2; + double X0 = M[0]*x + M[1]*(y + y1) + M[2]; + double Y0 = M[3]*x + M[4]*(y + y1) + M[5]; + double W0 = M[6]*x + M[7]*(y + y1) + M[8]; + short* alpha = A + y1*bw; + + for( x1 = 0; x1 < bw; x1++ ) { + + double W = W0 + M[6]*x1; + W = W ? INTER_TAB_SIZE/W : 0; + double fX = max((double)INT_MIN, min((double)INT_MAX, (X0 + M[0]*x1)*W)); + double fY = max((double)INT_MIN, min((double)INT_MAX, (Y0 + M[3]*x1)*W)); + + double _X = fX/double(INTER_TAB_SIZE); + double _Y = fY/double(INTER_TAB_SIZE); + + if( _X > margin && _X < width-1-margin && _Y > margin && _Y < height-1-margin ) + mask.at(y+y1, x+x1) = 1; + + int X = saturate_cast(fX); + int Y = saturate_cast(fY); + + xy[x1*2] = saturate_cast(X >> INTER_BITS); + xy[x1*2+1] = saturate_cast(Y >> INTER_BITS); + alpha[x1] = (short)((Y & (INTER_TAB_SIZE-1))*INTER_TAB_SIZE + (X & (INTER_TAB_SIZE-1))); + } + } + + Mat _matA(bh, bw, CV_16U, A); + remap( src, dpart, _XY, _matA, interpolation, borderType, borderValue ); + } + } + + for( y = 0; y < height; y++ ) { + const uchar* m = mask.ptr(y); + const uchar* s = prev_src.ptr(y); + uchar* d = dst.ptr(y); + for( x = 0; x < width; x++ ) { + if(m[x] == 0) + d[x] = s[x]; + } + } +} + +void ComputeMatch(const std::vector& prev_kpts, const std::vector& kpts, + const Mat& prev_desc, const Mat& desc, std::vector& prev_pts, std::vector& pts) +{ + prev_pts.clear(); + pts.clear(); + + if(prev_kpts.size() == 0 || kpts.size() == 0) + return; + + Mat mask = windowedMatchingMask(kpts, prev_kpts, 25, 25); + + BFMatcher desc_matcher(NORM_L2); + std::vector matches; + + desc_matcher.match(desc, prev_desc, matches, mask); + + prev_pts.reserve(matches.size()); + pts.reserve(matches.size()); + + for(size_t i = 0; i < matches.size(); i++) { + const DMatch& dmatch = matches[i]; + // get the point pairs that are successfully matched + prev_pts.push_back(prev_kpts[dmatch.trainIdx].pt); + pts.push_back(kpts[dmatch.queryIdx].pt); + } + + return; +} + +void MergeMatch(const std::vector& prev_pts1, const std::vector& pts1, + const std::vector& prev_pts2, const std::vector& pts2, + std::vector& prev_pts_all, std::vector& pts_all) +{ + prev_pts_all.clear(); + prev_pts_all.reserve(prev_pts1.size() + prev_pts2.size()); + + pts_all.clear(); + pts_all.reserve(pts1.size() + pts2.size()); + + for(size_t i = 0; i < prev_pts1.size(); i++) { + prev_pts_all.push_back(prev_pts1[i]); + pts_all.push_back(pts1[i]); + } + + for(size_t i = 0; i < prev_pts2.size(); i++) { + prev_pts_all.push_back(prev_pts2[i]); + pts_all.push_back(pts2[i]); + } + + return; +} + +void MatchFromFlow(const Mat& prev_grey, const Mat& flow, std::vector& prev_pts, std::vector& pts, const Mat& mask) +{ + int width = prev_grey.cols; + int height = prev_grey.rows; + prev_pts.clear(); + pts.clear(); + + const int MAX_COUNT = 1000; + goodFeaturesToTrack(prev_grey, prev_pts, MAX_COUNT, 0.001, 3, mask); + + if(prev_pts.size() == 0) + return; + + for(int i = 0; i < prev_pts.size(); i++) { + int x = std::min(std::max(cvRound(prev_pts[i].x), 0), width-1); + int y = std::min(std::max(cvRound(prev_pts[i].y), 0), height-1); + + const float* f = flow.ptr(y); + pts.push_back(Point2f(x+f[2*x], y+f[2*x+1])); + } +} + +void MatchFromFlow_copy(const Mat& prev_grey, const Mat& flow_x, const Mat& flow_y, std::vector& prev_pts, std::vector& pts, const Mat& mask) +{ + int width = prev_grey.cols; + int height = prev_grey.rows; + prev_pts.clear(); + pts.clear(); + + const int MAX_COUNT = 1000; + goodFeaturesToTrack(prev_grey, prev_pts, MAX_COUNT, 0.001, 3, mask); + + if(prev_pts.size() == 0) + return; + + for(int i = 0; i < prev_pts.size(); i++) { + int x = std::min(std::max(cvRound(prev_pts[i].x), 0), width-1); + int y = std::min(std::max(cvRound(prev_pts[i].y), 0), height-1); + + const float* f_x = flow_x.ptr(y); + const float* f_y = flow_y.ptr(y); + pts.push_back(Point2f(x+f_x[x], y+f_y[y])); + } +} + +#endif //DENSEFLOW_WARP_FLOW_H_H diff --git a/src/dense_warp_flow_gpu.cpp b/src/dense_warp_flow_gpu.cpp index 0697091..2dad457 100644 --- a/src/dense_warp_flow_gpu.cpp +++ b/src/dense_warp_flow_gpu.cpp @@ -15,190 +15,11 @@ #include #include -using namespace cv; -using namespace cv::gpu; - - -static void MyWarpPerspective(Mat& prev_src, Mat& src, Mat& dst, Mat& M0, int flags = INTER_LINEAR, - int borderType = BORDER_CONSTANT, const Scalar& borderValue = Scalar()) -{ - int width = src.cols; - int height = src.rows; - dst.create( height, width, CV_8UC1 ); - - Mat mask = Mat::zeros(height, width, CV_8UC1); - const int margin = 5; - - const int BLOCK_SZ = 32; - short XY[BLOCK_SZ*BLOCK_SZ*2], A[BLOCK_SZ*BLOCK_SZ]; - - int interpolation = flags & INTER_MAX; - if( interpolation == INTER_AREA ) - interpolation = INTER_LINEAR; - - double M[9]; - Mat matM(3, 3, CV_64F, M); - M0.convertTo(matM, matM.type()); - if( !(flags & WARP_INVERSE_MAP) ) - invert(matM, matM); - - int x, y, x1, y1; - - int bh0 = min(BLOCK_SZ/2, height); - int bw0 = min(BLOCK_SZ*BLOCK_SZ/bh0, width); - bh0 = min(BLOCK_SZ*BLOCK_SZ/bw0, height); - - for( y = 0; y < height; y += bh0 ) { - for( x = 0; x < width; x += bw0 ) { - int bw = min( bw0, width - x); - int bh = min( bh0, height - y); - - Mat _XY(bh, bw, CV_16SC2, XY); - Mat matA; - Mat dpart(dst, Rect(x, y, bw, bh)); - - for( y1 = 0; y1 < bh; y1++ ) { - - short* xy = XY + y1*bw*2; - double X0 = M[0]*x + M[1]*(y + y1) + M[2]; - double Y0 = M[3]*x + M[4]*(y + y1) + M[5]; - double W0 = M[6]*x + M[7]*(y + y1) + M[8]; - short* alpha = A + y1*bw; - - for( x1 = 0; x1 < bw; x1++ ) { - - double W = W0 + M[6]*x1; - W = W ? INTER_TAB_SIZE/W : 0; - double fX = max((double)INT_MIN, min((double)INT_MAX, (X0 + M[0]*x1)*W)); - double fY = max((double)INT_MIN, min((double)INT_MAX, (Y0 + M[3]*x1)*W)); - - double _X = fX/double(INTER_TAB_SIZE); - double _Y = fY/double(INTER_TAB_SIZE); - - if( _X > margin && _X < width-1-margin && _Y > margin && _Y < height-1-margin ) - mask.at(y+y1, x+x1) = 1; - - int X = saturate_cast(fX); - int Y = saturate_cast(fY); - - xy[x1*2] = saturate_cast(X >> INTER_BITS); - xy[x1*2+1] = saturate_cast(Y >> INTER_BITS); - alpha[x1] = (short)((Y & (INTER_TAB_SIZE-1))*INTER_TAB_SIZE + (X & (INTER_TAB_SIZE-1))); - } - } - - Mat _matA(bh, bw, CV_16U, A); - remap( src, dpart, _XY, _matA, interpolation, borderType, borderValue ); - } - } - - for( y = 0; y < height; y++ ) { - const uchar* m = mask.ptr(y); - const uchar* s = prev_src.ptr(y); - uchar* d = dst.ptr(y); - for( x = 0; x < width; x++ ) { - if(m[x] == 0) - d[x] = s[x]; - } - } -} - -void ComputeMatch(const std::vector& prev_kpts, const std::vector& kpts, - const Mat& prev_desc, const Mat& desc, std::vector& prev_pts, std::vector& pts) -{ - prev_pts.clear(); - pts.clear(); - - if(prev_kpts.size() == 0 || kpts.size() == 0) - return; - Mat mask = windowedMatchingMask(kpts, prev_kpts, 25, 25); +#include "warp_flow.h" - BFMatcher desc_matcher(NORM_L2); - std::vector matches; - - desc_matcher.match(desc, prev_desc, matches, mask); - - prev_pts.reserve(matches.size()); - pts.reserve(matches.size()); - - for(size_t i = 0; i < matches.size(); i++) { - const DMatch& dmatch = matches[i]; - // get the point pairs that are successfully matched - prev_pts.push_back(prev_kpts[dmatch.trainIdx].pt); - pts.push_back(kpts[dmatch.queryIdx].pt); - } - - return; -} - -void MergeMatch(const std::vector& prev_pts1, const std::vector& pts1, - const std::vector& prev_pts2, const std::vector& pts2, - std::vector& prev_pts_all, std::vector& pts_all) -{ - prev_pts_all.clear(); - prev_pts_all.reserve(prev_pts1.size() + prev_pts2.size()); - - pts_all.clear(); - pts_all.reserve(pts1.size() + pts2.size()); - - for(size_t i = 0; i < prev_pts1.size(); i++) { - prev_pts_all.push_back(prev_pts1[i]); - pts_all.push_back(pts1[i]); - } - - for(size_t i = 0; i < prev_pts2.size(); i++) { - prev_pts_all.push_back(prev_pts2[i]); - pts_all.push_back(pts2[i]); - } - - return; -} - -void MatchFromFlow(const Mat& prev_grey, const Mat& flow, std::vector& prev_pts, std::vector& pts, const Mat& mask) -{ - int width = prev_grey.cols; - int height = prev_grey.rows; - prev_pts.clear(); - pts.clear(); - - const int MAX_COUNT = 1000; - goodFeaturesToTrack(prev_grey, prev_pts, MAX_COUNT, 0.001, 3, mask); - - if(prev_pts.size() == 0) - return; - - for(int i = 0; i < prev_pts.size(); i++) { - int x = std::min(std::max(cvRound(prev_pts[i].x), 0), width-1); - int y = std::min(std::max(cvRound(prev_pts[i].y), 0), height-1); - - const float* f = flow.ptr(y); - pts.push_back(Point2f(x+f[2*x], y+f[2*x+1])); - } -} - -void MatchFromFlow_copy(const Mat& prev_grey, const Mat& flow_x, const Mat& flow_y, std::vector& prev_pts, std::vector& pts, const Mat& mask) -{ - int width = prev_grey.cols; - int height = prev_grey.rows; - prev_pts.clear(); - pts.clear(); - - const int MAX_COUNT = 1000; - goodFeaturesToTrack(prev_grey, prev_pts, MAX_COUNT, 0.001, 3, mask); - - if(prev_pts.size() == 0) - return; - - for(int i = 0; i < prev_pts.size(); i++) { - int x = std::min(std::max(cvRound(prev_pts[i].x), 0), width-1); - int y = std::min(std::max(cvRound(prev_pts[i].y), 0), height-1); - - const float* f_x = flow_x.ptr(y); - const float* f_y = flow_y.ptr(y); - pts.push_back(Point2f(x+f_x[x], y+f_y[y])); - } -} +using namespace cv; +using namespace cv::gpu; void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int dev_id, vector >& output_x, diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp index 32bc40c..53877d1 100644 --- a/src/py_denseflow.cpp +++ b/src/py_denseflow.cpp @@ -19,6 +19,7 @@ #include "opencv2/gpu/gpu.hpp" #include "opencv2/gpu/gpu.hpp" +#include "warp_flow.h" using namespace cv::gpu; using namespace cv; @@ -85,14 +86,7 @@ class TVL1FlowExtractor{ OpticalFlowDual_TVL1_GPU alg_tvl1; }; -extern void MyWarpPerspective(Mat& prev_src, Mat& src, Mat& dst, Mat& M0, int flags = INTER_LINEAR, - int borderType = BORDER_CONSTANT, const Scalar& borderValue = Scalar()); -extern void ComputeMatch(const std::vector& prev_kpts, const std::vector& kpts, - const Mat& prev_desc, const Mat& desc, std::vector& prev_pts, std::vector& pts); -extern void MergeMatch(const std::vector& prev_pts1, const std::vector& pts1, - const std::vector& prev_pts2, const std::vector& pts2, - std::vector& prev_pts_all, std::vector& pts_all); -extern void MatchFromFlow_copy(const Mat& prev_grey, const Mat& flow_x, const Mat& flow_y, std::vector& prev_pts, std::vector& pts, const Mat& mask); + class TVL1WarpFlowExtractor { public: From a282d619e27d88f4498531ffc53e9e17bbfbf303 Mon Sep 17 00:00:00 2001 From: Yuanjun Xiong Date: Wed, 25 May 2016 15:56:37 +0800 Subject: [PATCH 30/41] fix warp py interface --- src/py_denseflow.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp index 53877d1..26033f9 100644 --- a/src/py_denseflow.cpp +++ b/src/py_denseflow.cpp @@ -189,8 +189,8 @@ BOOST_PYTHON_MODULE(libpydenseflow){ .def("extract_flow", &TVL1FlowExtractor::extract_flow) .def("set_device", &TVL1FlowExtractor::set_device) .staticmethod("set_device"); - bp::class_("TVL1WarpFlowExtractor", bp::init()) + bp::class_("TVL1WarpFlowExtractor", bp::init()) .def("extract_warp_flow", &TVL1WarpFlowExtractor::extract_warp_flow) .def("set_device", &TVL1WarpFlowExtractor::set_device) .staticmethod("set_device"); -} \ No newline at end of file +} From a5db11baff750caf32966f581115d3db83ce77f2 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Mon, 1 Aug 2016 13:30:01 +0800 Subject: [PATCH 31/41] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2a0f1ce..0461885 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,9 @@ Extracting dense flow field given a video. - LibZip: to install on ubuntu ```apt-get install libzip-dev``` on mac ```brew install libzip``` +#### For OpenCV 3 Users +Please see the [opencv-3.1](https://github.com/yjxiong/dense_flow/tree/opencv-3.1) branch. Many thanks to @victorhcm for the contributions! + ###Install ``` git clone --recursive http://github.com/yjxiong/dense_flow From 60ac8c34d1c9871a2a912f46d3a4c1c786613233 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Mon, 1 Aug 2016 22:43:27 +0800 Subject: [PATCH 32/41] add resize before flow extraction --- include/dense_flow.h | 3 ++- src/dense_flow_gpu.cpp | 25 +++++++++++++++++++++---- tools/extract_flow_gpu.cpp | 4 ++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/include/dense_flow.h b/include/dense_flow.h index 3203b3c..950a4fe 100644 --- a/include/dense_flow.h +++ b/include/dense_flow.h @@ -15,7 +15,8 @@ void calcDenseFlow(string file_name, int bound, int type, int step, void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_id, vector >& output_x, vector >& output_y, - vector >& output_img); + vector >& output_img, + int new_width=0, int new_height=0); void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int dev_id, vector >& output_x, diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index e4b52e7..3b37bd8 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -9,7 +9,8 @@ using namespace cv::gpu; void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_id, vector >& output_x, vector >& output_y, - vector >& output_img){ + vector >& output_img, + int new_width, int new_height){ VideoCapture video_stream(file_name); CHECK(video_stream.isOpened())<<"Cannot open video stream \"" < 0) && (new_width > 0); + bool initialized = false; int cnt = 0; while(true){ @@ -34,9 +38,18 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i if (!initialized){ video_stream >> capture_frame; if (capture_frame.empty()) return; // read frames until end - initializeMats(capture_frame, capture_image, capture_gray, + + if (!do_resize){ + initializeMats(capture_frame, capture_image, capture_gray, prev_image, prev_gray); - capture_frame.copyTo(prev_image); + capture_frame.copyTo(prev_image); + }else{ + capture_image.create(new_size, CV_8UC3); + capture_gray.create(new_size, CV_8UC1); + prev_image.create(new_size, CV_8UC3); + prev_gray.create(new_size, CV_8UC1); + cv::resize(capture_frame, prev_image, new_size); + } cvtColor(prev_image, prev_gray, CV_BGR2GRAY); initialized = true; for(int s = 0; s < step; ++s){ @@ -45,7 +58,11 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i if (capture_frame.empty()) return; // read frames until end } }else { - capture_frame.copyTo(capture_image); + if (!do_resize) + capture_frame.copyTo(capture_image); + else + cv::resize(capture_frame, capture_image, new_size); + cvtColor(capture_image, capture_gray, CV_BGR2GRAY); d_frame_0.upload(prev_gray); d_frame_1.upload(capture_gray); diff --git a/tools/extract_flow_gpu.cpp b/tools/extract_flow_gpu.cpp index 89c7686..5cfcb39 100644 --- a/tools/extract_flow_gpu.cpp +++ b/tools/extract_flow_gpu.cpp @@ -18,6 +18,8 @@ int main(int argc, char** argv){ "{ d | device_id | 0 | set gpu id}" "{ s | step | 1 | specify the step for frame sampling}" "{ o | out | zip | output style}" + "{ w | newWidth | 0 | output style}" + "{ h | newHeight | 0 | output style}" }; CommandLineParser cmd(argc, argv, keys); @@ -30,6 +32,8 @@ int main(int argc, char** argv){ int type = cmd.get("type"); int device_id = cmd.get("device_id"); int step = cmd.get("step"); + int new_height = cmd.get("newHeight"); + int new_width = cmd.get("newWidth"); vector > out_vec_x, out_vec_y, out_vec_img; From 93918c08374f9b90cfc24d5bef18ad5c80f85ece Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 10 Aug 2016 15:02:24 +0800 Subject: [PATCH 33/41] fix resize command tool --- tools/extract_flow_gpu.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/extract_flow_gpu.cpp b/tools/extract_flow_gpu.cpp index 5cfcb39..1e74cfc 100644 --- a/tools/extract_flow_gpu.cpp +++ b/tools/extract_flow_gpu.cpp @@ -38,7 +38,7 @@ int main(int argc, char** argv){ vector > out_vec_x, out_vec_y, out_vec_img; calcDenseFlowGPU(vidFile, bound, type, step, device_id, - out_vec_x, out_vec_y, out_vec_img); + out_vec_x, out_vec_y, out_vec_img, new_width, new_height); if (output_style == "dir") { writeImages(out_vec_x, xFlowFile); From 98e511fc13038f2901ad77b65adc1ee69bf70ba0 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Thu, 11 Aug 2016 09:57:51 +0800 Subject: [PATCH 34/41] Update README.md --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 0461885..ef82628 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,26 @@ cmake .. && make -j - `test.avi`: input video - `tmp`: folder containing RGB images and optical flow images - `dir`: output generated images to folder. if set to `zip`, will write images to zip files instead. + +### Warp Flow +The warp optical flow is used in the following paper + +``` +@inproceedings{TSN2016ECCV, + author = {Limin Wang and + Yuanjun Xiong and + Zhe Wang and + Yu Qiao and + Dahua Lin and + Xiaoou Tang and + Luc {Val Gool}}, + title = {Temporal Segment Networks: Towards Good Practices for Deep Action Recognition}, + booktitle = {ECCV}, + year = {2016}, +} +``` + +To extract warp flow, use the command +``` +./extract_warp_gpu -f test.avi -x tmp/flow_x -y tmp/flow_y -i tmp/image -b 20 -t 1 -d 0 -s 1 -o dir +``` From 0df27a2fe48c1004350022373c60fbfc8a30e9cb Mon Sep 17 00:00:00 2001 From: yjxiong Date: Tue, 11 Oct 2016 22:46:00 +0200 Subject: [PATCH 35/41] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ef82628..ca241cb 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The warp optical flow is used in the following paper Yu Qiao and Dahua Lin and Xiaoou Tang and - Luc {Val Gool}}, + Luc {Van Gool}}, title = {Temporal Segment Networks: Towards Good Practices for Deep Action Recognition}, booktitle = {ECCV}, year = {2016}, From 41b0c7b8d1df94e0a8771b1270e1f2cbb3986038 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 4 Jan 2017 23:01:03 +0800 Subject: [PATCH 36/41] Fix CPU Flow Image Encoding --- src/dense_flow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp index fa2ee75..47dbced 100644 --- a/src/dense_flow.cpp +++ b/src/dense_flow.cpp @@ -40,7 +40,7 @@ void calcDenseFlow(string file_name, int bound, int type, int step, vector str_x, str_y, str_img; split(flow, flow_split); - encodeFlowMap(flow_split[0], flow_split[0], str_x, str_y, bound); + encodeFlowMap(flow_split[0], flow_split[1], str_x, str_y, bound); imencode(".jpg", capture_image, str_img); output_x.push_back(str_x); From 997977fbf0d1050cfece8b30b9f91772d0804123 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Wed, 17 May 2017 11:22:04 +0800 Subject: [PATCH 37/41] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ca241cb..0f34691 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ Extracting dense flow field given a video. -####Depencies: +#### Depencies: - LibZip: to install on ubuntu ```apt-get install libzip-dev``` on mac ```brew install libzip``` #### For OpenCV 3 Users Please see the [opencv-3.1](https://github.com/yjxiong/dense_flow/tree/opencv-3.1) branch. Many thanks to @victorhcm for the contributions! -###Install +### Install ``` git clone --recursive http://github.com/yjxiong/dense_flow mkdir build && cd build cmake .. && make -j ``` -###Usage +### Usage ``` ./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_y -i tmp/image -b 20 -t 1 -d 0 -s 1 -o dir ``` From c9369a32ea491001db5298dfda1fa227a912d34f Mon Sep 17 00:00:00 2001 From: yjxiong Date: Sun, 30 Jul 2017 11:51:07 +0800 Subject: [PATCH 38/41] Create LICENSE --- LICENSE | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..49c9b11 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2016 Multimedia Laboratory, The Chinese University of Hong Kong. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From a0b3ad91df4c750cd843072ff77188d593d10673 Mon Sep 17 00:00:00 2001 From: Yue Zhao Date: Thu, 27 Jun 2019 12:30:12 +0800 Subject: [PATCH 39/41] supports Opencv 4.1.0 (#4) * Add OpenCV 3.1.0 support and fix namespace issues * Fix CommandLineParser for OpenCV 3.1.0 * Clean code and fix usage example * Add Dual TVL1 CPU version * Add WarpFlow support to OpenCV 3.1.0 (#2) * Add std::vector * Add std::string * Add std:: and easylogging include * Update headers and add windowedMatchingMask func * Add cv:: namespace to countNonZero The compiler was reporting it to be ambiguous due to existing in both `cv::` and `cv::cuda::`. * Update call to SURF detector and extractor * Update calls to GPU dense flow functions * Fix namespace issues in extract warp flow * Replicate changes to Python interface * Clean code and add TODO * Fix command-line arguments to CV 3.1 * Change copyTo to download * Instantiate py_denseflow alg_tvl1 * Add @yjxiong usage example for libpydenseflow Use as $ python tools/action_flow.py * Extract from a video using pyflow * Rename action_flow.py to extract_flow.py * fit opencv3 command line parser * modification for opencv-4.1.0 --- README.md | 2 +- build_of.py | 2 +- include/common.h | 6 ++- include/dense_flow.h | 14 +++--- include/utils.h | 2 +- include/warp_flow.h | 19 ++++++++ src/common.cpp | 4 +- src/dense_flow.cpp | 37 ++++++++++---- src/dense_flow_gpu.cpp | 86 ++++++++++++++++++--------------- src/dense_warp_flow_gpu.cpp | 72 +++++++++++++++------------ src/py_denseflow.cpp | 70 +++++++++++++++------------ src/zip_utils.cpp | 2 +- tools/extract_flow.cpp | 28 ++++++----- tools/extract_flow.py | 72 +++++++++++++++++++++++++++ tools/extract_flow_gpu.cpp | 36 +++++++------- tools/extract_warp_flow_gpu.cpp | 28 +++++------ 16 files changed, 309 insertions(+), 171 deletions(-) create mode 100644 tools/extract_flow.py diff --git a/README.md b/README.md index 0f34691..1c549ae 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ cmake .. && make -j ### Usage ``` -./extract_gpu -f test.avi -x tmp/flow_x -y tmp/flow_y -i tmp/image -b 20 -t 1 -d 0 -s 1 -o dir +./extract_gpu -f=test.avi -x=tmp/flow_x -y=tmp/flow_y -i=tmp/image -b=20 -t=1 -d=0 -s=1 -o=dir ``` - `test.avi`: input video - `tmp`: folder containing RGB images and optical flow images diff --git a/build_of.py b/build_of.py index 83f6b13..a19af0e 100644 --- a/build_of.py +++ b/build_of.py @@ -45,7 +45,7 @@ def run_optical_flow(vid_item, dev_id=0): flow_x_path = '{}/flow_x'.format(out_full_path) flow_y_path = '{}/flow_y'.format(out_full_path) - cmd = './build/extract_gpu -f {} -x {} -y {} -i {} -b 20 -t 1 -d {} -s 1 -o zip'.format(vid_path, flow_x_path, flow_y_path, image_path, dev_id) + cmd = './build/extract_gpu -f={} -x={} -y={} -i={} -b=20 -t=1 -d={} -s=1 -o=zip'.format(vid_path, flow_x_path, flow_y_path, image_path, dev_id) os.system(cmd) print '{} {} done'.format(vid_id, vid_name) diff --git a/include/common.h b/include/common.h index b8717e5..be9d180 100644 --- a/include/common.h +++ b/include/common.h @@ -14,13 +14,15 @@ #include #include using namespace cv; +using std::string; +using std::vector; void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y, double lowerBound, double higherBound); void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scalar& color); void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, - vector& encoded_x, vector& encoded_y, + std::vector& encoded_x, std::vector& encoded_y, int bound, bool to_jpg=true); inline void initializeMats(const Mat& frame, @@ -33,7 +35,7 @@ inline void initializeMats(const Mat& frame, prev_gray.create(frame.size(), CV_8UC1); } -void writeImages(vector> images, string name_temp); +void writeImages(std::vector> images, std::string name_temp); #endif //DENSEFLOW_COMMON_H_H diff --git a/include/dense_flow.h b/include/dense_flow.h index 950a4fe..44369a5 100644 --- a/include/dense_flow.h +++ b/include/dense_flow.h @@ -18,14 +18,14 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i vector >& output_img, int new_width=0, int new_height=0); -void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int dev_id, - vector >& output_x, - vector >& output_y, - vector >& output_img); +void calcDenseFlowPureGPU(std::string file_name, int bound, int type, int step, int dev_id, + std::vector >& output_x, + std::vector >& output_y, + std::vector >& output_img); -void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int dev_id, - vector >& output_x, - vector >& output_y); +void calcDenseWarpFlowGPU(std::string file_name, int bound, int type, int step, int dev_id, + std::vector >& output_x, + std::vector >& output_y); void MatchFromFlow_copy(const Mat& prev_grey, const Mat& flow_x, const Mat& flow_y, std::vector& prev_pts, std::vector& pts, const Mat& mask); diff --git a/include/utils.h b/include/utils.h index d69e22c..63eb22e 100644 --- a/include/utils.h +++ b/include/utils.h @@ -7,6 +7,6 @@ #include "common.h" -void writeZipFile(vector >& data, string name_temp, string archive_name); +void writeZipFile(std::vector >& data, std::string name_temp, std::string archive_name); #endif //DENSEFLOW_UTILS_H diff --git a/include/warp_flow.h b/include/warp_flow.h index f0d560e..a442c68 100644 --- a/include/warp_flow.h +++ b/include/warp_flow.h @@ -5,6 +5,25 @@ #ifndef DENSEFLOW_WARP_FLOW_H_H #define DENSEFLOW_WARP_FLOW_H_H +cv::Mat windowedMatchingMask( const std::vector& keypoints1, const std::vector& keypoints2, + float maxDeltaX, float maxDeltaY ) +{ + if( keypoints1.empty() || keypoints2.empty() ) + return cv::Mat(); + + int n1 = (int)keypoints1.size(), n2 = (int)keypoints2.size(); + cv::Mat mask( n1, n2, CV_8UC1 ); + for( int i = 0; i < n1; i++ ) + { + for( int j = 0; j < n2; j++ ) + { + cv::Point2f diff = keypoints2[j].pt - keypoints1[i].pt; + mask.at(i, j) = std::abs(diff.x) < maxDeltaX && std::abs(diff.y) < maxDeltaY; + } + } + return mask; +} + void MyWarpPerspective(Mat& prev_src, Mat& src, Mat& dst, Mat& M0, int flags=INTER_LINEAR, int borderType=BORDER_CONSTANT, const Scalar& borderValue=Scalar()) { diff --git a/src/common.cpp b/src/common.cpp index 35fca2e..7e259a5 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -30,7 +30,7 @@ void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step,double, const Scala } void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, - vector& encoded_x, vector& encoded_y, + std::vector& encoded_x, std::vector& encoded_y, int bound, bool to_jpg){ Mat flow_img_x(flow_map_x.size(), CV_8UC1); Mat flow_img_y(flow_map_y.size(), CV_8UC1); @@ -49,7 +49,7 @@ void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y, } } -void writeImages(vector> images, string name_temp){ +void writeImages(std::vector> images, std::string name_temp){ for (int i = 0; i < images.size(); ++i){ char tmp[256]; sprintf(tmp, "_%05d.jpg", i+1); diff --git a/src/dense_flow.cpp b/src/dense_flow.cpp index 47dbced..15c0842 100644 --- a/src/dense_flow.cpp +++ b/src/dense_flow.cpp @@ -3,11 +3,12 @@ // #include "common.h" #include "dense_flow.h" +#include "opencv2/optflow.hpp" -void calcDenseFlow(string file_name, int bound, int type, int step, - vector >& output_x, - vector >& output_y, - vector >& output_img){ +void calcDenseFlow(std::string file_name, int bound, int type, int step, + std::vector >& output_x, + std::vector >& output_y, + std::vector >& output_img){ VideoCapture video_stream(file_name); CHECK(video_stream.isOpened())<<"Cannot open video stream \"" @@ -17,6 +18,7 @@ void calcDenseFlow(string file_name, int bound, int type, int step, Mat capture_frame, capture_image, prev_image, capture_gray, prev_gray; Mat flow, flow_split[2]; + cv::Ptr alg_tvl1 = cv::optflow::DualTVL1OpticalFlow::create(); bool initialized = false; for(int iter = 0;; iter++){ @@ -28,17 +30,32 @@ void calcDenseFlow(string file_name, int bound, int type, int step, initializeMats(capture_frame, capture_image, capture_gray, prev_image, prev_gray); capture_frame.copyTo(prev_image); - cvtColor(prev_image, prev_gray, CV_BGR2GRAY); + cvtColor(prev_image, prev_gray, cv::COLOR_BGR2GRAY); initialized = true; // LOG(INFO)<<"Initialized"; }else if(iter % step == 0){ capture_frame.copyTo(capture_image); - cvtColor(capture_image, capture_gray, CV_BGR2GRAY); - calcOpticalFlowFarneback(prev_gray, capture_gray, flow, - 0.702, 5, 10, 2, 7, 1.5, - cv::OPTFLOW_FARNEBACK_GAUSSIAN ); + cvtColor(capture_image, capture_gray, cv::COLOR_BGR2GRAY); - vector str_x, str_y, str_img; + switch(type){ + case 0: { + calcOpticalFlowFarneback(prev_gray, capture_gray, flow, + 0.702, 5, 10, 2, 7, 1.5, + cv::OPTFLOW_FARNEBACK_GAUSSIAN ); + break; + } + case 1: { + alg_tvl1->calc(prev_gray, capture_gray, flow); + break; + } + default: + LOG(WARNING)<<"Unknown optical method. Using Farneback"; + calcOpticalFlowFarneback(prev_gray, capture_gray, flow, + 0.702, 5, 10, 2, 7, 1.5, + cv::OPTFLOW_FARNEBACK_GAUSSIAN ); + } + + std::vector str_x, str_y, str_img; split(flow, flow_split); encodeFlowMap(flow_split[0], flow_split[1], str_x, str_y, bound); imencode(".jpg", capture_image, str_img); diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index 3b37bd8..c04e96b 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -2,8 +2,12 @@ // Created by yjxiong on 11/18/15. // #include "dense_flow.h" -#include "opencv2/gpu/gpu.hpp" -using namespace cv::gpu; +#include "opencv2/xfeatures2d.hpp" +#include "opencv2/cudaarithm.hpp" +#include "opencv2/cudaoptflow.hpp" +#include "opencv2/cudacodec.hpp" + +using namespace cv::cuda; void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_id, @@ -22,11 +26,11 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i Size new_size(new_width, new_height); GpuMat d_frame_0, d_frame_1; - GpuMat d_flow_x, d_flow_y; + GpuMat d_flow; - FarnebackOpticalFlow alg_farn; - OpticalFlowDual_TVL1_GPU alg_tvl1; - BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); + cv::Ptr alg_farn = cuda::FarnebackOpticalFlow::create(); + cv::Ptr alg_tvl1 = cuda::OpticalFlowDual_TVL1::create(); + cv::Ptr alg_brox = cuda::BroxOpticalFlow::create(0.197f, 50.0f, 0.8f, 10, 77, 10); bool do_resize = (new_height > 0) && (new_width > 0); @@ -50,11 +54,11 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i prev_gray.create(new_size, CV_8UC1); cv::resize(capture_frame, prev_image, new_size); } - cvtColor(prev_image, prev_gray, CV_BGR2GRAY); + cvtColor(prev_image, prev_gray, COLOR_BGR2GRAY); initialized = true; for(int s = 0; s < step; ++s){ video_stream >> capture_frame; - cnt ++; + cnt ++; if (capture_frame.empty()) return; // read frames until end } }else { @@ -63,24 +67,24 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i else cv::resize(capture_frame, capture_image, new_size); - cvtColor(capture_image, capture_gray, CV_BGR2GRAY); + cvtColor(capture_image, capture_gray, COLOR_BGR2GRAY); d_frame_0.upload(prev_gray); d_frame_1.upload(capture_gray); switch(type){ case 0: { - alg_farn(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_farn->calc(d_frame_0, d_frame_1, d_flow); break; } case 1: { - alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_tvl1->calc(d_frame_0, d_frame_1, d_flow); break; } case 2: { GpuMat d_buf_0, d_buf_1; d_frame_0.convertTo(d_buf_0, CV_32F, 1.0 / 255.0); d_frame_1.convertTo(d_buf_1, CV_32F, 1.0 / 255.0); - alg_brox(d_buf_0, d_buf_1, d_flow_x, d_flow_y); + alg_brox->calc(d_buf_0, d_buf_1, d_flow); break; } default: @@ -96,11 +100,14 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i // read frames until end } + GpuMat planes[2]; + cuda::split(d_flow, planes); + //get back flow map - d_flow_x.download(flow_x); - d_flow_y.download(flow_y); + Mat flow_x(planes[0]); + Mat flow_y(planes[1]); - vector str_x, str_y, str_img; + std::vector str_x, str_y, str_img; encodeFlowMap(flow_x, flow_y, str_x, str_y, bound); imencode(".jpg", capture_image, str_img); @@ -124,33 +131,33 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i /** * This function use pure GPU backend for video loading and optical flow calculation */ -void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int dev_id, - vector >& output_x, - vector >& output_y, - vector >& output_img){ +void calcDenseFlowPureGPU(std::string file_name, int bound, int type, int step, int dev_id, + std::vector >& output_x, + std::vector >& output_y, + std::vector >& output_img){ setDevice(dev_id); - VideoReader_GPU video_stream(file_name); + cv::Ptr video_stream = cudacodec::createVideoReader(file_name); // VideoCapture video_stream(file_name); - CHECK(video_stream.isOpened())<<"Cannot open video stream \"" - <isOpened())<<"Cannot open video stream \"" + // < alg_farn = cuda::FarnebackOpticalFlow::create(); + cv::Ptr alg_tvl1 = cuda::OpticalFlowDual_TVL1::create(); + cv::Ptr alg_brox = cuda::BroxOpticalFlow::create(0.197f, 50.0f, 0.8f, 10, 77, 10); bool initialized = false; while(true){ //build mats for the first frame if (!initialized){ - bool success = video_stream.read(capture_frame); + bool success = video_stream->nextFrame(capture_frame); if (!success) break; // read frames until end capture_image.create(capture_frame.size(), CV_8UC3); capture_gray.create(capture_frame.size(), CV_8UC1); @@ -159,30 +166,30 @@ void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int d prev_gray.create(capture_frame.size(), CV_8UC1); capture_frame.copyTo(prev_image); - cvtColor(prev_image, prev_gray, CV_BGR2GRAY); + cvtColor(prev_image, prev_gray, COLOR_BGR2GRAY); initialized = true; for (int s = 0; s < step; ++s){ - video_stream.read(capture_frame); + video_stream->nextFrame(capture_frame); } }else { capture_frame.copyTo(capture_image); - cvtColor(capture_image, capture_gray, CV_BGR2GRAY); + cvtColor(capture_image, capture_gray, COLOR_BGR2GRAY); switch(type){ case 0: { - alg_farn(prev_gray, capture_gray, d_flow_x, d_flow_y); + alg_farn->calc(prev_gray, capture_gray, d_flow); break; } case 1: { - alg_tvl1(prev_gray, capture_gray, d_flow_x, d_flow_y); + alg_tvl1->calc(prev_gray, capture_gray, d_flow); break; } case 2: { GpuMat d_buf_0, d_buf_1; prev_gray.convertTo(d_buf_0, CV_32F, 1.0 / 255.0); capture_gray.convertTo(d_buf_1, CV_32F, 1.0 / 255.0); - alg_brox(d_buf_0, d_buf_1, d_flow_x, d_flow_y); + alg_brox->calc(d_buf_0, d_buf_1, d_flow); break; } default: @@ -190,15 +197,18 @@ void calcDenseFlowPureGPU(string file_name, int bound, int type, int step, int d } for (int s = 0; s < step; ++s){ - if (!video_stream.read(capture_frame)) break; + if (!video_stream->nextFrame(capture_frame)) break; } + GpuMat planes[2]; + cuda::split(d_flow, planes); + //get back flow map - d_flow_x.download(flow_x); - d_flow_y.download(flow_y); + Mat flow_x(planes[0]); + Mat flow_y(planes[1]); capture_image.download(img); - vector str_x, str_y, str_img; + std::vector str_x, str_y, str_img; encodeFlowMap(flow_x, flow_y, str_x, str_y, bound); imencode(".jpg", img, str_img); diff --git a/src/dense_warp_flow_gpu.cpp b/src/dense_warp_flow_gpu.cpp index 2dad457..e95280a 100644 --- a/src/dense_warp_flow_gpu.cpp +++ b/src/dense_warp_flow_gpu.cpp @@ -9,9 +9,10 @@ #include "opencv2/imgproc/imgproc.hpp" #include "opencv2/features2d/features2d.hpp" #include "opencv2/core/core.hpp" -#include "opencv2/nonfree/nonfree.hpp" -#include "opencv2/gpu/gpu.hpp" -#include "opencv2/gpu/gpu.hpp" +#include "opencv2/xfeatures2d.hpp" +#include "opencv2/cudaarithm.hpp" +#include "opencv2/cudaoptflow.hpp" +#include "opencv2/cudacodec.hpp" #include #include @@ -19,18 +20,23 @@ #include "warp_flow.h" using namespace cv; -using namespace cv::gpu; +using namespace cv::cuda; +using namespace std; void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int dev_id, - vector >& output_x, - vector >& output_y){ + std::vector >& output_x, + std::vector >& output_y){ VideoCapture video_stream(file_name); CHECK(video_stream.isOpened())<<"Cannot open video stream \"" < detector_surf = xfeatures2d::SurfFeatureDetector::create(200); + cv::Ptr extractor_surf = xfeatures2d::SurfDescriptorExtractor::create(true, true); std::vector prev_pts_flow, pts_flow; std::vector prev_pts_surf, pts_surf; std::vector prev_pts_all, pts_all; @@ -42,11 +48,11 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d Mat flow_x, flow_y; GpuMat d_frame_0, d_frame_1; - GpuMat d_flow_x, d_flow_y; + GpuMat d_flow; - FarnebackOpticalFlow alg_farn; - OpticalFlowDual_TVL1_GPU alg_tvl1; - BroxOpticalFlow alg_brox(0.197f, 50.0f, 0.8f, 10, 77, 10); + cv::Ptr alg_farn = cuda::FarnebackOpticalFlow::create(); + cv::Ptr alg_tvl1 = cuda::OpticalFlowDual_TVL1::create(); + cv::Ptr alg_brox = cuda::BroxOpticalFlow::create(0.197f, 50.0f, 0.8f, 10, 77, 10); bool initialized = false; int cnt = 0; @@ -59,12 +65,13 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d initializeMats(capture_frame, capture_image, capture_gray, prev_image, prev_gray); capture_frame.copyTo(prev_image); - cvtColor(prev_image, prev_gray, CV_BGR2GRAY); + cvtColor(prev_image, prev_gray, COLOR_BGR2GRAY); //detect key points human_mask = Mat::ones(capture_frame.size(), CV_8UC1); - detector_surf.detect(prev_gray, prev_kpts_surf, human_mask); - extractor_surf.compute(prev_gray, prev_kpts_surf, prev_desc_surf); + detector_surf->detect(prev_gray, prev_kpts_surf, human_mask); + extractor_surf->compute(prev_gray, prev_kpts_surf, prev_desc_surf); + // TODO! check detector_surf->detectAndCompute() initialized = true; for(int s = 0; s < step; ++s){ @@ -74,39 +81,40 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d } }else { capture_frame.copyTo(capture_image); - cvtColor(capture_image, capture_gray, CV_BGR2GRAY); + cvtColor(capture_image, capture_gray, COLOR_BGR2GRAY); d_frame_0.upload(prev_gray); d_frame_1.upload(capture_gray); switch(type){ case 0: { - alg_farn(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_farn->calc(d_frame_0, d_frame_1, d_flow); break; } case 1: { - alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_tvl1->calc(d_frame_0, d_frame_1, d_flow); break; } case 2: { GpuMat d_buf_0, d_buf_1; d_frame_0.convertTo(d_buf_0, CV_32F, 1.0 / 255.0); d_frame_1.convertTo(d_buf_1, CV_32F, 1.0 / 255.0); - alg_brox(d_buf_0, d_buf_1, d_flow_x, d_flow_y); + alg_brox->calc(d_buf_0, d_buf_1, d_flow); break; } default: LOG(ERROR)<<"Unknown optical method: "<detect(capture_gray, kpts_surf, human_mask); + extractor_surf->compute(capture_gray, kpts_surf, desc_surf); ComputeMatch(prev_kpts_surf, kpts_surf, prev_desc_surf, desc_surf, prev_pts_surf, pts_surf); MatchFromFlow_copy(capture_gray, flow_x, flow_y, prev_pts_flow, pts_flow, human_mask); MergeMatch(prev_pts_flow, pts_flow, prev_pts_surf, pts_surf, prev_pts_all, pts_all); @@ -114,7 +122,7 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d if(pts_all.size() > 50) { std::vector match_mask; Mat temp = findHomography(prev_pts_all, pts_all, RANSAC, 1, match_mask); - if(countNonZero(Mat(match_mask)) > 25) + if(cv::countNonZero(Mat(match_mask)) > 25) H = temp; } @@ -128,18 +136,18 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d switch(type){ case 0: { - alg_farn(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_farn->calc(d_frame_0, d_frame_1, d_flow); break; } case 1: { - alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_tvl1->calc(d_frame_0, d_frame_1, d_flow); break; } case 2: { GpuMat d_buf_0, d_buf_1; d_frame_0.convertTo(d_buf_0, CV_32F, 1.0 / 255.0); d_frame_1.convertTo(d_buf_1, CV_32F, 1.0 / 255.0); - alg_brox(d_buf_0, d_buf_1, d_flow_x, d_flow_y); + alg_brox->calc(d_buf_0, d_buf_1, d_flow); break; } default: @@ -147,10 +155,10 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d } - //get back flow map - d_flow_x.download(flow_x); - d_flow_y.download(flow_y); + cuda::split(d_flow, planes); + planes[0].download(flow_x); + planes[1].download(flow_y); vector str_x, str_y; encodeFlowMap(flow_x, flow_y, str_x, str_y, bound); @@ -177,4 +185,4 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d } -} \ No newline at end of file +} diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp index 26033f9..4378c07 100644 --- a/src/py_denseflow.cpp +++ b/src/py_denseflow.cpp @@ -5,7 +5,9 @@ #include "common.h" -#include "opencv2/gpu/gpu.hpp" +#include "opencv2/cudaarithm.hpp" +#include "opencv2/cudaoptflow.hpp" +#include "opencv2/cudacodec.hpp" #include "opencv2/video/tracking.hpp" #include "opencv2/imgproc/imgproc.hpp" @@ -15,13 +17,11 @@ #include "opencv2/imgproc/imgproc.hpp" #include "opencv2/features2d/features2d.hpp" #include "opencv2/core/core.hpp" -#include "opencv2/nonfree/nonfree.hpp" -#include "opencv2/gpu/gpu.hpp" -#include "opencv2/gpu/gpu.hpp" +#include "opencv2/xfeatures2d.hpp" #include "warp_flow.h" -using namespace cv::gpu; +using namespace cv::cuda; using namespace cv; namespace bp = boost::python; @@ -30,6 +30,7 @@ class TVL1FlowExtractor{ public: TVL1FlowExtractor(int bound){ + alg_tvl1 = cuda::OpticalFlowDual_TVL1::create(); bound_ = bound; } @@ -51,21 +52,23 @@ class TVL1FlowExtractor{ initializeMats(input_frame, prev_frame, prev_gray, next_frame, next_gray); memcpy(prev_frame.data, first_data, bp::len(frames[0])); - cvtColor(prev_frame, prev_gray, CV_BGR2GRAY); + cvtColor(prev_frame, prev_gray, COLOR_BGR2GRAY); for (int idx = 1; idx < bp::len(frames); idx++){ const char* this_data = ((const char*)bp::extract(frames[idx])); memcpy(next_frame.data, this_data, bp::len(frames[0])); - cvtColor(next_frame, next_gray, CV_BGR2GRAY); + cvtColor(next_frame, next_gray, COLOR_BGR2GRAY); d_frame_0.upload(prev_gray); d_frame_1.upload(next_gray); - alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_tvl1->calc(d_frame_0, d_frame_1, d_flow); - d_flow_x.download(flow_x); - d_flow_y.download(flow_y); + GpuMat planes[2]; + cuda::split(d_flow, planes); + planes[0].download(flow_x); + planes[1].download(flow_y); - vector str_x, str_y; + std::vector str_x, str_y; encodeFlowMap(flow_x, flow_y, str_x, str_y, bound_, false); output.append( @@ -82,8 +85,8 @@ class TVL1FlowExtractor{ private: int bound_; GpuMat d_frame_0, d_frame_1; - GpuMat d_flow_x, d_flow_y; - OpticalFlowDual_TVL1_GPU alg_tvl1; + GpuMat d_flow; + cv::Ptr alg_tvl1; }; @@ -91,8 +94,10 @@ class TVL1FlowExtractor{ class TVL1WarpFlowExtractor { public: - TVL1WarpFlowExtractor(int bound) - :detector_surf(200), extractor_surf(true, true){ + TVL1WarpFlowExtractor(int bound) { + alg_tvl1 = cuda::OpticalFlowDual_TVL1::create(); + detector_surf = xfeatures2d::SurfFeatureDetector::create(200); + extractor_surf = xfeatures2d::SurfDescriptorExtractor::create(true, true); bound_ = bound; } @@ -112,23 +117,25 @@ class TVL1WarpFlowExtractor { human_mask = Mat::ones(input_frame.size(), CV_8UC1); memcpy(prev_frame.data, first_data, bp::len(frames[0])); - cvtColor(prev_frame, prev_gray, CV_BGR2GRAY); + cvtColor(prev_frame, prev_gray, COLOR_BGR2GRAY); for (int idx = 1; idx < bp::len(frames); idx++){ const char* this_data = ((const char*)bp::extract(frames[idx])); memcpy(next_frame.data, this_data, bp::len(frames[0])); - cvtColor(next_frame, next_gray, CV_BGR2GRAY); + cvtColor(next_frame, next_gray, COLOR_BGR2GRAY); d_frame_0.upload(prev_gray); d_frame_1.upload(next_gray); - alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_tvl1->calc(d_frame_0, d_frame_1, d_flow); - d_flow_x.download(flow_x); - d_flow_y.download(flow_y); + GpuMat planes[2]; + cuda::split(d_flow, planes); + planes[0].download(flow_x); + planes[1].download(flow_y); // warp to reduce holistic motion - detector_surf.detect(next_gray, kpts_surf, human_mask); - extractor_surf.compute(next_gray, kpts_surf, desc_surf); + detector_surf->detect(next_gray, kpts_surf, human_mask); + extractor_surf->compute(next_gray, kpts_surf, desc_surf); ComputeMatch(prev_kpts_surf, kpts_surf, prev_desc_surf, desc_surf, prev_pts_surf, pts_surf); MatchFromFlow_copy(next_gray, flow_x, flow_y, prev_pts_flow, pts_flow, human_mask); MergeMatch(prev_pts_flow, pts_flow, prev_pts_surf, pts_surf, prev_pts_all, pts_all); @@ -136,7 +143,7 @@ class TVL1WarpFlowExtractor { if(pts_all.size() > 50) { std::vector match_mask; Mat temp = findHomography(prev_pts_all, pts_all, RANSAC, 1, match_mask); - if(countNonZero(Mat(match_mask)) > 25) + if(cv::countNonZero(Mat(match_mask)) > 25) H = temp; } @@ -147,12 +154,13 @@ class TVL1WarpFlowExtractor { d_frame_0.upload(prev_gray); d_frame_1.upload(gray_warp); - alg_tvl1(d_frame_0, d_frame_1, d_flow_x, d_flow_y); + alg_tvl1->calc(d_frame_0, d_frame_1, d_flow); - d_flow_x.download(flow_x); - d_flow_y.download(flow_y); + cuda::split(d_flow, planes); + planes[0].download(flow_x); + planes[1].download(flow_y); - vector str_x, str_y; + std::vector str_x, str_y; encodeFlowMap(flow_x, flow_y, str_x, str_y, bound_, false); output.append( @@ -167,8 +175,8 @@ class TVL1WarpFlowExtractor { return output; } private: - SurfFeatureDetector detector_surf; - SurfDescriptorExtractor extractor_surf; + cv::Ptr detector_surf; + cv::Ptr extractor_surf; std::vector prev_pts_flow, pts_flow; std::vector prev_pts_surf, pts_surf; std::vector prev_pts_all, pts_all; @@ -176,9 +184,9 @@ class TVL1WarpFlowExtractor { Mat prev_desc_surf, desc_surf; GpuMat d_frame_0, d_frame_1; - GpuMat d_flow_x, d_flow_y; + GpuMat d_flow; - OpticalFlowDual_TVL1_GPU alg_tvl1; + cv::Ptr alg_tvl1; int bound_; }; diff --git a/src/zip_utils.cpp b/src/zip_utils.cpp index 6f2b626..32a7fa0 100644 --- a/src/zip_utils.cpp +++ b/src/zip_utils.cpp @@ -6,7 +6,7 @@ #include "zip.h" #include "easylogging++.h" -void writeZipFile(vector >& data, string name_temp, string archive_name){ +void writeZipFile(std::vector >& data, std::string name_temp, std::string archive_name){ int err=0; #ifdef USE_OBSEL_LIBZIP struct zip* archive = zip_open(archive_name.c_str(), ZIP_CREATE, &err); diff --git a/tools/extract_flow.cpp b/tools/extract_flow.cpp index b78cd0e..238af85 100644 --- a/tools/extract_flow.cpp +++ b/tools/extract_flow.cpp @@ -8,26 +8,28 @@ int main(int argc, char** argv) const char* keys = { - "{ f | vidFile | ex2.avi | filename of video }" - "{ x | xFlowFile | flow_x | filename of flow x component }" - "{ y | yFlowFile | flow_y | filename of flow x component }" - "{ i | imgFile | flow_i | filename of flow image}" - "{ b | bound | 15 | specify the maximum of optical flow}" - "{ o | out | zip | output style}" + "{ f vidFile | ex2.avi | filename of video }" + "{ x xFlowFile | flow_x | filename of flow x component }" + "{ y yFlowFile | flow_y | filename of flow x component }" + "{ i imgFile | flow_i | filename of flow image }" + "{ b bound | 15 | specify the maximum of optical flow}" + "{ t type | 0 | specify the optical flow algorithm }" + "{ o out | zip | output style }" }; CommandLineParser cmd(argc, argv, keys); - string vidFile = cmd.get("vidFile"); - string xFlowFile = cmd.get("xFlowFile"); - string yFlowFile = cmd.get("yFlowFile"); - string imgFile = cmd.get("imgFile"); - string output_style = cmd.get("out"); + std::string vidFile = cmd.get("vidFile"); + std::string xFlowFile = cmd.get("xFlowFile"); + std::string yFlowFile = cmd.get("yFlowFile"); + std::string imgFile = cmd.get("imgFile"); + std::string output_style = cmd.get("out"); int bound = cmd.get("bound"); + int type = cmd.get("type"); // LOG(INFO)<<"Starting extraction"; - vector > out_vec_x, out_vec_y, out_vec_img; + std::vector > out_vec_x, out_vec_y, out_vec_img; - calcDenseFlow(vidFile, bound, 0, 1, + calcDenseFlow(vidFile, bound, type, 1, out_vec_x, out_vec_y, out_vec_img); if (output_style == "dir") { diff --git a/tools/extract_flow.py b/tools/extract_flow.py new file mode 100644 index 0000000..615fbaf --- /dev/null +++ b/tools/extract_flow.py @@ -0,0 +1,72 @@ + +import sys + +sys.path.append('build/') + +import os +from libpydenseflow import TVL1FlowExtractor, TVL1WarpFlowExtractor +import numpy as np + +class FlowExtractor(object): + + def __init__(self, dev_id, bound=20): + TVL1FlowExtractor.set_device(dev_id) + self._et = TVL1FlowExtractor(bound) + + def extract_flow(self, frame_list, new_size=None): + """ + This function extracts the optical flow and interleave x and y channels + :param frame_list: + :return: + """ + frame_size = frame_list[0].shape[:2] + rst = self._et.extract_flow([x.tostring() for x in frame_list], frame_size[1], frame_size[0]) + n_out = len(rst) + if new_size is None: + ret = np.zeros((n_out*2, frame_size[0], frame_size[1])) + for i in xrange(n_out): + ret[2*i, :] = np.fromstring(rst[i][0], dtype='uint8').reshape(frame_size) + ret[2*i+1, :] = np.fromstring(rst[i][1], dtype='uint8').reshape(frame_size) + else: + import cv2 + ret = np.zeros((n_out*2, new_size[1], new_size[0])) + for i in xrange(n_out): + ret[2*i, :] = cv2.resize(np.fromstring(rst[i][0], dtype='uint8').reshape(frame_size), new_size) + ret[2*i+1, :] = cv2.resize(np.fromstring(rst[i][1], dtype='uint8').reshape(frame_size), new_size) + + return ret + +def save_optical_flow(output_folder, flow_frames): + try: + os.mkdir(output_folder) + except OSError: + pass + nframes = len(flow_frames) / 2 + for i in xrange(nframes): + out_x = '{0}/x_{1:04d}.jpg'.format(output_folder, i+1) + out_y = '{0}/y_{1:04d}.jpg'.format(output_folder, i+1) + cv2.imwrite(out_x, flow_frames[2*i]) + cv2.imwrite(out_y, flow_frames[2*i+1]) + +if __name__ == "__main__": + if len(sys.argv) < 3: # TODO! argparse + print ("Missing arguments.\n" + "Usage: \n" + " python tools/action_flow.py ") + sys.exit(-1) + + input_video = sys.argv[1] + output_folder = sys.argv[2] + + import cv2 + if os.path.exists(input_video): + frame_list = [] + cap = cv2.VideoCapture(input_video) + ret, frame = cap.read() + while ret: + frame_list.append(frame) + ret, frame = cap.read() + f = FlowExtractor(dev_id=0) + flow_frames = f.extract_flow(frame_list) + save_optical_flow(output_folder, flow_frames) + diff --git a/tools/extract_flow_gpu.cpp b/tools/extract_flow_gpu.cpp index 1e74cfc..c7cea87 100644 --- a/tools/extract_flow_gpu.cpp +++ b/tools/extract_flow_gpu.cpp @@ -3,31 +3,31 @@ INITIALIZE_EASYLOGGINGPP -using namespace cv::gpu; +using namespace cv::cuda; int main(int argc, char** argv){ // IO operation const char* keys = { - "{ f | vidFile | ex2.avi | filename of video }" - "{ x | xFlowFile | flow_x | filename of flow x component }" - "{ y | yFlowFile | flow_y | filename of flow x component }" - "{ i | imgFile | flow_i | filename of flow image}" - "{ b | bound | 15 | specify the maximum of optical flow}" - "{ t | type | 0 | specify the optical flow algorithm }" - "{ d | device_id | 0 | set gpu id}" - "{ s | step | 1 | specify the step for frame sampling}" - "{ o | out | zip | output style}" - "{ w | newWidth | 0 | output style}" - "{ h | newHeight | 0 | output style}" + "{ f vidFile | ex2.avi | filename of video }" + "{ x xFlowFile | flow_x | filename of flow x component }" + "{ y yFlowFile | flow_y | filename of flow x component }" + "{ i imgFile | flow_i | filename of flow image}" + "{ b bound | 15 | specify the maximum of optical flow}" + "{ t type | 0 | specify the optical flow algorithm }" + "{ d device_id | 0 | set gpu id}" + "{ s step | 1 | specify the step for frame sampling}" + "{ o out | zip | output style}" + "{ w newWidth | 0 | output style}" + "{ h newHeight | 0 | output style}" }; CommandLineParser cmd(argc, argv, keys); - string vidFile = cmd.get("vidFile"); - string xFlowFile = cmd.get("xFlowFile"); - string yFlowFile = cmd.get("yFlowFile"); - string imgFile = cmd.get("imgFile"); - string output_style = cmd.get("out"); + std::string vidFile = cmd.get("vidFile"); + std::string xFlowFile = cmd.get("xFlowFile"); + std::string yFlowFile = cmd.get("yFlowFile"); + std::string imgFile = cmd.get("imgFile"); + std::string output_style = cmd.get("out"); int bound = cmd.get("bound"); int type = cmd.get("type"); int device_id = cmd.get("device_id"); @@ -35,7 +35,7 @@ int main(int argc, char** argv){ int new_height = cmd.get("newHeight"); int new_width = cmd.get("newWidth"); - vector > out_vec_x, out_vec_y, out_vec_img; + std::vector > out_vec_x, out_vec_y, out_vec_img; calcDenseFlowGPU(vidFile, bound, type, step, device_id, out_vec_x, out_vec_y, out_vec_img, new_width, new_height); diff --git a/tools/extract_warp_flow_gpu.cpp b/tools/extract_warp_flow_gpu.cpp index d0bb92f..29af6cf 100644 --- a/tools/extract_warp_flow_gpu.cpp +++ b/tools/extract_warp_flow_gpu.cpp @@ -3,33 +3,33 @@ INITIALIZE_EASYLOGGINGPP -using namespace cv::gpu; +using namespace cv::cuda; int main(int argc, char** argv){ // IO operation const char* keys = { - "{ f | vidFile | ex2.avi | filename of video }" - "{ x | xFlowFile | flow_x | filename of flow x component }" - "{ y | yFlowFile | flow_y | filename of flow x component }" - "{ b | bound | 15 | specify the maximum of optical flow}" - "{ t | type | 0 | specify the optical flow algorithm }" - "{ d | device_id | 0 | set gpu id}" - "{ s | step | 1 | specify the step for frame sampling}" - "{ o | out | zip | output style}" + "{ f vidFile | ex2.avi | filename of video }" + "{ x xFlowFile | flow_x | filename of flow x component }" + "{ y yFlowFile | flow_y | filename of flow y component }" + "{ b bound | 15 | specify the maximum of optical flow}" + "{ t type | 0 | specify the optical flow algorithm }" + "{ d device_id | 0 | set gpu id}" + "{ s step | 1 | specify the step for frame sampling}" + "{ o out | zip | output style}" }; CommandLineParser cmd(argc, argv, keys); - string vidFile = cmd.get("vidFile"); - string xFlowFile = cmd.get("xFlowFile"); - string yFlowFile = cmd.get("yFlowFile"); - string output_style = cmd.get("out"); + std::string vidFile = cmd.get("vidFile"); + std::string xFlowFile = cmd.get("xFlowFile"); + std::string yFlowFile = cmd.get("yFlowFile"); + std::string output_style = cmd.get("out"); int bound = cmd.get("bound"); int type = cmd.get("type"); int device_id = cmd.get("device_id"); int step = cmd.get("step"); - vector > out_vec_x, out_vec_y; + std::vector > out_vec_x, out_vec_y; calcDenseWarpFlowGPU(vidFile, bound, type, step, device_id, out_vec_x, out_vec_y); From 4490b4e406fe7225a1bcdc55616d2a90bce83cd4 Mon Sep 17 00:00:00 2001 From: lizz Date: Tue, 20 Aug 2019 01:28:59 +0800 Subject: [PATCH 40/41] Fix typo & formatting (#6) Signed-off-by: lizz --- README.md | 4 ++-- include/common.h | 1 - matlab/extracOpticalFlow.m | 4 ++-- matlab/extractOpticalFlow.m | 4 ++-- matlab/extractOpticalFlow_gpu.m | 4 ++-- src/common.cpp | 1 - src/dense_flow_gpu.cpp | 2 +- src/dense_warp_flow_gpu.cpp | 4 ++-- src/py_denseflow.cpp | 4 ++-- tools/extract_flow.py | 1 - 10 files changed, 13 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 1c549ae..a3e7fda 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Extracting dense flow field given a video. -#### Depencies: -- LibZip: +#### Dependencies: +- LibZip: to install on ubuntu ```apt-get install libzip-dev``` on mac ```brew install libzip``` #### For OpenCV 3 Users diff --git a/include/common.h b/include/common.h index be9d180..6515c88 100644 --- a/include/common.h +++ b/include/common.h @@ -38,4 +38,3 @@ inline void initializeMats(const Mat& frame, void writeImages(std::vector> images, std::string name_temp); #endif //DENSEFLOW_COMMON_H_H - diff --git a/matlab/extracOpticalFlow.m b/matlab/extracOpticalFlow.m index 6a7fd82..ada79ce 100644 --- a/matlab/extracOpticalFlow.m +++ b/matlab/extracOpticalFlow.m @@ -16,7 +16,7 @@ for j = 1:length(filelist) if ~exist([path2,foldername{i},'/',filelist(j).name(1:end-4)],'dir') mkdir([path2,foldername{i},'/',filelist(j).name(1:end-4)]); - end + end file1 = [path1,foldername{i},'/',filelist(j).name]; file2 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_x']; file3 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_y']; @@ -26,4 +26,4 @@ end i end -end \ No newline at end of file +end diff --git a/matlab/extractOpticalFlow.m b/matlab/extractOpticalFlow.m index 1dd9df0..9fa2c1a 100644 --- a/matlab/extractOpticalFlow.m +++ b/matlab/extractOpticalFlow.m @@ -16,7 +16,7 @@ for j = 1:length(filelist) if ~exist([path2,foldername{i},'/',filelist(j).name(1:end-4)],'dir') mkdir([path2,foldername{i},'/',filelist(j).name(1:end-4)]); - end + end file1 = [path1,foldername{i},'/',filelist(j).name]; file2 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_x']; file3 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_y']; @@ -26,4 +26,4 @@ end i end -end \ No newline at end of file +end diff --git a/matlab/extractOpticalFlow_gpu.m b/matlab/extractOpticalFlow_gpu.m index 853d05d..91e0d6b 100644 --- a/matlab/extractOpticalFlow_gpu.m +++ b/matlab/extractOpticalFlow_gpu.m @@ -31,7 +31,7 @@ for j = 1:length(filelist) if ~exist([path2,foldername{i},'/',filelist(j).name(1:end-4)],'dir') mkdir([path2,foldername{i},'/',filelist(j).name(1:end-4)]); - end + end file1 = [path1,foldername{i},'/',filelist(j).name]; file2 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_x']; file3 = [path2,foldername{i},'/',filelist(j).name(1:end-4),'/','flow_y']; @@ -42,4 +42,4 @@ end i end -end \ No newline at end of file +end diff --git a/src/common.cpp b/src/common.cpp index 7e259a5..b39c2f3 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -59,4 +59,3 @@ void writeImages(std::vector> images, std::string name_temp){ fclose(fp); } } - diff --git a/src/dense_flow_gpu.cpp b/src/dense_flow_gpu.cpp index c04e96b..9b4ce51 100644 --- a/src/dense_flow_gpu.cpp +++ b/src/dense_flow_gpu.cpp @@ -66,7 +66,7 @@ void calcDenseFlowGPU(string file_name, int bound, int type, int step, int dev_i capture_frame.copyTo(capture_image); else cv::resize(capture_frame, capture_image, new_size); - + cvtColor(capture_image, capture_gray, COLOR_BGR2GRAY); d_frame_0.upload(prev_gray); d_frame_1.upload(capture_gray); diff --git a/src/dense_warp_flow_gpu.cpp b/src/dense_warp_flow_gpu.cpp index e95280a..ed5b382 100644 --- a/src/dense_warp_flow_gpu.cpp +++ b/src/dense_warp_flow_gpu.cpp @@ -33,7 +33,7 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d // OpenCV 3.1.0 SURF interface // - // source: http://stackoverflow.com/a/27533437/957997 + // source: http://stackoverflow.com/a/27533437/957997 // http://stackoverflow.com/questions/27533203/how-do-i-use-sift-in-opencv-3-0-with-c cv::Ptr detector_surf = xfeatures2d::SurfFeatureDetector::create(200); cv::Ptr extractor_surf = xfeatures2d::SurfDescriptorExtractor::create(true, true); @@ -69,7 +69,7 @@ void calcDenseWarpFlowGPU(string file_name, int bound, int type, int step, int d //detect key points human_mask = Mat::ones(capture_frame.size(), CV_8UC1); - detector_surf->detect(prev_gray, prev_kpts_surf, human_mask); + detector_surf->detect(prev_gray, prev_kpts_surf, human_mask); extractor_surf->compute(prev_gray, prev_kpts_surf, prev_desc_surf); // TODO! check detector_surf->detectAndCompute() diff --git a/src/py_denseflow.cpp b/src/py_denseflow.cpp index 4378c07..cfee077 100644 --- a/src/py_denseflow.cpp +++ b/src/py_denseflow.cpp @@ -175,8 +175,8 @@ class TVL1WarpFlowExtractor { return output; } private: - cv::Ptr detector_surf; - cv::Ptr extractor_surf; + cv::Ptr detector_surf; + cv::Ptr extractor_surf; std::vector prev_pts_flow, pts_flow; std::vector prev_pts_surf, pts_surf; std::vector prev_pts_all, pts_all; diff --git a/tools/extract_flow.py b/tools/extract_flow.py index 615fbaf..123ca9f 100644 --- a/tools/extract_flow.py +++ b/tools/extract_flow.py @@ -69,4 +69,3 @@ def save_optical_flow(output_folder, flow_frames): f = FlowExtractor(dev_id=0) flow_frames = f.extract_flow(frame_list) save_optical_flow(output_folder, flow_frames) - From 49f070134ecf3ea091a8f5df607527721f980458 Mon Sep 17 00:00:00 2001 From: yjxiong Date: Thu, 13 May 2021 15:45:03 -0700 Subject: [PATCH 41/41] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index a3e7fda..79032c5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +**Please Note**: This repository is no longer maintained. For extracting optical flow from videos please use [this latest tool](https://github.com/open-mmlab/denseflow) from Open-MMLAB. +---- + + Extracting dense flow field given a video. #### Dependencies: