Dense optical flow extraction dense_ Flow understanding

The main function ：

The input parameters are ：
-f Video file name
-x x Direction optical flow file prefix
-y y Direction optical flow file prefix
-i rgb Image file prefix
-b restrained distance
-t Optical flow extraction algorithm category （0- 1-）
-o Output format

std::vector<std::vector<uchar> > out_vec_x, out_vec_y, out_vec_img;

	calcDenseFlow(vidFile, bound, type, 1,
					 out_vec_x, out_vec_y, out_vec_img);

	if (output_style == "dir") {
    
		writeImages(out_vec_x, xFlowFile);
		writeImages(out_vec_y, yFlowFile);
		writeImages(out_vec_img, imgFile);
	}else{
    
// LOG(INFO)<<"Writing results to Zip archives";
		writeZipFile(out_vec_x, "x_%05d.jpg", xFlowFile+".zip");
		writeZipFile(out_vec_y, "y_%05d.jpg", yFlowFile+".zip");
		writeZipFile(out_vec_img, "img_%05d.jpg", imgFile+".zip");
	}

Optical flow calculation

According to the input video file name , Define a video capture object video_stream Read video . Define five vector Variable .
capture_frame-------t Frames captured at any time
capture_image-------t Always moving pictures
capture_gray---------t Graying of moving pictures at any time
pre_image-------------t-1 Always moving pictures
pre_gray---------------t-1 Always graying the moving picture
Read video frames to capture_frame in , If it is the first frame, initialize , Assign the above five variables to capture_frame Size memory . And assign the read first frame to pre_image And assign a value to pre_gay. From the second frame , Calculate the optical flow before the previous frame and the current frame , And stored in flow in , There are two optical flow calculation algorithms . The calculated optical flow split Divide into x and y Optical flow in two directions , First convert optical flow into picture form , Then it is encoded into a data stream , Stored in str_x and str_y in . Yes capture_image It is directly encoded into a data stream and stored in str_img in . Then add these three flows to the corresponding flow direction , The last video of every moment RGB Images and X、Y Optical flow data is stored in out_vec_x,out_vec_y,out_vec_img. Read these vectors , Write the data streams into the pictures respectively , A video optical flow extraction is completed .

Optical flow calculation function

void calcDenseFlow(std::string file_name, int bound, int type, int step,
                   std::vector<std::vector<uchar> >& output_x,
                   std::vector<std::vector<uchar> >& output_y,
                   std::vector<std::vector<uchar> >& output_img){
    
    # Read file_name Video file 
    VideoCapture video_stream(file_name);
    CHECK(video_stream.isOpened())<<"Cannot open video stream \""
                                  <<file_name
                                  <<"\" for optical flow extraction.";
    Mat capture_frame, capture_image, prev_image, capture_gray, prev_gray;
    Mat flow, flow_split[2];
    // Define a DualTVL object 
    cv::Ptr<cv::DualTVL1OpticalFlow> alg_tvl1 = cv::createOptFlow_DualTVL1();

    bool initialized = false;
    for(int iter = 0;; iter++){
    
        // Read the video frame to Mat Medium matrix 
        video_stream >> capture_frame;
        if (capture_frame.empty()) break; // read frames until end
        //build mats for the first frame
        if (!initialized){
    Mat
            // according to capture_frame, Create four of the same size 
            initializeMats(capture_frame, capture_image, capture_gray,
                           prev_image, prev_gray);
            // Assign the current frame to pre_image
            capture_frame.copyTo(prev_image);
            // Yes prev_image Assign a value to pre_gray
            cvtColor(prev_image, prev_gray, CV_BGR2GRAY);
            initialized = true;
// LOG(INFO)<<"Initialized";
        }else if(iter % step == 0){
    
            // Transmit the acquired frame data to capture_image, And carry out gray conversion 
            capture_frame.copyTo(capture_image);
            cvtColor(capture_image, capture_gray, CV_BGR2GRAY);
            switch(type){
    
                case 0: {
    
                    //Gunnar Farneback The algorithm is based on the assumption of constant image gradient and local optical flow .
                    calcOpticalFlowFarneback(prev_gray, capture_gray, flow,
                                             0.702, 5, 10, 2, 7, 1.5,
                                             cv::OPTFLOW_FARNEBACK_GAUSSIAN );
                    break;
                }
                case 1: {
    
                    // Use python Self contained TVL Extract optical flow 
                    alg_tvl1->calc(prev_gray, capture_gray, flow);
                    break;
                }
                default:
                    LOG(WARNING)<<"Unknown optical method. Using Farneback";
                    calcOpticalFlowFarneback(prev_gray, capture_gray, flow,
                                             0.702, 5, 10, 2, 7, 1.5,
                                             cv::OPTFLOW_FARNEBACK_GAUSSIAN );
            }
            // Define data flow 
            std::vector<uchar> str_x, str_y, str_img;
            // Divide the optical flow into x、y Direction exists flow_split in 
            split(flow, flow_split);
            // Opposite optical flow 、rgb Pictures are encoded into data streams 
            encodeFlowMap(flow_split[0], flow_split[1], str_x, str_y, bound);
            imencode(".jpg", capture_image, str_img);
            output_x.push_back(str_x);
            output_y.push_back(str_y);
            output_img.push_back(str_img);
			// to update pre_image、pre_gray
            std::swap(prev_gray, capture_gray);
            std::swap(prev_image, capture_image);
        }
    }
}

Light flow pictures

void convertFlowToImage(const Mat &flow_x, const Mat &flow_y, Mat &img_x, Mat &img_y,
                               double lowerBound, double higherBound) {
    
#define CAST(v, L, H) ((v) > (H) ? 255 : (v) < (L) ? 0 : cvRound(255*((v) - (L))/((H)-(L))))
    for (int i = 0; i < flow_x.rows; ++i) {
    
        for (int j = 0; j < flow_y.cols; ++j) {
    
            float x = flow_x.at<float>(i,j);
            float y = flow_y.at<float>(i,j);
            img_x.at<uchar>(i,j) = CAST(x, lowerBound, higherBound);
            img_y.at<uchar>(i,j) = CAST(y, lowerBound, higherBound);
        }
    }
#undef CAST
}

code ：Mat data -> Data flow

void encodeFlowMap(const Mat& flow_map_x, const Mat& flow_map_y,
                   std::vector<uchar>& encoded_x, std::vector<uchar>& encoded_y,
                   int bound, bool to_jpg){
    
    Mat flow_img_x(flow_map_x.size(), CV_8UC1);
    Mat flow_img_y(flow_map_y.size(), CV_8UC1);

    convertFlowToImage(flow_map_x, flow_map_y, flow_img_x, flow_img_y,
                       -bound, bound);

    if (to_jpg) {
    
        // take cv2:mat data flow_img_x Convert it into a data stream and store it in encoded_x in 
        imencode(".jpg", flow_img_x, encoded_x);
        imencode(".jpg", flow_img_y, encoded_y);
    }else {
    
        encoded_x.resize(flow_img_x.total());
        encoded_y.resize(flow_img_y.total());
        memcpy(encoded_x.data(), flow_img_x.data, flow_img_x.total());
        memcpy(encoded_y.data(), flow_img_y.data, flow_img_y.total());
    }
}

Data flow —> picture

// Intercept extract_flow
if (output_style == "dir") {
    
		writeImages(out_vec_x, xFlowFile);
		writeImages(out_vec_y, yFlowFile);
		writeImages(out_vec_img, imgFile);
	}else{
    
// LOG(INFO)<<"Writing results to Zip archives";
		writeZipFile(out_vec_x, "x_%05d.jpg", xFlowFile+".zip");
		writeZipFile(out_vec_y, "y_%05d.jpg", yFlowFile+".zip");
		writeZipFile(out_vec_img, "img_%05d.jpg", imgFile+".zip");
	}
// Input data stream and file prefix 
void writeImages(std::vector<std::vector<uchar>> images, std::string name_temp){
    
    for (int i = 0; i < images.size(); ++i){
    
        char tmp[256];
        sprintf(tmp, "_%05d.jpg", i+1);
        FILE* fp;
        fp = fopen((name_temp + tmp).c_str(), "wb");
        fwrite( images[i].data(), 1, images[i].size(), fp);
        fclose(fp);
    }
}