Source code of Lucas-Kanade method to calculate affine parameters between two images
Lucas-Kande method is one of the most famous image registration technique.
Optical flow based on Lucas-Kanade method is included in OpenCV Library.
However, I could not find any other Lucas-Kanade method such as estimating affine parameters between two images. I created a simple code which can estimate translation parameters (tx, ty), zoom ratio (zoom), rotation angle (theta).
This code can be apply to other warping function such as a perspective transform.
* Initial values of parameters are needed because Lucas-Kanade can only refine the parameter. This code do not include the process to calculate initial values.
-----------------------------------------------------------------------------------------------------------------------
#include <opencv/cv.h>
#include <opencv/cxcore.h>
#include <opencv/highgui.h>
#include <opencv/ml.h>
#include <vector>
#include <cmath>
#include <fstream>
using namespace std;
using namespace cv;
#define DIRECTORY_NAME1 "C:/test.avi"
#define FRAME_START 0
#define MARGIN_X 400
#define MARGIN_Y 300
#define STEP 8
/*************** Prototype *************************/
int LucasKanadeMethod(Mat frame_t1, Mat frame_t2, double *, double *, double *, double *);
int main()
{
// Parameters which are refined by Lucas-Kanade method.
double t_x, t_y;
double zoom, theta;
// Set a vido name.
VideoCapture cap1(DIRECTORY_NAME1);
// Set a start frame number.
cap1.set(CV_CAP_PROP_POS_FRAMES, FRAME_START);
// Get the number of imags
int frame_end = cap1.get(CV_CAP_PROP_FRAME_COUNT);
// previous images
Mat frame1_t1;
Mat frame1_t1_g;
// Load an initial image.
cap1 >> frame1_t2;
cvtColor( frame1_t2, frame1_t2_g, CV_BGR2GRAY);
frame1_t1 = frame1_t2.clone();
frame1_t1_g = frame1_t2_g.clone();
while (frame_end--) {
// current images
Mat frame1_t2;
Mat frame1_t2_g;
cap1 >> frame1_t2;
cvtColor( frame1_t2, frame1_t2_g, CV_BGR2GRAY);
// 'q': exit
if (waitKey(10) == 'q') {
break;
}
LucasKanadeMethod(frame1_t1_g, frame1_t2_g, &t_x, &t_y, &zoom, &theta);
// Save the current image as a previous image.
frame1_t1 = frame1_t2.clone();
frame1_t1_g = frame1_t2_g.clone();
}
return 0;
}
int LucasKanadeMethod(Mat frame_prev, Mat frame_current, double *t_x_out, double *t_y_out, double *zoom_out, double *theta_out){
// images for Lucas-Kanade mothod
Mat edgeh_img, edgev_img, edgeh_imgd, edgev_imgd, warped_img;
double theta = 0;
double zoom = 1.0;
double t_x = 0;
double t_y = 0;
int iter_num = 10;
while(iter_num--){
Mat affine_mat = (Mat_<double>(2,3) << zoom*cos(theta), zoom*(-sin(theta)), t_x, zoom*sin(theta), zoom*cos(theta), t_y);
warpAffine( frame_prev, warped_img, affine_mat, frame_prev.size(), CV_INTER_LINEAR);
Sobel(warped_img, edgeh_img, CV_16S, 1, 0 );
Sobel(warped_img, edgev_img, CV_16S, 0, 1 );
#ifdef _DEBUG
Mat diff_img;
absdiff(warped_img, frame_current, diff_img);
imshow("diff", diff_img);
waitKey(33);
#endif
Mat jac = (Mat_<double>(2,4) << 0, 0, 0, 0, 0, 0, 0, 0);
Mat hesse = (Mat_<double>(4,4) << 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
Mat jacb = (Mat_<double>(1,4) << 0, 0, 0, 0);
Mat b = (Mat_<double>(1,4) << 0, 0, 0, 0);
Mat dp = (Mat_<double>(4,1) << 0, 0, 0, 0);
Mat dI = (Mat_<double>(1,2) << 0, 0);
for( int loop_y = MARGIN_Y; loop_y<warped_img.rows-MARGIN_Y; loop_y+=STEP ){
for( int loop_x = MARGIN_X; loop_x<warped_img.cols-MARGIN_X; loop_x+=STEP ){
double dW_dzoom_x = cos(theta)*loop_x - sin(theta)*loop_y;
double dW_dzoom_y = sin(theta)*loop_x + cos(theta)*loop_y;
double dW_dtheta_x = zoom*(-sin(theta)*loop_x-cos(theta)*loop_y);
double dW_dtheta_y = zoom*( cos(theta)*loop_x-sin(theta)*loop_y);
double dW_dtx = 1;
double dW_dty = 1;
double diff = double(frame_current.at<unsigned char>(loop_y, loop_x)) - warped_img.at<unsigned char>(loop_y, loop_x);
dI.at<double>(0,0) = -edgeh_img.at<short>(loop_y, loop_x)/4;
dI.at<double>(0,1) = -edgev_img.at<short>(loop_y, loop_x)/4;
jac.at<double>(0,0) = dW_dzoom_x;
jac.at<double>(0,1) = dW_dtheta_x;
jac.at<double>(0,2) = dW_dtx;
jac.at<double>(0,3) = 0;
jac.at<double>(1,0) = dW_dzoom_y;
jac.at<double>(1,1) = dW_dtheta_y;
jac.at<double>(1,2) = 0;
jac.at<double>(1,3) = dW_dty;
jacb = dI*jac;
b += dI*jac*diff;
hesse += jacb.t()*jacb;
}
}
dp=hesse.inv()*b.t();
zoom += dp.at<double>(0,0);
theta += dp.at<double>(1,0);
t_x += dp.at<double>(2,0);
t_y += dp.at<double>(3,0);
}
*t_x_out = t_x;
*t_y_out = t_y;
*zoom_out = zoom;
*theta_out = theta;
return 0;
}
-----------------------------------------------------------------------------------------------------------------------
***Key words***
Lucas-Kanade mothod, Direct method, Source code, Open source, Free source, OpenCV, Computer Vision, Image registration, Affine transform
