C++ 强健的卡检测/迫害纠正OpenCV
目前,我有一种方法可以检测图像中的卡片,在大多数情况下,当光线相当一致且背景非常平静时,这种方法会起作用 下面是我用来执行此操作的代码:C++ 强健的卡检测/迫害纠正OpenCV,c++,opencv,image-processing,edge-detection,C++,Opencv,Image Processing,Edge Detection,目前,我有一种方法可以检测图像中的卡片,在大多数情况下,当光线相当一致且背景非常平静时,这种方法会起作用 下面是我用来执行此操作的代码: Mat img = inImg.clone(); outImg = Mat(inImg.size(), CV_8UC1); inImg.copyTo(outImg); Mat img_fullRes = img.clone(); pyrDown(img, img); Mat imgGray; cvt
Mat img = inImg.clone();
outImg = Mat(inImg.size(), CV_8UC1);
inImg.copyTo(outImg);
Mat img_fullRes = img.clone();
pyrDown(img, img);
Mat imgGray;
cvtColor(img, imgGray, CV_RGB2GRAY);
outImg_gray = imgGray.clone();
// Find Edges //
Mat detectedEdges = imgGray.clone();
bilateralFilter(imgGray, detectedEdges, 0, 185, 3, 0);
Canny( detectedEdges, detectedEdges, 20, 65, 3 );
dilate(detectedEdges, detectedEdges, Mat::ones(3,3,CV_8UC1));
Mat cdst = img.clone();
vector<Vec4i> lines;
HoughLinesP(detectedEdges, lines, 1, CV_PI/180, 60, 50, 3 );
for( size_t i = 0; i < lines.size(); i++ )
{
Vec4i l = lines[i];
// For debug
//line( cdst, cv::Point(l[0], l[1]), cv::Point(l[2], l[3]), Scalar(0,0,255), 1);
}
//cdst.copyTo(inImg);
// // Find points of intersection //
cv::Rect imgROI;
int ext = 10;
imgROI.x = ext;
imgROI.y = ext;
imgROI.width = img.size().width - ext;
imgROI.height = img.size().height - ext;
int N = lines.size();
// Creating N amount of points // N == lines.size()
cv::Point** poi = new cv::Point*[N];
for( int i = 0; i < N; i++ )
poi[i] = new cv::Point[N];
vector<cv::Point> poiList;
for( int i = 0; i < N; i++ )
{
poi[i][i] = cv::Point(-1,-1);
Vec4i line1 = lines[i];
for( int j = i + 1; j < N; j++ )
{
Vec4i line2 = lines[j];
cv::Point p = computeIntersect(line1, line2, imgROI);
if( p.x != -1 )
{
//line(cdst, p-cv::Point(2,0), p+cv::Point(2,0), Scalar(0,255,0));
//line(cdst, p-cv::Point(0,2), p+cv::Point(0,2), Scalar(0,255,0));
poiList.push_back(p);
}
poi[i][j] = p;
poi[j][i] = p;
}
}
cdst.copyTo(inImg);
if(poiList.size()==0)
{
outImg = inImg.clone();
//circle(outImg, cv::Point(100,100), 50, Scalar(255,0,0), -1);
return;
}
convexHull(poiList, poiList, false, true);
for( int i=0; i<poiList.size(); i++ )
{
cv::Point p = poiList[i];
//circle(cdst, p, 3, Scalar(255,0,0), 2);
}
//Evaluate all possible quadrilaterals
cv::Point cardCorners[4];
float metric_max = 0;
int Npoi = poiList.size();
for( int p1=0; p1<Npoi; p1++ )
{
cv::Point pts[4];
pts[0] = poiList[p1];
for( int p2=p1+1; p2<Npoi; p2++ )
{
pts[1] = poiList[p2];
if( isCloseBy(pts[1],pts[0]) )
continue;
for( int p3=p2+1; p3<Npoi; p3++ )
{
pts[2] = poiList[p3];
if( isCloseBy(pts[2],pts[1]) || isCloseBy(pts[2],pts[0]) )
continue;
for( int p4=p3+1; p4<Npoi; p4++ )
{
pts[3] = poiList[p4];
if( isCloseBy(pts[3],pts[0]) || isCloseBy(pts[3],pts[1])
|| isCloseBy(pts[3],pts[2]) )
continue;
// get the metrics
float area = getArea(pts);
cv::Point a = pts[0]-pts[1];
cv::Point b = pts[1]-pts[2];
cv::Point c = pts[2]-pts[3];
cv::Point d = pts[3]-pts[0];
float oppLenDiff = abs(a.dot(a)-c.dot(c)) + abs(b.dot(b)-d.dot(d));
float metric = area - 0.35*oppLenDiff;
if( metric > metric_max )
{
metric_max = metric;
cardCorners[0] = pts[0];
cardCorners[1] = pts[1];
cardCorners[2] = pts[2];
cardCorners[3] = pts[3];
}
}
}
}
}
// find the corners corresponding to the 4 corners of the physical card
sortPointsClockwise(cardCorners);
// Calculate Homography //
vector<Point2f> srcPts(4);
srcPts[0] = cardCorners[0]*2;
srcPts[1] = cardCorners[1]*2;
srcPts[2] = cardCorners[2]*2;
srcPts[3] = cardCorners[3]*2;
vector<Point2f> dstPts(4);
cv::Size outImgSize(1400,800);
dstPts[0] = Point2f(0,0);
dstPts[1] = Point2f(outImgSize.width-1,0);
dstPts[2] = Point2f(outImgSize.width-1,outImgSize.height-1);
dstPts[3] = Point2f(0,outImgSize.height-1);
Mat Homography = findHomography(srcPts, dstPts);
// Apply Homography
warpPerspective( img_fullRes, outImg, Homography, outImgSize, INTER_CUBIC );
outImg.copyTo(inImg);
排序点按时针方向定义为:
cv::Point computeIntersect(cv::Vec4i a, cv::Vec4i b, cv::Rect ROI)
{
int x1 = a[0], y1 = a[1], x2 = a[2], y2 = a[3];
int x3 = b[0], y3 = b[1], x4 = b[2], y4 = b[3];
cv::Point p1 = cv::Point (x1,y1);
cv::Point p2 = cv::Point (x2,y2);
cv::Point p3 = cv::Point (x3,y3);
cv::Point p4 = cv::Point (x4,y4);
// Check to make sure all points are within the image boundrys, if not reject them.
if( !ROI.contains(p1) || !ROI.contains(p2)
|| !ROI.contains(p3) || !ROI.contains(p4) )
return cv::Point (-1,-1);
cv::Point vec1 = p1-p2;
cv::Point vec2 = p3-p4;
float vec1_norm2 = vec1.x*vec1.x + vec1.y*vec1.y;
float vec2_norm2 = vec2.x*vec2.x + vec2.y*vec2.y;
float cosTheta = (vec1.dot(vec2))/sqrt(vec1_norm2*vec2_norm2);
float den = ((float)(x1-x2) * (y3-y4)) - ((y1-y2) * (x3-x4));
if(den != 0)
{
cv::Point2f pt;
pt.x = ((x1*y2 - y1*x2) * (x3-x4) - (x1-x2) * (x3*y4 - y3*x4)) / den;
pt.y = ((x1*y2 - y1*x2) * (y3-y4) - (y1-y2) * (x3*y4 - y3*x4)) / den;
if( !ROI.contains(pt) )
return cv::Point (-1,-1);
// no-confidence metric
float d1 = MIN( dist2(p1,pt), dist2(p2,pt) )/vec1_norm2;
float d2 = MIN( dist2(p3,pt), dist2(p4,pt) )/vec2_norm2;
float no_confidence_metric = MAX(sqrt(d1),sqrt(d2));
// If end point ratios are greater than .5 reject
if( no_confidence_metric < 0.5 && cosTheta < 0.707 )
return cv::Point (int(pt.x+0.5), int(pt.y+0.5));
}
return cv::Point(-1, -1);
}
void sortPointsClockwise(cv::Point a[])
{
cv::Point b[4];
cv::Point ctr = (a[0]+a[1]+a[2]+a[3]);
ctr.x /= 4;
ctr.y /= 4;
b[0] = a[0]-ctr;
b[1] = a[1]-ctr;
b[2] = a[2]-ctr;
b[3] = a[3]-ctr;
for( int i=0; i<4; i++ )
{
if( b[i].x < 0 )
{
if( b[i].y < 0 )
a[0] = b[i]+ctr;
else
a[3] = b[i]+ctr;
}
else
{
if( b[i].y < 0 )
a[1] = b[i]+ctr;
else
a[2] = b[i]+ctr;
}
}
}
float getArea(cv::Point arr[])
{
cv::Point diag1 = arr[0]-arr[2];
cv::Point diag2 = arr[1]-arr[3];
return 0.5*(diag1.cross(diag2));
}
bool isCloseBy( cv::Point p1, cv::Point p2 )
{
int D = 10;
// Checking that X values are within 10, same for Y values.
return ( abs(p1.x-p2.x)<=D && abs(p1.y-p2.y)<=D );
}
isCloseBy
定义为:
cv::Point computeIntersect(cv::Vec4i a, cv::Vec4i b, cv::Rect ROI)
{
int x1 = a[0], y1 = a[1], x2 = a[2], y2 = a[3];
int x3 = b[0], y3 = b[1], x4 = b[2], y4 = b[3];
cv::Point p1 = cv::Point (x1,y1);
cv::Point p2 = cv::Point (x2,y2);
cv::Point p3 = cv::Point (x3,y3);
cv::Point p4 = cv::Point (x4,y4);
// Check to make sure all points are within the image boundrys, if not reject them.
if( !ROI.contains(p1) || !ROI.contains(p2)
|| !ROI.contains(p3) || !ROI.contains(p4) )
return cv::Point (-1,-1);
cv::Point vec1 = p1-p2;
cv::Point vec2 = p3-p4;
float vec1_norm2 = vec1.x*vec1.x + vec1.y*vec1.y;
float vec2_norm2 = vec2.x*vec2.x + vec2.y*vec2.y;
float cosTheta = (vec1.dot(vec2))/sqrt(vec1_norm2*vec2_norm2);
float den = ((float)(x1-x2) * (y3-y4)) - ((y1-y2) * (x3-x4));
if(den != 0)
{
cv::Point2f pt;
pt.x = ((x1*y2 - y1*x2) * (x3-x4) - (x1-x2) * (x3*y4 - y3*x4)) / den;
pt.y = ((x1*y2 - y1*x2) * (y3-y4) - (y1-y2) * (x3*y4 - y3*x4)) / den;
if( !ROI.contains(pt) )
return cv::Point (-1,-1);
// no-confidence metric
float d1 = MIN( dist2(p1,pt), dist2(p2,pt) )/vec1_norm2;
float d2 = MIN( dist2(p3,pt), dist2(p4,pt) )/vec2_norm2;
float no_confidence_metric = MAX(sqrt(d1),sqrt(d2));
// If end point ratios are greater than .5 reject
if( no_confidence_metric < 0.5 && cosTheta < 0.707 )
return cv::Point (int(pt.x+0.5), int(pt.y+0.5));
}
return cv::Point(-1, -1);
}
void sortPointsClockwise(cv::Point a[])
{
cv::Point b[4];
cv::Point ctr = (a[0]+a[1]+a[2]+a[3]);
ctr.x /= 4;
ctr.y /= 4;
b[0] = a[0]-ctr;
b[1] = a[1]-ctr;
b[2] = a[2]-ctr;
b[3] = a[3]-ctr;
for( int i=0; i<4; i++ )
{
if( b[i].x < 0 )
{
if( b[i].y < 0 )
a[0] = b[i]+ctr;
else
a[3] = b[i]+ctr;
}
else
{
if( b[i].y < 0 )
a[1] = b[i]+ctr;
else
a[2] = b[i]+ctr;
}
}
}
float getArea(cv::Point arr[])
{
cv::Point diag1 = arr[0]-arr[2];
cv::Point diag2 = arr[1]-arr[3];
return 0.5*(diag1.cross(diag2));
}
bool isCloseBy( cv::Point p1, cv::Point p2 )
{
int D = 10;
// Checking that X values are within 10, same for Y values.
return ( abs(p1.x-p2.x)<=D && abs(p1.y-p2.y)<=D );
}
以下是几个测试图像及其结果:
抱歉,这篇文章太长了,不过我希望有人能建议一种方法,让我从图像中提取卡片的方法更加可靠。一个可以更好地处理破坏性的背景和不一致的照明
当一张卡片放在对比鲜明的背景上,光线很好时,我的方法几乎90%的时间都有效。但很明显,我需要一种更稳健的方法
有人有什么建议吗
谢谢
达努什卡的soloution尝试
Mat gray, bw; pyrDown(inImg, inImg);
cvtColor(inImg, gray, CV_RGB2GRAY);
int morph_size = 3;
Mat element = getStructuringElement( MORPH_ELLIPSE, cv::Size( 4*morph_size + 1, 2*morph_size+1 ), cv::Point( morph_size, morph_size ) );
morphologyEx(gray, gray, 2, element);
threshold(gray, bw, 160, 255, CV_THRESH_BINARY);
vector<vector<cv::Point> > contours;
vector<Vec4i> hierarchy;
findContours( bw, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cv::Point(0, 0) );
int largest_area=0;
int largest_contour_index=0;
cv::Rect bounding_rect;
for( int i = 0; i< contours.size(); i++ )
{
double a=contourArea( contours[i],false); // Find the area of contour
if(a>largest_area){
largest_area=a;
largest_contour_index=i; //Store the index of largest contour
bounding_rect=boundingRect(contours[i]);
}
}
//Scalar color( 255,255,255);
rectangle(inImg, bounding_rect, Scalar(0,255,0),1, 8,0);
Mat biggestRect = inImg(bounding_rect);
Mat card1 = biggestRect.clone();
Mat灰,bw;吡咯烷酮(inImg,inImg);
CVT颜色(inImg、灰色、CV_rgb2灰色);
int morp_size=3;
Mat元素=getStructuringElement(变形椭圆,cv::Size(4*变形大小+1,2*变形大小+1),cv::Point(变形大小,变形大小));
形态学(灰色,灰色,2,元素);
阈值(灰度、bw、160、255、CV_阈值_二进制);
矢量等值线;
向量层次;
findContours(bw、等高线、层次、CV_RETR_树、CV_链_近似_简单、CV::Point(0,0));
int最大面积=0;
int最大轮廓指数=0;
cv::Rect边界_Rect;
对于(int i=0;i最大面积){
最大面积=a;
最大轮廓指数=i;//存储最大轮廓指数
bounding_rect=boundingRect(等高线[i]);
}
}
//标量颜色(255255);
矩形(inImg,边界矩形,标量(0255,0),1,8,0);
Mat biggestRect=inImg(边界矩形);
Mat card1=biggestRect.clone();
图像处理的艺术(在我10多年的经验中)就是这样:一门艺术。没有一个单一的答案存在,而且总是有不止一种方法。在某些情况下,它肯定会失败
根据我在医学图像中自动检测特征的经验,建立可靠的算法需要很长时间,但事后看来,最好的结果是使用相对简单的算法。然而,要实现这个简单的算法需要很多时间
要做到这一点,一般方法始终是一样的:
- 入门是建立一个大型的测试图像数据库(至少100个)。这定义了“正常”图像,应该可以正常工作。通过收集图像,你已经开始思考这个问题
- 对图像进行注释以构建一种“基本真理”。在这种情况下,“基本事实”应该包含卡的4个角,因为这些是有趣的点
- 创建一个应用程序,在这些图像上运行一个算法,并将结果与基本事实进行比较。在这种情况下,“与地面真实值进行比较”将取找到的4个角点与地面真实值角点的平均距离
- 输出一个以制表符分隔的文件,该文件称为.xls,因此可以通过双击在Excel中打开(在Windows上)。很高兴能快速了解这些案例。先看看最坏的情况。然后手动打开这些案例,试图了解它们为什么不起作用
- 现在,您可以更改算法了。更改某些内容,然后重新运行。将新的Excel工作表与旧的Excel工作表进行比较。现在你开始意识到你必须做出的权衡
话虽如此,我认为您需要在调整算法时回答以下问题:
- 你允许折叠一些卡片吗?所以没有完全的直线?如果是这样,请更多地关注角,而不是线/边
- 你允许光线逐渐不同吗?如果是这样,局部对比度拉伸过滤器可能会有所帮助
- 你允许卡片的颜色与背景相同吗?如果是这样,你必须专注于卡片的内容,而不是卡片的边框
- 你允许不完美的镜头吗?如果是,扩展到什么程度
- 你们允许轮换卡片吗?如果是,扩展到什么程度
- 背景的颜色和/或纹理是否应一致
- 最小的可检测卡相对于图像大小应该有多小?如果您假设至少80%的宽度或高度应该被覆盖,那么您将获得健壮性
- 如果在图像中可以看到多张卡,那么算法是否应该稳健,只选择一张,或者任何输出是否正常
- 如果看不到卡,是否应检测到此情况?这种情况下的内置检测将使其更加用户友好(“未找到卡”),但也不太可靠
这些将对要获取的图像做出要求和假设。你可以依赖的假设非常强大:如果你选择正确的假设,它们会使算法快速、健壮、简单。也让这些需求和假设成为测试数据库的一部分
那么我会选择什么呢?根据您提供的三张图片,我将从以下内容开始:
- 假设卡片填充图像的比例从50%到100%
- 假设卡最多旋转10度左右
- 假设这些角很明显
- 假设卡的纵横比(高度除以宽度)介于1/3和3之间
- 假设背景中没有类似卡片的对象
然后,算法将如下所示:
- 使用角点过滤器在图像的每个象限中检测特定角点。因此,在图像的左上象限,卡的左上角。例如,查看或使用OpenCV函数,如
cornerHarris
- 为了更稳健,每个象限计算多个角点
- 试着建立平行线