Python 为了配合StereoBM使用，摄像机是否必须校准？

在我的情况下，无论参数如何调整，OpenCV StereoBM深度贴图返回的数据都没有意义

我正在为一个涉及OpenCV的设计项目做研究，并使用立体视觉生成深度图。我目前能够成功地加载我的网络摄像头，并使用StereoBM生成深度贴图。然而，正如我下面的截图所示，结果数据目前并不有用。因此，我创建了一个小型python应用程序，它可以帮助我调整StereoBM参数，但没有起到任何作用

我的问题是，为了配合StereoBM功能使用，摄像机是否必须进行校准

如果没有，有哪些替代方案可以帮助我改进结果（即提高分辨率、使用立体BGM等）

代码

import cv2
import time
import numpy as np
from Tkinter import *

oldVal = 15
def oddVals(n):
        global oldVal
        n = int(n)
        if not n % 2:
                window_size.set(n+1 if n > oldVal else n-1)
                oldVal = window_size.get()

minDispValues = [16,32,48,64]
def minDispCallback(n):
        n = int(n)
        newvalue = min(minDispValues, key=lambda x:abs(x-float(n)))
        min_disp.set(newvalue)

# Display the sliders to control the stereo vision 
master = Tk()

master.title("StereoBM Settings");

min_disp = Scale(master, from_=16, to=64, command=minDispCallback, length=600, orient=HORIZONTAL, label="Minimum Disparities")
min_disp.pack()
min_disp.set(16)

window_size = Scale(master, from_=5, to=255, command=oddVals, length=600, orient=HORIZONTAL, label="Window Size")
window_size.pack()
window_size.set(15)

Disp12MaxDiff = Scale(master, from_=5, to=30, length=600, orient=HORIZONTAL, label="Max Difference")
Disp12MaxDiff.pack()
Disp12MaxDiff.set(0)

UniquenessRatio = Scale(master, from_=0, to=30, length=600, orient=HORIZONTAL, label="Uniqueness Ratio")
UniquenessRatio.pack()
UniquenessRatio.set(15)

SpeckleRange = Scale(master, from_=0, to=60, length=600, orient=HORIZONTAL, label="Speckle Range")
SpeckleRange.pack()
SpeckleRange.set(34)

SpeckleWindowSize = Scale(master, from_=60, to=150, length=600, orient=HORIZONTAL, label="Speckle Window Size")
SpeckleWindowSize.pack()
SpeckleWindowSize.set(100)

master.update()

vcLeft = cv2.VideoCapture(0) # Load video campture for the left camera
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420);
#vcLeft.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340);
vcLeft.set(3,640) # Set camera width
vcLeft.set(4,480) # Set camera height

vcRight = cv2.VideoCapture(1) # Load video capture for the right camera
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH,420);
#vcRight.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT,340);

firstTime = time.time() # First time log

totalFramesPassed = 0 # Number of frames passed

if vcLeft.isOpened() and vcRight.isOpened():
        rvalLeft, frameLeft = vcLeft.read()
        rvalRight, frameRight = vcRight.read()

else:
        rvalLeft = False
        rvalRight = False

while rvalLeft and rvalRight: # If the cameras are opened

        rvalLeft, frameLeft = vcLeft.read()

        rvalRight, frameRight = vcRight.read()

        cv2.putText(frameLeft, "FPS : " + str(totalFramesPassed / (time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10)

        cv2.imshow("Left Camera", frameLeft)

        cv2.putText(frameRight, "FPS : " + str(totalFramesPassed / (time.time() - firstTime)),(40, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 150, 2, 10)

        cv2.imshow("Right Camera", frameRight)

        frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)

        frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)

        num_disp = 112 - min_disp.get()

        stereo = cv2.StereoBM_create(numDisparities = num_disp, blockSize = window_size.get())

        stereo.setMinDisparity(min_disp.get())

        stereo.setNumDisparities(num_disp)

        stereo.setBlockSize(window_size.get())

        stereo.setDisp12MaxDiff(Disp12MaxDiff.get())

        stereo.setUniquenessRatio(UniquenessRatio.get())

        stereo.setSpeckleRange(SpeckleRange.get())

        stereo.setSpeckleWindowSize(SpeckleWindowSize.get())

        disparity = stereo.compute(frameLeftNew, frameRightNew).astype(np.float32) / 16.0

        disp_map = (disparity - min_disp.get())/num_disp

        cv2.imshow("Disparity", disp_map)

        master.update() # Update the slider options

        key = cv2.waitKey(20)

        totalFramesPassed = totalFramesPassed + 1 # One frame passed, increment

        if key == 27:

                break


vcLeft.release()

vcRight.release()

---

正如StereoBM的opencv文档中所述，这两幅图像需要是“矫正立体对”

这意味着，在计算视差之前，需要校正两个摄像头

在计算视差之前，先看一下在哪里可以看到如何校正两个摄像头

当您使用stereoBM计算视差时，您看到的是两幅图像中平行极线的对应关系。 这意味着图像应以这样一种方式对齐，即两个图像中的相同行对应于空间中的相同行。整改过程会解决这个问题

---

有关更多信息，请查看

我发现，为了使用StereoBM函数，我们需要校正该对。此外，我还发现，尽管立体GBM功能更占用资源，但它给了我更优化的结果

如果将来有人需要校准摄像头，您可以使用以下代码来帮助您：

# Imports
import cv2
import numpy as np

# Constants
leftCameraNumber = 2 # Number for left camera
rightCameraNumber = 1 # Number for right camera

numberOfChessRows = 6
numberOfChessColumns = 8
chessSquareSize = 30 # Length of square in millimeters

numberOfChessColumns = numberOfChessColumns - 1 # Update to reflect how many corners are inside the chess board
numberOfChessRows = numberOfChessRows - 1

objp = np.zeros((numberOfChessColumns*numberOfChessRows,3), np.float32)
objp[:,:2] = np.mgrid[0:numberOfChessRows,0:numberOfChessColumns].T.reshape(-1,2)*chessSquareSize

objectPoints = []
leftImagePoints = []
rightImagePoints = []

parameterCriteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

# Code
print("Press \"n\" when you're done caputing checkerboards.")

vcLeft = cv2.VideoCapture(leftCameraNumber) # Load video campture for the left camera
vcLeft.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2);
vcLeft.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2);

vcRight = cv2.VideoCapture(rightCameraNumber) # Load video capture for the right camera
vcRight.set(cv2.CAP_PROP_FRAME_WIDTH,640*3/2);
vcRight.set(cv2.CAP_PROP_FRAME_HEIGHT,480*3/2);

if vcLeft.isOpened() and vcRight.isOpened():
    rvalLeft, frameLeft = vcLeft.read()
    rvalRight, frameRight = vcRight.read()

else:
    rvalLeft = False
    rvalRight = False

# Number of succesful recognitions
checkerboardRecognitions = 0

while rvalLeft and rvalRight: # If the cameras are opened

    vcLeft.grab();

    vcRight.grab();

    rvalLeft, frameLeft = vcLeft.retrieve()

    rvalRight, frameRight = vcRight.retrieve()

    frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)

    frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)

    foundPatternLeft, cornersLeft = cv2.findChessboardCorners(frameLeftNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK)

    foundPatternRight, cornersRight = cv2.findChessboardCorners(frameRightNew, (numberOfChessRows, numberOfChessColumns), None, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK)


    if foundPatternLeft and foundPatternRight: # If found corners in this frame

        # Process the images and display the count of checkboards in our array
        checkerboardRecognitions = checkerboardRecognitions + 1
        print("Checker board recognitions: " + str(checkerboardRecognitions))

        objectPoints.append(objp)

        exactCornersLeft = cv2.cornerSubPix(frameLeftNew, cornersLeft, (11, 11), (-1, -1), parameterCriteria);
        leftImagePoints.append(exactCornersLeft)

        exactCornersRight = cv2.cornerSubPix(frameRightNew, cornersRight, (11, 11), (-1, -1), parameterCriteria);
        rightImagePoints.append(exactCornersRight)

        frameLeft = cv2.drawChessboardCorners(frameLeft, (numberOfChessRows, numberOfChessColumns), (exactCornersLeft), True);

        frameRight = cv2.drawChessboardCorners(frameRight, (numberOfChessRows, numberOfChessColumns), (exactCornersRight), True);


    # Display current webcams regardless if board was found or not
    cv2.imshow("Left Camera", frameLeft)

    cv2.imshow("Right Camera", frameRight)


    key = cv2.waitKey(250) # Give the frame some time

    if key == ord('n'):

        break

cameraMatrixLeft = np.zeros( (3,3) )
cameraMatrixRight = np.zeros( (3,3) )
distortionLeft = np.zeros( (8,1) )
distortionRight = np.zeros( (8,1) )
height, width = frameLeft.shape[:2]

rms, leftMatrix, leftDistortion, rightMatrix, rightDistortion, R, T, E, F = cv2.stereoCalibrate(objectPoints, leftImagePoints, rightImagePoints,  cameraMatrixLeft, distortionLeft, cameraMatrixRight, distortionRight, (width, height),parameterCriteria, flags=0)

arr1 = np.arange(8).reshape(2, 4)
arr2 = np.arange(10).reshape(2, 5)
np.savez('camera_calibration.npz', leftMatrix=leftMatrix, leftDistortion=leftDistortion, rightMatrix=rightMatrix, rightDistortion=rightDistortion, R=R, T=T, E=E, F=F)
print("Calibration Settings Saved to File!")

print("RMS:")
print(rms)
print("Left Matrix:")
print(leftMatrix)
print("Left Distortion:")
print(leftDistortion)
print("Right Matrix:")
print(rightMatrix)
print("Right Distortion:")
print(rightDistortion)
print("R:")
print(R)
print("T:")
print(T)
print("E:")
print(E)
print("F:")
print(F)


leftRectTransform, rightRectTransform, leftProjMatrix, rightProjMatrix, _, _, _ = cv2.stereoRectify(leftMatrix, leftDistortion, rightMatrix, rightDistortion,  (width, height), R, T, alpha=-1);
leftMapX, leftMapY = cv2.initUndistortRectifyMap(leftMatrix, leftDistortion, leftRectTransform, leftProjMatrix, (width, height), cv2.CV_32FC1);
rightMapX, rightMapY = cv2.initUndistortRectifyMap(rightMatrix, rightDistortion, rightRectTransform, rightProjMatrix, (width, height), cv2.CV_32FC1);

minimumDisparities = 0
maximumDisparities = 128

stereo = cv2.StereoSGBM_create(minimumDisparities, maximumDisparities, 18)

while True: # If the cameras are opened
    vcLeft.grab();

    vcRight.grab();

    rvalLeft, frameLeft = vcLeft.retrieve()

    rvalRight, frameRight = vcRight.retrieve()

    frameLeftNew = cv2.cvtColor(frameLeft, cv2.COLOR_BGR2GRAY)

    frameRightNew = cv2.cvtColor(frameRight, cv2.COLOR_BGR2GRAY)

    leftRectified = cv2.remap(frameLeftNew, leftMapX, leftMapY, cv2.INTER_LINEAR);

    rightRectified = cv2.remap(frameRightNew, rightMapX, rightMapY, cv2.INTER_LINEAR);

    disparity = stereo.compute(leftRectified, rightRectified)

    cv2.filterSpeckles(disparity, 0, 6000, maximumDisparities);

    cv2.imshow("Normalized Disparity", (disparity/16.0 - minimumDisparities)/maximumDisparities);

    cv2.imshow("Left Camera", leftRectified)

    cv2.imshow("Right Camera", rightRectified)


    key = cv2.waitKey(10) # Give the frame some time

    if key == 27:

        break

print("Finished!")

---

校准摄像机——尤其是不失真的摄像机将有所帮助。您是否尝试过将StereoBM用于更多随机场景（文本、随机点等？）-可能是通信问题。我会尝试实现简单的相关算法，看看相关性有多好。@KamilSzelag谢谢你的建议。我将大胆尝试校准左右两幅图像，希望算法能更好地工作。我曾尝试在其他场景中使用StereoBM，但我会检查场景中的相关性是否良好。我希望校准是这里的主要问题，因为我目前的结果，即使场景中有很多对象，也是不可靠的。再次感谢。