Python Lucas Kanade光流实现中的错误

我使用numpy和OpenCV创建了一个两帧Lucas Kanade比例金字塔光流的实现，但是它的输出似乎不那么“清晰”，因为我使用的测试图像数据集显示的地面真实图像应该是这样的。我觉得算法似乎有一个小错误，导致它产生了“合理”的结果（例如，在没有太多运动的区域很少运动，而且似乎有一些方向一致性的感觉），以及“合理性”如果有错误的话，我很难找到我所犯的错误。代码如下：

类MultiResLucasKanade2：

def __init__(self, frames, num_pyramid_layers, scale_factor = 2):
    self.frames = frames
    self.scale_factor = scale_factor
    self.num_pyramid_layers = num_pyramid_layers
    self.calculate_flows()

def calculate_flows(self):
    '''function doesn't "save" any flows calculated right now -- just shows the "flow 
    image" of the flows it calculates'''
    for frame_index in range(1, self.frames.shape[0]):
        base_frame1 = self.frames[frame_index - 1]
        base_frame2 = self.frames[frame_index]
        frame1_pyr = self.build_pyramid(base_frame1)
        frame2_pyr = self.build_pyramid(base_frame2)
        flow_x_pyr = self.build_pyramid(np.zeros(base_frame1.shape))
        flow_y_pyr = self.build_pyramid(np.zeros(base_frame2.shape))

        flow_x_pyr[0], flow_y_pyr[0] = self.get_flows_between_frames(frame1_pyr[0], frame2_pyr[0])
        upscaled_previous_flow_x = cv2.resize(flow_x_pyr[0], frame1_pyr[1].shape[::-1])*self.scale_factor
        upscaled_previous_flow_y = cv2.resize(flow_y_pyr[0], frame1_pyr[1].shape[::-1])*self.scale_factor

        flow_x_sum = cv2.resize(upscaled_previous_flow_x, base_frame1.shape[::-1], cv2.INTER_LINEAR)
        flow_y_sum = cv2.resize(upscaled_previous_flow_y, base_frame1.shape[::-1], cv2.INTER_LINEAR)

        for pyr_index in range(1, len(frame1_pyr)):
            frame2 = frame2_pyr[pyr_index]
            frame1 = self.flow_interpolate_image(frame1_pyr[pyr_index], upscaled_previous_flow_x, upscaled_previous_flow_y)

            flow_x_pyr[pyr_index], flow_y_pyr[pyr_index] = self.get_flows_between_frames(frame1, frame2)
            if pyr_index < len(frame1_pyr) - 1:
                layer_scale_factor = (self.scale_factor)**(len(frame1_pyr) - 1 - pyr_index)
                upscaled_x_pyr = cv2.resize(flow_x_pyr[pyr_index], base_frame1.shape[::-1], cv2.INTER_LINEAR) * layer_scale_factor
                upscaled_y_pyr = cv2.resize(flow_y_pyr[pyr_index], base_frame1.shape[::-1], cv2.INTER_LINEAR) * layer_scale_factor
                flow_x_sum += upscaled_x_pyr
                flow_y_sum += upscaled_y_pyr

                downscale_to_next_layer_factor = 1.0/((self.scale_factor)**(len(frame1_pyr) - 2 - pyr_index))

                upscaled_previous_flow_x = cv2.resize(flow_x_sum, frame1_pyr[pyr_index+1].shape[::-1], cv2.INTER_LINEAR)*downscale_to_next_layer_factor
                upscaled_previous_flow_y = cv2.resize(flow_y_sum, frame1_pyr[pyr_index+1].shape[::-1], cv2.INTER_LINEAR)*downscale_to_next_layer_factor

        full_flow_image = self.get_flow_image(flow_x_sum, flow_y_sum)
        Image.fromarray(full_flow_image).show()
        Image.fromarray(self.flow_interpolate_image(frame1, flow_x_sum, flow_y_sum)).show()


def build_pyramid(self, image):
    pyramid = [image]
    for i in range(0, self.num_pyramid_layers - 1):
        prev_pyr_image = pyramid[len(pyramid)-1]
        append_pyr_image = cv2.GaussianBlur(prev_pyr_image, (5,5), 1.0)
        append_pyr_image = cv2.resize(append_pyr_image, (int((prev_pyr_image.shape[1]+1)/self.scale_factor), int((prev_pyr_image.shape[0]+1)/self.scale_factor)))
        pyramid.append(append_pyr_image)
    pyramid = list(reversed(pyramid))
    return pyramid

'''interpolates the first frame to fit the second'''
def flow_interpolate_image(self, image, x_flows, y_flows):
    x_indices_mat = np.array([np.arange(image.shape[1]) for j in range(0, image.shape[0])])
    y_indices_mat = np.array([np.arange(image.shape[0]) for j in range(0, image.shape[1])]).T

    x_map = (x_indices_mat - x_flows).astype(np.float32)
    y_map = (y_indices_mat - y_flows).astype(np.float32)
    warp_image = cv2.remap(image, x_map, y_map, cv2.INTER_LINEAR)
    return warp_image

def get_flows_between_frames(self, frame1, frame2):

    WINDOW_SIZE = 3
    WINDOW_MARGIN = (WINDOW_SIZE - 1)//2

    x_flows = np.zeros((frame1.shape[0], frame1.shape[1]))
    y_flows = np.zeros((frame1.shape[0], frame1.shape[1]))

    frame2_grad_x = cv2.Scharr(frame2, cv2.CV_32F, 1, 0)
    frame2_grad_y = cv2.Scharr(frame2, cv2.CV_32F, 0, 1)
    partial_frame2_partial_time = frame2 - frame1

    for x in range(WINDOW_MARGIN, x_flows.shape[1] - WINDOW_MARGIN):
        for y in range(WINDOW_MARGIN, x_flows.shape[0] - WINDOW_MARGIN):
            frame2_gradx_window = frame2_grad_x[y-WINDOW_MARGIN : y+WINDOW_MARGIN+1, x-WINDOW_MARGIN : x+WINDOW_MARGIN+1].flatten()
            frame2_grady_window = frame2_grad_y[y-WINDOW_MARGIN : y+WINDOW_MARGIN+1, x-WINDOW_MARGIN : x+WINDOW_MARGIN+1].flatten()
            frame2_gradients_window = np.zeros((frame2_gradx_window.shape[0], 2))
            frame2_gradients_window[:, 0] = frame2_gradx_window
            frame2_gradients_window[:, 1] = frame2_grady_window
            frame2_partial_time_window = partial_frame2_partial_time[y-WINDOW_MARGIN : y+WINDOW_MARGIN+1, x-WINDOW_MARGIN : x+WINDOW_MARGIN+1].flatten()

            covar_mat = frame2_gradients_window.T.dot(frame2_gradients_window)

            if MatrixHelper.is_invertible(covar_mat):
                mat_product = frame2_gradients_window.T.dot(-frame2_partial_time_window)
                flow_steps = np.linalg.inv(covar_mat).dot(mat_product)
                x_flows[y, x] = flow_steps[0]
                y_flows[y, x] = flow_steps[1]
    x_flows = cv2.resize(x_flows, frame1.shape[::-1], cv2.INTER_LINEAR)
    y_flows = cv2.resize(y_flows, frame1.shape[::-1], cv2.INTER_LINEAR)
    return x_flows, y_flows

def get_flow_image(self, x_flows, y_flows):
    flow_stacks = np.dstack((x_flows, y_flows))

    hsv_angles = np.arctan2(y_flows, x_flows)
    hsv_angles = np.rad2deg(hsv_angles)%360

    hsv_saturations = (np.linalg.norm(flow_stacks, axis = 2))
    hsv_saturations /= np.amax(hsv_saturations)

    flow_hsv_image = np.dstack((hsv_angles, hsv_saturations, np.ones((x_flows.shape[0], x_flows.shape[1])))).astype(np.float32)
    flow_rgb_image = np.uint8(255 * cv2.cvtColor(flow_hsv_image, cv2.COLOR_HSV2RGB))
    return flow_rgb_image

---

此代码非常慢，不确定是否需要使用这些

for

循环中的大多数

。我用Python编写了反向合成方法，运行它需要一秒钟（金字塔实现更不用说），但我的方法输出同音字，所以比较起来有点困难。不管怎样，稠密流模型也不能很好地处理这幅图像的同形图，因为有太多的像素是静止的（图像有点稀疏）。@AlexanderReynolds我明白了。所以我应该用几个分数来代替（通过角点反应或其他方式选择，否则数学基本保持不变）？另外（我假设您将“get_flows_Inter_frames”中的嵌套循环视为其速度慢的罪魁祸首），我寻找了一种使用numpy或其他类型的快捷方式实现此循环的方法，但没有想到或找到任何东西。我会再考虑一下。除了这些问题，算法是否应该正常工作？谢谢您的目标是什么，使用密集光流单独扭曲点，还是创建单应性将所有点扭曲在一起？你是从哪一张纸或哪一个来源创作的？我的目标不是解决同音字，只是找到每个像素的速度。我使用的来源是（链接到您必须下载的powerpoint）：robots.stanford.edu/cs223b04/CS%20223-B%20L9%20Optical%20Flow.ppt。如果您不想下载powerpoint，我觉得这张幻灯片最能代表我正在尝试做的事情：。然而，我不确定我的实现是否正确，因为我得到的输出图像（带有彩色斑点的白色图像）看起来不像真实情况（也不确定它是否应该看起来“完美”）。我同意你的结果看起来不正确。不知道问题出在哪里。为什么不试着创建一些您自己的测试图像来仔细检查它的工作情况呢？像一个简单的盐和胡椒的图像，然后只应用一些流（或单应性，无论什么）的图像，看看流场是否如预期。首先尝试简单的翻译，然后从那里缩小问题的范围。如果您看一看用于读取
.flo
文件的Matlab代码，那么将其翻译成Python并不难。
op_flow = MultiResLucasKanade2(np.array([bw_base_image, bw_fit_image]), 5, scale_factor = 2.0)