Hey there! Let's dive into your question about calculating block-based motion vectors between consecutive images using OpenCV.

Does OpenCV have a built-in function for this? #

First off, the short answer: OpenCV doesn't include a single dedicated function that directly spits out block-based motion vectors between two frames. But don't worry—it gives you all the building blocks you need to implement this easily, either via classic block matching or by repurposing optical flow tools with a little post-processing.

Method 1: Classic Block Matching (The Straightforward Approach) #

This is the traditional go-to for block-based motion estimation. Here's how it works:

1. Split the first frame into fixed-size blocks (16x16 is a common choice, inspired by video coding standards like MPEG).
2. For each block in the first frame, search a small neighborhood (search window) in the second frame to find the most similar block (using metrics like normalized squared difference or cross-correlation).
3. The offset between the original block's position and the best match's position is your motion vector for that block.

Python Implementation Example #

import cv2
import numpy as np

def compute_block_motion_vectors(frame1, frame2, block_size=16, search_window=16):
    # Convert to grayscale if frames are color
    if len(frame1.shape) == 3:
        frame1_gray = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
        frame2_gray = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
    else:
        frame1_gray = frame1
        frame2_gray = frame2

    h, w = frame1_gray.shape
    motion_vectors = []

    # Iterate over every block in the frame
    for y in range(0, h - block_size + 1, block_size):
        row_mvs = []
        for x in range(0, w - block_size + 1, block_size):
            # Extract current block from the first frame
            current_block = frame1_gray[y:y+block_size, x:x+block_size]
            
            # Define the search window boundaries in the second frame
            search_top = max(0, y - search_window // 2)
            search_bottom = min(h - block_size, y + search_window // 2)
            search_left = max(0, x - search_window // 2)
            search_right = min(w - block_size, x + search_window // 2)
            
            # Find the best matching block using normalized squared difference
            match_result = cv2.matchTemplate(
                frame2_gray[search_top:search_bottom+block_size, search_left:search_right+block_size],
                current_block,
                cv2.TM_SQDIFF_NORMED
            )
            min_val, _, min_loc, _ = cv2.minMaxLoc(match_result)
            
            # Calculate the motion vector (dx, dy)
            dx = (search_left + min_loc[0]) - x
            dy = (search_top + min_loc[1]) - y
            row_mvs.append((dx, dy))
        
        motion_vectors.append(row_mvs)
    
    return np.array(motion_vectors)

# Quick test usage
if __name__ == "__main__":
    frame1 = cv2.imread("frame1.jpg")
    frame2 = cv2.imread("frame2.jpg")
    motion_vecs = compute_block_motion_vectors(frame1, frame2, block_size=16, search_window=16)
    print(f"Motion vector grid shape: {motion_vecs.shape}")

Quick Notes:

• We use TM_SQDIFF_NORMED here—lower values mean a better match. You can swap this for TM_CCORR_NORMED if you prefer higher values = better matches.
• Adjust block_size and search_window based on your needs: smaller blocks give finer-grained motion but are more computationally heavy.
• For speed, you can swap the brute-force search for hierarchical search (searching coarse blocks first, then refining) instead of checking every pixel in the window.

Method 2: Dense Optical Flow + Block Aggregation #

If you want more robust motion vectors (especially in textureless areas where block matching might struggle), you can use OpenCV's dense optical flow and average vectors per block:

1. Compute pixel-level motion vectors between the two frames with cv2.calcOpticalFlowFarneback.
2. Split the dense flow map into blocks and calculate the average (or median) vector for each block.

Python Implementation Example #

import cv2
import numpy as np

def block_optical_flow(frame1, frame2, block_size=16):
    # Convert frames to grayscale
    frame1_gray = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
    frame2_gray = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
    
    # Compute dense optical flow (pixel-wise motion vectors)
    flow = cv2.calcOpticalFlowFarneback(
        frame1_gray, frame2_gray, None,
        pyr_scale=0.5, levels=3, winsize=15,
        iterations=3, poly_n=5, poly_sigma=1.2, flags=0
    )
    
    h, w = flow.shape[:2]
    motion_vectors = []
    
    # Aggregate flow vectors per block
    for y in range(0, h - block_size + 1, block_size):
        row_mvs = []
        for x in range(0, w - block_size + 1, block_size):
            block_flow = flow[y:y+block_size, x:x+block_size]
            avg_dx = np.mean(block_flow[..., 0])
            avg_dy = np.mean(block_flow[..., 1])
            row_mvs.append((avg_dx, avg_dy))
        
        motion_vectors.append(row_mvs)
    
    return np.array(motion_vectors)

# Quick test usage
if __name__ == "__main__":
    frame1 = cv2.imread("frame1.jpg")
    frame2 = cv2.imread("frame2.jpg")
    motion_vecs = block_optical_flow(frame1, frame2, block_size=16)
    print(f"Motion vector grid shape: {motion_vecs.shape}")

Quick Notes:

• cv2.calcOpticalFlowFarneback gives you a motion vector for every pixel, so averaging per block smooths out noise and gives you block-level motion.
• This method is more robust than brute-force block matching but uses more computational resources.

Which Method Should You Pick? #

• Go with block matching if you need a simple, interpretable implementation (great for learning or video coding-related tasks).
• Choose optical flow aggregation if you want more accurate motion vectors in complex or low-texture regions.

内容的提问来源于stack exchange，提问作者ESZ