image-matching-models/matching/viz.py

import sys
from matching import viz2d
import numpy as np
import cv2
import matplotlib
from kornia.utils import tensor_to_image
import torch

# This is to be able to use matplotlib also without a GUI
if not hasattr(sys, "ps1"):
    matplotlib.use("Agg")


def plot_matches(
    img0: np.ndarray,
    img1: np.ndarray,
    result_dict: dict,
    show_matched_kpts=True,
    show_all_kpts=False,
    save_path=None,
):
    """Plot matches between two images. Inlier matches are shown in green.

    Args:
        img0 (np.ndarray): img0 from matching procedure
        img1 (np.ndarray): img1 from matching procedure
        result_dict (dict): result from BaseMatcher, must contain keys: ['inlier_kpts0', 'inlier_kpts1', 'matched_kpts0', 'matched_kpts1']
        show_matched_kpts (bool, optional): Show matched kpts in addition to inliers. Matched kpts are blue. Defaults to True.
        show_all_kpts (bool, optional): Show all detected kpts in red. Defaults to False.
        save_path (str| Path, optional): path to save file to. Not saved if None. Defaults to None.

    Returns:
        List[plt.Axes]: plot axes
    """
    ax = viz2d.plot_images([img0, img1])

    if show_matched_kpts and "matched_kpts0" in result_dict.keys():
        viz2d.plot_matches(
            result_dict["matched_kpts0"],
            result_dict["matched_kpts1"],
            color="blue",
            lw=0.05,
            ps=2,
        )

    if show_all_kpts and result_dict["all_kpts0"] is not None:
        viz2d.plot_keypoints([result_dict["all_kpts0"], result_dict["all_kpts1"]], colors="red", ps=2)

    viz2d.plot_matches(result_dict["inlier_kpts0"], result_dict["inlier_kpts1"], color="lime", lw=0.2)

    viz2d.add_text(
        0,
        f"{len(result_dict['inlier_kpts0'])} inliers/{len(result_dict['matched_kpts1'])} matches\n({len(result_dict['inlier_kpts0'])/len(result_dict['matched_kpts1']):0.2f} inlier ratio)",
        fs=17,
        lwidth=2,
    )

    viz2d.add_text(0, "Img0", pos=(0.01, 0.01), va="bottom")
    viz2d.add_text(1, "Img1", pos=(0.01, 0.01), va="bottom")

    if save_path is not None:
        viz2d.save_plot(save_path)

    return ax


def plot_kpts(img0, result_dict, model_name="", save_path=None):
    """Plot keypoints in one image.

    Args:
        img0 (np.ndarray): img keypoints are detected from.
        result_dict (dict): return from BaseMatcher. Must contain ['all_kpts0']
        save_path (str| Path, optional): path to save file to. Not saved if None. Defaults to None.

    Returns:
        List[plt.Axes]: plot axes
    """
    if len(model_name):
        model_name = " - " + model_name
    ax = viz2d.plot_images([img0])
    viz2d.plot_keypoints([result_dict["all_kpts0"]], colors="orange", ps=10)
    viz2d.add_text(0, f"{len(result_dict['all_kpts0'])} kpts" + model_name, fs=20)
    if model_name is not None:
        viz2d.add_text(0, f"{len(result_dict['all_kpts0'])}", fs=20)

    if save_path is not None:
        viz2d.save_plot(save_path)

    return ax


def add_alpha_channel(img: np.ndarray) -> np.ndarray:
    """Add alpha channel to img using openCV

    Assumes incoming image in BGR order

    Args:
        img (np.ndarray): img without alpha

    Returns:
        np.ndarray: image with alpha channel (shape=[H,W,4])
    """
    if img.shape[2] == 3:
        # return np.concatenate(img, np.ones())
        return cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)
    else:
        return img


def stich(img0: np.ndarray | torch.Tensor, img1: np.ndarray | torch.Tensor, result_dict: dict) -> np.ndarray:
    """Stich two images together.

    Args:
        img0 (np.ndarray | torch.Tensor): img0 (to be warped as part of stitching)
        img1 (np.ndarray | torch.Tensor): img1 (is not warped in stitching)
        result_dict (dict): BaseMatcher return dict of results. Required keys: ["H"]

    Returns:
        np.ndarray: stitched images as array
    """
    # thanks to AHMAD-DOMA for stitching method (https://github.com/gmberton/image-matching-models/issues/7)
    if isinstance(img0, torch.Tensor):
        img0 = tensor_to_image(img0)
    if isinstance(img1, torch.Tensor):
        img1 = tensor_to_image(img1)

    img0, img1 = add_alpha_channel(img0), add_alpha_channel(img1)

    h0, w0 = img0.shape[:2]
    h1, w1 = img1.shape[:2]

    # Get the corners of the each image
    corners0 = np.float32([[0, 0], [0, h0], [w0, h0], [w0, 0]]).reshape(-1, 1, 2)
    corners1 = np.float32([[0, 0], [0, h1], [w1, h1], [w1, 0]]).reshape(-1, 1, 2)

    # Warp the corners of the first image using the homography
    warped_corners0 = cv2.perspectiveTransform(corners0, result_dict["H"])

    # Combine all corners and find min/max to find the overall bounding dimensions
    all_corners = np.concatenate((warped_corners0, corners1), axis=0)
    [x_min, y_min] = np.int32(all_corners.min(axis=0).ravel() - 0.5)
    [x_max, y_max] = np.int32(all_corners.max(axis=0).ravel() + 0.5)

    # compute translation offset
    translation = [-x_min, -y_min]
    H_translation = np.array([[1, 0, translation[0]], [0, 1, translation[1]], [0, 0, 1]])

    # warp first image onto stitch
    stiched_imgs = cv2.warpPerspective(img0, H_translation.dot(result_dict["H"]), (x_max - x_min, y_max - y_min))

    # overlay second image onto stitch
    stiched_imgs[translation[1] : translation[1] + h1, translation[0] : translation[0] + w1] = img1

    return stiched_imgs