utils/utils_heatmap.py

import copy
import torch
import numpy as np

from scipy.stats import linregress
from typing import List, Optional, Tuple


def generate_gaussian_matrix_vectorized(h, w, px, py, sigma=2):
    # Create a grid of indices
    x, y = np.meshgrid(np.arange(h), np.arange(w))

    # Calculate Gaussian values for the entire grid
    matrix = np.exp(-((x - px)**2 + (y - py)**2) / (2 * sigma**2))

    return matrix

def resize_keypoints(keypoints, original_size, new_size):

    ratio_h = new_size[0] / original_size[0]
    ratio_w = new_size[1] / original_size[1]

    resized_keypoints = {}
    for kp, values in keypoints.items():
        x_resized = int(values['x'] * ratio_w)
        y_resized = int(values['y'] * ratio_h)
        resized_keypoints[kp] = {'x': x_resized, 'y': y_resized, 'in_frame': values['in_frame']}
        if 'proj_err' in values.keys():
            resized_keypoints[kp]['proj_err'] = values['proj_err']

    return resized_keypoints


def generate_gaussian_array_vectorized(num_matrices, keypoints, original_size, down_ratio=2, sigma=2, proj_err_th=5.):

    new_size = tuple(ti/down_ratio for ti in original_size)
    resized_keypoints = resize_keypoints(keypoints, original_size, new_size)

    # Create an array of center points based on resized keypoints
    center_points = []

    for kp in range(1, num_matrices):
        if kp in resized_keypoints.keys():
            if (resized_keypoints[kp]['in_frame']):
                if 'proj_err' in resized_keypoints[kp].keys():
                    if resized_keypoints[kp]['proj_err'] <= proj_err_th:
                        center_points.append([resized_keypoints[kp]['x'], resized_keypoints[kp]['y']])
                    else:
                        center_points.append([np.inf, np.inf])
                else:
                    center_points.append([resized_keypoints[kp]['x'], resized_keypoints[kp]['y']])
            else:
                center_points.append([np.inf, np.inf])
        else:
            center_points.append([np.inf, np.inf])

    center_points = np.array(center_points)

    # Generate Gaussian matrices for all center points
    matrices = [generate_gaussian_matrix_vectorized(new_size[0], new_size[1], px, py, sigma) for px, py in center_points]
    matrices = np.array(matrices)
    matrices = np.concatenate((matrices, 1-matrices.sum(axis=0, keepdims=True)), axis=0)
    matrices = np.clip(matrices, 0, 1)

    return matrices


def resize_keypoints_l(keypoints, original_size, new_size):
    ratio_h = new_size[0] / original_size[0]
    ratio_w = new_size[1] / original_size[1]

    resized_keypoints = {}
    for kp, values in keypoints.items():
        x1_resized = int(values['x_1'] * ratio_w)
        y1_resized = int(values['y_1'] * ratio_h)
        x2_resized = int(values['x_2'] * ratio_w)
        y2_resized = int(values['y_2'] * ratio_h)

        resized_keypoints[kp] = {'x_1': x1_resized, 'y_1': y1_resized, 'x_2': x2_resized, 'y_2': y2_resized}

    return resized_keypoints

def generate_gaussian_array_vectorized_l(num_matrices, keypoints, original_size, down_ratio=2, sigma=2, sigma_mult=1):

    def sigma_f(px, py, size, sigma):
        #multiply sigma if point in image border
        if (px < 5 or px > size[0] - 5) | (py < 5 or py > size[1] - 5):
            return sigma_mult*sigma
        else:
            return sigma

    new_size = tuple(int(ti / down_ratio) for ti in original_size)
    resized_keypoints = resize_keypoints_l(keypoints, original_size, new_size)

    # Create an array of center points based on resized keypoints for both points
    center_points = []

    for kp in range(1, num_matrices+1):
        if kp in resized_keypoints.keys():
            center_points.append([resized_keypoints[kp]['x_1'], resized_keypoints[kp]['y_1']])
            center_points.append([resized_keypoints[kp]['x_2'], resized_keypoints[kp]['y_2']])

        else:
            center_points.append([np.inf, np.inf])
            center_points.append([np.inf, np.inf])

    center_points = np.array(center_points)

    # Generate Gaussian matrices for both points and sum them
    matrices1 = [generate_gaussian_matrix_vectorized(new_size[0], new_size[1], px, py, sigma_f(px, py, new_size, sigma)) for px, py in
                 center_points[::2]]
    matrices2 = [generate_gaussian_matrix_vectorized(new_size[0], new_size[1], px, py, sigma_f(px, py, new_size, sigma)) for px, py in
                 center_points[1::2]]
    matrices = np.array(matrices1) + np.array(matrices2)

    matrices_border = np.zeros((1, new_size[1], new_size[0]))

    for kp in range(1, num_matrices+1):
        if kp in resized_keypoints.keys():
            x1, y1 = resized_keypoints[kp]['x_1'], resized_keypoints[kp]['y_1']
            x2, y2 = resized_keypoints[kp]['x_2'], resized_keypoints[kp]['y_2']

            pixel_dist = np.linalg.norm(np.array([x2, y2]) - np.array([x1, y1]))
            num_gaussians = int(pixel_dist / (sigma))

            if num_gaussians != 1:
                for i in range(num_gaussians):
                    alpha = i / (num_gaussians - 1)
                    x = int(x1 + alpha * (x2 - x1))
                    y = int(y1 + alpha * (y2 - y1))
                    matrices_border[0, :, :] += generate_gaussian_matrix_vectorized(new_size[0], new_size[1], x, y, sigma)
            else:
                x, y = abs(x2 - x1) / 2, abs(y2 - y1) / 2
                matrices_border[0, :, :] += generate_gaussian_matrix_vectorized(new_size[0], new_size[1], x, y, sigma)

    matrices_border = np.clip(matrices_border, 0, 1)

    matrices_combined = np.concatenate((matrices, matrices_border), axis=0)

    return matrices_combined


def get_keypoints_from_heatmap_batch_maxpool(
        heatmap: torch.Tensor,
        scale: int = 2,
        max_keypoints: int = 1,
        min_keypoint_pixel_distance: int = 15,
        return_scores: bool = True,
) -> List[List[List[Tuple[int, int]]]]:
    """Fast extraction of keypoints from a batch of heatmaps using maxpooling.

    Inspired by mmdetection and CenterNet:
      https://mmdetection.readthedocs.io/en/v2.13.0/_modules/mmdet/models/utils/gaussian_target.html

    Args:
        heatmap (torch.Tensor): NxCxHxW heatmap batch
        max_keypoints (int, optional): max number of keypoints to extract, lowering will result in faster execution times. Defaults to 20.
        min_keypoint_pixel_distance (int, optional): _description_. Defaults to 1.

        Following thresholds can be used at inference time to select where you want to be on the AP curve. They should ofc. not be used for training
        abs_max_threshold (Optional[float], optional): _description_. Defaults to None.
        rel_max_threshold (Optional[float], optional): _description_. Defaults to None.

    Returns:
        The extracted keypoints for each batch, channel and heatmap; and their scores
    """
    batch_size, n_channels, _, width = heatmap.shape

    # obtain max_keypoints local maxima for each channel (w/ maxpool)

    kernel = min_keypoint_pixel_distance * 2 + 1
    pad = min_keypoint_pixel_distance
    # exclude border keypoints by padding with highest possible value
    # bc the borders are more susceptible to noise and could result in false positives
    padded_heatmap = torch.nn.functional.pad(heatmap, (pad, pad, pad, pad), mode="constant", value=1.0)
    max_pooled_heatmap = torch.nn.functional.max_pool2d(padded_heatmap, kernel, stride=1, padding=0)
    # if the value equals the original value, it is the local maximum
    local_maxima = max_pooled_heatmap == heatmap
    # all values to zero that are not local maxima
    heatmap = heatmap * local_maxima

    # extract top-k from heatmap (may include non-local maxima if there are less peaks than max_keypoints)
    scores, indices = torch.topk(heatmap.view(batch_size, n_channels, -1), max_keypoints, sorted=True)
    indices = torch.stack([torch.div(indices, width, rounding_mode="floor"), indices % width], dim=-1)
    # at this point either score > 0.0, in which case the index is a local maximum
    # or score is 0.0, in which case topk returned non-maxima, which will be filtered out later.

    #  remove top-k that are not local maxima and threshold (if required)
    # thresholding shouldn't be done during training

    #  moving them to CPU now to avoid multiple GPU-mem accesses!
    indices = indices.detach().cpu().numpy()
    scores = scores.detach().cpu().numpy()
    filtered_indices = [[[] for _ in range(n_channels)] for _ in range(batch_size)]
    filtered_scores = [[[] for _ in range(n_channels)] for _ in range(batch_size)]

    # have to do this manually as the number of maxima for each channel can be different
    for batch_idx in range(batch_size):
        for channel_idx in range(n_channels):
            candidates = indices[batch_idx, channel_idx]
            locs = []
            for candidate_idx in range(candidates.shape[0]):
                # convert to (u,v)
                loc = candidates[candidate_idx][::-1] * scale
                loc = loc.tolist()
                if return_scores:
                    loc.append(scores[batch_idx, channel_idx, candidate_idx])
                locs.append(loc)
            filtered_indices[batch_idx][channel_idx] = locs

    return torch.tensor(filtered_indices)


def get_keypoints_from_heatmap_batch_maxpool_l(
        heatmap: torch.Tensor,
        scale: int = 2,
        max_keypoints: int = 2,
        min_keypoint_pixel_distance: int = 10,
        return_scores: bool = True,
) -> List[List[List[Tuple[int, int]]]]:
    """Fast extraction of keypoints from a batch of heatmaps using maxpooling.

    Inspired by mmdetection and CenterNet:
      https://mmdetection.readthedocs.io/en/v2.13.0/_modules/mmdet/models/utils/gaussian_target.html

    Args:
        heatmap (torch.Tensor): NxCxHxW heatmap batch
        max_keypoints (int, optional): max number of keypoints to extract, lowering will result in faster execution times. Defaults to 20.
        min_keypoint_pixel_distance (int, optional): _description_. Defaults to 1.

        Following thresholds can be used at inference time to select where you want to be on the AP curve. They should ofc. not be used for training
        abs_max_threshold (Optional[float], optional): _description_. Defaults to None.
        rel_max_threshold (Optional[float], optional): _description_. Defaults to None.

    Returns:
        The extracted keypoints for each batch, channel and heatmap; and their scores
    """
    batch_size, n_channels, _, width = heatmap.shape
    kernel = min_keypoint_pixel_distance * 2 + 1
    pad = int((kernel-1)/2)

    max_pooled_heatmap = torch.nn.functional.max_pool2d(heatmap, kernel, stride=1, padding=pad)
    # if the value equals the original value, it is the local maximum
    local_maxima = max_pooled_heatmap == heatmap

    # all values to zero that are not local maxima
    heatmap = heatmap * local_maxima

    # extract top-k from heatmap (may include non-local maxima if there are less peaks than max_keypoints)
    scores, indices = torch.topk(heatmap.view(batch_size, n_channels, -1), max_keypoints, sorted=True)
    indices = torch.stack([torch.div(indices, width, rounding_mode="floor"), indices % width], dim=-1)
    # at this point either score > 0.0, in which case the index is a local maximum
    # or score is 0.0, in which case topk returned non-maxima, which will be filtered out later.

    #  remove top-k that are not local maxima and threshold (if required)
    # thresholding shouldn't be done during training

    #  moving them to CPU now to avoid multiple GPU-mem accesses!
    indices = indices.detach().cpu().numpy()
    scores = scores.detach().cpu().numpy()
    filtered_indices = [[[] for _ in range(n_channels)] for _ in range(batch_size)]
    filtered_scores = [[[] for _ in range(n_channels)] for _ in range(batch_size)]

    # have to do this manually as the number of maxima for each channel can be different
    for batch_idx in range(batch_size):
        for channel_idx in range(n_channels):
            candidates = indices[batch_idx, channel_idx]
            locs = []
            for candidate_idx in range(candidates.shape[0]):
                # convert to (u,v)
                loc = candidates[candidate_idx][::-1] * scale
                loc = loc.tolist()
                if return_scores:
                    loc.append(scores[batch_idx, channel_idx, candidate_idx])
                locs.append(loc)
            filtered_indices[batch_idx][channel_idx] = locs

    return torch.tensor(filtered_indices)


def coords_to_dict(coords, threshold=0.05, ground_plane_only=False):
    kp_list = []
    for batch in range(coords.size()[0]):
        keypoints = {}
        for count, c in enumerate(range(coords.size(1))):
            if coords.size(2) == 1:
                if ground_plane_only and c+1 in [12,15,16,19]:
                    continue
                if coords[batch, c, 0, -1] > threshold:
                    keypoints[count+1] = {'x': coords[batch, c, 0, 0].item(),
                                          'y': coords[batch, c, 0, 1].item(),
                                          'p': coords[batch, c, 0, 2].item()}
            else:
                if ground_plane_only and c+1 in [7,8,9,10,11,12]:
                    continue
                if coords[batch, c, 0, -1] > threshold and coords[batch, c, 1, -1] > threshold:
                    keypoints[count+1] = {'x_1': coords[batch, c, 0, 0].item(),
                                          'y_1': coords[batch, c, 0, 1].item(),
                                          'p_1': coords[batch, c, 0, 2].item(),
                                          'x_2': coords[batch, c, 1, 0].item(),
                                          'y_2': coords[batch, c, 1, 1].item(),
                                          'p_2': coords[batch, c, 1, 2].item()}

        kp_list.append(keypoints)
    return kp_list


def complete_keypoints(kp_dict, lines_dict, w, h, normalize=False):

    def line_intersection(x1, y1, x2, y2):
        #1e-7 sum in case there are two identical coordinate values
        x1[-1] += 1e-7
        x2[-1] += 1e-7
        slope1, intercept1, r1, p1, se1 = linregress(x1, y1)
        slope2, intercept2, r2, p2, se2 = linregress(x2, y2)

        x_intersection = (intercept2 - intercept1) / (slope1 - slope2 + 1e-7)
        y_intersection = slope1 * x_intersection + intercept1

        return x_intersection, y_intersection

    lines_list = ["Big rect. left bottom", "Big rect. left main", "Big rect. left top", "Big rect. right bottom",
                  "Big rect. right main", "Big rect. right top", "Goal left crossbar", "Goal left post left ",
                  "Goal left post right", "Goal right crossbar", "Goal right post left", "Goal right post right",
                  "Middle line", "Side line bottom", "Side line left", "Side line right", "Side line top",
                  "Small rect. left bottom", "Small rect. left main", "Small rect. left top", "Small rect. right bottom",
                  "Small rect. right main", "Small rect. right top"]


    keypoints_line_list = [['Side line top', 'Side line left'], ['Side line top', 'Middle line'],
                           ['Side line right', 'Side line top'], ['Side line left', 'Big rect. left top'],
                           ['Big rect. left top', 'Big rect. left main'], ['Big rect. right top', 'Big rect. right main'],
                           ['Side line right', 'Big rect. right top'], ['Side line left', 'Small rect. left top'],
                           ['Small rect. left top', 'Small rect. left main'], ['Small rect. right top', 'Small rect. right main'],
                           ['Side line right', 'Small rect. right top'], ['Goal left crossbar', 'Goal left post right'],
                           ['Side line left', 'Goal left post right'], ['Side line right', 'Goal right post left'],
                           ['Goal right crossbar', 'Goal right post left'], ['Goal left crossbar', 'Goal left post left '],
                           ['Side line left', 'Goal left post left '], ['Side line right', 'Goal right post right'],
                           ['Goal right crossbar', 'Goal right post right'], ['Side line left', 'Small rect. left bottom'],
                           ['Small rect. left bottom', 'Small rect. left main'], ['Small rect. right bottom', 'Small rect. right main'],
                           ['Side line right', 'Small rect. right bottom'], ['Side line left', 'Big rect. left bottom'],
                           ['Big rect. left bottom', 'Big rect. left main'], ['Big rect. right main', 'Big rect. right bottom'],
                           ['Side line right', 'Big rect. right bottom'], ['Side line left', 'Side line bottom'],
                           ['Side line bottom', 'Middle line'], ['Side line bottom', 'Side line right']]


    keypoint_aux_pair_list = [['Small rect. left main', 'Side line top'], ['Big rect. left main', 'Side line top'],
                              ['Big rect. right main', 'Side line top'], ['Small rect. right main', 'Side line top'],
                              ['Small rect. left main', 'Big rect. left top'], ['Big rect. right top', 'Small rect. right main'],
                              ['Small rect. left top', 'Big rect. left main'], ['Small rect. right top', 'Big rect. right main'],
                              ['Small rect. left bottom', 'Big rect. left main'], ['Small rect. right bottom', 'Big rect. right main'],
                              ['Small rect. left main', 'Big rect. left bottom'], ['Small rect. right main', 'Big rect. right bottom'],
                              ['Small rect. left main', 'Side line bottom'], ['Big rect. left main', 'Side line bottom'],
                              ['Big rect. right main', 'Side line bottom'], ['Small rect. right main', 'Side line bottom']]

    w_extra = 0. * w
    h_extra = 0. * h

    complete_dict = copy.deepcopy(kp_dict)
    for key in range(1, 31):
        if key not in kp_dict.keys():
            line_keys = keypoints_line_list[key-1]
            line_key1, line_key2 = lines_list.index(line_keys[0]) + 1, lines_list.index(line_keys[1]) + 1
            if all(line_key in lines_dict.keys() for line_key in [line_key1, line_key2]):
                x1 = [lines_dict[line_key1]['x_1'], lines_dict[line_key1]['x_2']]
                y1 = [lines_dict[line_key1]['y_1'], lines_dict[line_key1]['y_2']]
                x2 = [lines_dict[line_key2]['x_1'], lines_dict[line_key2]['x_2']]
                y2 = [lines_dict[line_key2]['y_1'], lines_dict[line_key2]['y_2']]
                new_kp = line_intersection(x1, y1, x2, y2)
                if -w_extra < new_kp[0] < w_extra + w and -h_extra < new_kp[1] < h_extra + h:
                    complete_dict[key] = {'x': round(new_kp[0], 0), 'y': round(new_kp[1], 0), 'p': 1.}

    for key in range(1, len(keypoint_aux_pair_list)):
        line_keys = keypoint_aux_pair_list[key-1]
        line_key1, line_key2 = lines_list.index(line_keys[0]) + 1, lines_list.index(line_keys[1]) + 1
        if all(line_key in lines_dict.keys() for line_key in [line_key1, line_key2]):
            x1 = [lines_dict[line_key1]['x_1'], lines_dict[line_key1]['x_2']]
            y1 = [lines_dict[line_key1]['y_1'], lines_dict[line_key1]['y_2']]
            x2 = [lines_dict[line_key2]['x_1'], lines_dict[line_key2]['x_2']]
            y2 = [lines_dict[line_key2]['y_1'], lines_dict[line_key2]['y_2']]
            new_kp = line_intersection(x1, y1, x2, y2)
            if -w_extra < new_kp[0] < w_extra + w and -h_extra < new_kp[1] < h_extra + h:
                complete_dict[key+57] = {'x': round(new_kp[0], 0), 'y': round(new_kp[1], 0), 'p': 1.}

    if normalize:
        for kp in complete_dict.keys():
            complete_dict[kp]['x'] /= w
            complete_dict[kp]['y'] /= h

        for line in lines_dict.keys():
            lines_dict[line]['x_1'] /= w
            lines_dict[line]['y_1'] /= h
            lines_dict[line]['x_2'] /= w
            lines_dict[line]['y_2'] /= h

    complete_dict = dict(sorted(complete_dict.items()))


    return complete_dict, lines_dict