seg_viz.py

import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg
from matplotlib.figure import Figure
import numpy as np
import torch
import torch.nn.functional as F


def preprocess_image_and_mask(image, seg_source):
    image = image.squeeze(0).cpu()  # Remove batch dimension
    mask = seg_source["segmentation"]

    if mask.shape[2:] != image.shape[1:]:
        mask = F.interpolate(mask, size=image.shape[1:], mode="bilinear")

    return image, mask


def apply_hard_edges(mask):
    return (torch.max(mask, dim=1, keepdim=True)[0] == mask).float()


def create_color_mask(mask, colormap="gist_rainbow"):
    colormap = plt.get_cmap(colormap)
    num_segments = mask.shape[1]
    mask = mask.squeeze(0).permute(1, 2, 0).cpu().numpy()
    color_mask = np.zeros((256, 256, 3), dtype=np.float32)
    patches = []

    for i in range(num_segments):
        color = (
            np.array(colormap((i - 1) / (num_segments - 1)))[:3]
            if i != 0
            else np.array((0, 0, 0))
        )
        patches.append(mpatches.Patch(color=color, label=str(i)))
        color_mask += mask[..., i : (i + 1)] * color.reshape(1, 1, 3)

    return color_mask, patches


def overlay_mask(image, color_mask, alpha=0.6):
    image_np = image.permute(1, 2, 0).numpy()
    if image_np.max() > 1:
        image_np = image_np / 255.0

    overlaid_image = image_np * (1 - alpha) + color_mask * alpha
    return np.clip(overlaid_image, 0, 1)


def create_figure(overlaid_image, patches):
    fig = Figure(figsize=(10, 10))
    canvas = FigureCanvasAgg(fig)
    ax = fig.add_subplot(111)

    ax.imshow(overlaid_image)
    ax.axis("off")
    ax.legend(
        handles=patches, loc="upper right", bbox_to_anchor=(1, 1), fontsize="small"
    )

    fig.tight_layout(pad=0)
    return fig, canvas


def render_figure_to_tensor(canvas):
    canvas.draw()
    result_image = np.frombuffer(canvas.tostring_rgb(), dtype="uint8")
    result_image = result_image.reshape(canvas.get_width_height()[::-1] + (3,))
    result_tensor = torch.from_numpy(result_image).unsqueeze(0).float() / 255.0
    return result_tensor


def visualize_frame(
    image, seg_source, hard_edges=True, colormap="gist_rainbow", alpha=0.6
):
    image, mask = preprocess_image_and_mask(image, seg_source)

    if hard_edges:
        mask = apply_hard_edges(mask)

    color_mask, patches = create_color_mask(mask, colormap)
    overlaid_image = overlay_mask(image, color_mask, alpha)
    fig, canvas = create_figure(overlaid_image, patches)
    result_tensor = render_figure_to_tensor(canvas)

    return result_tensor