clean_chart.py

from PIL import Image, ImageDraw, ImageStat
import numpy as np
import cv2

def polygon2bbox(polygon_dic):
    x_coords = []
    y_coords = []
    for item in polygon_dic.keys():
        if "x" in item:
            x_coords.append(polygon_dic[item])
        elif "y" in item:
            y_coords.append(polygon_dic[item])

    x0 = min(x_coords)
    y0 = min(y_coords)

    x1 = max(x_coords)
    y1 = max(y_coords)
    return (x0, y0, x1, y1)


def get_legend_boxes(annot):
    # ============================================================
    # This is used to extract the text bbox and text role from task3 field in gt
    text_role_dic_id = {}
    id_text_bb_dic = {}

    #Used role is what role you would like to extract, 
    #if you only want to process the legend area, then legend_label & legend_title is enough
    for role in ['legend_label', 'legend_title']:  
        text_role_dic_id[role] = []


    text_block_list = annot["task3"]["input"]["task2_output"]["text_blocks"]
    for item in text_block_list:
        item_id = item["id"]
        if "polygon" in item:
            polygon_dic = item["polygon"]
            # Convert the polygon to the bbox
            (bbo_x0, bbox_y0, bbo_x1, bbo_y1) = polygon2bbox(polygon_dic)
            # id_text_bb_dic[item_id] = [x0, y0, x1, y1]

            poly_x0,poly_x1,poly_x2,poly_x3,poly_y0,poly_y1,poly_y2,poly_y3 = polygon_dic.values()
            id_text_bb_dic[item_id] = {"bbox":[bbo_x0, bbox_y0, bbo_x1, bbo_y1], "polygon":[poly_x0,poly_x1,poly_x2,poly_x3,poly_y0,poly_y1,poly_y2,poly_y3]}
        else:
            # Handle cleaning of adobe synth data
            id_text_bb_dic[item_id] = {"bbox":[item['bb']['x0'], item['bb']['y0'],
             item['bb']['x0']+item['bb']['width']-1, item['bb']['y0']+item['bb']['height']-1]}

    text_role_list = annot["task3"]["output"]["text_roles"]
    for item in text_role_list:
        role = item["role"]
        item_id = item["id"]
        # if role in used_role:
        if role not in text_role_dic_id.keys():
            text_role_dic_id[role] = []
        text_role_dic_id[role].append(item_id) 

    # ============================================================

    # ============================================================
    # Handle the legend patch and generate the legend area bbox


    legend_area_bb_list = []
    legend_patch_list = []

    for item in annot["task5"]["output"]["legend_pairs"]:
        x0 = item["bb"]["x0"]
        y0 = item["bb"]["y0"]
        x1 = x0 + item["bb"]["width"]
        y1 = y0 + item["bb"]["height"]

        legend_patch_list.append({"bbox":[x0,y0,x1,y1]})


    for legend_id in text_role_dic_id["legend_title"] + text_role_dic_id["legend_label"]:
        legend_area_bb_list.append(id_text_bb_dic[legend_id])
    for bbox_item in legend_patch_list:
        legend_area_bb_list.append(bbox_item)

    return legend_area_bb_list


def get_plot_area(annot):
    plot_area = annot['task6']['input']['task4_output']['_plot_bb']
    img_plot_area = {key: max(value, 0) for key, value in plot_area.items()}
    return img_plot_area


def get_legend_area(bbox_item_list):
    if bbox_item_list == [] or {}:
        return ()
    bbox_list = []
    # print(bbox_item_list)
    for bbox_item in bbox_item_list:
        # print(bbox_item["bbox"])
        bbox_list.append(bbox_item["bbox"])
    x0 = sorted(bbox_list, key = lambda i:i[0])[0][0]
    y0 = sorted(bbox_list, key = lambda i:i[1])[0][1]
    x1 = sorted(bbox_list, key = lambda i:i[2])[-1][2]
    y1 = sorted(bbox_list, key = lambda i:i[3])[-1][3]
    # color = (random.random()*255,random.random()*255,random.random()*255)
    # cv2.rectangle(img,(int(x0),int(y0)),(int(x1),int(y1)),color,2)
    # cv2.putText(img, bbox_name, (int(x0),int(y0)), cv2.FONT_HERSHEY_PLAIN, 1.2, color, 1, cv2.LINE_AA)
    return (x0,y0,x1,y1)


def crop_to_plot_area(img, annot, crop_margin=1):
    plot_area = get_plot_area(annot)
    # print('crop_margin:', crop_margin)
    plot_x = plot_area['x0'] + crop_margin; plot_y = plot_area['y0'] + crop_margin;
    plot_w = plot_area['width'] - 2*crop_margin; plot_h = plot_area['height'] - 2*crop_margin
    # print(img.shape)
    # print(plot_area)
    # print(plot_x, plot_y, plot_w, plot_h)
    # crop out so we only have the plot area
    cropped_img = img[plot_y:plot_y+plot_h, plot_x:plot_x+plot_w].copy()
    # print(cropped_img.shape)
    return cropped_img, (plot_x, plot_y)


def clean_nonline_elements(img, annot, legend_margin=1):
    im = Image.fromarray(img)
    js_obj = annot
    imd = ImageDraw.Draw(im)
    if js_obj['task6']['input']['task4_output'] is not None:
        plot_bb = js_obj['task6']['input']['task4_output']['_plot_bb']
        ploth, plotw , x0, y0 = plot_bb['height'], plot_bb['width'], plot_bb['x0'], plot_bb['y0']
        ctp = js_obj['task6']['input']['task1_output']['chart_type'] 
        tb =  js_obj['task6']['input']['task2_output']['text_blocks']
        legend_boxes = get_legend_boxes(annot=js_obj)
        # lp =  js_obj['task6']['input']['task5_output']['legend_pairs']
        ln_data = js_obj['task6']['output']['visual elements']['lines']
        x_axis = js_obj['task6']['input']['task4_output']['axes']['x-axis']
        y_axis = js_obj['task6']['input']['task4_output']['axes']['y-axis']
        for pt in x_axis :
            x_, y_ = pt['tick_pt']['x'], pt['tick_pt']['y']
            shape = [(x_, y_), (x_+2, y_+5)]
            # print('axis tb', [(x_, y_), (x_+2, y_+5)])
            cl = ImageStat.Stat(im).median
            imd.rectangle(shape, fill =tuple(cl),outline=None)
        for pt in y_axis :
            x_, y_ = pt['tick_pt']['x'], pt['tick_pt']['y']
            shape = [(x_, y_), (x_+5, y_+2)]
            # print('axis tb', [(x_, y_), (x_+2, y_+5)])
            cl = ImageStat.Stat(im).median
            imd.rectangle(shape, fill =tuple(cl),outline=None)

        ## remove text box
        for bx in tb :
            poly = bx['polygon'] if 'polygon' in bx else bx['bb']
            # print(poly)
            # Handle adobe synth format..
            if 'height' in poly:
                x_min = poly['x0']
                x_max = poly['x0'] + poly['width'] - 1
                y_min = poly['y0']
                y_max = poly['y0'] + poly['height'] - 1
            else:
                x_min = min(int(poly['x0']), int(poly['x1']), int(poly['x2']), int(poly['x3']))
                x_max = max(int(poly['x0']), int(poly['x1']), int(poly['x2']), int(poly['x3']))
                y_min = min(int(poly['y0']), int(poly['y1']), int(poly['y2']), int(poly['y3']))
                y_max = max(int(poly['y0']), int(poly['y1']), int(poly['y2']), int(poly['y3']))
            # print(x_min,x_max, y_min,y_max)
            # img_[y_min:y_max, x_min :x_max, :] = 255
            shape = [(x_min, y_min), (x_max, y_max)]
            # print('removed tb', [(x_min, y_min), (x_max, y_max)])
            cl = ImageStat.Stat(im).median
            imd.rectangle(shape, fill =tuple(cl),outline=None)
        
        ## remove legend
        if legend_boxes:
            x_min, y_min, x_max, y_max = get_legend_area(legend_boxes)
            x_min -= legend_margin; y_min -= legend_margin
            x_max += legend_margin; y_max += legend_margin

            shape = [(x_min, y_min), (x_max, y_max)]
            # print('removed leg', [(x_min, y_min), (x_max, y_max)])
            cl = ImageStat.Stat(im).median
            imd.rectangle(shape, fill = tuple(cl),outline=None)

        # imd.rectangle(shape, fill=(0,0,0),outline=None)
        # print('crop', (x0, y0, x0+plotw, y0+ploth))
        # im = im.crop((x0, y0, x0+plotw, y0+ploth))
        return np.array(im)
    
    return img

def resize(image, width=None, height=None, inter=cv2.INTER_AREA):
    # (Borrowed from imutils)initialize the dimensions of the image to be resized and
    # grab the image size
    dim = None
    (h, w) = image.shape[:2]

    # if both the width and height are None, then return the
    # original image
    if width is None and height is None:
        return image

    # check to see if the width is None
    if width is None:
        # calculate the ratio of the height and construct the
        # dimensions
        r = height / float(h)
        dim = (int(w * r), height)

    # otherwise, the height is None
    else:
        # calculate the ratio of the width and construct the
        # dimensions
        r = width / float(w)
        dim = (width, int(h * r))

    # resize the image
    resized = cv2.resize(image, dim, interpolation=inter)

    # return the resized image
    return resized


def _get_interpolation(inter_string):
    
    # In case it's already an interpolation code..
    if isinstance(inter_string, int):
        return inter_string
    
    inter_string = inter_string.lower().strip()
    inter_methods = {'area':cv2.INTER_AREA,
                     'linear':cv2.INTER_LINEAR,
                     'cubic':cv2.INTER_CUBIC,
                     'nearest':cv2.INTER_NEAREST}
    
    if inter_string not in inter_methods:
        raise Exception("Unknown Interpolation Method: '{}'".format(inter_string))
    
    return inter_methods[inter_string]


def padd_square(img, desired_size, padd_color=255):
    """
        resize and square padd
        img: np.array of image shaped (h,w,c)
        desired_size: int size of the image after resize and padding
    """
    
    if padd_color==255 and img.ndim == 3:
        padd_color = [255, 255, 255]
    
    size = img.shape[:2]
    delta_w = desired_size - size[1]
    delta_h = desired_size - size[0]
    top, bottom = delta_h//2, delta_h-(delta_h//2)
    left, right = delta_w//2, delta_w-(delta_w//2)

    new_img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT,
    value=padd_color)

    return new_img, (left, top)


def get_clean_input(img, annot, crop_to_plot=True, remove_text_legend=True, legend_margin=1, crop_margin=1, max_size=512, padd=True):
    """
        img: rgb image of line chart
        annot: json obj of PMC groundtruth
        max_size: resize max dimension of input to this size (maintaining aspect ratio)
        padd: whether to square padd the image after resizing
        crop_to_plot: whether to crop the chart image to plot area based on annotation provided
        remove_text_legend: whether to remove the text and legend boxes from chart image based on annotation provided
        returns: rgb image of cleaned line chart with plot area cropped
    """

    clean_img = img if not remove_text_legend else clean_nonline_elements(img, annot, legend_margin)
    
    if crop_to_plot:
        clean_img, (tx_crop, ty_crop)= crop_to_plot_area(clean_img, annot, crop_margin)
    else:
        tx_crop, ty_crop = 0,0
    
    sx, sy= 1,1
    tx_padd, ty_padd = 0,0

    h_cropped, w_cropped = clean_img.shape[:2]
    if max_size:
        if clean_img.shape[0] > clean_img.shape[1]:
            clean_img = resize(clean_img, height=max_size)
        else:
            clean_img = resize(clean_img, width=max_size)
        
        sx, sy = float(clean_img.shape[1])/w_cropped, float(clean_img.shape[0])/h_cropped

    if padd:
        clean_img, (tx_padd, ty_padd) = padd_square(clean_img, max_size)

    transformation = (sx, sy, tx_crop, ty_crop, tx_padd, ty_padd)
    
    return clean_img, transformation

# with open(annot_path, 'r') as f:
    # annot = json.load(f)