Trjtdtk

This is what I mean as document text image:

I want to label the texts in image as separate blocks and my model should detect these labels as classes.

NOTE:

This is how the end result should be like:

The labels like Block 1, Block 2, Block 3,.. should be Logo, Title, Date,.. Others, etc.

Work done:

First approach : I tried to implement this method via Object Detection, it didn't work. It didn't even detect any text.

Second approach : Then I tried it using PixelLink. As this model is build for scene text detection, it detected each and every text in the image. But this method can detect multiple lines of text if the threshold values are increased.
But I have no idea how do I add labels to the text blocks.

PIXEL_CLS_WEIGHT_all_ones = 'PIXEL_CLS_WEIGHT_all_ones' 
PIXEL_CLS_WEIGHT_bbox_balanced = 'PIXEL_CLS_WEIGHT_bbox_balanced'
PIXEL_NEIGHBOUR_TYPE_4 = 'PIXEL_NEIGHBOUR_TYPE_4'
PIXEL_NEIGHBOUR_TYPE_8 = 'PIXEL_NEIGHBOUR_TYPE_8'

DECODE_METHOD_join = 'DECODE_METHOD_join'


def get_neighbours_8(x, y):
 """
 Get 8 neighbours of point(x, y)
 """
 return [(x - 1, y - 1), (x, y - 1), (x + 1, y - 1), 
 (x - 1, y), (x + 1, y), 
 (x - 1, y + 1), (x, y + 1), (x + 1, y + 1)]


def get_neighbours_4(x, y):
 return [(x - 1, y), (x + 1, y), (x, y + 1), (x, y - 1)]


def get_neighbours(x, y):
 import config
 neighbour_type = config.pixel_neighbour_type
 if neighbour_type == PIXEL_NEIGHBOUR_TYPE_4:
 return get_neighbours_4(x, y)
 else:
 return get_neighbours_8(x, y)

def get_neighbours_fn():
 import config
 neighbour_type = config.pixel_neighbour_type
 if neighbour_type == PIXEL_NEIGHBOUR_TYPE_4:
 return get_neighbours_4, 4
 else:
 return get_neighbours_8, 8



def is_valid_cord(x, y, w, h):
 """
 Tell whether the 2D coordinate (x, y) is valid or not.
 If valid, it should be on an h x w image
 """
 return x >=0 and x < w and y >= 0 and y < h;

#=====================Ground Truth Calculation Begin==================
def tf_cal_gt_for_single_image(xs, ys, labels):
 pixel_cls_label, pixel_cls_weight, 
 pixel_link_label, pixel_link_weight = 
 tf.py_func(
 cal_gt_for_single_image, 
 [xs, ys, labels],
 [tf.int32, tf.float32, tf.int32, tf.float32]
 )
 import config
 score_map_shape = config.score_map_shape
 num_neighbours = config.num_neighbours
 h, w = score_map_shape
 pixel_cls_label.set_shape(score_map_shape)
 pixel_cls_weight.set_shape(score_map_shape)
 pixel_link_label.set_shape([h, w, num_neighbours])
 pixel_link_weight.set_shape([h, w, num_neighbours])
 return pixel_cls_label, pixel_cls_weight, 
 pixel_link_label, pixel_link_weight


def cal_gt_for_single_image(normed_xs, normed_ys, labels):
 """
 Args:
 xs, ys: both in shape of (N, 4), 
 and N is the number of bboxes,
 their values are normalized to [0,1]
 labels: shape = (N,), only two values are allowed:
 -1: ignored
 1: text
 Return:
 pixel_cls_label
 pixel_cls_weight
 pixel_link_label
 pixel_link_weight
 """
 import config
 score_map_shape = config.score_map_shape
 pixel_cls_weight_method = config.pixel_cls_weight_method
 h, w = score_map_shape
 text_label = config.text_label
 ignore_label = config.ignore_label
 background_label = config.background_label
 num_neighbours = config.num_neighbours
 bbox_border_width = config.bbox_border_width
 pixel_cls_border_weight_lambda = config.pixel_cls_border_weight_lambda

 # validate the args
 assert np.ndim(normed_xs) == 2
 assert np.shape(normed_xs)[-1] == 4
 assert np.shape(normed_xs) == np.shape(normed_ys)
 assert len(normed_xs) == len(labels)

# assert set(labels).issubset(set([text_label, ignore_label, background_label]))

 num_positive_bboxes = np.sum(np.asarray(labels) == text_label)
 # rescale normalized xys to absolute values
 xs = normed_xs * w
 ys = normed_ys * h

 # initialize ground truth values
 mask = np.zeros(score_map_shape, dtype = np.int32)
 pixel_cls_label = np.ones(score_map_shape, dtype = np.int32) * background_label
 pixel_cls_weight = np.zeros(score_map_shape, dtype = np.float32)

 pixel_link_label = np.zeros((h, w, num_neighbours), dtype = np.int32)
 pixel_link_weight = np.ones((h, w, num_neighbours), dtype = np.float32)

 # find overlapped pixels, and consider them as ignored in pixel_cls_weight
 # and pixels in ignored bboxes are ignored as well
 # That is to say, only the weights of not ignored pixels are set to 1

 ## get the masks of all bboxes
 bbox_masks = []
 pos_mask = mask.copy()
 for bbox_idx, (bbox_xs, bbox_ys) in enumerate(zip(xs, ys)):
 if labels[bbox_idx] == background_label:
 continue

 bbox_mask = mask.copy()

 bbox_points = zip(bbox_xs, bbox_ys)
 bbox_contours = util.img.points_to_contours(bbox_points)
 util.img.draw_contours(bbox_mask, bbox_contours, idx = -1, 
 color = 1, border_width = -1)

 bbox_masks.append(bbox_mask)

 if labels[bbox_idx] == text_label:
 pos_mask += bbox_mask

 # treat overlapped in-bbox pixels as negative, 
 # and non-overlapped ones as positive
 pos_mask = np.asarray(pos_mask == 1, dtype = np.int32)
 num_positive_pixels = np.sum(pos_mask)

 ## add all bbox_maskes, find non-overlapping pixels
 sum_mask = np.sum(bbox_masks, axis = 0)
 not_overlapped_mask = sum_mask == 1


 ## gt and weight calculation
 for bbox_idx, bbox_mask in enumerate(bbox_masks):
 bbox_label = labels[bbox_idx]
 if bbox_label == ignore_label:
 # for ignored bboxes, only non-overlapped pixels are encoded as ignored 
 bbox_ignore_pixel_mask = bbox_mask * not_overlapped_mask
 pixel_cls_label += bbox_ignore_pixel_mask * ignore_label
 continue

 if labels[bbox_idx] == background_label:
 continue
 # from here on, only text boxes left.

 # for positive bboxes, all pixels within it and pos_mask are positive
 bbox_positive_pixel_mask = bbox_mask * pos_mask
 # background or text is encoded into cls gt
 pixel_cls_label += bbox_positive_pixel_mask * bbox_label

 # for the pixel cls weights, only positive pixels are set to ones
 if pixel_cls_weight_method == PIXEL_CLS_WEIGHT_all_ones:
 pixel_cls_weight += bbox_positive_pixel_mask
 elif pixel_cls_weight_method == PIXEL_CLS_WEIGHT_bbox_balanced:
 # let N denote num_positive_pixels
 # weight per pixel = N /num_positive_bboxes / n_pixels_in_bbox
 # so all pixel weights in this bbox sum to N/num_positive_bboxes
 # and all pixels weights in this image sum to N, the same
 # as setting all weights to 1
 num_bbox_pixels = np.sum(bbox_positive_pixel_mask)
 if num_bbox_pixels > 0: 
 per_bbox_weight = num_positive_pixels * 1.0 / num_positive_bboxes
 per_pixel_weight = per_bbox_weight / num_bbox_pixels
 pixel_cls_weight += bbox_positive_pixel_mask * per_pixel_weight
 else:
 raise ValueError, 'pixel_cls_weight_method not supported:%s'
 %(pixel_cls_weight_method)


 ## calculate the labels and weights of links
 ### for all pixels in bboxes, all links are positive at first
 bbox_point_cords = np.where(bbox_positive_pixel_mask)
 pixel_link_label[bbox_point_cords] = 1


 ## the border of bboxes might be distored because of overlapping
 ## so recalculate it, and find the border mask 
 new_bbox_contours = util.img.find_contours(bbox_positive_pixel_mask)
 bbox_border_mask = mask.copy()
 util.img.draw_contours(bbox_border_mask, new_bbox_contours, -1, 
 color = 1, border_width = bbox_border_width * 2 + 1)
 bbox_border_mask *= bbox_positive_pixel_mask
 bbox_border_cords = np.where(bbox_border_mask)

 ## give more weight to the border pixels if configured
 pixel_cls_weight[bbox_border_cords] *= pixel_cls_border_weight_lambda

 ### change link labels according to their neighbour status
 border_points = zip(*bbox_border_cords)
 def in_bbox(nx, ny):
 return bbox_positive_pixel_mask[ny, nx]

 for y, x in border_points:
 neighbours = get_neighbours(x, y)
 for n_idx, (nx, ny) in enumerate(neighbours):
 if not is_valid_cord(nx, ny, w, h) or not in_bbox(nx, ny):
 pixel_link_label[y, x, n_idx] = 0

 pixel_cls_weight = np.asarray(pixel_cls_weight, dtype = np.float32) 
 pixel_link_weight *= np.expand_dims(pixel_cls_weight, axis = -1)

# try:
# np.testing.assert_almost_equal(np.sum(pixel_cls_weight), num_positive_pixels, decimal = 1)
# except:
# print num_positive_pixels, np.sum(pixel_cls_label), np.sum(pixel_cls_weight)
# import pdb
# pdb.set_trace()
 return pixel_cls_label, pixel_cls_weight, pixel_link_label, pixel_link_weight

#=====================Ground Truth Calculation End====================


#============================Decode Begin=============================

def tf_decode_score_map_to_mask_in_batch(pixel_cls_scores, pixel_link_scores):
 masks = tf.py_func(decode_batch, 
 [pixel_cls_scores, pixel_link_scores], tf.int32)
 b, h, w = pixel_cls_scores.shape.as_list()
 masks.set_shape([b, h, w])
 return masks



def decode_batch(pixel_cls_scores, pixel_link_scores, 
 pixel_conf_threshold = None, link_conf_threshold = None):
 import config

 if pixel_conf_threshold is None:
 pixel_conf_threshold = config.pixel_conf_threshold

 if link_conf_threshold is None:
 link_conf_threshold = config.link_conf_threshold

 batch_size = pixel_cls_scores.shape[0]
 batch_mask = []
 for image_idx in xrange(batch_size):
 image_pos_pixel_scores = pixel_cls_scores[image_idx, :, :]
 image_pos_link_scores = pixel_link_scores[image_idx, :, :, :] 
 mask = decode_image(
 image_pos_pixel_scores, image_pos_link_scores, 
 pixel_conf_threshold, link_conf_threshold
 )
 batch_mask.append(mask)
 return np.asarray(batch_mask, np.int32)

# @util.dec.print_calling_in_short
# @util.dec.timeit
def decode_image(pixel_scores, link_scores, 
 pixel_conf_threshold, link_conf_threshold):
 import config
 if config.decode_method == DECODE_METHOD_join:
 mask = decode_image_by_join(pixel_scores, link_scores, 
 pixel_conf_threshold, link_conf_threshold)
 return mask
 elif config.decode_method == DECODE_METHOD_border_split:
 return decode_image_by_border(pixel_scores, link_scores, 
 pixel_conf_threshold, link_conf_threshold)
 else:
 raise ValueError('Unknow decode method:%s'%(config.decode_method))


import pyximport; pyximport.install() 
from pixel_link_decode import decode_image_by_join

def min_area_rect(cnt):
 """
 Args:
 xs: numpy ndarray with shape=(N,4). N is the number of oriented bboxes. 4 contains [x1, x2, x3, x4]
 ys: numpy ndarray with shape=(N,4), [y1, y2, y3, y4]
 Note that [(x1, y1), (x2, y2), (x3, y3), (x4, y4)] can represent an oriented bbox.
 Return:
 the oriented rects sorrounding the box, in the format:[cx, cy, w, h, theta]. 
 """
 rect = cv2.minAreaRect(cnt)
 cx, cy = rect[0]
 w, h = rect[1]
 theta = rect[2]
 box = [cx, cy, w, h, theta]
 return box, w * h

def rect_to_xys(rect, image_shape):
 """Convert rect to xys, i.e., eight points
 The `image_shape` is used to to make sure all points return are valid, i.e., within image area
 """
 h, w = image_shape[0:2]
 def get_valid_x(x):
 if x < 0:
 return 0
 if x >= w:
 return w - 1
 return x

 def get_valid_y(y):
 if y < 0:
 return 0
 if y >= h:
 return h - 1
 return y

 rect = ((rect[0], rect[1]), (rect[2], rect[3]), rect[4])
 points = cv2.cv.BoxPoints(rect)
 points = np.int0(points)
 for i_xy, (x, y) in enumerate(points):
 x = get_valid_x(x)
 y = get_valid_y(y)
 points[i_xy, :] = [x, y]
 points = np.reshape(points, -1)
 return points

# @util.dec.print_calling_in_short
# @util.dec.timeit
def mask_to_bboxes(mask, image_shape = None, min_area = None, 
 min_height = None, min_aspect_ratio = None):
 import config
 feed_shape = config.train_image_shape

 if image_shape is None:
 image_shape = feed_shape

 image_h, image_w = image_shape[0:2]

 if min_area is None:
 min_area = config.min_area

 if min_height is None:
 min_height = config.min_height
 bboxes = []
 max_bbox_idx = mask.max()
 mask = util.img.resize(img = mask, size = (image_w, image_h), 
 interpolation = cv2.INTER_NEAREST)

 for bbox_idx in xrange(1, max_bbox_idx + 1):
 bbox_mask = mask == bbox_idx
# if bbox_mask.sum() < 10:
# continue
 cnts = util.img.find_contours(bbox_mask)
 if len(cnts) == 0:
 continue
 cnt = cnts[0]
 rect, rect_area = min_area_rect(cnt)

 w, h = rect[2:-1]
 if min(w, h) < min_height:
 continue

 if rect_area < min_area:
 continue

# if max(w, h) * 1.0 / min(w, h) < 2:
# continue
 xys = rect_to_xys(rect, image_shape)
 bboxes.append(xys)

 return bboxes

Any suggestions?

Is there any approach that is more suitable for the problem I'm trying to solve?

If you want something like that, you can try to test it with already existing & online app like the following :

https://jinapdf.com/image-to-text-file.php

If this is what you want, you can create one :

1) First locate boundaries for text

2) Convert it to text

This is quite complicated bu you can still use what is called OCR (Optical Character Recognition) and therefore search for it in github and use open source projects as a starter like the following :

https://github.com/prabhakar267/ocr-convert-image-to-text