This commit is contained in:
191
tools/end2end/eval_end2end.py
Normal file
191
tools/end2end/eval_end2end.py
Normal file
@@ -0,0 +1,191 @@
|
||||
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import shapely
|
||||
from shapely.geometry import Polygon
|
||||
import numpy as np
|
||||
from collections import defaultdict
|
||||
import operator
|
||||
import editdistance
|
||||
|
||||
|
||||
def strQ2B(ustring):
|
||||
rstring = ""
|
||||
for uchar in ustring:
|
||||
inside_code = ord(uchar)
|
||||
if inside_code == 12288:
|
||||
inside_code = 32
|
||||
elif inside_code >= 65281 and inside_code <= 65374:
|
||||
inside_code -= 65248
|
||||
rstring += chr(inside_code)
|
||||
return rstring
|
||||
|
||||
|
||||
def polygon_from_str(polygon_points):
|
||||
"""
|
||||
Create a shapely polygon object from gt or dt line.
|
||||
"""
|
||||
polygon_points = np.array(polygon_points).reshape(4, 2)
|
||||
polygon = Polygon(polygon_points).convex_hull
|
||||
return polygon
|
||||
|
||||
|
||||
def polygon_iou(poly1, poly2):
|
||||
"""
|
||||
Intersection over union between two shapely polygons.
|
||||
"""
|
||||
if not poly1.intersects(poly2): # this test is fast and can accelerate calculation
|
||||
iou = 0
|
||||
else:
|
||||
try:
|
||||
inter_area = poly1.intersection(poly2).area
|
||||
union_area = poly1.area + poly2.area - inter_area
|
||||
iou = float(inter_area) / union_area
|
||||
except shapely.geos.TopologicalError:
|
||||
# except Exception as e:
|
||||
# print(e)
|
||||
print("shapely.geos.TopologicalError occurred, iou set to 0")
|
||||
iou = 0
|
||||
return iou
|
||||
|
||||
|
||||
def ed(str1, str2):
|
||||
return editdistance.eval(str1, str2)
|
||||
|
||||
|
||||
def e2e_eval(gt_dir, res_dir, ignore_blank=False):
|
||||
print("start testing...")
|
||||
iou_thresh = 0.5
|
||||
val_names = os.listdir(gt_dir)
|
||||
num_gt_chars = 0
|
||||
gt_count = 0
|
||||
dt_count = 0
|
||||
hit = 0
|
||||
ed_sum = 0
|
||||
|
||||
for i, val_name in enumerate(val_names):
|
||||
with open(os.path.join(gt_dir, val_name), encoding="utf-8") as f:
|
||||
gt_lines = [o.strip() for o in f.readlines()]
|
||||
gts = []
|
||||
ignore_masks = []
|
||||
for line in gt_lines:
|
||||
parts = line.strip().split("\t")
|
||||
# ignore illegal data
|
||||
if len(parts) < 9:
|
||||
continue
|
||||
assert len(parts) < 11
|
||||
if len(parts) == 9:
|
||||
gts.append(parts[:8] + [""])
|
||||
else:
|
||||
gts.append(parts[:8] + [parts[-1]])
|
||||
|
||||
ignore_masks.append(parts[8])
|
||||
|
||||
val_path = os.path.join(res_dir, val_name)
|
||||
if not os.path.exists(val_path):
|
||||
dt_lines = []
|
||||
else:
|
||||
with open(val_path, encoding="utf-8") as f:
|
||||
dt_lines = [o.strip() for o in f.readlines()]
|
||||
dts = []
|
||||
for line in dt_lines:
|
||||
# print(line)
|
||||
parts = line.strip().split("\t")
|
||||
assert len(parts) < 10, "line error: {}".format(line)
|
||||
if len(parts) == 8:
|
||||
dts.append(parts + [""])
|
||||
else:
|
||||
dts.append(parts)
|
||||
|
||||
dt_match = [False] * len(dts)
|
||||
gt_match = [False] * len(gts)
|
||||
all_ious = defaultdict(tuple)
|
||||
for index_gt, gt in enumerate(gts):
|
||||
gt_coors = [float(gt_coor) for gt_coor in gt[0:8]]
|
||||
gt_poly = polygon_from_str(gt_coors)
|
||||
for index_dt, dt in enumerate(dts):
|
||||
dt_coors = [float(dt_coor) for dt_coor in dt[0:8]]
|
||||
dt_poly = polygon_from_str(dt_coors)
|
||||
iou = polygon_iou(dt_poly, gt_poly)
|
||||
if iou >= iou_thresh:
|
||||
all_ious[(index_gt, index_dt)] = iou
|
||||
sorted_ious = sorted(all_ious.items(), key=operator.itemgetter(1), reverse=True)
|
||||
sorted_gt_dt_pairs = [item[0] for item in sorted_ious]
|
||||
|
||||
# matched gt and dt
|
||||
for gt_dt_pair in sorted_gt_dt_pairs:
|
||||
index_gt, index_dt = gt_dt_pair
|
||||
if gt_match[index_gt] == False and dt_match[index_dt] == False:
|
||||
gt_match[index_gt] = True
|
||||
dt_match[index_dt] = True
|
||||
if ignore_blank:
|
||||
gt_str = strQ2B(gts[index_gt][8]).replace(" ", "")
|
||||
dt_str = strQ2B(dts[index_dt][8]).replace(" ", "")
|
||||
else:
|
||||
gt_str = strQ2B(gts[index_gt][8])
|
||||
dt_str = strQ2B(dts[index_dt][8])
|
||||
if ignore_masks[index_gt] == "0":
|
||||
ed_sum += ed(gt_str, dt_str)
|
||||
num_gt_chars += len(gt_str)
|
||||
if gt_str == dt_str:
|
||||
hit += 1
|
||||
gt_count += 1
|
||||
dt_count += 1
|
||||
|
||||
# unmatched dt
|
||||
for tindex, dt_match_flag in enumerate(dt_match):
|
||||
if dt_match_flag == False:
|
||||
dt_str = dts[tindex][8]
|
||||
gt_str = ""
|
||||
ed_sum += ed(dt_str, gt_str)
|
||||
dt_count += 1
|
||||
|
||||
# unmatched gt
|
||||
for tindex, gt_match_flag in enumerate(gt_match):
|
||||
if gt_match_flag == False and ignore_masks[tindex] == "0":
|
||||
dt_str = ""
|
||||
gt_str = gts[tindex][8]
|
||||
ed_sum += ed(gt_str, dt_str)
|
||||
num_gt_chars += len(gt_str)
|
||||
gt_count += 1
|
||||
|
||||
eps = 1e-9
|
||||
print("hit, dt_count, gt_count", hit, dt_count, gt_count)
|
||||
precision = hit / (dt_count + eps)
|
||||
recall = hit / (gt_count + eps)
|
||||
fmeasure = 2.0 * precision * recall / (precision + recall + eps)
|
||||
avg_edit_dist_img = ed_sum / len(val_names)
|
||||
avg_edit_dist_field = ed_sum / (gt_count + eps)
|
||||
character_acc = 1 - ed_sum / (num_gt_chars + eps)
|
||||
|
||||
print("character_acc: %.2f" % (character_acc * 100) + "%")
|
||||
print("avg_edit_dist_field: %.2f" % (avg_edit_dist_field))
|
||||
print("avg_edit_dist_img: %.2f" % (avg_edit_dist_img))
|
||||
print("precision: %.2f" % (precision * 100) + "%")
|
||||
print("recall: %.2f" % (recall * 100) + "%")
|
||||
print("fmeasure: %.2f" % (fmeasure * 100) + "%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# if len(sys.argv) != 3:
|
||||
# print("python3 ocr_e2e_eval.py gt_dir res_dir")
|
||||
# exit(-1)
|
||||
# gt_folder = sys.argv[1]
|
||||
# pred_folder = sys.argv[2]
|
||||
gt_folder = sys.argv[1]
|
||||
pred_folder = sys.argv[2]
|
||||
e2e_eval(gt_folder, pred_folder)
|
||||
Reference in New Issue
Block a user