This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Time : 2019/12/5 15:36
|
||||
# @Author : zhoujun
|
||||
|
||||
from .quad_metric import QuadMetric
|
||||
@@ -0,0 +1,474 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
import math
|
||||
from collections import namedtuple
|
||||
import numpy as np
|
||||
from shapely.geometry import Polygon
|
||||
|
||||
|
||||
class DetectionDetEvalEvaluator(object):
|
||||
def __init__(
|
||||
self,
|
||||
area_recall_constraint=0.8,
|
||||
area_precision_constraint=0.4,
|
||||
ev_param_ind_center_diff_thr=1,
|
||||
mtype_oo_o=1.0,
|
||||
mtype_om_o=0.8,
|
||||
mtype_om_m=1.0,
|
||||
):
|
||||
self.area_recall_constraint = area_recall_constraint
|
||||
self.area_precision_constraint = area_precision_constraint
|
||||
self.ev_param_ind_center_diff_thr = ev_param_ind_center_diff_thr
|
||||
self.mtype_oo_o = mtype_oo_o
|
||||
self.mtype_om_o = mtype_om_o
|
||||
self.mtype_om_m = mtype_om_m
|
||||
|
||||
def evaluate_image(self, gt, pred):
|
||||
def get_union(pD, pG):
|
||||
return Polygon(pD).union(Polygon(pG)).area
|
||||
|
||||
def get_intersection_over_union(pD, pG):
|
||||
return get_intersection(pD, pG) / get_union(pD, pG)
|
||||
|
||||
def get_intersection(pD, pG):
|
||||
return Polygon(pD).intersection(Polygon(pG)).area
|
||||
|
||||
def one_to_one_match(row, col):
|
||||
cont = 0
|
||||
for j in range(len(recallMat[0])):
|
||||
if (
|
||||
recallMat[row, j] >= self.area_recall_constraint
|
||||
and precisionMat[row, j] >= self.area_precision_constraint
|
||||
):
|
||||
cont = cont + 1
|
||||
if cont != 1:
|
||||
return False
|
||||
cont = 0
|
||||
for i in range(len(recallMat)):
|
||||
if (
|
||||
recallMat[i, col] >= self.area_recall_constraint
|
||||
and precisionMat[i, col] >= self.area_precision_constraint
|
||||
):
|
||||
cont = cont + 1
|
||||
if cont != 1:
|
||||
return False
|
||||
|
||||
if (
|
||||
recallMat[row, col] >= self.area_recall_constraint
|
||||
and precisionMat[row, col] >= self.area_precision_constraint
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
def num_overlaps_gt(gtNum):
|
||||
cont = 0
|
||||
for detNum in range(len(detRects)):
|
||||
if detNum not in detDontCareRectsNum:
|
||||
if recallMat[gtNum, detNum] > 0:
|
||||
cont = cont + 1
|
||||
return cont
|
||||
|
||||
def num_overlaps_det(detNum):
|
||||
cont = 0
|
||||
for gtNum in range(len(recallMat)):
|
||||
if gtNum not in gtDontCareRectsNum:
|
||||
if recallMat[gtNum, detNum] > 0:
|
||||
cont = cont + 1
|
||||
return cont
|
||||
|
||||
def is_single_overlap(row, col):
|
||||
if num_overlaps_gt(row) == 1 and num_overlaps_det(col) == 1:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def one_to_many_match(gtNum):
|
||||
many_sum = 0
|
||||
detRects = []
|
||||
for detNum in range(len(recallMat[0])):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and detNum not in detDontCareRectsNum
|
||||
):
|
||||
if precisionMat[gtNum, detNum] >= self.area_precision_constraint:
|
||||
many_sum += recallMat[gtNum, detNum]
|
||||
detRects.append(detNum)
|
||||
if round(many_sum, 4) >= self.area_recall_constraint:
|
||||
return True, detRects
|
||||
else:
|
||||
return False, []
|
||||
|
||||
def many_to_one_match(detNum):
|
||||
many_sum = 0
|
||||
gtRects = []
|
||||
for gtNum in range(len(recallMat)):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and gtNum not in gtDontCareRectsNum
|
||||
):
|
||||
if recallMat[gtNum, detNum] >= self.area_recall_constraint:
|
||||
many_sum += precisionMat[gtNum, detNum]
|
||||
gtRects.append(gtNum)
|
||||
if round(many_sum, 4) >= self.area_precision_constraint:
|
||||
return True, gtRects
|
||||
else:
|
||||
return False, []
|
||||
|
||||
def center_distance(r1, r2):
|
||||
return ((np.mean(r1, axis=0) - np.mean(r2, axis=0)) ** 2).sum() ** 0.5
|
||||
|
||||
def diag(r):
|
||||
r = np.array(r)
|
||||
return (
|
||||
(r[:, 0].max() - r[:, 0].min()) ** 2
|
||||
+ (r[:, 1].max() - r[:, 1].min()) ** 2
|
||||
) ** 0.5
|
||||
|
||||
perSampleMetrics = {}
|
||||
|
||||
recall = 0
|
||||
precision = 0
|
||||
hmean = 0
|
||||
recallAccum = 0.0
|
||||
precisionAccum = 0.0
|
||||
gtRects = []
|
||||
detRects = []
|
||||
gtPolPoints = []
|
||||
detPolPoints = []
|
||||
gtDontCareRectsNum = (
|
||||
[]
|
||||
) # Array of Ground Truth Rectangles' keys marked as don't Care
|
||||
detDontCareRectsNum = (
|
||||
[]
|
||||
) # Array of Detected Rectangles' matched with a don't Care GT
|
||||
pairs = []
|
||||
evaluationLog = ""
|
||||
|
||||
recallMat = np.empty([1, 1])
|
||||
precisionMat = np.empty([1, 1])
|
||||
|
||||
for n in range(len(gt)):
|
||||
points = gt[n]["points"]
|
||||
# transcription = gt[n]['text']
|
||||
dontCare = gt[n]["ignore"]
|
||||
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
gtRects.append(points)
|
||||
gtPolPoints.append(points)
|
||||
if dontCare:
|
||||
gtDontCareRectsNum.append(len(gtRects) - 1)
|
||||
|
||||
evaluationLog += (
|
||||
"GT rectangles: "
|
||||
+ str(len(gtRects))
|
||||
+ (
|
||||
" (" + str(len(gtDontCareRectsNum)) + " don't care)\n"
|
||||
if len(gtDontCareRectsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
for n in range(len(pred)):
|
||||
points = pred[n]["points"]
|
||||
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
detRect = points
|
||||
detRects.append(detRect)
|
||||
detPolPoints.append(points)
|
||||
if len(gtDontCareRectsNum) > 0:
|
||||
for dontCareRectNum in gtDontCareRectsNum:
|
||||
dontCareRect = gtRects[dontCareRectNum]
|
||||
intersected_area = get_intersection(dontCareRect, detRect)
|
||||
rdDimensions = Polygon(detRect).area
|
||||
if rdDimensions == 0:
|
||||
precision = 0
|
||||
else:
|
||||
precision = intersected_area / rdDimensions
|
||||
if precision > self.area_precision_constraint:
|
||||
detDontCareRectsNum.append(len(detRects) - 1)
|
||||
break
|
||||
|
||||
evaluationLog += (
|
||||
"DET rectangles: "
|
||||
+ str(len(detRects))
|
||||
+ (
|
||||
" (" + str(len(detDontCareRectsNum)) + " don't care)\n"
|
||||
if len(detDontCareRectsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
if len(gtRects) == 0:
|
||||
recall = 1
|
||||
precision = 0 if len(detRects) > 0 else 1
|
||||
|
||||
if len(detRects) > 0:
|
||||
# Calculate recall and precision matrixes
|
||||
outputShape = [len(gtRects), len(detRects)]
|
||||
recallMat = np.empty(outputShape)
|
||||
precisionMat = np.empty(outputShape)
|
||||
gtRectMat = np.zeros(len(gtRects), np.int8)
|
||||
detRectMat = np.zeros(len(detRects), np.int8)
|
||||
for gtNum in range(len(gtRects)):
|
||||
for detNum in range(len(detRects)):
|
||||
rG = gtRects[gtNum]
|
||||
rD = detRects[detNum]
|
||||
intersected_area = get_intersection(rG, rD)
|
||||
rgDimensions = Polygon(rG).area
|
||||
rdDimensions = Polygon(rD).area
|
||||
recallMat[gtNum, detNum] = (
|
||||
0 if rgDimensions == 0 else intersected_area / rgDimensions
|
||||
)
|
||||
precisionMat[gtNum, detNum] = (
|
||||
0 if rdDimensions == 0 else intersected_area / rdDimensions
|
||||
)
|
||||
|
||||
# Find one-to-one matches
|
||||
evaluationLog += "Find one-to-one matches\n"
|
||||
for gtNum in range(len(gtRects)):
|
||||
for detNum in range(len(detRects)):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and gtNum not in gtDontCareRectsNum
|
||||
and detNum not in detDontCareRectsNum
|
||||
):
|
||||
match = one_to_one_match(gtNum, detNum)
|
||||
if match is True:
|
||||
# in deteval we have to make other validation before mark as one-to-one
|
||||
if is_single_overlap(gtNum, detNum) is True:
|
||||
rG = gtRects[gtNum]
|
||||
rD = detRects[detNum]
|
||||
normDist = center_distance(rG, rD)
|
||||
normDist /= diag(rG) + diag(rD)
|
||||
normDist *= 2.0
|
||||
if normDist < self.ev_param_ind_center_diff_thr:
|
||||
gtRectMat[gtNum] = 1
|
||||
detRectMat[detNum] = 1
|
||||
recallAccum += self.mtype_oo_o
|
||||
precisionAccum += self.mtype_oo_o
|
||||
pairs.append(
|
||||
{"gt": gtNum, "det": detNum, "type": "OO"}
|
||||
)
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ "\n"
|
||||
)
|
||||
else:
|
||||
evaluationLog += (
|
||||
"Match Discarded GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ " normDist: "
|
||||
+ str(normDist)
|
||||
+ " \n"
|
||||
)
|
||||
else:
|
||||
evaluationLog += (
|
||||
"Match Discarded GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ " not single overlap\n"
|
||||
)
|
||||
# Find one-to-many matches
|
||||
evaluationLog += "Find one-to-many matches\n"
|
||||
for gtNum in range(len(gtRects)):
|
||||
if gtNum not in gtDontCareRectsNum:
|
||||
match, matchesDet = one_to_many_match(gtNum)
|
||||
if match is True:
|
||||
evaluationLog += "num_overlaps_gt=" + str(
|
||||
num_overlaps_gt(gtNum)
|
||||
)
|
||||
# in deteval we have to make other validation before mark as one-to-one
|
||||
if num_overlaps_gt(gtNum) >= 2:
|
||||
gtRectMat[gtNum] = 1
|
||||
recallAccum += (
|
||||
self.mtype_oo_o
|
||||
if len(matchesDet) == 1
|
||||
else self.mtype_om_o
|
||||
)
|
||||
precisionAccum += (
|
||||
self.mtype_oo_o
|
||||
if len(matchesDet) == 1
|
||||
else self.mtype_om_o * len(matchesDet)
|
||||
)
|
||||
pairs.append(
|
||||
{
|
||||
"gt": gtNum,
|
||||
"det": matchesDet,
|
||||
"type": "OO" if len(matchesDet) == 1 else "OM",
|
||||
}
|
||||
)
|
||||
for detNum in matchesDet:
|
||||
detRectMat[detNum] = 1
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(matchesDet)
|
||||
+ "\n"
|
||||
)
|
||||
else:
|
||||
evaluationLog += (
|
||||
"Match Discarded GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(matchesDet)
|
||||
+ " not single overlap\n"
|
||||
)
|
||||
|
||||
# Find many-to-one matches
|
||||
evaluationLog += "Find many-to-one matches\n"
|
||||
for detNum in range(len(detRects)):
|
||||
if detNum not in detDontCareRectsNum:
|
||||
match, matchesGt = many_to_one_match(detNum)
|
||||
if match is True:
|
||||
# in deteval we have to make other validation before mark as one-to-one
|
||||
if num_overlaps_det(detNum) >= 2:
|
||||
detRectMat[detNum] = 1
|
||||
recallAccum += (
|
||||
self.mtype_oo_o
|
||||
if len(matchesGt) == 1
|
||||
else self.mtype_om_m * len(matchesGt)
|
||||
)
|
||||
precisionAccum += (
|
||||
self.mtype_oo_o
|
||||
if len(matchesGt) == 1
|
||||
else self.mtype_om_m
|
||||
)
|
||||
pairs.append(
|
||||
{
|
||||
"gt": matchesGt,
|
||||
"det": detNum,
|
||||
"type": "OO" if len(matchesGt) == 1 else "MO",
|
||||
}
|
||||
)
|
||||
for gtNum in matchesGt:
|
||||
gtRectMat[gtNum] = 1
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(matchesGt)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ "\n"
|
||||
)
|
||||
else:
|
||||
evaluationLog += (
|
||||
"Match Discarded GT #"
|
||||
+ str(matchesGt)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ " not single overlap\n"
|
||||
)
|
||||
|
||||
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
||||
if numGtCare == 0:
|
||||
recall = float(1)
|
||||
precision = float(0) if len(detRects) > 0 else float(1)
|
||||
else:
|
||||
recall = float(recallAccum) / numGtCare
|
||||
precision = (
|
||||
float(0)
|
||||
if (len(detRects) - len(detDontCareRectsNum)) == 0
|
||||
else float(precisionAccum)
|
||||
/ (len(detRects) - len(detDontCareRectsNum))
|
||||
)
|
||||
hmean = (
|
||||
0
|
||||
if (precision + recall) == 0
|
||||
else 2.0 * precision * recall / (precision + recall)
|
||||
)
|
||||
|
||||
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
||||
numDetCare = len(detRects) - len(detDontCareRectsNum)
|
||||
|
||||
perSampleMetrics = {
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"hmean": hmean,
|
||||
"pairs": pairs,
|
||||
"recallMat": [] if len(detRects) > 100 else recallMat.tolist(),
|
||||
"precisionMat": [] if len(detRects) > 100 else precisionMat.tolist(),
|
||||
"gtPolPoints": gtPolPoints,
|
||||
"detPolPoints": detPolPoints,
|
||||
"gtCare": numGtCare,
|
||||
"detCare": numDetCare,
|
||||
"gtDontCare": gtDontCareRectsNum,
|
||||
"detDontCare": detDontCareRectsNum,
|
||||
"recallAccum": recallAccum,
|
||||
"precisionAccum": precisionAccum,
|
||||
"evaluationLog": evaluationLog,
|
||||
}
|
||||
|
||||
return perSampleMetrics
|
||||
|
||||
def combine_results(self, results):
|
||||
numGt = 0
|
||||
numDet = 0
|
||||
methodRecallSum = 0
|
||||
methodPrecisionSum = 0
|
||||
|
||||
for result in results:
|
||||
numGt += result["gtCare"]
|
||||
numDet += result["detCare"]
|
||||
methodRecallSum += result["recallAccum"]
|
||||
methodPrecisionSum += result["precisionAccum"]
|
||||
|
||||
methodRecall = 0 if numGt == 0 else methodRecallSum / numGt
|
||||
methodPrecision = 0 if numDet == 0 else methodPrecisionSum / numDet
|
||||
methodHmean = (
|
||||
0
|
||||
if methodRecall + methodPrecision == 0
|
||||
else 2 * methodRecall * methodPrecision / (methodRecall + methodPrecision)
|
||||
)
|
||||
|
||||
methodMetrics = {
|
||||
"precision": methodPrecision,
|
||||
"recall": methodRecall,
|
||||
"hmean": methodHmean,
|
||||
}
|
||||
|
||||
return methodMetrics
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
evaluator = DetectionDetEvalEvaluator()
|
||||
gts = [
|
||||
[
|
||||
{
|
||||
"points": [(0, 0), (1, 0), (1, 1), (0, 1)],
|
||||
"text": 1234,
|
||||
"ignore": False,
|
||||
},
|
||||
{
|
||||
"points": [(2, 2), (3, 2), (3, 3), (2, 3)],
|
||||
"text": 5678,
|
||||
"ignore": True,
|
||||
},
|
||||
]
|
||||
]
|
||||
preds = [
|
||||
[
|
||||
{
|
||||
"points": [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
|
||||
"text": 123,
|
||||
"ignore": False,
|
||||
}
|
||||
]
|
||||
]
|
||||
results = []
|
||||
for gt, pred in zip(gts, preds):
|
||||
results.append(evaluator.evaluate_image(gt, pred))
|
||||
metrics = evaluator.combine_results(results)
|
||||
print(metrics)
|
||||
@@ -0,0 +1,417 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
import math
|
||||
from collections import namedtuple
|
||||
import numpy as np
|
||||
from shapely.geometry import Polygon
|
||||
|
||||
|
||||
class DetectionICDAR2013Evaluator(object):
|
||||
def __init__(
|
||||
self,
|
||||
area_recall_constraint=0.8,
|
||||
area_precision_constraint=0.4,
|
||||
ev_param_ind_center_diff_thr=1,
|
||||
mtype_oo_o=1.0,
|
||||
mtype_om_o=0.8,
|
||||
mtype_om_m=1.0,
|
||||
):
|
||||
self.area_recall_constraint = area_recall_constraint
|
||||
self.area_precision_constraint = area_precision_constraint
|
||||
self.ev_param_ind_center_diff_thr = ev_param_ind_center_diff_thr
|
||||
self.mtype_oo_o = mtype_oo_o
|
||||
self.mtype_om_o = mtype_om_o
|
||||
self.mtype_om_m = mtype_om_m
|
||||
|
||||
def evaluate_image(self, gt, pred):
|
||||
def get_union(pD, pG):
|
||||
return Polygon(pD).union(Polygon(pG)).area
|
||||
|
||||
def get_intersection_over_union(pD, pG):
|
||||
return get_intersection(pD, pG) / get_union(pD, pG)
|
||||
|
||||
def get_intersection(pD, pG):
|
||||
return Polygon(pD).intersection(Polygon(pG)).area
|
||||
|
||||
def one_to_one_match(row, col):
|
||||
cont = 0
|
||||
for j in range(len(recallMat[0])):
|
||||
if (
|
||||
recallMat[row, j] >= self.area_recall_constraint
|
||||
and precisionMat[row, j] >= self.area_precision_constraint
|
||||
):
|
||||
cont = cont + 1
|
||||
if cont != 1:
|
||||
return False
|
||||
cont = 0
|
||||
for i in range(len(recallMat)):
|
||||
if (
|
||||
recallMat[i, col] >= self.area_recall_constraint
|
||||
and precisionMat[i, col] >= self.area_precision_constraint
|
||||
):
|
||||
cont = cont + 1
|
||||
if cont != 1:
|
||||
return False
|
||||
|
||||
if (
|
||||
recallMat[row, col] >= self.area_recall_constraint
|
||||
and precisionMat[row, col] >= self.area_precision_constraint
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
def one_to_many_match(gtNum):
|
||||
many_sum = 0
|
||||
detRects = []
|
||||
for detNum in range(len(recallMat[0])):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and detNum not in detDontCareRectsNum
|
||||
):
|
||||
if precisionMat[gtNum, detNum] >= self.area_precision_constraint:
|
||||
many_sum += recallMat[gtNum, detNum]
|
||||
detRects.append(detNum)
|
||||
if round(many_sum, 4) >= self.area_recall_constraint:
|
||||
return True, detRects
|
||||
else:
|
||||
return False, []
|
||||
|
||||
def many_to_one_match(detNum):
|
||||
many_sum = 0
|
||||
gtRects = []
|
||||
for gtNum in range(len(recallMat)):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and gtNum not in gtDontCareRectsNum
|
||||
):
|
||||
if recallMat[gtNum, detNum] >= self.area_recall_constraint:
|
||||
many_sum += precisionMat[gtNum, detNum]
|
||||
gtRects.append(gtNum)
|
||||
if round(many_sum, 4) >= self.area_precision_constraint:
|
||||
return True, gtRects
|
||||
else:
|
||||
return False, []
|
||||
|
||||
def center_distance(r1, r2):
|
||||
return ((np.mean(r1, axis=0) - np.mean(r2, axis=0)) ** 2).sum() ** 0.5
|
||||
|
||||
def diag(r):
|
||||
r = np.array(r)
|
||||
return (
|
||||
(r[:, 0].max() - r[:, 0].min()) ** 2
|
||||
+ (r[:, 1].max() - r[:, 1].min()) ** 2
|
||||
) ** 0.5
|
||||
|
||||
perSampleMetrics = {}
|
||||
|
||||
recall = 0
|
||||
precision = 0
|
||||
hmean = 0
|
||||
recallAccum = 0.0
|
||||
precisionAccum = 0.0
|
||||
gtRects = []
|
||||
detRects = []
|
||||
gtPolPoints = []
|
||||
detPolPoints = []
|
||||
gtDontCareRectsNum = (
|
||||
[]
|
||||
) # Array of Ground Truth Rectangles' keys marked as don't Care
|
||||
detDontCareRectsNum = (
|
||||
[]
|
||||
) # Array of Detected Rectangles' matched with a don't Care GT
|
||||
pairs = []
|
||||
evaluationLog = ""
|
||||
|
||||
recallMat = np.empty([1, 1])
|
||||
precisionMat = np.empty([1, 1])
|
||||
|
||||
for n in range(len(gt)):
|
||||
points = gt[n]["points"]
|
||||
# transcription = gt[n]['text']
|
||||
dontCare = gt[n]["ignore"]
|
||||
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
gtRects.append(points)
|
||||
gtPolPoints.append(points)
|
||||
if dontCare:
|
||||
gtDontCareRectsNum.append(len(gtRects) - 1)
|
||||
|
||||
evaluationLog += (
|
||||
"GT rectangles: "
|
||||
+ str(len(gtRects))
|
||||
+ (
|
||||
" (" + str(len(gtDontCareRectsNum)) + " don't care)\n"
|
||||
if len(gtDontCareRectsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
for n in range(len(pred)):
|
||||
points = pred[n]["points"]
|
||||
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
detRect = points
|
||||
detRects.append(detRect)
|
||||
detPolPoints.append(points)
|
||||
if len(gtDontCareRectsNum) > 0:
|
||||
for dontCareRectNum in gtDontCareRectsNum:
|
||||
dontCareRect = gtRects[dontCareRectNum]
|
||||
intersected_area = get_intersection(dontCareRect, detRect)
|
||||
rdDimensions = Polygon(detRect).area
|
||||
if rdDimensions == 0:
|
||||
precision = 0
|
||||
else:
|
||||
precision = intersected_area / rdDimensions
|
||||
if precision > self.area_precision_constraint:
|
||||
detDontCareRectsNum.append(len(detRects) - 1)
|
||||
break
|
||||
|
||||
evaluationLog += (
|
||||
"DET rectangles: "
|
||||
+ str(len(detRects))
|
||||
+ (
|
||||
" (" + str(len(detDontCareRectsNum)) + " don't care)\n"
|
||||
if len(detDontCareRectsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
if len(gtRects) == 0:
|
||||
recall = 1
|
||||
precision = 0 if len(detRects) > 0 else 1
|
||||
|
||||
if len(detRects) > 0:
|
||||
# Calculate recall and precision matrixes
|
||||
outputShape = [len(gtRects), len(detRects)]
|
||||
recallMat = np.empty(outputShape)
|
||||
precisionMat = np.empty(outputShape)
|
||||
gtRectMat = np.zeros(len(gtRects), np.int8)
|
||||
detRectMat = np.zeros(len(detRects), np.int8)
|
||||
for gtNum in range(len(gtRects)):
|
||||
for detNum in range(len(detRects)):
|
||||
rG = gtRects[gtNum]
|
||||
rD = detRects[detNum]
|
||||
intersected_area = get_intersection(rG, rD)
|
||||
rgDimensions = Polygon(rG).area
|
||||
rdDimensions = Polygon(rD).area
|
||||
recallMat[gtNum, detNum] = (
|
||||
0 if rgDimensions == 0 else intersected_area / rgDimensions
|
||||
)
|
||||
precisionMat[gtNum, detNum] = (
|
||||
0 if rdDimensions == 0 else intersected_area / rdDimensions
|
||||
)
|
||||
|
||||
# Find one-to-one matches
|
||||
evaluationLog += "Find one-to-one matches\n"
|
||||
for gtNum in range(len(gtRects)):
|
||||
for detNum in range(len(detRects)):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and gtNum not in gtDontCareRectsNum
|
||||
and detNum not in detDontCareRectsNum
|
||||
):
|
||||
match = one_to_one_match(gtNum, detNum)
|
||||
if match is True:
|
||||
# in deteval we have to make other validation before mark as one-to-one
|
||||
rG = gtRects[gtNum]
|
||||
rD = detRects[detNum]
|
||||
normDist = center_distance(rG, rD)
|
||||
normDist /= diag(rG) + diag(rD)
|
||||
normDist *= 2.0
|
||||
if normDist < self.ev_param_ind_center_diff_thr:
|
||||
gtRectMat[gtNum] = 1
|
||||
detRectMat[detNum] = 1
|
||||
recallAccum += self.mtype_oo_o
|
||||
precisionAccum += self.mtype_oo_o
|
||||
pairs.append({"gt": gtNum, "det": detNum, "type": "OO"})
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ "\n"
|
||||
)
|
||||
else:
|
||||
evaluationLog += (
|
||||
"Match Discarded GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ " normDist: "
|
||||
+ str(normDist)
|
||||
+ " \n"
|
||||
)
|
||||
# Find one-to-many matches
|
||||
evaluationLog += "Find one-to-many matches\n"
|
||||
for gtNum in range(len(gtRects)):
|
||||
if gtNum not in gtDontCareRectsNum:
|
||||
match, matchesDet = one_to_many_match(gtNum)
|
||||
if match is True:
|
||||
evaluationLog += "num_overlaps_gt=" + str(
|
||||
num_overlaps_gt(gtNum)
|
||||
)
|
||||
gtRectMat[gtNum] = 1
|
||||
recallAccum += (
|
||||
self.mtype_oo_o if len(matchesDet) == 1 else self.mtype_om_o
|
||||
)
|
||||
precisionAccum += (
|
||||
self.mtype_oo_o
|
||||
if len(matchesDet) == 1
|
||||
else self.mtype_om_o * len(matchesDet)
|
||||
)
|
||||
pairs.append(
|
||||
{
|
||||
"gt": gtNum,
|
||||
"det": matchesDet,
|
||||
"type": "OO" if len(matchesDet) == 1 else "OM",
|
||||
}
|
||||
)
|
||||
for detNum in matchesDet:
|
||||
detRectMat[detNum] = 1
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(matchesDet)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
# Find many-to-one matches
|
||||
evaluationLog += "Find many-to-one matches\n"
|
||||
for detNum in range(len(detRects)):
|
||||
if detNum not in detDontCareRectsNum:
|
||||
match, matchesGt = many_to_one_match(detNum)
|
||||
if match is True:
|
||||
detRectMat[detNum] = 1
|
||||
recallAccum += (
|
||||
self.mtype_oo_o
|
||||
if len(matchesGt) == 1
|
||||
else self.mtype_om_m * len(matchesGt)
|
||||
)
|
||||
precisionAccum += (
|
||||
self.mtype_oo_o if len(matchesGt) == 1 else self.mtype_om_m
|
||||
)
|
||||
pairs.append(
|
||||
{
|
||||
"gt": matchesGt,
|
||||
"det": detNum,
|
||||
"type": "OO" if len(matchesGt) == 1 else "MO",
|
||||
}
|
||||
)
|
||||
for gtNum in matchesGt:
|
||||
gtRectMat[gtNum] = 1
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(matchesGt)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
||||
if numGtCare == 0:
|
||||
recall = float(1)
|
||||
precision = float(0) if len(detRects) > 0 else float(1)
|
||||
else:
|
||||
recall = float(recallAccum) / numGtCare
|
||||
precision = (
|
||||
float(0)
|
||||
if (len(detRects) - len(detDontCareRectsNum)) == 0
|
||||
else float(precisionAccum)
|
||||
/ (len(detRects) - len(detDontCareRectsNum))
|
||||
)
|
||||
hmean = (
|
||||
0
|
||||
if (precision + recall) == 0
|
||||
else 2.0 * precision * recall / (precision + recall)
|
||||
)
|
||||
|
||||
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
||||
numDetCare = len(detRects) - len(detDontCareRectsNum)
|
||||
|
||||
perSampleMetrics = {
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"hmean": hmean,
|
||||
"pairs": pairs,
|
||||
"recallMat": [] if len(detRects) > 100 else recallMat.tolist(),
|
||||
"precisionMat": [] if len(detRects) > 100 else precisionMat.tolist(),
|
||||
"gtPolPoints": gtPolPoints,
|
||||
"detPolPoints": detPolPoints,
|
||||
"gtCare": numGtCare,
|
||||
"detCare": numDetCare,
|
||||
"gtDontCare": gtDontCareRectsNum,
|
||||
"detDontCare": detDontCareRectsNum,
|
||||
"recallAccum": recallAccum,
|
||||
"precisionAccum": precisionAccum,
|
||||
"evaluationLog": evaluationLog,
|
||||
}
|
||||
|
||||
return perSampleMetrics
|
||||
|
||||
def combine_results(self, results):
|
||||
numGt = 0
|
||||
numDet = 0
|
||||
methodRecallSum = 0
|
||||
methodPrecisionSum = 0
|
||||
|
||||
for result in results:
|
||||
numGt += result["gtCare"]
|
||||
numDet += result["detCare"]
|
||||
methodRecallSum += result["recallAccum"]
|
||||
methodPrecisionSum += result["precisionAccum"]
|
||||
|
||||
methodRecall = 0 if numGt == 0 else methodRecallSum / numGt
|
||||
methodPrecision = 0 if numDet == 0 else methodPrecisionSum / numDet
|
||||
methodHmean = (
|
||||
0
|
||||
if methodRecall + methodPrecision == 0
|
||||
else 2 * methodRecall * methodPrecision / (methodRecall + methodPrecision)
|
||||
)
|
||||
|
||||
methodMetrics = {
|
||||
"precision": methodPrecision,
|
||||
"recall": methodRecall,
|
||||
"hmean": methodHmean,
|
||||
}
|
||||
|
||||
return methodMetrics
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
evaluator = DetectionICDAR2013Evaluator()
|
||||
gts = [
|
||||
[
|
||||
{
|
||||
"points": [(0, 0), (1, 0), (1, 1), (0, 1)],
|
||||
"text": 1234,
|
||||
"ignore": False,
|
||||
},
|
||||
{
|
||||
"points": [(2, 2), (3, 2), (3, 3), (2, 3)],
|
||||
"text": 5678,
|
||||
"ignore": True,
|
||||
},
|
||||
]
|
||||
]
|
||||
preds = [
|
||||
[
|
||||
{
|
||||
"points": [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
|
||||
"text": 123,
|
||||
"ignore": False,
|
||||
}
|
||||
]
|
||||
]
|
||||
results = []
|
||||
for gt, pred in zip(gts, preds):
|
||||
results.append(evaluator.evaluate_image(gt, pred))
|
||||
metrics = evaluator.combine_results(results)
|
||||
print(metrics)
|
||||
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
from collections import namedtuple
|
||||
import numpy as np
|
||||
from shapely.geometry import Polygon
|
||||
import cv2
|
||||
|
||||
|
||||
def iou_rotate(box_a, box_b, method="union"):
|
||||
rect_a = cv2.minAreaRect(box_a)
|
||||
rect_b = cv2.minAreaRect(box_b)
|
||||
r1 = cv2.rotatedRectangleIntersection(rect_a, rect_b)
|
||||
if r1[0] == 0:
|
||||
return 0
|
||||
else:
|
||||
inter_area = cv2.contourArea(r1[1])
|
||||
area_a = cv2.contourArea(box_a)
|
||||
area_b = cv2.contourArea(box_b)
|
||||
union_area = area_a + area_b - inter_area
|
||||
if union_area == 0 or inter_area == 0:
|
||||
return 0
|
||||
if method == "union":
|
||||
iou = inter_area / union_area
|
||||
elif method == "intersection":
|
||||
iou = inter_area / min(area_a, area_b)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
return iou
|
||||
|
||||
|
||||
class DetectionIoUEvaluator(object):
|
||||
def __init__(
|
||||
self, is_output_polygon=False, iou_constraint=0.5, area_precision_constraint=0.5
|
||||
):
|
||||
self.is_output_polygon = is_output_polygon
|
||||
self.iou_constraint = iou_constraint
|
||||
self.area_precision_constraint = area_precision_constraint
|
||||
|
||||
def evaluate_image(self, gt, pred):
|
||||
def get_union(pD, pG):
|
||||
return Polygon(pD).union(Polygon(pG)).area
|
||||
|
||||
def get_intersection_over_union(pD, pG):
|
||||
return get_intersection(pD, pG) / get_union(pD, pG)
|
||||
|
||||
def get_intersection(pD, pG):
|
||||
return Polygon(pD).intersection(Polygon(pG)).area
|
||||
|
||||
def compute_ap(confList, matchList, numGtCare):
|
||||
correct = 0
|
||||
AP = 0
|
||||
if len(confList) > 0:
|
||||
confList = np.array(confList)
|
||||
matchList = np.array(matchList)
|
||||
sorted_ind = np.argsort(-confList)
|
||||
confList = confList[sorted_ind]
|
||||
matchList = matchList[sorted_ind]
|
||||
for n in range(len(confList)):
|
||||
match = matchList[n]
|
||||
if match:
|
||||
correct += 1
|
||||
AP += float(correct) / (n + 1)
|
||||
|
||||
if numGtCare > 0:
|
||||
AP /= numGtCare
|
||||
|
||||
return AP
|
||||
|
||||
perSampleMetrics = {}
|
||||
|
||||
matchedSum = 0
|
||||
|
||||
Rectangle = namedtuple("Rectangle", "xmin ymin xmax ymax")
|
||||
|
||||
numGlobalCareGt = 0
|
||||
numGlobalCareDet = 0
|
||||
|
||||
arrGlobalConfidences = []
|
||||
arrGlobalMatches = []
|
||||
|
||||
recall = 0
|
||||
precision = 0
|
||||
hmean = 0
|
||||
|
||||
detMatched = 0
|
||||
|
||||
iouMat = np.empty([1, 1])
|
||||
|
||||
gtPols = []
|
||||
detPols = []
|
||||
|
||||
gtPolPoints = []
|
||||
detPolPoints = []
|
||||
|
||||
# Array of Ground Truth Polygons' keys marked as don't Care
|
||||
gtDontCarePolsNum = []
|
||||
# Array of Detected Polygons' matched with a don't Care GT
|
||||
detDontCarePolsNum = []
|
||||
|
||||
pairs = []
|
||||
detMatchedNums = []
|
||||
|
||||
arrSampleConfidences = []
|
||||
arrSampleMatch = []
|
||||
|
||||
evaluationLog = ""
|
||||
|
||||
for n in range(len(gt)):
|
||||
points = gt[n]["points"]
|
||||
# transcription = gt[n]['text']
|
||||
dontCare = gt[n]["ignore"]
|
||||
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
gtPol = points
|
||||
gtPols.append(gtPol)
|
||||
gtPolPoints.append(points)
|
||||
if dontCare:
|
||||
gtDontCarePolsNum.append(len(gtPols) - 1)
|
||||
|
||||
evaluationLog += (
|
||||
"GT polygons: "
|
||||
+ str(len(gtPols))
|
||||
+ (
|
||||
" (" + str(len(gtDontCarePolsNum)) + " don't care)\n"
|
||||
if len(gtDontCarePolsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
for n in range(len(pred)):
|
||||
points = pred[n]["points"]
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
detPol = points
|
||||
detPols.append(detPol)
|
||||
detPolPoints.append(points)
|
||||
if len(gtDontCarePolsNum) > 0:
|
||||
for dontCarePol in gtDontCarePolsNum:
|
||||
dontCarePol = gtPols[dontCarePol]
|
||||
intersected_area = get_intersection(dontCarePol, detPol)
|
||||
pdDimensions = Polygon(detPol).area
|
||||
precision = (
|
||||
0 if pdDimensions == 0 else intersected_area / pdDimensions
|
||||
)
|
||||
if precision > self.area_precision_constraint:
|
||||
detDontCarePolsNum.append(len(detPols) - 1)
|
||||
break
|
||||
|
||||
evaluationLog += (
|
||||
"DET polygons: "
|
||||
+ str(len(detPols))
|
||||
+ (
|
||||
" (" + str(len(detDontCarePolsNum)) + " don't care)\n"
|
||||
if len(detDontCarePolsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
if len(gtPols) > 0 and len(detPols) > 0:
|
||||
# Calculate IoU and precision matrixs
|
||||
outputShape = [len(gtPols), len(detPols)]
|
||||
iouMat = np.empty(outputShape)
|
||||
gtRectMat = np.zeros(len(gtPols), np.int8)
|
||||
detRectMat = np.zeros(len(detPols), np.int8)
|
||||
if self.is_output_polygon:
|
||||
for gtNum in range(len(gtPols)):
|
||||
for detNum in range(len(detPols)):
|
||||
pG = gtPols[gtNum]
|
||||
pD = detPols[detNum]
|
||||
iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG)
|
||||
else:
|
||||
# gtPols = np.float32(gtPols)
|
||||
# detPols = np.float32(detPols)
|
||||
for gtNum in range(len(gtPols)):
|
||||
for detNum in range(len(detPols)):
|
||||
pG = np.float32(gtPols[gtNum])
|
||||
pD = np.float32(detPols[detNum])
|
||||
iouMat[gtNum, detNum] = iou_rotate(pD, pG)
|
||||
for gtNum in range(len(gtPols)):
|
||||
for detNum in range(len(detPols)):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and gtNum not in gtDontCarePolsNum
|
||||
and detNum not in detDontCarePolsNum
|
||||
):
|
||||
if iouMat[gtNum, detNum] > self.iou_constraint:
|
||||
gtRectMat[gtNum] = 1
|
||||
detRectMat[detNum] = 1
|
||||
detMatched += 1
|
||||
pairs.append({"gt": gtNum, "det": detNum})
|
||||
detMatchedNums.append(detNum)
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
numGtCare = len(gtPols) - len(gtDontCarePolsNum)
|
||||
numDetCare = len(detPols) - len(detDontCarePolsNum)
|
||||
if numGtCare == 0:
|
||||
recall = float(1)
|
||||
precision = float(0) if numDetCare > 0 else float(1)
|
||||
else:
|
||||
recall = float(detMatched) / numGtCare
|
||||
precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare
|
||||
|
||||
hmean = (
|
||||
0
|
||||
if (precision + recall) == 0
|
||||
else 2.0 * precision * recall / (precision + recall)
|
||||
)
|
||||
|
||||
matchedSum += detMatched
|
||||
numGlobalCareGt += numGtCare
|
||||
numGlobalCareDet += numDetCare
|
||||
|
||||
perSampleMetrics = {
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"hmean": hmean,
|
||||
"pairs": pairs,
|
||||
"iouMat": [] if len(detPols) > 100 else iouMat.tolist(),
|
||||
"gtPolPoints": gtPolPoints,
|
||||
"detPolPoints": detPolPoints,
|
||||
"gtCare": numGtCare,
|
||||
"detCare": numDetCare,
|
||||
"gtDontCare": gtDontCarePolsNum,
|
||||
"detDontCare": detDontCarePolsNum,
|
||||
"detMatched": detMatched,
|
||||
"evaluationLog": evaluationLog,
|
||||
}
|
||||
|
||||
return perSampleMetrics
|
||||
|
||||
def combine_results(self, results):
|
||||
numGlobalCareGt = 0
|
||||
numGlobalCareDet = 0
|
||||
matchedSum = 0
|
||||
for result in results:
|
||||
numGlobalCareGt += result["gtCare"]
|
||||
numGlobalCareDet += result["detCare"]
|
||||
matchedSum += result["detMatched"]
|
||||
|
||||
methodRecall = (
|
||||
0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt
|
||||
)
|
||||
methodPrecision = (
|
||||
0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet
|
||||
)
|
||||
methodHmean = (
|
||||
0
|
||||
if methodRecall + methodPrecision == 0
|
||||
else 2 * methodRecall * methodPrecision / (methodRecall + methodPrecision)
|
||||
)
|
||||
|
||||
methodMetrics = {
|
||||
"precision": methodPrecision,
|
||||
"recall": methodRecall,
|
||||
"hmean": methodHmean,
|
||||
}
|
||||
|
||||
return methodMetrics
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
evaluator = DetectionIoUEvaluator()
|
||||
preds = [
|
||||
[
|
||||
{
|
||||
"points": [(0.1, 0.1), (0.5, 0), (0.5, 1), (0, 1)],
|
||||
"text": 1234,
|
||||
"ignore": False,
|
||||
},
|
||||
{
|
||||
"points": [(0.5, 0.1), (1, 0), (1, 1), (0.5, 1)],
|
||||
"text": 5678,
|
||||
"ignore": False,
|
||||
},
|
||||
]
|
||||
]
|
||||
gts = [
|
||||
[
|
||||
{
|
||||
"points": [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
|
||||
"text": 123,
|
||||
"ignore": False,
|
||||
}
|
||||
]
|
||||
]
|
||||
results = []
|
||||
for gt, pred in zip(gts, preds):
|
||||
results.append(evaluator.evaluate_image(gt, pred))
|
||||
metrics = evaluator.combine_results(results)
|
||||
print(metrics)
|
||||
@@ -0,0 +1,398 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
import math
|
||||
from collections import namedtuple
|
||||
import numpy as np
|
||||
from shapely.geometry import Polygon
|
||||
|
||||
|
||||
class DetectionMTWI2018Evaluator(object):
|
||||
def __init__(
|
||||
self,
|
||||
area_recall_constraint=0.7,
|
||||
area_precision_constraint=0.7,
|
||||
ev_param_ind_center_diff_thr=1,
|
||||
):
|
||||
self.area_recall_constraint = area_recall_constraint
|
||||
self.area_precision_constraint = area_precision_constraint
|
||||
self.ev_param_ind_center_diff_thr = ev_param_ind_center_diff_thr
|
||||
|
||||
def evaluate_image(self, gt, pred):
|
||||
def get_union(pD, pG):
|
||||
return Polygon(pD).union(Polygon(pG)).area
|
||||
|
||||
def get_intersection_over_union(pD, pG):
|
||||
return get_intersection(pD, pG) / get_union(pD, pG)
|
||||
|
||||
def get_intersection(pD, pG):
|
||||
return Polygon(pD).intersection(Polygon(pG)).area
|
||||
|
||||
def one_to_one_match(row, col):
|
||||
cont = 0
|
||||
for j in range(len(recallMat[0])):
|
||||
if (
|
||||
recallMat[row, j] >= self.area_recall_constraint
|
||||
and precisionMat[row, j] >= self.area_precision_constraint
|
||||
):
|
||||
cont = cont + 1
|
||||
if cont != 1:
|
||||
return False
|
||||
cont = 0
|
||||
for i in range(len(recallMat)):
|
||||
if (
|
||||
recallMat[i, col] >= self.area_recall_constraint
|
||||
and precisionMat[i, col] >= self.area_precision_constraint
|
||||
):
|
||||
cont = cont + 1
|
||||
if cont != 1:
|
||||
return False
|
||||
|
||||
if (
|
||||
recallMat[row, col] >= self.area_recall_constraint
|
||||
and precisionMat[row, col] >= self.area_precision_constraint
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
def one_to_many_match(gtNum):
|
||||
many_sum = 0
|
||||
detRects = []
|
||||
for detNum in range(len(recallMat[0])):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and detNum not in detDontCareRectsNum
|
||||
):
|
||||
if precisionMat[gtNum, detNum] >= self.area_precision_constraint:
|
||||
many_sum += recallMat[gtNum, detNum]
|
||||
detRects.append(detNum)
|
||||
if round(many_sum, 4) >= self.area_recall_constraint:
|
||||
return True, detRects
|
||||
else:
|
||||
return False, []
|
||||
|
||||
def many_to_one_match(detNum):
|
||||
many_sum = 0
|
||||
gtRects = []
|
||||
for gtNum in range(len(recallMat)):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and gtNum not in gtDontCareRectsNum
|
||||
):
|
||||
if recallMat[gtNum, detNum] >= self.area_recall_constraint:
|
||||
many_sum += precisionMat[gtNum, detNum]
|
||||
gtRects.append(gtNum)
|
||||
if round(many_sum, 4) >= self.area_precision_constraint:
|
||||
return True, gtRects
|
||||
else:
|
||||
return False, []
|
||||
|
||||
def center_distance(r1, r2):
|
||||
return ((np.mean(r1, axis=0) - np.mean(r2, axis=0)) ** 2).sum() ** 0.5
|
||||
|
||||
def diag(r):
|
||||
r = np.array(r)
|
||||
return (
|
||||
(r[:, 0].max() - r[:, 0].min()) ** 2
|
||||
+ (r[:, 1].max() - r[:, 1].min()) ** 2
|
||||
) ** 0.5
|
||||
|
||||
perSampleMetrics = {}
|
||||
|
||||
recall = 0
|
||||
precision = 0
|
||||
hmean = 0
|
||||
recallAccum = 0.0
|
||||
precisionAccum = 0.0
|
||||
gtRects = []
|
||||
detRects = []
|
||||
gtPolPoints = []
|
||||
detPolPoints = []
|
||||
gtDontCareRectsNum = (
|
||||
[]
|
||||
) # Array of Ground Truth Rectangles' keys marked as don't Care
|
||||
detDontCareRectsNum = (
|
||||
[]
|
||||
) # Array of Detected Rectangles' matched with a don't Care GT
|
||||
pairs = []
|
||||
evaluationLog = ""
|
||||
|
||||
recallMat = np.empty([1, 1])
|
||||
precisionMat = np.empty([1, 1])
|
||||
|
||||
for n in range(len(gt)):
|
||||
points = gt[n]["points"]
|
||||
# transcription = gt[n]['text']
|
||||
dontCare = gt[n]["ignore"]
|
||||
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
gtRects.append(points)
|
||||
gtPolPoints.append(points)
|
||||
if dontCare:
|
||||
gtDontCareRectsNum.append(len(gtRects) - 1)
|
||||
|
||||
evaluationLog += (
|
||||
"GT rectangles: "
|
||||
+ str(len(gtRects))
|
||||
+ (
|
||||
" (" + str(len(gtDontCareRectsNum)) + " don't care)\n"
|
||||
if len(gtDontCareRectsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
for n in range(len(pred)):
|
||||
points = pred[n]["points"]
|
||||
|
||||
if not Polygon(points).is_valid or not Polygon(points).is_simple:
|
||||
continue
|
||||
|
||||
detRect = points
|
||||
detRects.append(detRect)
|
||||
detPolPoints.append(points)
|
||||
if len(gtDontCareRectsNum) > 0:
|
||||
for dontCareRectNum in gtDontCareRectsNum:
|
||||
dontCareRect = gtRects[dontCareRectNum]
|
||||
intersected_area = get_intersection(dontCareRect, detRect)
|
||||
rdDimensions = Polygon(detRect).area
|
||||
if rdDimensions == 0:
|
||||
precision = 0
|
||||
else:
|
||||
precision = intersected_area / rdDimensions
|
||||
if precision > 0.5:
|
||||
detDontCareRectsNum.append(len(detRects) - 1)
|
||||
break
|
||||
|
||||
evaluationLog += (
|
||||
"DET rectangles: "
|
||||
+ str(len(detRects))
|
||||
+ (
|
||||
" (" + str(len(detDontCareRectsNum)) + " don't care)\n"
|
||||
if len(detDontCareRectsNum) > 0
|
||||
else "\n"
|
||||
)
|
||||
)
|
||||
|
||||
if len(gtRects) == 0:
|
||||
recall = 1
|
||||
precision = 0 if len(detRects) > 0 else 1
|
||||
|
||||
if len(detRects) > 0:
|
||||
# Calculate recall and precision matrixs
|
||||
outputShape = [len(gtRects), len(detRects)]
|
||||
recallMat = np.empty(outputShape)
|
||||
precisionMat = np.empty(outputShape)
|
||||
gtRectMat = np.zeros(len(gtRects), np.int8)
|
||||
detRectMat = np.zeros(len(detRects), np.int8)
|
||||
for gtNum in range(len(gtRects)):
|
||||
for detNum in range(len(detRects)):
|
||||
rG = gtRects[gtNum]
|
||||
rD = detRects[detNum]
|
||||
intersected_area = get_intersection(rG, rD)
|
||||
rgDimensions = Polygon(rG).area
|
||||
rdDimensions = Polygon(rD).area
|
||||
recallMat[gtNum, detNum] = (
|
||||
0 if rgDimensions == 0 else intersected_area / rgDimensions
|
||||
)
|
||||
precisionMat[gtNum, detNum] = (
|
||||
0 if rdDimensions == 0 else intersected_area / rdDimensions
|
||||
)
|
||||
|
||||
# Find one-to-one matches
|
||||
evaluationLog += "Find one-to-one matches\n"
|
||||
for gtNum in range(len(gtRects)):
|
||||
for detNum in range(len(detRects)):
|
||||
if (
|
||||
gtRectMat[gtNum] == 0
|
||||
and detRectMat[detNum] == 0
|
||||
and gtNum not in gtDontCareRectsNum
|
||||
and detNum not in detDontCareRectsNum
|
||||
):
|
||||
match = one_to_one_match(gtNum, detNum)
|
||||
if match is True:
|
||||
# in deteval we have to make other validation before mark as one-to-one
|
||||
rG = gtRects[gtNum]
|
||||
rD = detRects[detNum]
|
||||
normDist = center_distance(rG, rD)
|
||||
normDist /= diag(rG) + diag(rD)
|
||||
normDist *= 2.0
|
||||
if normDist < self.ev_param_ind_center_diff_thr:
|
||||
gtRectMat[gtNum] = 1
|
||||
detRectMat[detNum] = 1
|
||||
recallAccum += 1.0
|
||||
precisionAccum += 1.0
|
||||
pairs.append({"gt": gtNum, "det": detNum, "type": "OO"})
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ "\n"
|
||||
)
|
||||
else:
|
||||
evaluationLog += (
|
||||
"Match Discarded GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ " normDist: "
|
||||
+ str(normDist)
|
||||
+ " \n"
|
||||
)
|
||||
# Find one-to-many matches
|
||||
evaluationLog += "Find one-to-many matches\n"
|
||||
for gtNum in range(len(gtRects)):
|
||||
if gtNum not in gtDontCareRectsNum:
|
||||
match, matchesDet = one_to_many_match(gtNum)
|
||||
if match is True:
|
||||
gtRectMat[gtNum] = 1
|
||||
recallAccum += 1.0
|
||||
precisionAccum += len(matchesDet) / (
|
||||
1 + math.log(len(matchesDet))
|
||||
)
|
||||
pairs.append(
|
||||
{
|
||||
"gt": gtNum,
|
||||
"det": matchesDet,
|
||||
"type": "OO" if len(matchesDet) == 1 else "OM",
|
||||
}
|
||||
)
|
||||
for detNum in matchesDet:
|
||||
detRectMat[detNum] = 1
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(gtNum)
|
||||
+ " with Det #"
|
||||
+ str(matchesDet)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
# Find many-to-one matches
|
||||
evaluationLog += "Find many-to-one matches\n"
|
||||
for detNum in range(len(detRects)):
|
||||
if detNum not in detDontCareRectsNum:
|
||||
match, matchesGt = many_to_one_match(detNum)
|
||||
if match is True:
|
||||
detRectMat[detNum] = 1
|
||||
recallAccum += len(matchesGt) / (1 + math.log(len(matchesGt)))
|
||||
precisionAccum += 1.0
|
||||
pairs.append(
|
||||
{
|
||||
"gt": matchesGt,
|
||||
"det": detNum,
|
||||
"type": "OO" if len(matchesGt) == 1 else "MO",
|
||||
}
|
||||
)
|
||||
for gtNum in matchesGt:
|
||||
gtRectMat[gtNum] = 1
|
||||
evaluationLog += (
|
||||
"Match GT #"
|
||||
+ str(matchesGt)
|
||||
+ " with Det #"
|
||||
+ str(detNum)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
||||
if numGtCare == 0:
|
||||
recall = float(1)
|
||||
precision = float(0) if len(detRects) > 0 else float(1)
|
||||
else:
|
||||
recall = float(recallAccum) / numGtCare
|
||||
precision = (
|
||||
float(0)
|
||||
if (len(detRects) - len(detDontCareRectsNum)) == 0
|
||||
else float(precisionAccum)
|
||||
/ (len(detRects) - len(detDontCareRectsNum))
|
||||
)
|
||||
hmean = (
|
||||
0
|
||||
if (precision + recall) == 0
|
||||
else 2.0 * precision * recall / (precision + recall)
|
||||
)
|
||||
|
||||
numGtCare = len(gtRects) - len(gtDontCareRectsNum)
|
||||
numDetCare = len(detRects) - len(detDontCareRectsNum)
|
||||
|
||||
perSampleMetrics = {
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"hmean": hmean,
|
||||
"pairs": pairs,
|
||||
"recallMat": [] if len(detRects) > 100 else recallMat.tolist(),
|
||||
"precisionMat": [] if len(detRects) > 100 else precisionMat.tolist(),
|
||||
"gtPolPoints": gtPolPoints,
|
||||
"detPolPoints": detPolPoints,
|
||||
"gtCare": numGtCare,
|
||||
"detCare": numDetCare,
|
||||
"gtDontCare": gtDontCareRectsNum,
|
||||
"detDontCare": detDontCareRectsNum,
|
||||
"recallAccum": recallAccum,
|
||||
"precisionAccum": precisionAccum,
|
||||
"evaluationLog": evaluationLog,
|
||||
}
|
||||
|
||||
return perSampleMetrics
|
||||
|
||||
def combine_results(self, results):
|
||||
numGt = 0
|
||||
numDet = 0
|
||||
methodRecallSum = 0
|
||||
methodPrecisionSum = 0
|
||||
|
||||
for result in results:
|
||||
numGt += result["gtCare"]
|
||||
numDet += result["detCare"]
|
||||
methodRecallSum += result["recallAccum"]
|
||||
methodPrecisionSum += result["precisionAccum"]
|
||||
|
||||
methodRecall = 0 if numGt == 0 else methodRecallSum / numGt
|
||||
methodPrecision = 0 if numDet == 0 else methodPrecisionSum / numDet
|
||||
methodHmean = (
|
||||
0
|
||||
if methodRecall + methodPrecision == 0
|
||||
else 2 * methodRecall * methodPrecision / (methodRecall + methodPrecision)
|
||||
)
|
||||
|
||||
methodMetrics = {
|
||||
"precision": methodPrecision,
|
||||
"recall": methodRecall,
|
||||
"hmean": methodHmean,
|
||||
}
|
||||
|
||||
return methodMetrics
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
evaluator = DetectionICDAR2013Evaluator()
|
||||
gts = [
|
||||
[
|
||||
{
|
||||
"points": [(0, 0), (1, 0), (1, 1), (0, 1)],
|
||||
"text": 1234,
|
||||
"ignore": False,
|
||||
},
|
||||
{
|
||||
"points": [(2, 2), (3, 2), (3, 3), (2, 3)],
|
||||
"text": 5678,
|
||||
"ignore": True,
|
||||
},
|
||||
]
|
||||
]
|
||||
preds = [
|
||||
[
|
||||
{
|
||||
"points": [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
|
||||
"text": 123,
|
||||
"ignore": False,
|
||||
}
|
||||
]
|
||||
]
|
||||
results = []
|
||||
for gt, pred in zip(gts, preds):
|
||||
results.append(evaluator.evaluate_image(gt, pred))
|
||||
metrics = evaluator.combine_results(results)
|
||||
print(metrics)
|
||||
@@ -0,0 +1,100 @@
|
||||
import numpy as np
|
||||
|
||||
from .detection.iou import DetectionIoUEvaluator
|
||||
|
||||
|
||||
class AverageMeter(object):
|
||||
"""Computes and stores the average and current value"""
|
||||
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.val = 0
|
||||
self.avg = 0
|
||||
self.sum = 0
|
||||
self.count = 0
|
||||
|
||||
def update(self, val, n=1):
|
||||
self.val = val
|
||||
self.sum += val * n
|
||||
self.count += n
|
||||
self.avg = self.sum / self.count
|
||||
return self
|
||||
|
||||
|
||||
class QuadMetric:
|
||||
def __init__(self, is_output_polygon=False):
|
||||
self.is_output_polygon = is_output_polygon
|
||||
self.evaluator = DetectionIoUEvaluator(is_output_polygon=is_output_polygon)
|
||||
|
||||
def measure(self, batch, output, box_thresh=0.6):
|
||||
"""
|
||||
batch: (image, polygons, ignore_tags
|
||||
batch: a dict produced by dataloaders.
|
||||
image: tensor of shape (N, C, H, W).
|
||||
polygons: tensor of shape (N, K, 4, 2), the polygons of objective regions.
|
||||
ignore_tags: tensor of shape (N, K), indicates whether a region is ignorable or not.
|
||||
shape: the original shape of images.
|
||||
filename: the original filenames of images.
|
||||
output: (polygons, ...)
|
||||
"""
|
||||
results = []
|
||||
gt_polyons_batch = batch["text_polys"]
|
||||
ignore_tags_batch = batch["ignore_tags"]
|
||||
pred_polygons_batch = np.array(output[0])
|
||||
pred_scores_batch = np.array(output[1])
|
||||
for polygons, pred_polygons, pred_scores, ignore_tags in zip(
|
||||
gt_polyons_batch, pred_polygons_batch, pred_scores_batch, ignore_tags_batch
|
||||
):
|
||||
gt = [
|
||||
dict(points=np.int64(polygons[i]), ignore=ignore_tags[i])
|
||||
for i in range(len(polygons))
|
||||
]
|
||||
if self.is_output_polygon:
|
||||
pred = [
|
||||
dict(points=pred_polygons[i]) for i in range(len(pred_polygons))
|
||||
]
|
||||
else:
|
||||
pred = []
|
||||
# print(pred_polygons.shape)
|
||||
for i in range(pred_polygons.shape[0]):
|
||||
if pred_scores[i] >= box_thresh:
|
||||
# print(pred_polygons[i,:,:].tolist())
|
||||
pred.append(
|
||||
dict(points=pred_polygons[i, :, :].astype(np.int32))
|
||||
)
|
||||
# pred = [dict(points=pred_polygons[i,:,:].tolist()) if pred_scores[i] >= box_thresh for i in range(pred_polygons.shape[0])]
|
||||
results.append(self.evaluator.evaluate_image(gt, pred))
|
||||
return results
|
||||
|
||||
def validate_measure(self, batch, output, box_thresh=0.6):
|
||||
return self.measure(batch, output, box_thresh)
|
||||
|
||||
def evaluate_measure(self, batch, output):
|
||||
return (
|
||||
self.measure(batch, output),
|
||||
np.linspace(0, batch["image"].shape[0]).tolist(),
|
||||
)
|
||||
|
||||
def gather_measure(self, raw_metrics):
|
||||
raw_metrics = [
|
||||
image_metrics
|
||||
for batch_metrics in raw_metrics
|
||||
for image_metrics in batch_metrics
|
||||
]
|
||||
|
||||
result = self.evaluator.combine_results(raw_metrics)
|
||||
|
||||
precision = AverageMeter()
|
||||
recall = AverageMeter()
|
||||
fmeasure = AverageMeter()
|
||||
|
||||
precision.update(result["precision"], n=len(raw_metrics))
|
||||
recall.update(result["recall"], n=len(raw_metrics))
|
||||
fmeasure_score = (
|
||||
2 * precision.val * recall.val / (precision.val + recall.val + 1e-8)
|
||||
)
|
||||
fmeasure.update(fmeasure_score)
|
||||
|
||||
return {"precision": precision, "recall": recall, "fmeasure": fmeasure}
|
||||
Reference in New Issue
Block a user