first commit

ppocr/metrics/__init__.py

@@ -0,0 +1,62 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import copy
+
+__all__ = ["build_metric"]
+
+from .det_metric import DetMetric, DetFCEMetric
+from .rec_metric import RecMetric, CNTMetric, CANMetric, LaTeXOCRMetric
+from .cls_metric import ClsMetric
+from .e2e_metric import E2EMetric
+from .distillation_metric import DistillationMetric
+from .table_metric import TableMetric
+from .kie_metric import KIEMetric
+from .vqa_token_ser_metric import VQASerTokenMetric
+from .vqa_token_re_metric import VQAReTokenMetric
+from .sr_metric import SRMetric
+from .ct_metric import CTMetric
+
+
+def build_metric(config):
+    support_dict = [
+        "DetMetric",
+        "DetFCEMetric",
+        "RecMetric",
+        "ClsMetric",
+        "E2EMetric",
+        "DistillationMetric",
+        "TableMetric",
+        "KIEMetric",
+        "VQASerTokenMetric",
+        "VQAReTokenMetric",
+        "SRMetric",
+        "CTMetric",
+        "CNTMetric",
+        "CANMetric",
+        "LaTeXOCRMetric",
+    ]
+
+    config = copy.deepcopy(config)
+    module_name = config.pop("name")
+    assert module_name in support_dict, Exception(
+        "metric only support {}".format(support_dict)
+    )
+    module_class = eval(module_name)(**config)
+    return module_class

ppocr/metrics/bleu.py

@@ -0,0 +1,243 @@
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This code is refer from:
+https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py
+"""
+
+import re
+import math
+import collections
+from functools import lru_cache
+
+
+def _get_ngrams(segment, max_order):
+    """Extracts all n-grams upto a given maximum order from an input segment.
+
+    Args:
+      segment: text segment from which n-grams will be extracted.
+      max_order: maximum length in tokens of the n-grams returned by this
+          methods.
+
+    Returns:
+      The Counter containing all n-grams upto max_order in segment
+      with a count of how many times each n-gram occurred.
+    """
+    ngram_counts = collections.Counter()
+    for order in range(1, max_order + 1):
+        for i in range(0, len(segment) - order + 1):
+            ngram = tuple(segment[i : i + order])
+            ngram_counts[ngram] += 1
+    return ngram_counts
+
+
+def compute_bleu(reference_corpus, translation_corpus, max_order=4, smooth=False):
+    """Computes BLEU score of translated segments against one or more references.
+
+    Args:
+      reference_corpus: list of lists of references for each translation. Each
+          reference should be tokenized into a list of tokens.
+      translation_corpus: list of translations to score. Each translation
+          should be tokenized into a list of tokens.
+      max_order: Maximum n-gram order to use when computing BLEU score.
+      smooth: Whether or not to apply Lin et al. 2004 smoothing.
+
+    Returns:
+      3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram
+      precisions and brevity penalty.
+    """
+    matches_by_order = [0] * max_order
+    possible_matches_by_order = [0] * max_order
+    reference_length = 0
+    translation_length = 0
+    for references, translation in zip(reference_corpus, translation_corpus):
+        reference_length += min(len(r) for r in references)
+        translation_length += len(translation)
+
+        merged_ref_ngram_counts = collections.Counter()
+        for reference in references:
+            merged_ref_ngram_counts |= _get_ngrams(reference, max_order)
+        translation_ngram_counts = _get_ngrams(translation, max_order)
+        overlap = translation_ngram_counts & merged_ref_ngram_counts
+        for ngram in overlap:
+            matches_by_order[len(ngram) - 1] += overlap[ngram]
+        for order in range(1, max_order + 1):
+            possible_matches = len(translation) - order + 1
+            if possible_matches > 0:
+                possible_matches_by_order[order - 1] += possible_matches
+
+    precisions = [0] * max_order
+    for i in range(0, max_order):
+        if smooth:
+            precisions[i] = (matches_by_order[i] + 1.0) / (
+                possible_matches_by_order[i] + 1.0
+            )
+        else:
+            if possible_matches_by_order[i] > 0:
+                precisions[i] = (
+                    float(matches_by_order[i]) / possible_matches_by_order[i]
+                )
+            else:
+                precisions[i] = 0.0
+
+    if min(precisions) > 0:
+        p_log_sum = sum((1.0 / max_order) * math.log(p) for p in precisions)
+        geo_mean = math.exp(p_log_sum)
+    else:
+        geo_mean = 0
+
+    if float(translation_length) == 0 or float(reference_length) == 0:
+        ratio = 1e-5
+    else:
+        ratio = float(translation_length) / reference_length
+
+    if ratio > 1.0:
+        bp = 1.0
+    else:
+        bp = math.exp(1 - 1.0 / ratio)
+
+    bleu = geo_mean * bp
+
+    return (bleu, precisions, bp, ratio, translation_length, reference_length)
+
+
+class BaseTokenizer:
+    """A base dummy tokenizer to derive from."""
+
+    def signature(self):
+        """
+        Returns a signature for the tokenizer.
+        :return: signature string
+        """
+        return "none"
+
+    def __call__(self, line):
+        """
+        Tokenizes an input line with the tokenizer.
+        :param line: a segment to tokenize
+        :return: the tokenized line
+        """
+        return line
+
+
+class TokenizerRegexp(BaseTokenizer):
+    def signature(self):
+        return "re"
+
+    def __init__(self):
+        self._re = [
+            # language-dependent part (assuming Western languages)
+            (re.compile(r"([\{-\~\[-\` -\&\(-\+\:-\@\/])"), r" \1 "),
+            # tokenize period and comma unless preceded by a digit
+            (re.compile(r"([^0-9])([\.,])"), r"\1 \2 "),
+            # tokenize period and comma unless followed by a digit
+            (re.compile(r"([\.,])([^0-9])"), r" \1 \2"),
+            # tokenize dash when preceded by a digit
+            (re.compile(r"([0-9])(-)"), r"\1 \2 "),
+            # one space only between words
+            # NOTE: Doing this in Python (below) is faster
+            # (re.compile(r'\s+'), r' '),
+        ]
+
+    @lru_cache(maxsize=2**16)
+    def __call__(self, line):
+        """Common post-processing tokenizer for `13a` and `zh` tokenizers.
+        :param line: a segment to tokenize
+        :return: the tokenized line
+        """
+        for _re, repl in self._re:
+            line = _re.sub(repl, line)
+
+        # no leading or trailing spaces, single space within words
+        # return ' '.join(line.split())
+        # This line is changed with regards to the original tokenizer (seen above) to return individual words
+        return line.split()
+
+
+class Tokenizer13a(BaseTokenizer):
+    def signature(self):
+        return "13a"
+
+    def __init__(self):
+        self._post_tokenizer = TokenizerRegexp()
+
+    @lru_cache(maxsize=2**16)
+    def __call__(self, line):
+        """Tokenizes an input line using a relatively minimal tokenization
+        that is however equivalent to mteval-v13a, used by WMT.
+
+        :param line: a segment to tokenize
+        :return: the tokenized line
+        """
+
+        # language-independent part:
+        line = line.replace("<skipped>", "")
+        line = line.replace("-\n", "")
+        line = line.replace("\n", " ")
+
+        if "&" in line:
+            line = line.replace("&quot;", '"')
+            line = line.replace("&amp;", "&")
+            line = line.replace("&lt;", "<")
+            line = line.replace("&gt;", ">")
+
+        return self._post_tokenizer(f" {line} ")
+
+
+def compute_bleu_score(
+    predictions, references, tokenizer=Tokenizer13a(), max_order=4, smooth=False
+):
+    # if only one reference is provided make sure we still use list of lists
+    if isinstance(references[0], str):
+        references = [[ref] for ref in references]
+
+    references = [[tokenizer(r) for r in ref] for ref in references]
+    predictions = [tokenizer(p) for p in predictions]
+    score = compute_bleu(
+        reference_corpus=references,
+        translation_corpus=predictions,
+        max_order=max_order,
+        smooth=smooth,
+    )
+    (bleu, precisions, bp, ratio, translation_length, reference_length) = score
+    return bleu
+
+
+def cal_distance(word1, word2):
+    m = len(word1)
+    n = len(word2)
+    if m * n == 0:
+        return m + n
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+    for i in range(m + 1):
+        dp[i][0] = i
+    for j in range(n + 1):
+        dp[0][j] = j
+    for i in range(1, m + 1):
+        for j in range(1, n + 1):
+            a = dp[i - 1][j] + 1
+            b = dp[i][j - 1] + 1
+            c = dp[i - 1][j - 1]
+            if word1[i - 1] != word2[j - 1]:
+                c += 1
+            dp[i][j] = min(a, b, c)
+    return dp[m][n]
+
+
+def compute_edit_distance(prediction, label):
+    prediction = prediction.strip().split(" ")
+    label = label.strip().split(" ")
+    distance = cal_distance(prediction, label)
+    return distance

ppocr/metrics/cls_metric.py

@@ -0,0 +1,48 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class ClsMetric(object):
+    def __init__(self, main_indicator="acc", **kwargs):
+        self.main_indicator = main_indicator
+        self.eps = 1e-5
+        self.reset()
+
+    def __call__(self, pred_label, *args, **kwargs):
+        preds, labels = pred_label
+        correct_num = 0
+        all_num = 0
+        for (pred, pred_conf), (target, _) in zip(preds, labels):
+            if pred == target:
+                correct_num += 1
+            all_num += 1
+        self.correct_num += correct_num
+        self.all_num += all_num
+        return {
+            "acc": correct_num / (all_num + self.eps),
+        }
+
+    def get_metric(self):
+        """
+        return metrics {
+                 'acc': 0
+            }
+        """
+        acc = self.correct_num / (self.all_num + self.eps)
+        self.reset()
+        return {"acc": acc}
+
+    def reset(self):
+        self.correct_num = 0
+        self.all_num = 0

ppocr/metrics/ct_metric.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+from scipy import io
+import numpy as np
+
+from ppocr.utils.e2e_metric.Deteval import combine_results, get_score_C
+
+
+class CTMetric(object):
+    def __init__(self, main_indicator, delimiter="\t", **kwargs):
+        self.delimiter = delimiter
+        self.main_indicator = main_indicator
+        self.reset()
+
+    def reset(self):
+        self.results = []  # clear results
+
+    def __call__(self, preds, batch, **kwargs):
+        # NOTE: only support bs=1 now, as the label length of different sample is Unequal
+        assert len(preds) == 1, "CentripetalText test now only support batch_size=1."
+        label = batch[2]
+        text = batch[3]
+        pred = preds[0]["points"]
+        result = get_score_C(label, text, pred)
+
+        self.results.append(result)
+
+    def get_metric(self):
+        """
+        Input format: y0,x0, ..... yn,xn. Each detection is separated by the end of line token ('\n')'
+        """
+        metrics = combine_results(self.results, rec_flag=False)
+        self.reset()
+        return metrics

ppocr/metrics/det_metric.py

@@ -0,0 +1,153 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+__all__ = ["DetMetric", "DetFCEMetric"]
+
+from .eval_det_iou import DetectionIoUEvaluator
+
+
+class DetMetric(object):
+    def __init__(self, main_indicator="hmean", **kwargs):
+        self.evaluator = DetectionIoUEvaluator()
+        self.main_indicator = main_indicator
+        self.reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        """
+        batch: a list produced by dataloaders.
+            image: np.ndarray  of shape (N, C, H, W).
+            ratio_list: np.ndarray  of shape(N,2)
+            polygons: np.ndarray  of shape (N, K, 4, 2), the polygons of objective regions.
+            ignore_tags: np.ndarray  of shape (N, K), indicates whether a region is ignorable or not.
+        preds: a list of dict produced by post process
+             points: np.ndarray of shape (N, K, 4, 2), the polygons of objective regions.
+        """
+        gt_polyons_batch = batch[2]
+        ignore_tags_batch = batch[3]
+        for pred, gt_polyons, ignore_tags in zip(
+            preds, gt_polyons_batch, ignore_tags_batch
+        ):
+            # prepare gt
+            gt_info_list = [
+                {"points": gt_polyon, "text": "", "ignore": ignore_tag}
+                for gt_polyon, ignore_tag in zip(gt_polyons, ignore_tags)
+            ]
+            # prepare det
+            det_info_list = [
+                {"points": det_polyon, "text": ""} for det_polyon in pred["points"]
+            ]
+            result = self.evaluator.evaluate_image(gt_info_list, det_info_list)
+            self.results.append(result)
+
+    def get_metric(self):
+        """
+        return metrics {
+                 'precision': 0,
+                 'recall': 0,
+                 'hmean': 0
+            }
+        """
+
+        metrics = self.evaluator.combine_results(self.results)
+        self.reset()
+        return metrics
+
+    def reset(self):
+        self.results = []  # clear results
+
+
+class DetFCEMetric(object):
+    def __init__(self, main_indicator="hmean", **kwargs):
+        self.evaluator = DetectionIoUEvaluator()
+        self.main_indicator = main_indicator
+        self.reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        """
+        batch: a list produced by dataloaders.
+            image: np.ndarray  of shape (N, C, H, W).
+            ratio_list: np.ndarray  of shape(N,2)
+            polygons: np.ndarray  of shape (N, K, 4, 2), the polygons of objective regions.
+            ignore_tags: np.ndarray  of shape (N, K), indicates whether a region is ignorable or not.
+        preds: a list of dict produced by post process
+             points: np.ndarray of shape (N, K, 4, 2), the polygons of objective regions.
+        """
+        gt_polyons_batch = batch[2]
+        ignore_tags_batch = batch[3]
+
+        for pred, gt_polyons, ignore_tags in zip(
+            preds, gt_polyons_batch, ignore_tags_batch
+        ):
+            # prepare gt
+            gt_info_list = [
+                {"points": gt_polyon, "text": "", "ignore": ignore_tag}
+                for gt_polyon, ignore_tag in zip(gt_polyons, ignore_tags)
+            ]
+            # prepare det
+            det_info_list = [
+                {"points": det_polyon, "text": "", "score": score}
+                for det_polyon, score in zip(pred["points"], pred["scores"])
+            ]
+
+            for score_thr in self.results.keys():
+                det_info_list_thr = [
+                    det_info
+                    for det_info in det_info_list
+                    if det_info["score"] >= score_thr
+                ]
+                result = self.evaluator.evaluate_image(gt_info_list, det_info_list_thr)
+                self.results[score_thr].append(result)
+
+    def get_metric(self):
+        """
+        return metrics {'heman':0,
+            'thr 0.3':'precision: 0 recall: 0 hmean: 0',
+            'thr 0.4':'precision: 0 recall: 0 hmean: 0',
+            'thr 0.5':'precision: 0 recall: 0 hmean: 0',
+            'thr 0.6':'precision: 0 recall: 0 hmean: 0',
+            'thr 0.7':'precision: 0 recall: 0 hmean: 0',
+            'thr 0.8':'precision: 0 recall: 0 hmean: 0',
+            'thr 0.9':'precision: 0 recall: 0 hmean: 0',
+            }
+        """
+        metrics = {}
+        hmean = 0
+        for score_thr in self.results.keys():
+            metric = self.evaluator.combine_results(self.results[score_thr])
+            # for key, value in metric.items():
+            #     metrics['{}_{}'.format(key, score_thr)] = value
+            metric_str = "precision:{:.5f} recall:{:.5f} hmean:{:.5f}".format(
+                metric["precision"], metric["recall"], metric["hmean"]
+            )
+            metrics["thr {}".format(score_thr)] = metric_str
+            hmean = max(hmean, metric["hmean"])
+        metrics["hmean"] = hmean
+
+        self.reset()
+        return metrics
+
+    def reset(self):
+        self.results = {
+            0.3: [],
+            0.4: [],
+            0.5: [],
+            0.6: [],
+            0.7: [],
+            0.8: [],
+            0.9: [],
+        }  # clear results

ppocr/metrics/distillation_metric.py

@@ -0,0 +1,72 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import copy
+
+from .rec_metric import RecMetric
+from .det_metric import DetMetric
+from .e2e_metric import E2EMetric
+from .cls_metric import ClsMetric
+from .vqa_token_ser_metric import VQASerTokenMetric
+from .vqa_token_re_metric import VQAReTokenMetric
+
+
+class DistillationMetric(object):
+    def __init__(self, key=None, base_metric_name=None, main_indicator=None, **kwargs):
+        self.main_indicator = main_indicator
+        self.key = key
+        self.main_indicator = main_indicator
+        self.base_metric_name = base_metric_name
+        self.kwargs = kwargs
+        self.metrics = None
+
+    def _init_metrcis(self, preds):
+        self.metrics = dict()
+        mod = importlib.import_module(__name__)
+        for key in preds:
+            self.metrics[key] = getattr(mod, self.base_metric_name)(
+                main_indicator=self.main_indicator, **self.kwargs
+            )
+            self.metrics[key].reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        assert isinstance(preds, dict)
+        if self.metrics is None:
+            self._init_metrcis(preds)
+        output = dict()
+        for key in preds:
+            self.metrics[key].__call__(preds[key], batch, **kwargs)
+
+    def get_metric(self):
+        """
+        return metrics {
+                 'acc': 0,
+                 'norm_edit_dis': 0,
+            }
+        """
+        output = dict()
+        for key in self.metrics:
+            metric = self.metrics[key].get_metric()
+            # main indicator
+            if key == self.key:
+                output.update(metric)
+            else:
+                for sub_key in metric:
+                    output["{}_{}".format(key, sub_key)] = metric[sub_key]
+        return output
+
+    def reset(self):
+        for key in self.metrics:
+            self.metrics[key].reset()

ppocr/metrics/e2e_metric.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+__all__ = ["E2EMetric"]
+
+from ppocr.utils.e2e_metric.Deteval import get_socre_A, get_socre_B, combine_results
+from ppocr.utils.e2e_utils.extract_textpoint_slow import get_dict
+
+
+class E2EMetric(object):
+    def __init__(
+        self,
+        mode,
+        gt_mat_dir,
+        character_dict_path,
+        main_indicator="f_score_e2e",
+        **kwargs,
+    ):
+        self.mode = mode
+        self.gt_mat_dir = gt_mat_dir
+        self.label_list = get_dict(character_dict_path)
+        self.max_index = len(self.label_list)
+        self.main_indicator = main_indicator
+        self.reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        if self.mode == "A":
+            gt_polyons_batch = batch[2]
+            temp_gt_strs_batch = batch[3][0]
+            ignore_tags_batch = batch[4]
+            gt_strs_batch = []
+
+            for temp_list in temp_gt_strs_batch:
+                t = ""
+                for index in temp_list:
+                    if index < self.max_index:
+                        t += self.label_list[index]
+                gt_strs_batch.append(t)
+
+            for pred, gt_polyons, gt_strs, ignore_tags in zip(
+                [preds], gt_polyons_batch, [gt_strs_batch], ignore_tags_batch
+            ):
+                # prepare gt
+                gt_info_list = [
+                    {"points": gt_polyon, "text": gt_str, "ignore": ignore_tag}
+                    for gt_polyon, gt_str, ignore_tag in zip(
+                        gt_polyons, gt_strs, ignore_tags
+                    )
+                ]
+                # prepare det
+                e2e_info_list = [
+                    {"points": det_polyon, "texts": pred_str}
+                    for det_polyon, pred_str in zip(pred["points"], pred["texts"])
+                ]
+
+                result = get_socre_A(gt_info_list, e2e_info_list)
+                self.results.append(result)
+        else:
+            img_id = batch[5][0]
+            e2e_info_list = [
+                {"points": det_polyon, "texts": pred_str}
+                for det_polyon, pred_str in zip(preds["points"], preds["texts"])
+            ]
+            result = get_socre_B(self.gt_mat_dir, img_id, e2e_info_list)
+            self.results.append(result)
+
+    def get_metric(self):
+        metrics = combine_results(self.results)
+        self.reset()
+        return metrics
+
+    def reset(self):
+        self.results = []  # clear results

ppocr/metrics/eval_det_iou.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from collections import namedtuple
+import numpy as np
+from shapely.geometry import Polygon
+
+"""
+reference from :
+https://github.com/MhLiao/DB/blob/3c32b808d4412680310d3d28eeb6a2d5bf1566c5/concern/icdar2015_eval/detection/iou.py#L8
+"""
+
+
+class DetectionIoUEvaluator(object):
+    def __init__(self, iou_constraint=0.5, area_precision_constraint=0.5):
+        self.iou_constraint = iou_constraint
+        self.area_precision_constraint = area_precision_constraint
+
+    def evaluate_image(self, gt, pred):
+        def get_union(pD, pG):
+            return Polygon(pD).union(Polygon(pG)).area
+
+        def get_intersection_over_union(pD, pG):
+            return get_intersection(pD, pG) / get_union(pD, pG)
+
+        def get_intersection(pD, pG):
+            return Polygon(pD).intersection(Polygon(pG)).area
+
+        def compute_ap(confList, matchList, numGtCare):
+            correct = 0
+            AP = 0
+            if len(confList) > 0:
+                confList = np.array(confList)
+                matchList = np.array(matchList)
+                sorted_ind = np.argsort(-confList)
+                confList = confList[sorted_ind]
+                matchList = matchList[sorted_ind]
+                for n in range(len(confList)):
+                    match = matchList[n]
+                    if match:
+                        correct += 1
+                        AP += float(correct) / (n + 1)
+
+                if numGtCare > 0:
+                    AP /= numGtCare
+
+            return AP
+
+        perSampleMetrics = {}
+
+        matchedSum = 0
+
+        Rectangle = namedtuple("Rectangle", "xmin ymin xmax ymax")
+
+        numGlobalCareGt = 0
+        numGlobalCareDet = 0
+
+        arrGlobalConfidences = []
+        arrGlobalMatches = []
+
+        recall = 0
+        precision = 0
+        hmean = 0
+
+        detMatched = 0
+
+        iouMat = np.empty([1, 1])
+
+        gtPols = []
+        detPols = []
+
+        gtPolPoints = []
+        detPolPoints = []
+
+        # Array of Ground Truth Polygons' keys marked as don't Care
+        gtDontCarePolsNum = []
+        # Array of Detected Polygons' matched with a don't Care GT
+        detDontCarePolsNum = []
+
+        pairs = []
+        detMatchedNums = []
+
+        arrSampleConfidences = []
+        arrSampleMatch = []
+
+        evaluationLog = ""
+
+        for n in range(len(gt)):
+            points = gt[n]["points"]
+            dontCare = gt[n]["ignore"]
+            if not Polygon(points).is_valid:
+                continue
+
+            gtPol = points
+            gtPols.append(gtPol)
+            gtPolPoints.append(points)
+            if dontCare:
+                gtDontCarePolsNum.append(len(gtPols) - 1)
+
+        evaluationLog += (
+            "GT polygons: "
+            + str(len(gtPols))
+            + (
+                " (" + str(len(gtDontCarePolsNum)) + " don't care)\n"
+                if len(gtDontCarePolsNum) > 0
+                else "\n"
+            )
+        )
+
+        for n in range(len(pred)):
+            points = pred[n]["points"]
+            if not Polygon(points).is_valid:
+                continue
+
+            detPol = points
+            detPols.append(detPol)
+            detPolPoints.append(points)
+            if len(gtDontCarePolsNum) > 0:
+                for dontCarePol in gtDontCarePolsNum:
+                    dontCarePol = gtPols[dontCarePol]
+                    intersected_area = get_intersection(dontCarePol, detPol)
+                    pdDimensions = Polygon(detPol).area
+                    precision = (
+                        0 if pdDimensions == 0 else intersected_area / pdDimensions
+                    )
+                    if precision > self.area_precision_constraint:
+                        detDontCarePolsNum.append(len(detPols) - 1)
+                        break
+
+        evaluationLog += (
+            "DET polygons: "
+            + str(len(detPols))
+            + (
+                " (" + str(len(detDontCarePolsNum)) + " don't care)\n"
+                if len(detDontCarePolsNum) > 0
+                else "\n"
+            )
+        )
+
+        if len(gtPols) > 0 and len(detPols) > 0:
+            # Calculate IoU and precision matrixs
+            outputShape = [len(gtPols), len(detPols)]
+            iouMat = np.empty(outputShape)
+            gtRectMat = np.zeros(len(gtPols), np.int8)
+            detRectMat = np.zeros(len(detPols), np.int8)
+            for gtNum in range(len(gtPols)):
+                for detNum in range(len(detPols)):
+                    pG = gtPols[gtNum]
+                    pD = detPols[detNum]
+                    iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG)
+
+            for gtNum in range(len(gtPols)):
+                for detNum in range(len(detPols)):
+                    if (
+                        gtRectMat[gtNum] == 0
+                        and detRectMat[detNum] == 0
+                        and gtNum not in gtDontCarePolsNum
+                        and detNum not in detDontCarePolsNum
+                    ):
+                        if iouMat[gtNum, detNum] > self.iou_constraint:
+                            gtRectMat[gtNum] = 1
+                            detRectMat[detNum] = 1
+                            detMatched += 1
+                            pairs.append({"gt": gtNum, "det": detNum})
+                            detMatchedNums.append(detNum)
+                            evaluationLog += (
+                                "Match GT #"
+                                + str(gtNum)
+                                + " with Det #"
+                                + str(detNum)
+                                + "\n"
+                            )
+
+        numGtCare = len(gtPols) - len(gtDontCarePolsNum)
+        numDetCare = len(detPols) - len(detDontCarePolsNum)
+        if numGtCare == 0:
+            recall = float(1)
+            precision = float(0) if numDetCare > 0 else float(1)
+        else:
+            recall = float(detMatched) / numGtCare
+            precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare
+
+        hmean = (
+            0
+            if (precision + recall) == 0
+            else 2.0 * precision * recall / (precision + recall)
+        )
+
+        matchedSum += detMatched
+        numGlobalCareGt += numGtCare
+        numGlobalCareDet += numDetCare
+
+        perSampleMetrics = {
+            "gtCare": numGtCare,
+            "detCare": numDetCare,
+            "detMatched": detMatched,
+        }
+        return perSampleMetrics
+
+    def combine_results(self, results):
+        numGlobalCareGt = 0
+        numGlobalCareDet = 0
+        matchedSum = 0
+        for result in results:
+            numGlobalCareGt += result["gtCare"]
+            numGlobalCareDet += result["detCare"]
+            matchedSum += result["detMatched"]
+
+        methodRecall = (
+            0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt
+        )
+        methodPrecision = (
+            0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet
+        )
+        methodHmean = (
+            0
+            if methodRecall + methodPrecision == 0
+            else 2 * methodRecall * methodPrecision / (methodRecall + methodPrecision)
+        )
+        methodMetrics = {
+            "precision": methodPrecision,
+            "recall": methodRecall,
+            "hmean": methodHmean,
+        }
+
+        return methodMetrics
+
+
+if __name__ == "__main__":
+    evaluator = DetectionIoUEvaluator()
+    gts = [
+        [
+            {
+                "points": [(0, 0), (1, 0), (1, 1), (0, 1)],
+                "text": 1234,
+                "ignore": False,
+            },
+            {
+                "points": [(2, 2), (3, 2), (3, 3), (2, 3)],
+                "text": 5678,
+                "ignore": False,
+            },
+        ]
+    ]
+    preds = [
+        [
+            {
+                "points": [(0.1, 0.1), (1, 0), (1, 1), (0, 1)],
+                "text": 123,
+                "ignore": False,
+            }
+        ]
+    ]
+    results = []
+    for gt, pred in zip(gts, preds):
+        results.append(evaluator.evaluate_image(gt, pred))
+    metrics = evaluator.combine_results(results)
+    print(metrics)

ppocr/metrics/kie_metric.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# The code is refer from: https://github.com/open-mmlab/mmocr/blob/main/mmocr/core/evaluation/kie_metric.py
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import paddle
+
+__all__ = ["KIEMetric"]
+
+
+class KIEMetric(object):
+    def __init__(self, main_indicator="hmean", **kwargs):
+        self.main_indicator = main_indicator
+        self.reset()
+        self.node = []
+        self.gt = []
+
+    def __call__(self, preds, batch, **kwargs):
+        nodes, _ = preds
+        gts, tag = batch[4].squeeze(0), batch[5].tolist()[0]
+        gts = gts[: tag[0], :1].reshape([-1])
+        self.node.append(nodes.numpy())
+        self.gt.append(gts)
+        # result = self.compute_f1_score(nodes, gts)
+        # self.results.append(result)
+
+    def compute_f1_score(self, preds, gts):
+        ignores = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 25]
+        C = preds.shape[1]
+        classes = np.array(sorted(set(range(C)) - set(ignores)))
+        hist = (
+            np.bincount((gts * C).astype("int64") + preds.argmax(1), minlength=C**2)
+            .reshape([C, C])
+            .astype("float32")
+        )
+        diag = np.diag(hist)
+        recalls = diag / hist.sum(1).clip(min=1)
+        precisions = diag / hist.sum(0).clip(min=1)
+        f1 = 2 * recalls * precisions / (recalls + precisions).clip(min=1e-8)
+        return f1[classes]
+
+    def combine_results(self, results):
+        node = np.concatenate(self.node, 0)
+        gts = np.concatenate(self.gt, 0)
+        results = self.compute_f1_score(node, gts)
+        data = {"hmean": results.mean()}
+        return data
+
+    def get_metric(self):
+        metrics = self.combine_results(self.results)
+        self.reset()
+        return metrics
+
+    def reset(self):
+        self.results = []  # clear results
+        self.node = []
+        self.gt = []

ppocr/metrics/rec_metric.py

@@ -0,0 +1,297 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from rapidfuzz.distance import Levenshtein
+from difflib import SequenceMatcher
+
+import numpy as np
+import string
+from .bleu import compute_bleu_score, compute_edit_distance
+
+
+class RecMetric(object):
+    def __init__(
+        self, main_indicator="acc", is_filter=False, ignore_space=True, **kwargs
+    ):
+        self.main_indicator = main_indicator
+        self.is_filter = is_filter
+        self.ignore_space = ignore_space
+        self.eps = 1e-5
+        self.reset()
+
+    def _normalize_text(self, text):
+        text = "".join(
+            filter(lambda x: x in (string.digits + string.ascii_letters), text)
+        )
+        return text.lower()
+
+    def __call__(self, pred_label, *args, **kwargs):
+        preds, labels = pred_label
+        correct_num = 0
+        all_num = 0
+        norm_edit_dis = 0.0
+        for (pred, pred_conf), (target, _) in zip(preds, labels):
+            if self.ignore_space:
+                pred = pred.replace(" ", "")
+                target = target.replace(" ", "")
+            if self.is_filter:
+                pred = self._normalize_text(pred)
+                target = self._normalize_text(target)
+            norm_edit_dis += Levenshtein.normalized_distance(pred, target)
+            if pred == target:
+                correct_num += 1
+            all_num += 1
+        self.correct_num += correct_num
+        self.all_num += all_num
+        self.norm_edit_dis += norm_edit_dis
+        return {
+            "acc": correct_num / (all_num + self.eps),
+            "norm_edit_dis": 1 - norm_edit_dis / (all_num + self.eps),
+        }
+
+    def get_metric(self):
+        """
+        return metrics {
+                 'acc': 0,
+                 'norm_edit_dis': 0,
+            }
+        """
+        acc = 1.0 * self.correct_num / (self.all_num + self.eps)
+        norm_edit_dis = 1 - self.norm_edit_dis / (self.all_num + self.eps)
+        self.reset()
+        return {"acc": acc, "norm_edit_dis": norm_edit_dis}
+
+    def reset(self):
+        self.correct_num = 0
+        self.all_num = 0
+        self.norm_edit_dis = 0
+
+
+class CNTMetric(object):
+    def __init__(self, main_indicator="acc", **kwargs):
+        self.main_indicator = main_indicator
+        self.eps = 1e-5
+        self.reset()
+
+    def __call__(self, pred_label, *args, **kwargs):
+        preds, labels = pred_label
+        correct_num = 0
+        all_num = 0
+        for pred, target in zip(preds, labels):
+            if pred == target:
+                correct_num += 1
+            all_num += 1
+        self.correct_num += correct_num
+        self.all_num += all_num
+        return {
+            "acc": correct_num / (all_num + self.eps),
+        }
+
+    def get_metric(self):
+        """
+        return metrics {
+                 'acc': 0,
+            }
+        """
+        acc = 1.0 * self.correct_num / (self.all_num + self.eps)
+        self.reset()
+        return {"acc": acc}
+
+    def reset(self):
+        self.correct_num = 0
+        self.all_num = 0
+
+
+class CANMetric(object):
+    def __init__(self, main_indicator="exp_rate", **kwargs):
+        self.main_indicator = main_indicator
+        self.word_right = []
+        self.exp_right = []
+        self.word_total_length = 0
+        self.exp_total_num = 0
+        self.word_rate = 0
+        self.exp_rate = 0
+        self.reset()
+        self.epoch_reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        for k, v in kwargs.items():
+            epoch_reset = v
+            if epoch_reset:
+                self.epoch_reset()
+        word_probs = preds
+        word_label, word_label_mask = batch
+        line_right = 0
+        if word_probs is not None:
+            word_pred = word_probs.argmax(2)
+        word_pred = word_pred.cpu().detach().numpy()
+        word_scores = [
+            SequenceMatcher(
+                None, s1[: int(np.sum(s3))], s2[: int(np.sum(s3))], autojunk=False
+            ).ratio()
+            * (len(s1[: int(np.sum(s3))]) + len(s2[: int(np.sum(s3))]))
+            / len(s1[: int(np.sum(s3))])
+            / 2
+            for s1, s2, s3 in zip(word_label, word_pred, word_label_mask)
+        ]
+        batch_size = len(word_scores)
+        for i in range(batch_size):
+            if word_scores[i] == 1:
+                line_right += 1
+        self.word_rate = np.mean(word_scores)  # float
+        self.exp_rate = line_right / batch_size  # float
+        exp_length, word_length = word_label.shape[:2]
+        self.word_right.append(self.word_rate * word_length)
+        self.exp_right.append(self.exp_rate * exp_length)
+        self.word_total_length = self.word_total_length + word_length
+        self.exp_total_num = self.exp_total_num + exp_length
+
+    def get_metric(self):
+        """
+        return {
+            'word_rate': 0,
+            "exp_rate": 0,
+        }
+        """
+        cur_word_rate = sum(self.word_right) / self.word_total_length
+        cur_exp_rate = sum(self.exp_right) / self.exp_total_num
+        self.reset()
+        return {"word_rate": cur_word_rate, "exp_rate": cur_exp_rate}
+
+    def reset(self):
+        self.word_rate = 0
+        self.exp_rate = 0
+
+    def epoch_reset(self):
+        self.word_right = []
+        self.exp_right = []
+        self.word_total_length = 0
+        self.exp_total_num = 0
+
+
+class LaTeXOCRMetric(object):
+    def __init__(self, main_indicator="exp_rate", cal_bleu_score=False, **kwargs):
+        self.main_indicator = main_indicator
+        self.cal_bleu_score = cal_bleu_score
+        self.edit_right = []
+        self.exp_right = []
+        self.bleu_right = []
+        self.e1_right = []
+        self.e2_right = []
+        self.e3_right = []
+        self.exp_total_num = 0
+        self.edit_dist = 0
+        self.exp_rate = 0
+        if self.cal_bleu_score:
+            self.bleu_score = 0
+        self.e1 = 0
+        self.e2 = 0
+        self.e3 = 0
+        self.reset()
+        self.epoch_reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        for k, v in kwargs.items():
+            epoch_reset = v
+            if epoch_reset:
+                self.epoch_reset()
+        word_pred = preds
+        word_label = batch
+        line_right, e1, e2, e3 = 0, 0, 0, 0
+        bleu_list, lev_dist = [], []
+        for labels, prediction in zip(word_label, word_pred):
+            if prediction == labels:
+                line_right += 1
+            distance = compute_edit_distance(prediction, labels)
+            bleu_list.append(compute_bleu_score([prediction], [labels]))
+            lev_dist.append(Levenshtein.normalized_distance(prediction, labels))
+            if distance <= 1:
+                e1 += 1
+            if distance <= 2:
+                e2 += 1
+            if distance <= 3:
+                e3 += 1
+
+        batch_size = len(lev_dist)
+
+        self.edit_dist = sum(lev_dist)  # float
+        self.exp_rate = line_right  # float
+        if self.cal_bleu_score:
+            self.bleu_score = sum(bleu_list)
+            self.bleu_right.append(self.bleu_score)
+        self.e1 = e1
+        self.e2 = e2
+        self.e3 = e3
+        exp_length = len(word_label)
+        self.edit_right.append(self.edit_dist)
+        self.exp_right.append(self.exp_rate)
+        self.e1_right.append(self.e1)
+        self.e2_right.append(self.e2)
+        self.e3_right.append(self.e3)
+        self.exp_total_num = self.exp_total_num + exp_length
+
+    def get_metric(self):
+        """
+        return {
+            'edit distance': 0,
+            "bleu_score": 0,
+            "exp_rate": 0,
+        }
+        """
+        cur_edit_distance = sum(self.edit_right) / self.exp_total_num
+        cur_exp_rate = sum(self.exp_right) / self.exp_total_num
+        if self.cal_bleu_score:
+            cur_bleu_score = sum(self.bleu_right) / self.exp_total_num
+        cur_exp_1 = sum(self.e1_right) / self.exp_total_num
+        cur_exp_2 = sum(self.e2_right) / self.exp_total_num
+        cur_exp_3 = sum(self.e3_right) / self.exp_total_num
+        self.reset()
+        if self.cal_bleu_score:
+            return {
+                "bleu_score": cur_bleu_score,
+                "edit distance": cur_edit_distance,
+                "exp_rate": cur_exp_rate,
+                "exp_rate<=1 ": cur_exp_1,
+                "exp_rate<=2 ": cur_exp_2,
+                "exp_rate<=3 ": cur_exp_3,
+            }
+        else:
+
+            return {
+                "edit distance": cur_edit_distance,
+                "exp_rate": cur_exp_rate,
+                "exp_rate<=1 ": cur_exp_1,
+                "exp_rate<=2 ": cur_exp_2,
+                "exp_rate<=3 ": cur_exp_3,
+            }
+
+    def reset(self):
+        self.edit_dist = 0
+        self.exp_rate = 0
+        if self.cal_bleu_score:
+            self.bleu_score = 0
+        self.e1 = 0
+        self.e2 = 0
+        self.e3 = 0
+
+    def epoch_reset(self):
+        self.edit_right = []
+        self.exp_right = []
+        if self.cal_bleu_score:
+            self.bleu_right = []
+        self.e1_right = []
+        self.e2_right = []
+        self.e3_right = []
+        self.editdistance_total_length = 0
+        self.exp_total_num = 0

ppocr/metrics/sr_metric.py

@@ -0,0 +1,161 @@
+# copyright (c) 2022 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+https://github.com/FudanVI/FudanOCR/blob/main/text-gestalt/utils/ssim_psnr.py
+"""
+
+from math import exp
+
+import paddle
+import paddle.nn.functional as F
+import paddle.nn as nn
+import string
+
+
+class SSIM(nn.Layer):
+    def __init__(self, window_size=11, size_average=True):
+        super(SSIM, self).__init__()
+        self.window_size = window_size
+        self.size_average = size_average
+        self.channel = 1
+        self.window = self.create_window(window_size, self.channel)
+
+    def gaussian(self, window_size, sigma):
+        gauss = paddle.to_tensor(
+            [
+                exp(-((x - window_size // 2) ** 2) / float(2 * sigma**2))
+                for x in range(window_size)
+            ]
+        )
+        return gauss / gauss.sum()
+
+    def create_window(self, window_size, channel):
+        _1D_window = self.gaussian(window_size, 1.5).unsqueeze(1)
+        _2D_window = _1D_window.mm(_1D_window.t()).unsqueeze(0).unsqueeze(0)
+        window = _2D_window.expand([channel, 1, window_size, window_size])
+        return window
+
+    def _ssim(self, img1, img2, window, window_size, channel, size_average=True):
+        mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel)
+        mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel)
+
+        mu1_sq = mu1.pow(2)
+        mu2_sq = mu2.pow(2)
+        mu1_mu2 = mu1 * mu2
+
+        sigma1_sq = (
+            F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel)
+            - mu1_sq
+        )
+        sigma2_sq = (
+            F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel)
+            - mu2_sq
+        )
+        sigma12 = (
+            F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel)
+            - mu1_mu2
+        )
+
+        C1 = 0.01**2
+        C2 = 0.03**2
+
+        ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / (
+            (mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2)
+        )
+
+        if size_average:
+            return ssim_map.mean()
+        else:
+            return ssim_map.mean([1, 2, 3])
+
+    def ssim(self, img1, img2, window_size=11, size_average=True):
+        (_, channel, _, _) = img1.shape
+        window = self.create_window(window_size, channel)
+
+        return self._ssim(img1, img2, window, window_size, channel, size_average)
+
+    def forward(self, img1, img2):
+        (_, channel, _, _) = img1.shape
+
+        if channel == self.channel and self.window.dtype == img1.dtype:
+            window = self.window
+        else:
+            window = self.create_window(self.window_size, channel)
+
+            self.window = window
+            self.channel = channel
+
+        return self._ssim(
+            img1, img2, window, self.window_size, channel, self.size_average
+        )
+
+
+class SRMetric(object):
+    def __init__(self, main_indicator="all", **kwargs):
+        self.main_indicator = main_indicator
+        self.eps = 1e-5
+        self.psnr_result = []
+        self.ssim_result = []
+        self.calculate_ssim = SSIM()
+        self.reset()
+
+    def reset(self):
+        self.correct_num = 0
+        self.all_num = 0
+        self.norm_edit_dis = 0
+        self.psnr_result = []
+        self.ssim_result = []
+
+    def calculate_psnr(self, img1, img2):
+        # img1 and img2 have range [0, 1]
+        mse = ((img1 * 255 - img2 * 255) ** 2).mean()
+        if mse == 0:
+            return float("inf")
+        return 20 * paddle.log10(255.0 / paddle.sqrt(mse))
+
+    def _normalize_text(self, text):
+        text = "".join(
+            filter(lambda x: x in (string.digits + string.ascii_letters), text)
+        )
+        return text.lower()
+
+    def __call__(self, pred_label, *args, **kwargs):
+        metric = {}
+        images_sr = pred_label["sr_img"]
+        images_hr = pred_label["hr_img"]
+        psnr = self.calculate_psnr(images_sr, images_hr)
+        ssim = self.calculate_ssim(images_sr, images_hr)
+        self.psnr_result.append(psnr)
+        self.ssim_result.append(ssim)
+
+    def get_metric(self):
+        """
+        return metrics {
+                 'acc': 0,
+                 'norm_edit_dis': 0,
+            }
+        """
+        self.psnr_avg = sum(self.psnr_result) / len(self.psnr_result)
+        self.psnr_avg = round(self.psnr_avg.item(), 6)
+        self.ssim_avg = sum(self.ssim_result) / len(self.ssim_result)
+        self.ssim_avg = round(self.ssim_avg.item(), 6)
+
+        self.all_avg = self.psnr_avg + self.ssim_avg
+
+        self.reset()
+        return {
+            "psnr_avg": self.psnr_avg,
+            "ssim_avg": self.ssim_avg,
+            "all": self.all_avg,
+        }

ppocr/metrics/table_metric.py

@@ -0,0 +1,161 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+from ppocr.metrics.det_metric import DetMetric
+
+
+class TableStructureMetric(object):
+    def __init__(self, main_indicator="acc", eps=1e-6, del_thead_tbody=False, **kwargs):
+        self.main_indicator = main_indicator
+        self.eps = eps
+        self.del_thead_tbody = del_thead_tbody
+        self.reset()
+
+    def __call__(self, pred_label, batch=None, *args, **kwargs):
+        preds, labels = pred_label
+        pred_structure_batch_list = preds["structure_batch_list"]
+        gt_structure_batch_list = labels["structure_batch_list"]
+        correct_num = 0
+        all_num = 0
+        for (pred, pred_conf), target in zip(
+            pred_structure_batch_list, gt_structure_batch_list
+        ):
+            pred_str = "".join(pred)
+            target_str = "".join(target)
+            if self.del_thead_tbody:
+                pred_str = (
+                    pred_str.replace("<thead>", "")
+                    .replace("</thead>", "")
+                    .replace("<tbody>", "")
+                    .replace("</tbody>", "")
+                )
+                target_str = (
+                    target_str.replace("<thead>", "")
+                    .replace("</thead>", "")
+                    .replace("<tbody>", "")
+                    .replace("</tbody>", "")
+                )
+            if pred_str == target_str:
+                correct_num += 1
+            all_num += 1
+        self.correct_num += correct_num
+        self.all_num += all_num
+
+    def get_metric(self):
+        """
+        return metrics {
+                 'acc': 0,
+            }
+        """
+        acc = 1.0 * self.correct_num / (self.all_num + self.eps)
+        self.reset()
+        return {"acc": acc}
+
+    def reset(self):
+        self.correct_num = 0
+        self.all_num = 0
+        self.len_acc_num = 0
+        self.token_nums = 0
+        self.anys_dict = dict()
+
+
+class TableMetric(object):
+    def __init__(
+        self,
+        main_indicator="acc",
+        compute_bbox_metric=False,
+        box_format="xyxy",
+        del_thead_tbody=False,
+        **kwargs,
+    ):
+        """
+
+        @param sub_metrics: configs of sub_metric
+        @param main_matric: main_matric for save best_model
+        @param kwargs:
+        """
+        self.structure_metric = TableStructureMetric(del_thead_tbody=del_thead_tbody)
+        self.bbox_metric = DetMetric() if compute_bbox_metric else None
+        self.main_indicator = main_indicator
+        self.box_format = box_format
+        self.reset()
+
+    def __call__(self, pred_label, batch=None, *args, **kwargs):
+        self.structure_metric(pred_label)
+        if self.bbox_metric is not None:
+            self.bbox_metric(*self.prepare_bbox_metric_input(pred_label))
+
+    def prepare_bbox_metric_input(self, pred_label):
+        pred_bbox_batch_list = []
+        gt_ignore_tags_batch_list = []
+        gt_bbox_batch_list = []
+        preds, labels = pred_label
+
+        batch_num = len(preds["bbox_batch_list"])
+        for batch_idx in range(batch_num):
+            # pred
+            pred_bbox_list = [
+                self.format_box(pred_box)
+                for pred_box in preds["bbox_batch_list"][batch_idx]
+            ]
+            pred_bbox_batch_list.append({"points": pred_bbox_list})
+
+            # gt
+            gt_bbox_list = []
+            gt_ignore_tags_list = []
+            for gt_box in labels["bbox_batch_list"][batch_idx]:
+                gt_bbox_list.append(self.format_box(gt_box))
+                gt_ignore_tags_list.append(0)
+            gt_bbox_batch_list.append(gt_bbox_list)
+            gt_ignore_tags_batch_list.append(gt_ignore_tags_list)
+
+        return [
+            pred_bbox_batch_list,
+            [0, 0, gt_bbox_batch_list, gt_ignore_tags_batch_list],
+        ]
+
+    def get_metric(self):
+        structure_metric = self.structure_metric.get_metric()
+        if self.bbox_metric is None:
+            return structure_metric
+        bbox_metric = self.bbox_metric.get_metric()
+        if self.main_indicator == self.bbox_metric.main_indicator:
+            output = bbox_metric
+            for sub_key in structure_metric:
+                output["structure_metric_{}".format(sub_key)] = structure_metric[
+                    sub_key
+                ]
+        else:
+            output = structure_metric
+            for sub_key in bbox_metric:
+                output["bbox_metric_{}".format(sub_key)] = bbox_metric[sub_key]
+        return output
+
+    def reset(self):
+        self.structure_metric.reset()
+        if self.bbox_metric is not None:
+            self.bbox_metric.reset()
+
+    def format_box(self, box):
+        if self.box_format == "xyxy":
+            x1, y1, x2, y2 = box
+            box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
+        elif self.box_format == "xywh":
+            x, y, w, h = box
+            x1, y1, x2, y2 = x - w // 2, y - h // 2, x + w // 2, y + h // 2
+            box = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
+        elif self.box_format == "xyxyxyxy":
+            x1, y1, x2, y2, x3, y3, x4, y4 = box
+            box = [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+        return box

ppocr/metrics/vqa_token_re_metric.py

@@ -0,0 +1,191 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import paddle
+
+__all__ = ["VQAReTokenMetric"]
+
+
+class VQAReTokenMetric(object):
+    def __init__(self, main_indicator="hmean", **kwargs):
+        self.main_indicator = main_indicator
+        self.reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        pred_relations, relations, entities = preds
+        self.pred_relations_list.extend(pred_relations)
+        self.relations_list.extend(relations)
+        self.entities_list.extend(entities)
+
+    def get_metric(self):
+        gt_relations = []
+        for b in range(len(self.relations_list)):
+            rel_sent = []
+            relation_list = self.relations_list[b]
+            entitie_list = self.entities_list[b]
+            head_len = relation_list[0, 0]
+            if head_len > 0:
+                entitie_start_list = entitie_list[1 : entitie_list[0, 0] + 1, 0]
+                entitie_end_list = entitie_list[1 : entitie_list[0, 1] + 1, 1]
+                entitie_label_list = entitie_list[1 : entitie_list[0, 2] + 1, 2]
+                for head, tail in zip(
+                    relation_list[1 : head_len + 1, 0],
+                    relation_list[1 : head_len + 1, 1],
+                ):
+                    rel = {}
+                    rel["head_id"] = head
+                    rel["head"] = (entitie_start_list[head], entitie_end_list[head])
+                    rel["head_type"] = entitie_label_list[head]
+
+                    rel["tail_id"] = tail
+                    rel["tail"] = (entitie_start_list[tail], entitie_end_list[tail])
+                    rel["tail_type"] = entitie_label_list[tail]
+
+                    rel["type"] = 1
+                    rel_sent.append(rel)
+            gt_relations.append(rel_sent)
+        re_metrics = self.re_score(
+            self.pred_relations_list, gt_relations, mode="boundaries"
+        )
+        metrics = {
+            "precision": re_metrics["ALL"]["p"],
+            "recall": re_metrics["ALL"]["r"],
+            "hmean": re_metrics["ALL"]["f1"],
+        }
+        self.reset()
+        return metrics
+
+    def reset(self):
+        self.pred_relations_list = []
+        self.relations_list = []
+        self.entities_list = []
+
+    def re_score(self, pred_relations, gt_relations, mode="strict"):
+        """Evaluate RE predictions
+
+        Args:
+            pred_relations (list) :  list of list of predicted relations (several relations in each sentence)
+            gt_relations (list) :    list of list of ground truth relations
+
+                rel = { "head": (start_idx (inclusive), end_idx (exclusive)),
+                        "tail": (start_idx (inclusive), end_idx (exclusive)),
+                        "head_type": ent_type,
+                        "tail_type": ent_type,
+                        "type": rel_type}
+
+            vocab (Vocab) :         dataset vocabulary
+            mode (str) :            in 'strict' or 'boundaries'"""
+
+        assert mode in ["strict", "boundaries"]
+
+        relation_types = [v for v in [0, 1] if not v == 0]
+        scores = {rel: {"tp": 0, "fp": 0, "fn": 0} for rel in relation_types + ["ALL"]}
+
+        # Count GT relations and Predicted relations
+        n_sents = len(gt_relations)
+        n_rels = sum([len([rel for rel in sent]) for sent in gt_relations])
+        n_found = sum([len([rel for rel in sent]) for sent in pred_relations])
+
+        # Count TP, FP and FN per type
+        for pred_sent, gt_sent in zip(pred_relations, gt_relations):
+            for rel_type in relation_types:
+                # strict mode takes argument types into account
+                if mode == "strict":
+                    pred_rels = {
+                        (rel["head"], rel["head_type"], rel["tail"], rel["tail_type"])
+                        for rel in pred_sent
+                        if rel["type"] == rel_type
+                    }
+                    gt_rels = {
+                        (rel["head"], rel["head_type"], rel["tail"], rel["tail_type"])
+                        for rel in gt_sent
+                        if rel["type"] == rel_type
+                    }
+
+                # boundaries mode only takes argument spans into account
+                elif mode == "boundaries":
+                    pred_rels = {
+                        (rel["head"], rel["tail"])
+                        for rel in pred_sent
+                        if rel["type"] == rel_type
+                    }
+                    gt_rels = {
+                        (rel["head"], rel["tail"])
+                        for rel in gt_sent
+                        if rel["type"] == rel_type
+                    }
+
+                scores[rel_type]["tp"] += len(pred_rels & gt_rels)
+                scores[rel_type]["fp"] += len(pred_rels - gt_rels)
+                scores[rel_type]["fn"] += len(gt_rels - pred_rels)
+
+        # Compute per entity Precision / Recall / F1
+        for rel_type in scores.keys():
+            if scores[rel_type]["tp"]:
+                scores[rel_type]["p"] = scores[rel_type]["tp"] / (
+                    scores[rel_type]["fp"] + scores[rel_type]["tp"]
+                )
+                scores[rel_type]["r"] = scores[rel_type]["tp"] / (
+                    scores[rel_type]["fn"] + scores[rel_type]["tp"]
+                )
+            else:
+                scores[rel_type]["p"], scores[rel_type]["r"] = 0, 0
+
+            if not scores[rel_type]["p"] + scores[rel_type]["r"] == 0:
+                scores[rel_type]["f1"] = (
+                    2
+                    * scores[rel_type]["p"]
+                    * scores[rel_type]["r"]
+                    / (scores[rel_type]["p"] + scores[rel_type]["r"])
+                )
+            else:
+                scores[rel_type]["f1"] = 0
+
+        # Compute micro F1 Scores
+        tp = sum([scores[rel_type]["tp"] for rel_type in relation_types])
+        fp = sum([scores[rel_type]["fp"] for rel_type in relation_types])
+        fn = sum([scores[rel_type]["fn"] for rel_type in relation_types])
+
+        if tp:
+            precision = tp / (tp + fp)
+            recall = tp / (tp + fn)
+            f1 = 2 * precision * recall / (precision + recall)
+
+        else:
+            precision, recall, f1 = 0, 0, 0
+
+        scores["ALL"]["p"] = precision
+        scores["ALL"]["r"] = recall
+        scores["ALL"]["f1"] = f1
+        scores["ALL"]["tp"] = tp
+        scores["ALL"]["fp"] = fp
+        scores["ALL"]["fn"] = fn
+
+        # Compute Macro F1 Scores
+        scores["ALL"]["Macro_f1"] = np.mean(
+            [scores[ent_type]["f1"] for ent_type in relation_types]
+        )
+        scores["ALL"]["Macro_p"] = np.mean(
+            [scores[ent_type]["p"] for ent_type in relation_types]
+        )
+        scores["ALL"]["Macro_r"] = np.mean(
+            [scores[ent_type]["r"] for ent_type in relation_types]
+        )
+
+        return scores

ppocr/metrics/vqa_token_ser_metric.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import paddle
+
+__all__ = ["VQASerTokenMetric"]
+
+
+class VQASerTokenMetric(object):
+    def __init__(self, main_indicator="hmean", **kwargs):
+        self.main_indicator = main_indicator
+        self.reset()
+
+    def __call__(self, preds, batch, **kwargs):
+        preds, labels = preds
+        self.pred_list.extend(preds)
+        self.gt_list.extend(labels)
+
+    def get_metric(self):
+        from seqeval.metrics import f1_score, precision_score, recall_score
+
+        metrics = {
+            "precision": precision_score(self.gt_list, self.pred_list),
+            "recall": recall_score(self.gt_list, self.pred_list),
+            "hmean": f1_score(self.gt_list, self.pred_list),
+        }
+        self.reset()
+        return metrics
+
+    def reset(self):
+        self.pred_list = []
+        self.gt_list = []