first commit

ppocr/data/imaug/vqa/token/__init__.py

@@ -0,0 +1,18 @@
+# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .vqa_token_chunk import VQASerTokenChunk, VQAReTokenChunk
+from .vqa_token_pad import VQATokenPad
+from .vqa_token_relation import VQAReTokenRelation
+from .vqa_re_convert import TensorizeEntitiesRelations

ppocr/data/imaug/vqa/token/vqa_re_convert.py

@@ -0,0 +1,49 @@
+# copyright (c) 2022 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+
+class TensorizeEntitiesRelations(object):
+    def __init__(self, max_seq_len=512, infer_mode=False, **kwargs):
+        self.max_seq_len = max_seq_len
+        self.infer_mode = infer_mode
+
+    def __call__(self, data):
+        entities = data["entities"]
+        relations = data["relations"]
+
+        entities_new = np.full(
+            shape=[self.max_seq_len + 1, 3], fill_value=-1, dtype="int64"
+        )
+        entities_new[0, 0] = len(entities["start"])
+        entities_new[0, 1] = len(entities["end"])
+        entities_new[0, 2] = len(entities["label"])
+        entities_new[1 : len(entities["start"]) + 1, 0] = np.array(entities["start"])
+        entities_new[1 : len(entities["end"]) + 1, 1] = np.array(entities["end"])
+        entities_new[1 : len(entities["label"]) + 1, 2] = np.array(entities["label"])
+
+        relations_new = np.full(
+            shape=[self.max_seq_len * self.max_seq_len + 1, 2],
+            fill_value=-1,
+            dtype="int64",
+        )
+        relations_new[0, 0] = len(relations["head"])
+        relations_new[0, 1] = len(relations["tail"])
+        relations_new[1 : len(relations["head"]) + 1, 0] = np.array(relations["head"])
+        relations_new[1 : len(relations["tail"]) + 1, 1] = np.array(relations["tail"])
+
+        data["entities"] = entities_new
+        data["relations"] = relations_new
+        return data

ppocr/data/imaug/vqa/token/vqa_token_chunk.py

@@ -0,0 +1,134 @@
+# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import defaultdict
+
+
+class VQASerTokenChunk(object):
+    def __init__(self, max_seq_len=512, infer_mode=False, **kwargs):
+        self.max_seq_len = max_seq_len
+        self.infer_mode = infer_mode
+
+    def __call__(self, data):
+        encoded_inputs_all = []
+        seq_len = len(data["input_ids"])
+        for index in range(0, seq_len, self.max_seq_len):
+            chunk_beg = index
+            chunk_end = min(index + self.max_seq_len, seq_len)
+            encoded_inputs_example = {}
+            for key in data:
+                if key in [
+                    "label",
+                    "input_ids",
+                    "labels",
+                    "token_type_ids",
+                    "bbox",
+                    "attention_mask",
+                ]:
+                    if self.infer_mode and key == "labels":
+                        encoded_inputs_example[key] = data[key]
+                    else:
+                        encoded_inputs_example[key] = data[key][chunk_beg:chunk_end]
+                else:
+                    encoded_inputs_example[key] = data[key]
+
+            encoded_inputs_all.append(encoded_inputs_example)
+        if len(encoded_inputs_all) == 0:
+            return None
+        return encoded_inputs_all[0]
+
+
+class VQAReTokenChunk(object):
+    def __init__(
+        self, max_seq_len=512, entities_labels=None, infer_mode=False, **kwargs
+    ):
+        self.max_seq_len = max_seq_len
+        self.entities_labels = (
+            {"HEADER": 0, "QUESTION": 1, "ANSWER": 2}
+            if entities_labels is None
+            else entities_labels
+        )
+        self.infer_mode = infer_mode
+
+    def __call__(self, data):
+        # prepare data
+        entities = data.pop("entities")
+        relations = data.pop("relations")
+        encoded_inputs_all = []
+        for index in range(0, len(data["input_ids"]), self.max_seq_len):
+            item = {}
+            for key in data:
+                if key in [
+                    "label",
+                    "input_ids",
+                    "labels",
+                    "token_type_ids",
+                    "bbox",
+                    "attention_mask",
+                ]:
+                    if self.infer_mode and key == "labels":
+                        item[key] = data[key]
+                    else:
+                        item[key] = data[key][index : index + self.max_seq_len]
+                else:
+                    item[key] = data[key]
+            # select entity in current chunk
+            entities_in_this_span = []
+            global_to_local_map = {}  #
+            for entity_id, entity in enumerate(entities):
+                if (
+                    index <= entity["start"] < index + self.max_seq_len
+                    and index <= entity["end"] < index + self.max_seq_len
+                ):
+                    entity["start"] = entity["start"] - index
+                    entity["end"] = entity["end"] - index
+                    global_to_local_map[entity_id] = len(entities_in_this_span)
+                    entities_in_this_span.append(entity)
+
+            # select relations in current chunk
+            relations_in_this_span = []
+            for relation in relations:
+                if (
+                    index <= relation["start_index"] < index + self.max_seq_len
+                    and index <= relation["end_index"] < index + self.max_seq_len
+                ):
+                    relations_in_this_span.append(
+                        {
+                            "head": global_to_local_map[relation["head"]],
+                            "tail": global_to_local_map[relation["tail"]],
+                            "start_index": relation["start_index"] - index,
+                            "end_index": relation["end_index"] - index,
+                        }
+                    )
+            item.update(
+                {
+                    "entities": self.reformat(entities_in_this_span),
+                    "relations": self.reformat(relations_in_this_span),
+                }
+            )
+            if len(item["entities"]) > 0:
+                item["entities"]["label"] = [
+                    self.entities_labels[x] for x in item["entities"]["label"]
+                ]
+                encoded_inputs_all.append(item)
+        if len(encoded_inputs_all) == 0:
+            return None
+        return encoded_inputs_all[0]
+
+    def reformat(self, data):
+        new_data = defaultdict(list)
+        for item in data:
+            for k, v in item.items():
+                new_data[k].append(v)
+        return new_data

ppocr/data/imaug/vqa/token/vqa_token_pad.py

@@ -0,0 +1,117 @@
+# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import paddle
+import numpy as np
+
+
+class VQATokenPad(object):
+    def __init__(
+        self,
+        max_seq_len=512,
+        pad_to_max_seq_len=True,
+        return_attention_mask=True,
+        return_token_type_ids=True,
+        truncation_strategy="longest_first",
+        return_overflowing_tokens=False,
+        return_special_tokens_mask=False,
+        infer_mode=False,
+        **kwargs,
+    ):
+        self.max_seq_len = max_seq_len
+        self.pad_to_max_seq_len = max_seq_len
+        self.return_attention_mask = return_attention_mask
+        self.return_token_type_ids = return_token_type_ids
+        self.truncation_strategy = truncation_strategy
+        self.return_overflowing_tokens = return_overflowing_tokens
+        self.return_special_tokens_mask = return_special_tokens_mask
+        self.pad_token_label_id = paddle.nn.CrossEntropyLoss().ignore_index
+        self.infer_mode = infer_mode
+
+    def __call__(self, data):
+        needs_to_be_padded = (
+            self.pad_to_max_seq_len and len(data["input_ids"]) < self.max_seq_len
+        )
+
+        if needs_to_be_padded:
+            if "tokenizer_params" in data:
+                tokenizer_params = data.pop("tokenizer_params")
+            else:
+                tokenizer_params = dict(
+                    padding_side="right", pad_token_type_id=0, pad_token_id=1
+                )
+
+            difference = self.max_seq_len - len(data["input_ids"])
+            if tokenizer_params["padding_side"] == "right":
+                if self.return_attention_mask:
+                    data["attention_mask"] = [1] * len(data["input_ids"]) + [
+                        0
+                    ] * difference
+                if self.return_token_type_ids:
+                    data["token_type_ids"] = (
+                        data["token_type_ids"]
+                        + [tokenizer_params["pad_token_type_id"]] * difference
+                    )
+                if self.return_special_tokens_mask:
+                    data["special_tokens_mask"] = (
+                        data["special_tokens_mask"] + [1] * difference
+                    )
+                data["input_ids"] = (
+                    data["input_ids"] + [tokenizer_params["pad_token_id"]] * difference
+                )
+                if not self.infer_mode:
+                    data["labels"] = (
+                        data["labels"] + [self.pad_token_label_id] * difference
+                    )
+                data["bbox"] = data["bbox"] + [[0, 0, 0, 0]] * difference
+            elif tokenizer_params["padding_side"] == "left":
+                if self.return_attention_mask:
+                    data["attention_mask"] = [0] * difference + [1] * len(
+                        data["input_ids"]
+                    )
+                if self.return_token_type_ids:
+                    data["token_type_ids"] = [
+                        tokenizer_params["pad_token_type_id"]
+                    ] * difference + data["token_type_ids"]
+                if self.return_special_tokens_mask:
+                    data["special_tokens_mask"] = [1] * difference + data[
+                        "special_tokens_mask"
+                    ]
+                data["input_ids"] = [
+                    tokenizer_params["pad_token_id"]
+                ] * difference + data["input_ids"]
+                if not self.infer_mode:
+                    data["labels"] = [self.pad_token_label_id] * difference + data[
+                        "labels"
+                    ]
+                data["bbox"] = [[0, 0, 0, 0]] * difference + data["bbox"]
+        else:
+            if self.return_attention_mask:
+                data["attention_mask"] = [1] * len(data["input_ids"])
+
+        for key in data:
+            if key in [
+                "input_ids",
+                "labels",
+                "token_type_ids",
+                "bbox",
+                "attention_mask",
+            ]:
+                if self.infer_mode:
+                    if key != "labels":
+                        length = min(len(data[key]), self.max_seq_len)
+                        data[key] = data[key][:length]
+                    else:
+                        continue
+                data[key] = np.array(data[key], dtype="int64")
+        return data

ppocr/data/imaug/vqa/token/vqa_token_relation.py

@@ -0,0 +1,76 @@
+# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class VQAReTokenRelation(object):
+    def __init__(self, **kwargs):
+        pass
+
+    def __call__(self, data):
+        """
+        build relations
+        """
+        entities = data["entities"]
+        relations = data["relations"]
+        id2label = data.pop("id2label")
+        empty_entity = data.pop("empty_entity")
+        entity_id_to_index_map = data.pop("entity_id_to_index_map")
+
+        relations = list(set(relations))
+        relations = [
+            rel
+            for rel in relations
+            if rel[0] not in empty_entity and rel[1] not in empty_entity
+        ]
+        kv_relations = []
+        for rel in relations:
+            pair = [id2label[rel[0]], id2label[rel[1]]]
+            if pair == ["question", "answer"]:
+                kv_relations.append(
+                    {
+                        "head": entity_id_to_index_map[rel[0]],
+                        "tail": entity_id_to_index_map[rel[1]],
+                    }
+                )
+            elif pair == ["answer", "question"]:
+                kv_relations.append(
+                    {
+                        "head": entity_id_to_index_map[rel[1]],
+                        "tail": entity_id_to_index_map[rel[0]],
+                    }
+                )
+            else:
+                continue
+        relations = sorted(
+            [
+                {
+                    "head": rel["head"],
+                    "tail": rel["tail"],
+                    "start_index": self.get_relation_span(rel, entities)[0],
+                    "end_index": self.get_relation_span(rel, entities)[1],
+                }
+                for rel in kv_relations
+            ],
+            key=lambda x: x["head"],
+        )
+
+        data["relations"] = relations
+        return data
+
+    def get_relation_span(self, rel, entities):
+        bound = []
+        for entity_index in [rel["head"], rel["tail"]]:
+            bound.append(entities[entity_index]["start"])
+            bound.append(entities[entity_index]["end"])
+        return min(bound), max(bound)