This commit is contained in:
492
ppocr/modeling/necks/db_fpn.py
Normal file
492
ppocr/modeling/necks/db_fpn.py
Normal file
@@ -0,0 +1,492 @@
|
||||
# copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import paddle
|
||||
from paddle import nn
|
||||
import paddle.nn.functional as F
|
||||
from paddle import ParamAttr
|
||||
import os
|
||||
import sys
|
||||
from ppocr.modeling.necks.intracl import IntraCLBlock
|
||||
|
||||
__dir__ = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.append(__dir__)
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(__dir__, "../../..")))
|
||||
|
||||
from ppocr.modeling.backbones.det_mobilenet_v3 import SEModule
|
||||
|
||||
|
||||
class DSConv(nn.Layer):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels,
|
||||
out_channels,
|
||||
kernel_size,
|
||||
padding,
|
||||
stride=1,
|
||||
groups=None,
|
||||
if_act=True,
|
||||
act="relu",
|
||||
**kwargs,
|
||||
):
|
||||
super(DSConv, self).__init__()
|
||||
if groups == None:
|
||||
groups = in_channels
|
||||
self.if_act = if_act
|
||||
self.act = act
|
||||
self.conv1 = nn.Conv2D(
|
||||
in_channels=in_channels,
|
||||
out_channels=in_channels,
|
||||
kernel_size=kernel_size,
|
||||
stride=stride,
|
||||
padding=padding,
|
||||
groups=groups,
|
||||
bias_attr=False,
|
||||
)
|
||||
|
||||
self.bn1 = nn.BatchNorm(num_channels=in_channels, act=None)
|
||||
|
||||
self.conv2 = nn.Conv2D(
|
||||
in_channels=in_channels,
|
||||
out_channels=int(in_channels * 4),
|
||||
kernel_size=1,
|
||||
stride=1,
|
||||
bias_attr=False,
|
||||
)
|
||||
|
||||
self.bn2 = nn.BatchNorm(num_channels=int(in_channels * 4), act=None)
|
||||
|
||||
self.conv3 = nn.Conv2D(
|
||||
in_channels=int(in_channels * 4),
|
||||
out_channels=out_channels,
|
||||
kernel_size=1,
|
||||
stride=1,
|
||||
bias_attr=False,
|
||||
)
|
||||
self._c = [in_channels, out_channels]
|
||||
if in_channels != out_channels:
|
||||
self.conv_end = nn.Conv2D(
|
||||
in_channels=in_channels,
|
||||
out_channels=out_channels,
|
||||
kernel_size=1,
|
||||
stride=1,
|
||||
bias_attr=False,
|
||||
)
|
||||
|
||||
def forward(self, inputs):
|
||||
x = self.conv1(inputs)
|
||||
x = self.bn1(x)
|
||||
|
||||
x = self.conv2(x)
|
||||
x = self.bn2(x)
|
||||
if self.if_act:
|
||||
if self.act == "relu":
|
||||
x = F.relu(x)
|
||||
elif self.act == "hardswish":
|
||||
x = F.hardswish(x)
|
||||
else:
|
||||
print(
|
||||
"The activation function({}) is selected incorrectly.".format(
|
||||
self.act
|
||||
)
|
||||
)
|
||||
exit()
|
||||
|
||||
x = self.conv3(x)
|
||||
if self._c[0] != self._c[1]:
|
||||
x = x + self.conv_end(inputs)
|
||||
return x
|
||||
|
||||
|
||||
class DBFPN(nn.Layer):
|
||||
def __init__(self, in_channels, out_channels, use_asf=False, **kwargs):
|
||||
super(DBFPN, self).__init__()
|
||||
self.out_channels = out_channels
|
||||
self.use_asf = use_asf
|
||||
weight_attr = paddle.nn.initializer.KaimingUniform()
|
||||
|
||||
self.in2_conv = nn.Conv2D(
|
||||
in_channels=in_channels[0],
|
||||
out_channels=self.out_channels,
|
||||
kernel_size=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.in3_conv = nn.Conv2D(
|
||||
in_channels=in_channels[1],
|
||||
out_channels=self.out_channels,
|
||||
kernel_size=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.in4_conv = nn.Conv2D(
|
||||
in_channels=in_channels[2],
|
||||
out_channels=self.out_channels,
|
||||
kernel_size=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.in5_conv = nn.Conv2D(
|
||||
in_channels=in_channels[3],
|
||||
out_channels=self.out_channels,
|
||||
kernel_size=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.p5_conv = nn.Conv2D(
|
||||
in_channels=self.out_channels,
|
||||
out_channels=self.out_channels // 4,
|
||||
kernel_size=3,
|
||||
padding=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.p4_conv = nn.Conv2D(
|
||||
in_channels=self.out_channels,
|
||||
out_channels=self.out_channels // 4,
|
||||
kernel_size=3,
|
||||
padding=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.p3_conv = nn.Conv2D(
|
||||
in_channels=self.out_channels,
|
||||
out_channels=self.out_channels // 4,
|
||||
kernel_size=3,
|
||||
padding=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.p2_conv = nn.Conv2D(
|
||||
in_channels=self.out_channels,
|
||||
out_channels=self.out_channels // 4,
|
||||
kernel_size=3,
|
||||
padding=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
|
||||
if self.use_asf is True:
|
||||
self.asf = ASFBlock(self.out_channels, self.out_channels // 4)
|
||||
|
||||
def forward(self, x):
|
||||
c2, c3, c4, c5 = x
|
||||
|
||||
in5 = self.in5_conv(c5)
|
||||
in4 = self.in4_conv(c4)
|
||||
in3 = self.in3_conv(c3)
|
||||
in2 = self.in2_conv(c2)
|
||||
|
||||
out4 = in4 + F.upsample(
|
||||
in5, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/16
|
||||
out3 = in3 + F.upsample(
|
||||
out4, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/8
|
||||
out2 = in2 + F.upsample(
|
||||
out3, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/4
|
||||
|
||||
p5 = self.p5_conv(in5)
|
||||
p4 = self.p4_conv(out4)
|
||||
p3 = self.p3_conv(out3)
|
||||
p2 = self.p2_conv(out2)
|
||||
p5 = F.upsample(p5, scale_factor=8, mode="nearest", align_mode=1)
|
||||
p4 = F.upsample(p4, scale_factor=4, mode="nearest", align_mode=1)
|
||||
p3 = F.upsample(p3, scale_factor=2, mode="nearest", align_mode=1)
|
||||
|
||||
fuse = paddle.concat([p5, p4, p3, p2], axis=1)
|
||||
|
||||
if self.use_asf is True:
|
||||
fuse = self.asf(fuse, [p5, p4, p3, p2])
|
||||
|
||||
return fuse
|
||||
|
||||
|
||||
class RSELayer(nn.Layer):
|
||||
def __init__(self, in_channels, out_channels, kernel_size, shortcut=True):
|
||||
super(RSELayer, self).__init__()
|
||||
weight_attr = paddle.nn.initializer.KaimingUniform()
|
||||
self.out_channels = out_channels
|
||||
self.in_conv = nn.Conv2D(
|
||||
in_channels=in_channels,
|
||||
out_channels=self.out_channels,
|
||||
kernel_size=kernel_size,
|
||||
padding=int(kernel_size // 2),
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
self.se_block = SEModule(self.out_channels)
|
||||
self.shortcut = shortcut
|
||||
|
||||
def forward(self, ins):
|
||||
x = self.in_conv(ins)
|
||||
if self.shortcut:
|
||||
out = x + self.se_block(x)
|
||||
else:
|
||||
out = self.se_block(x)
|
||||
return out
|
||||
|
||||
|
||||
class RSEFPN(nn.Layer):
|
||||
def __init__(self, in_channels, out_channels, shortcut=True, **kwargs):
|
||||
super(RSEFPN, self).__init__()
|
||||
self.out_channels = out_channels
|
||||
self.ins_conv = nn.LayerList()
|
||||
self.inp_conv = nn.LayerList()
|
||||
self.intracl = False
|
||||
if "intracl" in kwargs.keys() and kwargs["intracl"] is True:
|
||||
self.intracl = kwargs["intracl"]
|
||||
self.incl1 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
self.incl2 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
self.incl3 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
self.incl4 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
|
||||
for i in range(len(in_channels)):
|
||||
self.ins_conv.append(
|
||||
RSELayer(in_channels[i], out_channels, kernel_size=1, shortcut=shortcut)
|
||||
)
|
||||
self.inp_conv.append(
|
||||
RSELayer(
|
||||
out_channels, out_channels // 4, kernel_size=3, shortcut=shortcut
|
||||
)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
c2, c3, c4, c5 = x
|
||||
|
||||
in5 = self.ins_conv[3](c5)
|
||||
in4 = self.ins_conv[2](c4)
|
||||
in3 = self.ins_conv[1](c3)
|
||||
in2 = self.ins_conv[0](c2)
|
||||
|
||||
out4 = in4 + F.upsample(
|
||||
in5, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/16
|
||||
out3 = in3 + F.upsample(
|
||||
out4, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/8
|
||||
out2 = in2 + F.upsample(
|
||||
out3, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/4
|
||||
|
||||
p5 = self.inp_conv[3](in5)
|
||||
p4 = self.inp_conv[2](out4)
|
||||
p3 = self.inp_conv[1](out3)
|
||||
p2 = self.inp_conv[0](out2)
|
||||
|
||||
if self.intracl is True:
|
||||
p5 = self.incl4(p5)
|
||||
p4 = self.incl3(p4)
|
||||
p3 = self.incl2(p3)
|
||||
p2 = self.incl1(p2)
|
||||
|
||||
p5 = F.upsample(p5, scale_factor=8, mode="nearest", align_mode=1)
|
||||
p4 = F.upsample(p4, scale_factor=4, mode="nearest", align_mode=1)
|
||||
p3 = F.upsample(p3, scale_factor=2, mode="nearest", align_mode=1)
|
||||
|
||||
fuse = paddle.concat([p5, p4, p3, p2], axis=1)
|
||||
return fuse
|
||||
|
||||
|
||||
class LKPAN(nn.Layer):
|
||||
def __init__(self, in_channels, out_channels, mode="large", **kwargs):
|
||||
super(LKPAN, self).__init__()
|
||||
self.out_channels = out_channels
|
||||
weight_attr = paddle.nn.initializer.KaimingUniform()
|
||||
|
||||
self.ins_conv = nn.LayerList()
|
||||
self.inp_conv = nn.LayerList()
|
||||
# pan head
|
||||
self.pan_head_conv = nn.LayerList()
|
||||
self.pan_lat_conv = nn.LayerList()
|
||||
|
||||
if mode.lower() == "lite":
|
||||
p_layer = DSConv
|
||||
elif mode.lower() == "large":
|
||||
p_layer = nn.Conv2D
|
||||
else:
|
||||
raise ValueError(
|
||||
"mode can only be one of ['lite', 'large'], but received {}".format(
|
||||
mode
|
||||
)
|
||||
)
|
||||
|
||||
for i in range(len(in_channels)):
|
||||
self.ins_conv.append(
|
||||
nn.Conv2D(
|
||||
in_channels=in_channels[i],
|
||||
out_channels=self.out_channels,
|
||||
kernel_size=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
)
|
||||
|
||||
self.inp_conv.append(
|
||||
p_layer(
|
||||
in_channels=self.out_channels,
|
||||
out_channels=self.out_channels // 4,
|
||||
kernel_size=9,
|
||||
padding=4,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
)
|
||||
|
||||
if i > 0:
|
||||
self.pan_head_conv.append(
|
||||
nn.Conv2D(
|
||||
in_channels=self.out_channels // 4,
|
||||
out_channels=self.out_channels // 4,
|
||||
kernel_size=3,
|
||||
padding=1,
|
||||
stride=2,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
)
|
||||
self.pan_lat_conv.append(
|
||||
p_layer(
|
||||
in_channels=self.out_channels // 4,
|
||||
out_channels=self.out_channels // 4,
|
||||
kernel_size=9,
|
||||
padding=4,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
bias_attr=False,
|
||||
)
|
||||
)
|
||||
|
||||
self.intracl = False
|
||||
if "intracl" in kwargs.keys() and kwargs["intracl"] is True:
|
||||
self.intracl = kwargs["intracl"]
|
||||
self.incl1 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
self.incl2 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
self.incl3 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
self.incl4 = IntraCLBlock(self.out_channels // 4, reduce_factor=2)
|
||||
|
||||
def forward(self, x):
|
||||
c2, c3, c4, c5 = x
|
||||
|
||||
in5 = self.ins_conv[3](c5)
|
||||
in4 = self.ins_conv[2](c4)
|
||||
in3 = self.ins_conv[1](c3)
|
||||
in2 = self.ins_conv[0](c2)
|
||||
|
||||
out4 = in4 + F.upsample(
|
||||
in5, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/16
|
||||
out3 = in3 + F.upsample(
|
||||
out4, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/8
|
||||
out2 = in2 + F.upsample(
|
||||
out3, scale_factor=2, mode="nearest", align_mode=1
|
||||
) # 1/4
|
||||
|
||||
f5 = self.inp_conv[3](in5)
|
||||
f4 = self.inp_conv[2](out4)
|
||||
f3 = self.inp_conv[1](out3)
|
||||
f2 = self.inp_conv[0](out2)
|
||||
|
||||
pan3 = f3 + self.pan_head_conv[0](f2)
|
||||
pan4 = f4 + self.pan_head_conv[1](pan3)
|
||||
pan5 = f5 + self.pan_head_conv[2](pan4)
|
||||
|
||||
p2 = self.pan_lat_conv[0](f2)
|
||||
p3 = self.pan_lat_conv[1](pan3)
|
||||
p4 = self.pan_lat_conv[2](pan4)
|
||||
p5 = self.pan_lat_conv[3](pan5)
|
||||
|
||||
if self.intracl is True:
|
||||
p5 = self.incl4(p5)
|
||||
p4 = self.incl3(p4)
|
||||
p3 = self.incl2(p3)
|
||||
p2 = self.incl1(p2)
|
||||
|
||||
p5 = F.upsample(p5, scale_factor=8, mode="nearest", align_mode=1)
|
||||
p4 = F.upsample(p4, scale_factor=4, mode="nearest", align_mode=1)
|
||||
p3 = F.upsample(p3, scale_factor=2, mode="nearest", align_mode=1)
|
||||
|
||||
fuse = paddle.concat([p5, p4, p3, p2], axis=1)
|
||||
return fuse
|
||||
|
||||
|
||||
class ASFBlock(nn.Layer):
|
||||
"""
|
||||
This code is referred from:
|
||||
https://github.com/MhLiao/DB/blob/master/decoders/feature_attention.py
|
||||
"""
|
||||
|
||||
def __init__(self, in_channels, inter_channels, out_features_num=4):
|
||||
"""
|
||||
Adaptive Scale Fusion (ASF) block of DBNet++
|
||||
Args:
|
||||
in_channels: the number of channels in the input data
|
||||
inter_channels: the number of middle channels
|
||||
out_features_num: the number of fused stages
|
||||
"""
|
||||
super(ASFBlock, self).__init__()
|
||||
weight_attr = paddle.nn.initializer.KaimingUniform()
|
||||
self.in_channels = in_channels
|
||||
self.inter_channels = inter_channels
|
||||
self.out_features_num = out_features_num
|
||||
self.conv = nn.Conv2D(in_channels, inter_channels, 3, padding=1)
|
||||
|
||||
self.spatial_scale = nn.Sequential(
|
||||
# Nx1xHxW
|
||||
nn.Conv2D(
|
||||
in_channels=1,
|
||||
out_channels=1,
|
||||
kernel_size=3,
|
||||
bias_attr=False,
|
||||
padding=1,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
),
|
||||
nn.ReLU(),
|
||||
nn.Conv2D(
|
||||
in_channels=1,
|
||||
out_channels=1,
|
||||
kernel_size=1,
|
||||
bias_attr=False,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
),
|
||||
nn.Sigmoid(),
|
||||
)
|
||||
|
||||
self.channel_scale = nn.Sequential(
|
||||
nn.Conv2D(
|
||||
in_channels=inter_channels,
|
||||
out_channels=out_features_num,
|
||||
kernel_size=1,
|
||||
bias_attr=False,
|
||||
weight_attr=ParamAttr(initializer=weight_attr),
|
||||
),
|
||||
nn.Sigmoid(),
|
||||
)
|
||||
|
||||
def forward(self, fuse_features, features_list):
|
||||
fuse_features = self.conv(fuse_features)
|
||||
spatial_x = paddle.mean(fuse_features, axis=1, keepdim=True)
|
||||
attention_scores = self.spatial_scale(spatial_x) + fuse_features
|
||||
attention_scores = self.channel_scale(attention_scores)
|
||||
assert len(features_list) == self.out_features_num
|
||||
|
||||
out_list = []
|
||||
for i in range(self.out_features_num):
|
||||
out_list.append(attention_scores[:, i : i + 1] * features_list[i])
|
||||
return paddle.concat(out_list, axis=1)
|
||||
Reference in New Issue
Block a user