acon.py

import numpy as np

import paddle
import paddle.nn as nn
import paddle.nn.functional as F


class AconC(nn.Layer):
    r""" ACON activation (activate or not).
    # AconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is a learnable parameter
    # according to "Activate or Not: Learning Customized Activation" <https://arxiv.org/pdf/2009.04759.pdf>.
    """

    def __init__(self, width):
        super().__init__()
        self.p1 = self.create_parameter(
            shape=(1, width, 1, 1),
            default_initializer=nn.initializer.Normal())
        self.p2 = self.create_parameter(
            shape=(1, width, 1, 1),
            default_initializer=nn.initializer.Normal())
        self.beta = self.create_parameter(
            shape=(1, width, 1, 1),
            default_initializer=nn.initializer.Constant(1.0))

    def forward(self, x):
        return (self.p1 * x - self.p2 * x) * F.sigmoid(self.beta * (
            self.p1 * x - self.p2 * x)) + self.p2 * x


class MetaAconC(nn.Layer):
    r""" ACON activation (activate or not).
    # MetaAconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is generated by a small network
    # according to "Activate or Not: Learning Customized Activation" <https://arxiv.org/pdf/2009.04759.pdf>.
    """

    def __init__(self, width, r=16):
        super().__init__()
        self.fc1 = nn.Conv2D(
            width, max(r, width // r), kernel_size=1, stride=1, bias_attr=True)
        self.bn1 = nn.BatchNorm2D(max(r, width // r))
        self.fc2 = nn.Conv2D(
            max(r, width // r), width, kernel_size=1, stride=1, bias_attr=True)
        self.bn2 = nn.BatchNorm2D(width)

        self.p1 = self.create_parameter(
            shape=(1, width, 1, 1),
            default_initializer=nn.initializer.Normal())
        self.p2 = self.create_parameter(
            shape=(1, width, 1, 1),
            default_initializer=nn.initializer.Normal())

    def forward(self, x):
        beta = F.sigmoid(
            self.bn2(
                self.fc2(
                    self.bn1(
                        self.fc1(
                            x.mean(
                                axis=2, keepdim=True).mean(
                                    axis=3, keepdim=True))))))
        return (self.p1 * x - self.p2 * x) * F.sigmoid(beta * (
            self.p1 * x - self.p2 * x)) + self.p2 * x


if __name__ == "__main__":
    paddle.set_device("cpu")
    x = paddle.rand([2, 32, 8, 10])
    acon_func = AconC(32)
    meta_acon_func = MetaAconC(32)
    y1 = acon_func(x)
    print(y1)
    y2 = meta_acon_func(x)
    print(y2)