athena__activation_selu Module

Module containing implementation of the SELU activation function

This module implements Scaled Exponential Linear Unit (SELU), which has self-normalizing properties for deep networks.

Mathematical operation:

where and preserve mean=0, variance=1

Derivative:

Properties: Self-normalizing, enables very deep networks Requires: Lecun Normal initialisation, alpha dropout Reference: Klambauer et al. (2017), NeurIPS


Uses


Interfaces

public interface selu_actv_type

  • private function initialise(scale, alpha, lambda, attributes) result(activation)

    Initialise a SELU activation function

    Arguments

    Type IntentOptional Attributes Name
    real(kind=real32), intent(in), optional :: scale

    Optional scale factor for activation output
    real(kind=real32), intent(in), optional :: alpha

    Optional alpha parameter (default: 1.67326)
    real(kind=real32), intent(in), optional :: lambda

    Optional lambda parameter (default: 1.0507)
    type(onnx_attribute_type), intent(in), optional :: attributes(:)

    Optional ONNX attributes

    Return Value type(selu_actv_type)

    SELU activation type


Derived Types

type, public, extends(relu_actv_type) ::  selu_actv_type

Type for SELU activation function with overloaded procedures

Components

Type Visibility Attributes Name Initial
real(kind=real32), public :: alpha = 1.6732632423543772848170429916717_real32

Alpha parameter for SELU
logical, public :: apply_scaling = .false.

Boolean to apply scaling or not
real(kind=real32), public :: lambda = 1.0507009873554804934193349852946_real32

Lambda parameter for SELU
character(len=10), public :: name

Name of the activation function
real(kind=real32), public :: scale = 1._real32

Scale of the activation function
real(kind=real32), public :: threshold

Threshold of the activation function

Constructor

private function initialise (scale, alpha, lambda, attributes)

Initialise a SELU activation function

Type-Bound Procedures

procedure, public, pass(this) :: apply => apply_selu
procedure, public, pass(this) :: apply_attributes => apply_attributes_selu
procedure, public, pass(this) :: export_attributes => export_attributes_selu
procedure, public, pass(this) :: print_to_unit => print_to_unit_actv
procedure, public, pass(this) :: reset => reset_selu

Functions

public function create_from_onnx_selu_activation(attributes) result(activation)

Create SELU activation function from ONNX attributes

Arguments

Type IntentOptional Attributes Name
type(onnx_attribute_type), intent(in), dimension(:) :: attributes

Array of ONNX attributes

Return Value class(base_actv_type), allocatable

Instance of activation type

private function apply_selu(this, val) result(output)

Apply SELU activation to array

Read more…

Arguments

Type IntentOptional Attributes Name
class(selu_actv_type), intent(in) :: this

SELU activation type
type(array_type), intent(in) :: val

Input values

Return Value type(array_type), pointer

Activated output values

private pure function export_attributes_selu(this) result(attributes)

Export SELU activation function attributes as ONNX attributes

Arguments

Type IntentOptional Attributes Name
class(selu_actv_type), intent(in) :: this

SELU activation type

Return Value type(onnx_attribute_type), allocatable, dimension(:)

Array of ONNX attributes

private function initialise(scale, alpha, lambda, attributes) result(activation)

Initialise a SELU activation function

Arguments

Type IntentOptional Attributes Name
real(kind=real32), intent(in), optional :: scale

Optional scale factor for activation output
real(kind=real32), intent(in), optional :: alpha

Optional alpha parameter (default: 1.67326)
real(kind=real32), intent(in), optional :: lambda

Optional lambda parameter (default: 1.0507)
type(onnx_attribute_type), intent(in), optional :: attributes(:)

Optional ONNX attributes

Return Value type(selu_actv_type)

SELU activation type


Subroutines

private subroutine apply_attributes_selu(this, attributes)

Load ONNX attributes into SELU activation function

Arguments

Type IntentOptional Attributes Name
class(selu_actv_type), intent(inout) :: this

SELU activation type
type(onnx_attribute_type), intent(in), dimension(:) :: attributes

Array of ONNX attributes

private pure subroutine reset_selu(this)

Reset SELU activation function attributes and variables

Arguments

Type IntentOptional Attributes Name
class(selu_actv_type), intent(inout) :: this

SELU activation type