athena__initialiser_he Module

Module containing the implementation of the He initialiser

This module implements He (Kaiming/MSRA) initialisation, designed for layers with ReLU activation to prevent vanishing/exploding gradients.

Mathematical operation:

Uniform variant:

Normal variant:

where is the number of input units (fan-in).

Rationale: Maintains variance through ReLU layers

Best for: ReLU, Leaky ReLU, PReLU activations Reference: He et al. (2015), ICCV, arXiv:1502.01852


Uses


Interfaces

public interface he_normal_init_type

  • private module function initialiser_normal_setup(scale, mode) result(initialiser)

    Interface for the He normal initialiser

    Arguments

    Type IntentOptional Attributes Name
    real(kind=real32), intent(in), optional :: scale

    Scaling factor (default: 1.0)
    character(len=*), intent(in), optional :: mode

    Mode for calculating the scaling factor (default: "fan_in")

    Return Value type(he_normal_init_type)

    He normal initialiser object

public interface he_uniform_init_type

  • private module function initialiser_uniform_setup(scale, mode) result(initialiser)

    Interface for the He uniform initialiser

    Arguments

    Type IntentOptional Attributes Name
    real(kind=real32), intent(in), optional :: scale

    Scaling factor (default: 1.0)
    character(len=*), intent(in), optional :: mode

    Mode for calculating the scaling factor (default: "fan_in")

    Return Value type(he_uniform_init_type)

    He uniform initialiser object


Derived Types

type, public, extends(base_init_type) ::  he_normal_init_type

Type for the He initialiser (normal)

Components

Type Visibility Attributes Name Initial
real(kind=real32), public :: mean = 1._real32

Scale, mean, and standard deviation of the initialiser
character(len=20), public :: name

Name of the initialiser
real(kind=real32), public :: scale = 1._real32

Scale, mean, and standard deviation of the initialiser
real(kind=real32), public :: std = 0.01_real32

Scale, mean, and standard deviation of the initialiser
integer, private :: mode = 1

Constructor

private module function initialiser_normal_setup (scale, mode)

Interface for the He normal initialiser

Type-Bound Procedures

procedure, public, pass(this) :: initialise => he_normal_initialise

Initialise the weights and biases using the He normal distribution

type, public, extends(base_init_type) ::  he_uniform_init_type

Type for the He initialiser (uniform)

Components

Type Visibility Attributes Name Initial
real(kind=real32), public :: mean = 1._real32

Scale, mean, and standard deviation of the initialiser
character(len=20), public :: name

Name of the initialiser
real(kind=real32), public :: scale = 1._real32

Scale, mean, and standard deviation of the initialiser
real(kind=real32), public :: std = 0.01_real32

Scale, mean, and standard deviation of the initialiser
integer, private :: mode = 1

Constructor

private module function initialiser_uniform_setup (scale, mode)

Interface for the He uniform initialiser

Type-Bound Procedures

procedure, public, pass(this) :: initialise => he_uniform_initialise

Initialise the weights and biases using the He uniform distribution

Functions

private module function initialiser_normal_setup(scale, mode) result(initialiser)

Arguments

Type IntentOptional Attributes Name
real(kind=real32), intent(in), optional :: scale

Scaling factor (default: 1.0)
character(len=*), intent(in), optional :: mode

Mode for calculating the scaling factor (default: "fan_in")

Return Value type(he_normal_init_type)

He normal initialiser object

private module function initialiser_uniform_setup(scale, mode) result(initialiser)

Arguments

Type IntentOptional Attributes Name
real(kind=real32), intent(in), optional :: scale

Scaling factor (default: 1.0)
character(len=*), intent(in), optional :: mode

Mode for calculating the scaling factor (default: "fan_in")

Return Value type(he_uniform_init_type)

He uniform initialiser object


Subroutines

private subroutine he_normal_initialise(this, input, fan_in, fan_out, spacing)

Initialise the weights and biases using the He normal distribution

Arguments

Type IntentOptional Attributes Name
class(he_normal_init_type), intent(inout) :: this

Instance of the He initialiser
real(kind=real32), intent(out), dimension(..) :: input

Weights and biases to initialise
integer, intent(in), optional :: fan_in

Number of input and output parameters
integer, intent(in), optional :: fan_out

Number of input and output parameters
integer, intent(in), optional, dimension(:) :: spacing

Spacing of the input and output units (not used)

private subroutine he_uniform_initialise(this, input, fan_in, fan_out, spacing)

Initialise the weights and biases using the He uniform distribution

Arguments

Type IntentOptional Attributes Name
class(he_uniform_init_type), intent(inout) :: this

Instance of the Glorot initialiser
real(kind=real32), intent(out), dimension(..) :: input

Weights and biases to initialise
integer, intent(in), optional :: fan_in

Number of input and output units
integer, intent(in), optional :: fan_out

Number of input and output units
integer, intent(in), optional, dimension(:) :: spacing

Spacing of the input and output units (not used)