athena__batchnorm1d

Module containing implementation of 0D and 1D batch normalisation layers

This module implements batch normalisation for 3D convolutional layers, normalizing activations across the batch dimension.

Mathematical operation (training): $\mu_\mathcal{B} = \frac{1}{m}\sum_{i=1}^{m} x_i$ $\sigma^2_\mathcal{B} = \frac{1}{m}\sum_{i=1}^{m} (x_i - \mu_\mathcal{B})^2$ $\hat{x}_i = \frac{x_i - \mu_\mathcal{B}}{\sqrt{\sigma^2_\mathcal{B} + \epsilon}}$ $y_i = \gamma \hat{x}_i + \beta$

where $\gamma, \beta$ are learnable parameters, $\epsilon$ is stability constant

Inference: uses running statistics $\mu_{\text{running}}, \sigma^2_{\text{running}}$ from training

Benefits: Reduces internal covariate shift, enables higher learning rates, acts as regularisation, reduces dependence on initialisation Reference: Ioffe & Szegedy (2015), ICML

Uses

- diffstruc
- athena__misc_types
- coreutils
- athena__diffstruc_extd
- athena__base_layer

Interfaces

public interface batchnorm1d_layer_type

Interface for setting up the 1D batch normalisation layer

private module function layer_setup(input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose) result(layer)

Set up the 1D batch normalisation layer

Arguments

Type	Intent	Optional	Attributes		Name
integer,	intent(in),	optional,	dimension(:)	::	input_shape	Input shape
integer,	intent(in),	optional		::	num_channels	Number of channels and inputs
integer,	intent(in),	optional		::	num_inputs	Number of channels and inputs
real(kind=real32),	intent(in),	optional		::	momentum	Momentum and epsilon
real(kind=real32),	intent(in),	optional		::	epsilon	Momentum and epsilon
real(kind=real32),	intent(in),	optional		::	gamma_init_mean	Gamma initialisation mean and standard deviation
real(kind=real32),	intent(in),	optional		::	gamma_init_std	Gamma initialisation mean and standard deviation
real(kind=real32),	intent(in),	optional		::	beta_init_mean	Beta initialisation mean and standard deviation
real(kind=real32),	intent(in),	optional		::	beta_init_std	Beta initialisation mean and standard deviation
class(*),	intent(in),	optional		::	gamma_initialiser	Initialisers
class(*),	intent(in),	optional		::	beta_initialiser	Initialisers
class(*),	intent(in),	optional		::	moving_mean_initialiser	Initialisers
class(*),	intent(in),	optional		::	moving_variance_initialiser	Initialisers
integer,	intent(in),	optional		::	verbose	Verbosity level

Return Value type(batchnorm1d_layer_type)

Instance of the 1D batch normalisation layer

Derived Types

type, public, extends(batch_layer_type) :: batchnorm1d_layer_type

Type for 0D or 1D batch normalisation layer with overloaded procedures

Components

Type	Visibility	Attributes		Name		Initial
class(base_actv_type),	public,	allocatable	::	activation			Activation function
real(kind=real32),	public		::	beta_init_mean	=	0._real32	Initialisation parameters for beta
real(kind=real32),	public		::	beta_init_std	=	0.01_real32	Initialisation parameters for beta
class(base_init_type),	public,	allocatable	::	bias_init			Initialisers for kernel and bias
character(len=14),	public		::	bias_initialiser	=	''	Initialisers for kernel and bias
integer,	public,	allocatable, dimension(:)	::	bias_shape			Shape of biases
real(kind=real32),	public		::	epsilon	=	0.001_real32	Epsilon factor
real(kind=real32),	public		::	gamma_init_mean	=	1._real32	Initialisation parameters for gamma
real(kind=real32),	public		::	gamma_init_std	=	0.01_real32	Initialisation parameters for gamma
type(graph_type),	public,	allocatable, dimension(:)	::	graph			Graph structure of input data
integer,	public		::	id			Unique identifier
logical,	public		::	inference	=	.false.	Inference mode
integer,	public		::	input_rank	=	0	Rank of input data
integer,	public,	allocatable, dimension(:)	::	input_shape			Input shape
class(base_init_type),	public,	allocatable	::	kernel_init			Initialisers for kernel and bias
character(len=14),	public		::	kernel_initialiser	=	''	Initialisers for kernel and bias
real(kind=real32),	public,	allocatable, dimension(:)	::	mean			Mean and variance (not learnable)
real(kind=real32),	public		::	momentum	=	0.99_real32	Momentum factor NOTE: if momentum = 0, mean and variance batch-dependent values NOTE: if momentum > 0, mean and variance are running averages
class(base_init_type),	public,	allocatable	::	moving_mean_init			Initialisers for moving mean and variance
class(base_init_type),	public,	allocatable	::	moving_variance_init			Initialisers for moving mean and variance
character(len=:),	public,	allocatable	::	name			Layer name
integer,	public		::	num_channels			Number of channels
integer,	public		::	num_params	=	0	Number of learnable parameters
class(array_type),	public,	allocatable, dimension(:,:)	::	output			Output
integer,	public		::	output_rank	=	0	Rank of output data
integer,	public,	allocatable, dimension(:)	::	output_shape			Output shape
type(array_type),	public,	allocatable, dimension(:)	::	params			Learnable parameters
character(len=20),	public		::	subtype	=	repeat(" ", 20)
character(len=4),	public		::	type	=	'base'	Layer type
logical,	public		::	use_bias	=	.false.	Layer has bias
logical,	public		::	use_graph_input	=	.false.	Use graph input
logical,	public		::	use_graph_output	=	.false.	Use graph output
real(kind=real32),	public,	allocatable, dimension(:)	::	variance			Mean and variance (not learnable)
integer,	public,	allocatable, dimension(:,:)	::	weight_shape			Shape of weights

Constructor

Interface for setting up the 1D batch normalisation layer

private module function layer_setup (input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose)	Set up the 1D batch normalisation layer

Finalizations Procedures

final :: finalise_batchnorm1d	Finalise 1D batch normalisation layer

Type-Bound Procedures

procedure, public :: add_t_t => add_learnable	Add two layers
procedure, public, pass(this) :: build_from_onnx => build_from_onnx_base	Build layer from ONNX node and initialiser
procedure, public, pass(this) :: emit_onnx_graph_inputs => emit_onnx_graph_inputs_base	Emit graph input tensor declarations for this layer
procedure, public, pass(this) :: emit_onnx_nodes => emit_onnx_nodes_base	Emit ONNX JSON nodes for this layer (format-aware and polymorphic)
procedure, public, pass(this) :: extract_output => extract_output_base	Extract the output of the layer as a standard real array
procedure, public, pass(this) :: forward => forward_batchnorm1d	Forward propagation derived type handler
procedure, public, pass(this) :: forward_eval => forward_eval_base	Forward pass of layer and return output for evaluation
procedure, public, pass(this) :: get_attributes => get_attributes_batch	Get the attributes of the layer (for ONNX export)
procedure, public, pass(this) :: get_gradients	Get parameter gradients of layer
procedure, public, pass(this) :: get_num_params => get_num_params_batch	Get the number of parameters in the layer
procedure, public, pass(this) :: get_params	Get learnable parameters of layer
procedure, public, pass(this) :: init => init_batch	Initialise the layer
procedure, public, pass(this) :: nullify_graph => nullify_graph_base	Nullify the forward pass data of the layer to free memory Read more…
generic, public :: operator(+) => add_t_t	Operator overloading for addition
procedure, public, pass(this) :: print => print_base	Print the layer to a file with additional information
procedure, public, pass(this) :: print_to_unit => print_to_unit_batch	Print layer to unit
procedure, public, pass(this) :: read => read_batchnorm1d	Read 1D batch normalisation layer from file
procedure, public, pass(this) :: reduce => reduce_learnable	Merge another learnable layer into this one
procedure, public, pass(this) :: set_gradients	Set learnable parameters of layer
procedure, public, pass(this) :: set_graph => set_graph_base	Set the graph structure of the input data !! this is adjacency and edge weighting
procedure, public, pass(this) :: set_hyperparams => set_hyperparams_batchnorm1d	Set hyperparameters for 1D batch normalisation layer
procedure, public, pass(this) :: set_params	Set learnable parameters of layer
procedure, public, pass(this) :: set_rank => set_rank_base	Set the input and output ranks of the layer
procedure, public, pass(this) :: set_shape => set_shape_base	Set the input shape of the layer

Functions

public function read_batchnorm1d_layer(unit, verbose) result(layer)

Read 1D batch normalisation layer from file and return layer

Arguments

Type	Intent	Optional	Attributes		Name
integer,	intent(in)			::	unit	File unit
integer,	intent(in),	optional		::	verbose	Verbosity level

Return Value class(base_layer_type), allocatable

Allocatable instance of the base layer

private module function layer_setup(input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose) result(layer)

Set up the 1D batch normalisation layer

Arguments

Type	Intent	Optional	Attributes		Name
integer,	intent(in),	optional,	dimension(:)	::	input_shape	Input shape
integer,	intent(in),	optional		::	num_channels	Number of channels and inputs
integer,	intent(in),	optional		::	num_inputs	Number of channels and inputs
real(kind=real32),	intent(in),	optional		::	momentum	Momentum and epsilon
real(kind=real32),	intent(in),	optional		::	epsilon	Momentum and epsilon
real(kind=real32),	intent(in),	optional		::	gamma_init_mean	Gamma initialisation mean and standard deviation
real(kind=real32),	intent(in),	optional		::	gamma_init_std	Gamma initialisation mean and standard deviation
real(kind=real32),	intent(in),	optional		::	beta_init_mean	Beta initialisation mean and standard deviation
real(kind=real32),	intent(in),	optional		::	beta_init_std	Beta initialisation mean and standard deviation
class(*),	intent(in),	optional		::	gamma_initialiser	Initialisers
class(*),	intent(in),	optional		::	beta_initialiser	Initialisers
class(*),	intent(in),	optional		::	moving_mean_initialiser	Initialisers
class(*),	intent(in),	optional		::	moving_variance_initialiser	Initialisers
integer,	intent(in),	optional		::	verbose	Verbosity level

Return Value type(batchnorm1d_layer_type)

Instance of the 1D batch normalisation layer

Subroutines

private subroutine build_from_onnx_batchnorm1d(this, node, initialisers, value_info, verbose)

Read ONNX attributes for 1D batch normalisation layer

Arguments

Type	Intent	Attributes		Name
class(batchnorm1d_layer_type),	intent(inout)		::	this	Instance of the 1D batch normalisation layer
type(onnx_node_type),	intent(in)		::	node	ONNX node information
type(onnx_initialiser_type),	intent(in),	dimension(:)	::	initialisers	ONNX initialiser information
type(onnx_tensor_type),	intent(in),	dimension(:)	::	value_info	ONNX value info
integer,	intent(in)		::	verbose	Verbosity level

private subroutine finalise_batchnorm1d(this)

Finalise 1D batch normalisation layer

Arguments

Type	Intent	Optional	Attributes		Name
type(batchnorm1d_layer_type),	intent(inout)			::	this	Instance of the 1D batch normalisation layer

private subroutine forward_batchnorm1d(this, input)

Forward propagation

Arguments

Type	Intent	Optional	Attributes		Name
class(batchnorm1d_layer_type),	intent(inout)			::	this	Instance of the 1D batch normalisation layer
class(array_type),	intent(in),		dimension(:,:)	::	input	Input values

private subroutine read_batchnorm1d(this, unit, verbose)

Read 1D batch normalisation layer from file

Arguments

Type	Intent	Optional		Name
class(batchnorm1d_layer_type),	intent(inout)		::	this	Instance of the 1D batch normalisation layer
integer,	intent(in)		::	unit	File unit
integer,	intent(in),	optional	::	verbose	Verbosity level

private subroutine set_hyperparams_batchnorm1d(this, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose)

Set hyperparameters for 1D batch normalisation layer

Arguments

Type	Intent	Optional	Attributes		Name
class(batchnorm1d_layer_type),	intent(inout)			::	this	Instance of the 1D batch normalisation layer
real(kind=real32),	intent(in)			::	momentum	Momentum and epsilon
real(kind=real32),	intent(in)			::	epsilon	Momentum and epsilon
real(kind=real32),	intent(in)			::	gamma_init_mean	Gamma initialisation mean and standard deviation
real(kind=real32),	intent(in)			::	gamma_init_std	Gamma initialisation mean and standard deviation
real(kind=real32),	intent(in)			::	beta_init_mean	Beta initialisation mean and standard deviation
real(kind=real32),	intent(in)			::	beta_init_std	Beta initialisation mean and standard deviation
class(base_init_type),	intent(in),		allocatable	::	gamma_initialiser	Gamma and beta initialisers
class(base_init_type),	intent(in),		allocatable	::	beta_initialiser	Gamma and beta initialisers
class(base_init_type),	intent(in),		allocatable	::	moving_mean_initialiser	Moving mean and variance initialisers
class(base_init_type),	intent(in),		allocatable	::	moving_variance_initialiser	Moving mean and variance initialisers
integer,	intent(in),	optional		::	verbose	Verbosity level

athena__batchnorm1d_layer Module

Contents

Interfaces

Derived Types

Functions

Subroutines

Uses

Interfaces

public interface batchnorm1d_layer_type

private module function layer_setup(input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose) result(layer)

Arguments

Return Value type(batchnorm1d_layer_type)

Derived Types

type, public, extends(batch_layer_type) :: batchnorm1d_layer_type

Components

Constructor

Finalizations Procedures

Type-Bound Procedures

Functions

public function read_batchnorm1d_layer(unit, verbose) result(layer)

Arguments

Return Value class(base_layer_type), allocatable

private module function layer_setup(input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose) result(layer)

Arguments

Return Value type(batchnorm1d_layer_type)

Subroutines

private subroutine build_from_onnx_batchnorm1d(this, node, initialisers, value_info, verbose)

Arguments

private subroutine finalise_batchnorm1d(this)

Arguments

private subroutine forward_batchnorm1d(this, input)

Arguments

private subroutine read_batchnorm1d(this, unit, verbose)

Arguments

private subroutine set_hyperparams_batchnorm1d(this, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose)

Arguments

athena__batchnorm1d_layer Module

Contents

Interfaces

Derived Types

Functions

Subroutines

Uses

Interfaces

public interface batchnorm1d_layer_type

private module function layer_setup(input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose) result(layer) Implementation →

Arguments

Return Value type(batchnorm1d_layer_type)

Derived Types

type, public, extends(batch_layer_type) :: batchnorm1d_layer_type

Components

Constructor

Finalizations Procedures

Type-Bound Procedures

Functions

public function read_batchnorm1d_layer(unit, verbose) result(layer)

Arguments

Return Value class(base_layer_type), allocatable

private module function layer_setup(input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose) result(layer)

Arguments

Return Value type(batchnorm1d_layer_type)

Subroutines

private subroutine build_from_onnx_batchnorm1d(this, node, initialisers, value_info, verbose)

Arguments

private subroutine finalise_batchnorm1d(this)

Arguments

private subroutine forward_batchnorm1d(this, input)

Arguments

private subroutine read_batchnorm1d(this, unit, verbose)

Arguments

private subroutine set_hyperparams_batchnorm1d(this, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose)

Arguments

private module function layer_setup(input_shape, num_channels, num_inputs, momentum, epsilon, gamma_init_mean, gamma_init_std, beta_init_mean, beta_init_std, gamma_initialiser, beta_initialiser, moving_mean_initialiser, moving_variance_initialiser, verbose) result(layer)