Struct MultiHeadAttentionConfig 

pub struct MultiHeadAttentionConfig {
    pub d_model: usize,
    pub n_heads: usize,
    pub dropout: f64,
    pub min_float: f64,
    pub quiet_softmax: bool,
    pub initializer: Initializer,
    pub key_bias: bool,
    pub value_bias: bool,
    pub query_bias: bool,
    pub output_bias: bool,
}

Configuration to create a Multi Head Attention layer using the init function.

Fields

d_model: usize

The size of each linear layer.

n_heads: usize

The number of heads.

dropout: f64

The dropout rate. Default: 0.1

min_float: f64

The minimum value a float can take. Default: -1.0e4 This is used to mask attention scores before calculating attention weights. A value too low might result in NaN.

quiet_softmax: bool

Use “quiet softmax” instead of regular softmax.

• Usage may improve performance by allowing attention heads to deposit no information (if the sequence contains no information relevant to that head).
• Usage may reduce the entropy of weights in the model, enhancing quantization and compression.

Reference: https://www.evanmiller.org/attention-is-off-by-one.html

initializer: Initializer

The type of function used to initialize neural network parameters

key_bias: bool

Enable bias for the key Linear layer.

value_bias: bool

Enable bias for the value Linear layer.

query_bias: bool

Enable bias for the query Linear layer.

output_bias: bool

Enable bias for the output Linear layer.

Implementations

impl MultiHeadAttentionConfig

pub fn new(d_model: usize, n_heads: usize) -> MultiHeadAttentionConfig

Create a new instance of the config.

Arguments

Required Arguments

d_model

The size of each linear layer.

n_heads

The number of heads.

Default Arguments

dropout

The dropout rate. Default: 0.1

• Defaults to 0.1

min_float

The minimum value a float can take. Default: -1.0e4 This is used to mask attention scores before calculating attention weights. A value too low might result in NaN.

• Defaults to -1.0e4

quiet_softmax

Use “quiet softmax” instead of regular softmax.

• Usage may improve performance by allowing attention heads to deposit no information (if the sequence contains no information relevant to that head).
• Usage may reduce the entropy of weights in the model, enhancing quantization and compression.

Reference: https://www.evanmiller.org/attention-is-off-by-one.html

• Defaults to false

initializer

The type of function used to initialize neural network parameters

• Defaults to "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}"

key_bias

Enable bias for the key Linear layer.

• Defaults to true

value_bias

Enable bias for the value Linear layer.

• Defaults to true

query_bias

Enable bias for the query Linear layer.

• Defaults to true

output_bias

Enable bias for the output Linear layer.

• Defaults to true

impl MultiHeadAttentionConfig

pub fn with_dropout(self, dropout: f64) -> MultiHeadAttentionConfig

Sets the value for the field dropout.

The dropout rate. Default: 0.1

• Defaults to 0.1

pub fn with_min_float(self, min_float: f64) -> MultiHeadAttentionConfig

Sets the value for the field min_float.

The minimum value a float can take. Default: -1.0e4 This is used to mask attention scores before calculating attention weights. A value too low might result in NaN.

• Defaults to -1.0e4

pub fn with_quiet_softmax(self, quiet_softmax: bool) -> MultiHeadAttentionConfig

Sets the value for the field quiet_softmax.

Use “quiet softmax” instead of regular softmax.

• Usage may improve performance by allowing attention heads to deposit no information (if the sequence contains no information relevant to that head).
• Usage may reduce the entropy of weights in the model, enhancing quantization and compression.

Reference: https://www.evanmiller.org/attention-is-off-by-one.html

• Defaults to false

pub fn with_initializer( self, initializer: Initializer, ) -> MultiHeadAttentionConfig

Sets the value for the field initializer.

The type of function used to initialize neural network parameters

• Defaults to "Initializer::KaimingUniform{gain:1.0/num_traits::Float::sqrt(3.0), fan_out_only:false}"

pub fn with_key_bias(self, key_bias: bool) -> MultiHeadAttentionConfig

Sets the value for the field key_bias.

Enable bias for the key Linear layer.

• Defaults to true

pub fn with_value_bias(self, value_bias: bool) -> MultiHeadAttentionConfig

Sets the value for the field value_bias.

Enable bias for the value Linear layer.

• Defaults to true

pub fn with_query_bias(self, query_bias: bool) -> MultiHeadAttentionConfig

Sets the value for the field query_bias.

Enable bias for the query Linear layer.

• Defaults to true

pub fn with_output_bias(self, output_bias: bool) -> MultiHeadAttentionConfig

Sets the value for the field output_bias.

Enable bias for the output Linear layer.

• Defaults to true

impl MultiHeadAttentionConfig

pub fn init(&self, device: &Device) -> MultiHeadAttention

Initialize a new multihead attention module.

Trait Implementations

impl Clone for MultiHeadAttentionConfig

fn clone(&self) -> MultiHeadAttentionConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Config for MultiHeadAttentionConfig

fn save<P>(&self, file: P) -> Result<(), Error>

where P: AsRef<Path>,

Saves the configuration to a file.

fn load<P>(file: P) -> Result<Self, ConfigError>

where P: AsRef<Path>,

Loads the configuration from a file.

fn load_binary(data: &[u8]) -> Result<Self, ConfigError>

Loads the configuration from a binary buffer.

impl Debug for MultiHeadAttentionConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for MultiHeadAttentionConfig

fn deserialize<D>( deserializer: D, ) -> Result<MultiHeadAttentionConfig, <D as Deserializer<'de>>::Error>

where D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for MultiHeadAttentionConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Serialize for MultiHeadAttentionConfig

fn serialize<S>( &self, serializer: S, ) -> Result<<S as Serializer>::Ok, <S as Serializer>::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.