Struct TransformerEncoderConfig

pub struct TransformerEncoderConfig {
    pub d_model: usize,
    pub d_ff: usize,
    pub n_heads: usize,
    pub n_layers: usize,
    pub dropout: f64,
    pub norm_first: bool,
    pub quiet_softmax: bool,
    pub initializer: Initializer,
}

Configuration to create a Transformer Encoder layer using the init function.

Fields

d_model: usize

The size of the model.

d_ff: usize

The size of the position-wise feed-forward network.

n_heads: usize

The number of attention heads.

n_layers: usize

The number of layers.

dropout: f64

The dropout rate. Default: 0.1

norm_first: bool

Layer norm will be applied first instead of after the other modules.

quiet_softmax: bool

Use “quiet softmax” instead of regular softmax.

• Usage may improve performance by allowing attention heads to deposit no information (if the sequence contains no information relevant to that head).
• Usage may reduce the entropy of weights in the model, enhancing quantization and compression.

Reference: https://www.evanmiller.org/attention-is-off-by-one.html

initializer: Initializer

The type of function used to initialize neural network parameters

Implementations

impl TransformerEncoderConfig

pub fn new( d_model: usize, d_ff: usize, n_heads: usize, n_layers: usize, ) -> TransformerEncoderConfig

Create a new instance of the config.

impl TransformerEncoderConfig

pub fn with_dropout(self, dropout: f64) -> TransformerEncoderConfig

The dropout rate. Default: 0.1

pub fn with_norm_first(self, norm_first: bool) -> TransformerEncoderConfig

Layer norm will be applied first instead of after the other modules.

pub fn with_quiet_softmax(self, quiet_softmax: bool) -> TransformerEncoderConfig

Use “quiet softmax” instead of regular softmax.

pub fn with_initializer( self, initializer: Initializer, ) -> TransformerEncoderConfig

The type of function used to initialize neural network parameters

impl TransformerEncoderConfig

pub fn init<B>(&self, device: &<B as Backend>::Device) -> TransformerEncoder<B>

where B: Backend,

Initialize a new transformer encoder module.

Trait Implementations

impl Clone for TransformerEncoderConfig

fn clone(&self) -> TransformerEncoderConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Config for TransformerEncoderConfig

fn save<P>(&self, file: P) -> Result<(), Error>

where P: AsRef<Path>,

Saves the configuration to a file.

fn load<P>(file: P) -> Result<Self, ConfigError>

where P: AsRef<Path>,

Loads the configuration from a file.

fn load_binary(data: &[u8]) -> Result<Self, ConfigError>

Loads the configuration from a binary buffer.

impl Debug for TransformerEncoderConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for TransformerEncoderConfig

fn deserialize<D>( deserializer: D, ) -> Result<TransformerEncoderConfig, <D as Deserializer<'de>>::Error>

where D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for TransformerEncoderConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Serialize for TransformerEncoderConfig

fn serialize<S>( &self, serializer: S, ) -> Result<<S as Serializer>::Ok, <S as Serializer>::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.