Struct PositionWiseFeedForward 

pub struct PositionWiseFeedForward<B>
where
    B: Backend,
{
    pub linear_inner: Linear<B>,
    pub linear_outer: Linear<B>,
    pub dropout: Dropout,
    pub activation: Activation<B>,
}

Applies the position-wise feed-forward network to the input tensor from the paper Attention Is All You Need.

Params

• linear inner: Linear layer with d_model input features and d_ff output features.
• linear outer: Linear layer with d_ff input features and d_model output features.

FFN(x) = max(0, xW1 + b1)W2 + b2

Should be created using PositionWiseFeedForwardConfig

Notes

The activation field is currently marked #[module(skip)] for backward compatibility with records saved before this field was introduced (when the activation was always Gelu and had no state). This means activation state is not persisted when saving or loading records.

For stateless activations (GELU, ReLU, etc.) this has no effect. If you are using SwiGLU, its learnable parameters will not be saved or loaded correctly.

Fields

linear_inner: Linear<B>

Linear layer with d_model input features and d_ff output features.

linear_outer: Linear<B>

Linear layer with d_ff input features and d_model output features.

dropout: Dropout

Dropout layer.

activation: Activation<B>

Activation function.

Implementations

impl<B> PositionWiseFeedForward<B>

where B: Backend,

pub fn forward<const D: usize>(&self, input: Tensor<B, D>) -> Tensor<B, D>

Applies the forward pass on the input tensor.

Shapes

• tensor: [batch_size, seq_length, d_model]
• output: [batch_size, seq_length, d_model]

Trait Implementations

impl<B> AutodiffModule<B> for PositionWiseFeedForward<B>

where B: Backend + AutodiffBackend, <B as AutodiffBackend>::InnerBackend: Backend,

type InnerModule = PositionWiseFeedForward<<B as AutodiffBackend>::InnerBackend>

Inner module without auto-differentiation.

fn valid( &self, ) -> <PositionWiseFeedForward<B> as AutodiffModule<B>>::InnerModule

Returns the same module, but on the inner backend without auto-differentiation.

fn from_inner( module: <PositionWiseFeedForward<B> as AutodiffModule<B>>::InnerModule, ) -> PositionWiseFeedForward<B>

Wraps an inner module back into an auto-diff module.

impl<B> Clone for PositionWiseFeedForward<B>

where B: Backend,

fn clone(&self) -> PositionWiseFeedForward<B>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<B> Debug for PositionWiseFeedForward<B>

where B: Debug + Backend,

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<B> Display for PositionWiseFeedForward<B>

where B: Backend,

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<B> HasAutodiffModule<B> for PositionWiseFeedForward<<B as AutodiffBackend>::InnerBackend>

where B: Backend + AutodiffBackend, <B as AutodiffBackend>::InnerBackend: Backend,

type TrainModule = PositionWiseFeedForward<B>

The module with auto-differentiation.

impl<B> Module<B> for PositionWiseFeedForward<B>

where B: Backend,

type Record = PositionWiseFeedForwardRecord<B>

Type to save and load the module.

fn load_record( self, record: <PositionWiseFeedForward<B> as Module<B>>::Record, ) -> PositionWiseFeedForward<B>

Load the module state from a record.

fn into_record(self) -> <PositionWiseFeedForward<B> as Module<B>>::Record

Convert the module into a record containing the state.

fn num_params(&self) -> usize

Get the number of parameters the module has, including all of its sub-modules.

fn visit<Visitor>(&self, visitor: &mut Visitor)

where Visitor: ModuleVisitor<B>,

Visit each tensor parameter in the module with a visitor.

fn map<Mapper>(self, mapper: &mut Mapper) -> PositionWiseFeedForward<B>

where Mapper: ModuleMapper<B>,

Map each tensor parameter in the module with a mapper.

fn collect_devices( &self, devices: Vec<<B as BackendTypes>::Device>, ) -> Vec<<B as BackendTypes>::Device>

Return all the devices found in the underneath module tree added to the given vector without duplicates.

fn to_device( self, device: &<B as BackendTypes>::Device, ) -> PositionWiseFeedForward<B>

Move the module and all of its sub-modules to the given device.

fn fork( self, device: &<B as BackendTypes>::Device, ) -> PositionWiseFeedForward<B>

Fork the module and all of its sub-modules to the given device.

fn devices(&self) -> Vec<<B as BackendTypes>::Device>

Return all the devices found in the underneath module tree without duplicates.

fn no_grad(self) -> Self

Each tensor in the module tree will not require grad.

fn train<AB>(self) -> Self::TrainModule

where AB: AutodiffBackend<InnerBackend = B>, Self: HasAutodiffModule<AB>,

Move the module and all of its sub-modules to the autodiff backend.

fn save_file<FR, PB>( self, file_path: PB, recorder: &FR, ) -> Result<(), RecorderError>

where FR: FileRecorder<B>, PB: Into<PathBuf>,

Save the module to a file using the provided file recorder.

fn load_file<FR, PB>( self, file_path: PB, recorder: &FR, device: &<B as BackendTypes>::Device, ) -> Result<Self, RecorderError>

where FR: FileRecorder<B>, PB: Into<PathBuf>,

Load the module from a file using the provided file recorder.

fn quantize_weights(self, quantizer: &mut Quantizer) -> Self

Quantize the weights of the module.

impl<B> ModuleDisplay for PositionWiseFeedForward<B>

where B: Backend,

fn custom_settings(&self) -> Option<DisplaySettings>

Custom display settings for the module.

fn custom_content(&self, content: Content) -> Option<Content>

Custom attributes for the module.

fn format(&self, passed_settings: DisplaySettings) -> String

Formats the module with provided display settings.

impl<B> ModuleDisplayDefault for PositionWiseFeedForward<B>

where B: Backend,

fn content(&self, content: Content) -> Option<Content>

Attributes of the module used for display purposes.

fn num_params(&self) -> usize

Gets the number of the parameters of the module.