Struct RotaryPositionalEncoding

pub struct RotaryPositionalEncoding {
    pub dimensions: i32,
    pub traditional: bool,
    pub base: f32,
    pub scale: f32,
}

Implements the rotary positional encoding.

The traditional implementation rotates consecutive pairs of elements in the feature dimension while the default implementation rotates pairs with stride half the feature dimensions for efficiency.

For more details see RoFormer: Enhanced Transformer with Rotary Position Embedding (https://arxiv.org/abs/2104.09864)

Fields

dimensions: i32

The feature dimensions to be rotated. If the input feature is larger than dims then the rest is left unchanged

traditional: bool

If true choose the traditional implementation which is slightly less efficient

base: f32

The base used to compute angular frequency for each dimension in the positional encodings

scale: f32

scale used to scale the positions

Implementations

impl RotaryPositionalEncoding

pub fn new(dimensions: impl Into<i32>) -> Self

Creates a new instance of RotaryPositionalEncoding.

impl RotaryPositionalEncoding

pub const DEFAULT_TRADITIONAL: bool = false

Default value for traditional field.

pub const DEFAULT_BASE: f32 = 1.0E+4f32

Default value for base field.

pub const DEFAULT_SCALE: f32 = 1f32

Default value for scale field.

Trait Implementations

impl Buildable for RotaryPositionalEncoding

type Builder = RotaryPositionalEncodingBuilder

The builder type for this buildable type

impl Builder<RotaryPositionalEncoding> for RotaryPositionalEncodingBuilder

type Error = Infallible

Error with building

fn build(self) -> Result<RotaryPositionalEncoding, Self::Error>

Build the type

impl Clone for RotaryPositionalEncoding

fn clone(&self) -> RotaryPositionalEncoding

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for RotaryPositionalEncoding

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'a, Input> Module<Input> for RotaryPositionalEncoding

where Input: Into<RopeInput<'a>>,

type Error = Exception

Error type for the module.

type Output = Array

Output type of the module.

fn forward(&mut self, input: Input) -> Result<Self::Output, Self::Error>

Forward pass of the module.

fn training_mode(&mut self, _mode: bool)

Set whether the module is in training mode.

impl ModuleParameters for RotaryPositionalEncoding

fn freeze_parameters(&mut self, recursive: bool)

Freeze all parameters in the module.

fn unfreeze_parameters(&mut self, recursive: bool)

Unfreeze all parameters in the module.

fn parameters(&self) -> ModuleParamRef<'_>

Get references to the module parameters.

fn parameters_mut(&mut self) -> ModuleParamMut<'_>

Get mutable references to the module parameters.

fn trainable_parameters(&self) -> ModuleParamRef<'_>

Get references to the trainable parameters. A parameter is trainable if it is NOT frozen.

fn all_frozen(&self) -> Option<bool>

Check if all parameters in the module are frozen. Returns None if there are no parameters.

fn any_frozen(&self) -> Option<bool>

Check if any parameter in the module is frozen. Returns None if there are no parameters.

fn update(&mut self, parameters: ModuleParam)

Update the module parameters.

fn update_flattened(&mut self, flattened_parameters: FlattenedModuleParam)

Update the module parameters from a flattened representation.