Struct Adafactor

pub struct Adafactor {
    pub lr: Option<f32>,
    pub eps: (Array, Array),
    pub clip_threshold: Array,
    pub decay_rate: Array,
    pub beta1: AdafactorBeta1,
    pub weight_decay: f32,
    pub scale_parameter: bool,
    pub relative_step: bool,
    pub warmup_init: bool,
    pub state: State<AdafactorState>,
}

The Adafactor optimizer.

Our Adafactor implementation follows the original paper: `Adafactor: Adaptive Learning Rates with Sublinear Memory Cost https://arxiv.org/abs/1804.04235

Fields

lr: Option<f32>

The learning rate.

eps: (Array, Array)

The first term is added to the square of the gradients to improve numerical stability. Default to Adafactor::DEFAULT_EPS.

clip_threshold: Array

Clips the unscaled update. Default to Adafactor::DEFAULT_CLIP_THRESHOLD.

decay_rate: Array

Coefficient for the running average of the squared gradient. Default to Adafactor::DEFAULT_DECAY_RATE.

beta1: AdafactorBeta1

If set then the first moment will be used.

weight_decay: f32

The weight decay. Default to Adafactor::DEFAULT_WEIGHT_DECAY.

scale_parameter: bool

If true the learningRate will be scaled by max(eps.0, RMS(parameter)). Default to Adafactor::DEFAULT_SCALE_PARAMETER.

relative_step: bool

If true the learningRate will be ignored and the relative step size will be computed. Default to Adafactor::DEFAULT_RELATIVE_STEP.

warmup_init: bool

If true the relative step size will be calculated by the current step. Default to Adafactor::DEFAULT_WARMUP_INIT.

state: State<AdafactorState>

Inner state.

Implementations

impl Adafactor

pub fn new() -> Result<Self, <<Self as Buildable>::Builder as Builder<Self>>::Error>

Creates a new instance of Adafactor.

impl Adafactor

pub const DEFAULT_LR: Option<f32> = None

Default value for lr

pub const DEFAULT_EPS: (f32, f32) = _

Default values for eps

pub const DEFAULT_CLIP_THRESHOLD: f32 = 1f32

Default value for clip_threshold

pub const DEFAULT_DECAY_RATE: f32 = -0.800000011f32

Default value for decay_rate

pub const DEFAULT_WEIGHT_DECAY: f32 = 0f32

Default value for weight_decay

pub const DEFAULT_SCALE_PARAMETER: bool = true

Default value for scale_parameter

pub const DEFAULT_RELATIVE_STEP: bool = true

Default value for relative_step

pub const DEFAULT_WARMUP_INIT: bool = false

Default value for warmup_init

pub const DEFAULT_BETA1: Option<f32> = None

Default value for beta1

Trait Implementations

impl Buildable for Adafactor

type Builder = AdafactorBuilder

The builder type for this buildable type

impl Builder<Adafactor> for AdafactorBuilder

type Error = AdafactorBuildError

Error with building

fn build(self) -> Result<Adafactor, Self::Error>

Build the type

impl Clone for Adafactor

fn clone(&self) -> Adafactor

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Adafactor

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Optimizer for Adafactor

type State = HashMap<Rc<str>, AdafactorState>

State of the optimizer.

fn state(&self) -> &Self::State

Get the state of the optimizer.

fn state_mut(&mut self) -> &mut Self::State

Get the mutable state of the optimizer.

fn update_single( &mut self, key: &Rc<str>, gradient: &Array, parameter: &mut Array, ) -> Result<()>

Update a single parameter with the given gradient.

fn update<M>( &mut self, model: &mut M, gradients: impl Borrow<FlattenedModuleParam>, ) -> Result<()>

where M: ModuleParameters,

Apply the gradients to the parameters of the model and update the model with the new parameters.

impl Updatable for &mut Adafactor

fn updatable_states(&self) -> impl IntoIterator<Item = &Array>

Returns a list of references to the updatable states.

fn updatable_states_mut(&mut self) -> impl IntoIterator<Item = &mut Array>

Returns a list of mutable references to the updatable states.

impl Updatable for Adafactor

fn updatable_states(&self) -> impl IntoIterator<Item = &Array>

Returns a list of references to the updatable states.

fn updatable_states_mut(&mut self) -> impl IntoIterator<Item = &mut Array>

Returns a list of mutable references to the updatable states.