Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Fix] Ranger optimizer #43

Merged
merged 7 commits into from
Jan 28, 2022
Merged

[Fix] Ranger optimizer #43

Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
838d117
fix: Ranger
kozistr Jan 28, 2022
b5f3da2
update: version to 0.3.2
kozistr Jan 28, 2022
3ef421b
update: Ranger
kozistr Jan 28, 2022
856fa72
docs: Range21 docstring
kozistr Jan 28, 2022
5f68bcf
refactor: add_
kozistr Jan 28, 2022
18a5fda
fix: Ranger21
kozistr Jan 28, 2022
9c2650e
update: Ranger21
kozistr Jan 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix: Ranger
  • Loading branch information
@kozistr
kozistr committed Jan 28, 2022
commit 838d1176af9126ed84b4431cd9d4837ac820a02c
14 changes: 7 additions & 7 deletions pytorch_optimizer/ranger.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,15 +121,15 @@ def step(self, _: CLOSURE = None) -> LOSS:
exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']

bias_correction1 = 1 - beta1 ** state['step']

if self.use_gc and grad.dim() > self.gc_gradient_threshold:
grad = centralize_gradient(grad, gc_conv_only=False)

state['step'] += 1

exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
exp_avg.mul_(beta1).add_(1 - beta1, grad)
exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)

bias_correction1 = 1 - beta1 ** state['step']

buffered = self.buffer[int(state['step'] % 10)]

Expand Down Expand Up @@ -166,15 +166,15 @@ def step(self, _: CLOSURE = None) -> LOSS:

if n_sma > self.n_sma_threshold:
denom = exp_avg_sq.sqrt().add_(group['eps'])
p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)
p_data_fp32.addcdiv_(exp_avg, denom, value=-step_size * group['lr'])
else:
p_data_fp32.add_(-step_size * group['lr'], exp_avg)
p_data_fp32.add_(exp_avg, alpha=-step_size * group['lr'])

p.data.copy_(p_data_fp32)

if state['step'] % group['k'] == 0:
slow_p = state['slow_buffer']
slow_p.add_(self.alpha, p.data - slow_p)
slow_p.add_(p.data - slow_p, alpha=self.alpha)
p.data.copy_(slow_p)

return loss