remove gradient clipping i dont think its needed at this small scale

2022-08-20 00:33:28 +00:00 · 2022-08-20 00:33:28 +00:00 · 4e0137ddf6
parent 35435ec087
commit 4e0137ddf6
1 changed files with 0 additions and 1 deletions
--- a/makemore.py
+++ b/makemore.py
@ -382,7 +382,6 @@ if __name__ == '__main__':
        # calculate the gradient, clip it, update the weights
        model.zero_grad(set_to_none=True)
        loss.backward()
-        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        optimizer.step()
        if args.device.startswith('cuda'):
            torch.cuda.synchronize()