Added gradient clipping.

2026-09-09 11:21:11 +08:00 · 2022-10-05 11:29:49 -07:00
parent 98f2eeac35
commit 2ac3f94a80
1 changed files with 1 additions and 0 deletions
@@ -845,6 +845,7 @@ class Trainer(object):

                    self.accelerator.backward(loss)

+                accelerator.clip_grad_norm_(self.model.parameters(), 1.0)
                pbar.set_description(f'loss: {total_loss:.4f}')

                accelerator.wait_for_everyone()