From 6201a7321f197bbbb6c2e15d39c5e7833fc11c70 Mon Sep 17 00:00:00 2001 From: eknag Date: Sat, 23 Sep 2023 13:05:03 -0700 Subject: [PATCH 1/4] changed to proper Xavier initialization, existing implementation was resulting in a large negative bias, which was killing all gradients through the following relu. https://paperswithcode.com/method/xavier-initialization --- torchbenchmark/models/dlrm/dlrm_s_pytorch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torchbenchmark/models/dlrm/dlrm_s_pytorch.py b/torchbenchmark/models/dlrm/dlrm_s_pytorch.py index 584c7a53b2..419d49dcd0 100644 --- a/torchbenchmark/models/dlrm/dlrm_s_pytorch.py +++ b/torchbenchmark/models/dlrm/dlrm_s_pytorch.py @@ -149,8 +149,7 @@ def create_mlp(self, ln, sigmoid_layer): mean = 0.0 # std_dev = np.sqrt(variance) std_dev = np.sqrt(2 / (m + n)) # np.sqrt(1 / m) # np.sqrt(1 / n) W = np.random.normal(mean, std_dev, size=(m, n)).astype(np.float32) - std_dev = np.sqrt(1 / m) # np.sqrt(2 / (m + 1)) - bt = np.random.normal(mean, std_dev, size=m).astype(np.float32) + bt = np.zeros(m).astype(np.float32) # approach 1 LL.weight.data = torch.tensor(W, requires_grad=True) LL.bias.data = torch.tensor(bt, requires_grad=True) From e7566ec8aea08766752fbc8805bc934909dbe059 Mon Sep 17 00:00:00 2001 From: eknag Date: Sat, 23 Sep 2023 13:21:46 -0700 Subject: [PATCH 2/4] call zero_grad in dlrm training loop to stop gradients from accumulating --- torchbenchmark/models/dlrm/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchbenchmark/models/dlrm/__init__.py b/torchbenchmark/models/dlrm/__init__.py index 5f6843b1b7..d9e5f31e36 100644 --- a/torchbenchmark/models/dlrm/__init__.py +++ b/torchbenchmark/models/dlrm/__init__.py @@ -223,6 +223,7 @@ def train(self): loss.backward() self.optimizer.step() self.lr_scheduler.step() + self.optimizer.zero_grad() # get_optimizer override is important! This model has both a self.opt # _and_ a self.optimizer and we want just the optimizer From 9f4b92da088aba89a319ba50a3687b0af075a221 Mon Sep 17 00:00:00 2001 From: eknag Date: Sat, 23 Sep 2023 16:07:21 -0700 Subject: [PATCH 3/4] Revert "call zero_grad in dlrm training loop to stop gradients from accumulating" This reverts commit e7566ec8aea08766752fbc8805bc934909dbe059. --- torchbenchmark/models/dlrm/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torchbenchmark/models/dlrm/__init__.py b/torchbenchmark/models/dlrm/__init__.py index d9e5f31e36..5f6843b1b7 100644 --- a/torchbenchmark/models/dlrm/__init__.py +++ b/torchbenchmark/models/dlrm/__init__.py @@ -223,7 +223,6 @@ def train(self): loss.backward() self.optimizer.step() self.lr_scheduler.step() - self.optimizer.zero_grad() # get_optimizer override is important! This model has both a self.opt # _and_ a self.optimizer and we want just the optimizer From 5d2bf0f308177fdc8a04ff038a1ebfee70aca937 Mon Sep 17 00:00:00 2001 From: eknag Date: Wed, 27 Sep 2023 10:29:30 -0700 Subject: [PATCH 4/4] added link to upstream PR --- torchbenchmark/models/dlrm/dlrm_s_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchbenchmark/models/dlrm/dlrm_s_pytorch.py b/torchbenchmark/models/dlrm/dlrm_s_pytorch.py index 419d49dcd0..59b24045d3 100644 --- a/torchbenchmark/models/dlrm/dlrm_s_pytorch.py +++ b/torchbenchmark/models/dlrm/dlrm_s_pytorch.py @@ -149,7 +149,7 @@ def create_mlp(self, ln, sigmoid_layer): mean = 0.0 # std_dev = np.sqrt(variance) std_dev = np.sqrt(2 / (m + n)) # np.sqrt(1 / m) # np.sqrt(1 / n) W = np.random.normal(mean, std_dev, size=(m, n)).astype(np.float32) - bt = np.zeros(m).astype(np.float32) + bt = np.zeros(m).astype(np.float32) # see upstream PR at https://github.com/facebookresearch/dlrm/pull/358 # approach 1 LL.weight.data = torch.tensor(W, requires_grad=True) LL.bias.data = torch.tensor(bt, requires_grad=True)