change configuration

hyunwoongko · Dec 22, 2019 · 1d076f1 · 1d076f1
1 parent 1a4aecf
commit 1d076f1
Show file tree

Hide file tree

Showing 10 changed files with 34 additions and 30 deletions.
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
diff --git a/README.md b/README.md
@@ -346,20 +346,21 @@ I follow original paper's parameter settings. (below) <br>
 #### 2.1.1 configuration
 
 * batch_size = 128
-* max_len = 50
+* max_len = 256
 * d_model = 512
 * n_layers = 6
 * n_heads = 8
 * ffn_hidden = 2048
 * drop_prob = 0.1
-* init_lr = 1e-5
-* factor = 0.8
+* init_lr = 0.1
+* factor = 0.9
 * min_lr = init_lr * 1e-12
-* patience = 7
+* patience = 10
 * warmup = 100
-* weight_decay = 1e-3
+* adam_eps = 5e-9
 * epoch = 1000
 * clip = 1
+* weight_decay = 5e-4
 
 #### 2.1.2 Training Result
 
@@ -383,20 +384,21 @@ I follow original paper's parameter settings. (below) <br>
 #### 2.2.1 configuration
 
 * batch_size = 32
-* max_len = 50
+* max_len = 256
 * d_model = 1024
 * n_layers = 6
 * n_heads = 16
 * ffn_hidden = 4096
 * drop_prob = 0.3
-* init_lr = 1e-5
-* factor = 0.8
+* init_lr = 0.1
+* factor = 0.9
 * min_lr = init_lr * 1e-12
-* patience = 7
+* patience = 10
 * warmup = 300
-* weight_decay = 1e-3
+* adam_eps = 5e-9
 * epoch = 3000
 * clip = 1
+* weight_decay = 5e-4
 
 
 #### 2.2.2 Training Result

diff --git a/__pycache__/conf.cpython-36.pyc b/__pycache__/conf.cpython-36.pyc
diff --git a/conf.py b/conf.py
@@ -10,7 +10,7 @@
 
 # model parameter setting
 batch_size = 128
-max_len = 50
+max_len = 256
 d_model = 512
 n_layers = 6
 n_heads = 8
@@ -19,11 +19,12 @@
 
 # optimizer parameter setting
 init_lr = 1e-5
-factor = 0.8
+factor = 0.9
+adam_eps = 5e-9
 min_lr = init_lr * 1e-12
-patience = 7
+patience = 10
 warmup = 100
-weight_decay = 1e-3
 epoch = 1000
-clip = 1
+clip = 1.0
+weight_decay = 5e-4
 inf = float('inf')
diff --git a/result/bleu.txt b/result/bleu.txt
@@ -1 +1 @@
-[0.07622405577310074, 4.063269288356802]
+[2.827831523488076, 6.194122239433678, 4.606590271720176, 4.685197853298899, 6.045665341358408, 7.125261812202991, 8.25697743905172, 7.580444916387854, 8.338297951580502, 8.453383483339628, 9.183913177314075, 8.180751547212271, 7.621637534712988, 9.1451628254334, 6.079991866539901, 8.621405495293304, 9.055088751270226, 7.286257529722675, 8.33328847834003, 8.918739296087466, 10.388830175948957, 8.802745932760116, 7.507820219254959, 6.505035365385905, 7.663984675438474, 8.358961099691795, 10.04285147132403, 9.879443037755467, 9.515200628820612, 9.216006388400029, 10.876757153102968, 10.895384135728484, 10.883231083772795, 10.68614836740474, 11.144889380486772, 11.786201130604315, 12.035120315080444, 11.27757781731269, 11.928482712373574, 12.608963338491668, 11.38765611048646, 12.924190184604155, 12.754199949922185, 12.183623455821934, 13.115491886042456, 12.535383621717713, 13.105809315009006, 13.311987912050801, 13.640648941076009]
diff --git a/result/test_loss.txt b/result/test_loss.txt
@@ -1 +1 @@
-[6.120936930179596, 5.668061196804047]
+[6.051182329654694, 5.601898372173309, 5.471410274505615, 5.412692308425903, 5.343479812145233, 5.222925186157227, 5.227630913257599, 5.26406067609787, 5.214920699596405, 5.20239919424057, 5.199727714061737, 5.212658584117889, 5.215712606906891, 5.19791579246521, 5.244936943054199, 5.21477746963501, 5.227468132972717, 5.258215069770813, 5.368910908699036, 5.341602087020874, 5.256721675395966, 5.238627552986145, 5.19616973400116, 5.239213228225708, 5.278360068798065, 5.263343691825867, 5.071344375610352, 5.050609529018402, 5.094151437282562, 5.071705460548401, 4.875069797039032, 4.8230626583099365, 4.7614569664001465, 5.02071487903595, 4.728554964065552, 4.7205312848091125, 4.702261924743652, 4.717274785041809, 4.70519757270813, 4.631350159645081, 4.614207565784454, 4.654602944850922, 4.544328689575195, 4.557780861854553, 4.580122411251068, 4.542717456817627, 4.555516839027405, 4.5319549441337585, 4.5201539397239685]
diff --git a/result/train_loss.txt b/result/train_loss.txt
@@ -1 +1 @@
-[7.244170436775106, 6.017328854699492]
+[7.179653346275968, 5.925660166971484, 5.623609406307406, 5.497337597582309, 5.371063736566888, 5.292561930181696, 5.24227236230993, 5.216812644235889, 5.189904116323866, 5.171018995377461, 5.155494324436272, 5.144120651194703, 5.127561531402991, 5.111723859929828, 5.103064745008158, 5.093994800214725, 5.082707213935348, 5.064601412953785, 5.029171657982377, 4.973260522413884, 4.924090748841542, 4.866061557232021, 4.83097792722055, 4.79862921458509, 4.770209068768875, 4.744393138633425, 4.721462331679424, 4.702160049640135, 4.681777636910325, 4.6629403068105555, 4.644884678760814, 4.630824891481105, 4.6187271882784, 4.604299488571772, 4.59801833535081, 4.579300288061739, 4.565715075589487, 4.5534290343129165, 4.542830326483639, 4.536759973097478, 4.520962454674002, 4.514251122915797, 4.507030797949971, 4.486862909426248, 4.473883872515304, 4.46828231097318, 4.456066211414758, 4.438996747726911, 4.420722213610678]
diff --git a/train.py b/train.py
@@ -21,7 +21,7 @@ def count_parameters(model):
 
 def initialize_weights(m):
  if hasattr(m, 'weight') and m.weight.dim() > 1:
- nn.init.kaiming_normal_(m.weight.data)
+ nn.init.kaiming_uniform(m.weight.data)
 
 
 model = Transformer(src_pad_idx=src_pad_idx,
@@ -41,7 +41,8 @@ def initialize_weights(m):
 model.apply(initialize_weights)
 optimizer = Adam(params=model.parameters(),
  lr=init_lr,
- weight_decay=weight_decay)
+ weight_decay=weight_decay,
+ eps=adam_eps)
 
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer=optimizer,
  verbose=True,

diff --git a/util/__pycache__/data_loader.cpython-36.pyc b/util/__pycache__/data_loader.cpython-36.pyc
diff --git a/util/data_loader.py b/util/data_loader.py
@@ -4,7 +4,7 @@
 @homepage : https://github.com/gusdnd852
 """
 from torchtext.data import Field, BucketIterator
-from torchtext.datasets import Multi30k
+from torchtext.datasets.translation import Multi30k
 
 
 class DataLoader: