Skip to content

Commit

Permalink
Stateful RNNs now support recurrent_dropout
Browse files Browse the repository at this point in the history
  • Loading branch information
@ageron
ageron committed Apr 21, 2019
1 parent a2117b6 commit 1aa6650
Showing 1 changed file with 42 additions and 45 deletions.
87 changes: 42 additions & 45 deletions 16_nlp_with_rnns_and_attention.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -643,11 +643,10 @@
"source": [
"model = keras.models.Sequential([\n",
" keras.layers.GRU(128, return_sequences=True, stateful=True,\n",
"# dropout=0.2, recurrent_dropout=0.2, # see TF issue #27829\n",
" dropout=0.2, recurrent_dropout=0.2,\n",
" batch_input_shape=[batch_size, None, max_id]),\n",
" keras.layers.GRU(128, return_sequences=True, stateful=True\n",
"# dropout=0.2, recurrent_dropout=0.2 # see TF issue #27829\n",
" ),\n",
" keras.layers.GRU(128, return_sequences=True, stateful=True,\n",
" dropout=0.2, recurrent_dropout=0.2),\n",
" keras.layers.TimeDistributed(keras.layers.Dense(max_id,\n",
" activation=\"softmax\"))\n",
"])"
Expand All @@ -674,82 +673,82 @@
"output_type": "stream",
"text": [
"Epoch 1/50\n",
"313/313 [==============================] - 37s 118ms/step - loss: 2.4841\n",
"313/313 [==============================] - 101s 322ms/step - loss: 2.6180\n",
"Epoch 2/50\n",
"313/313 [==============================] - 35s 110ms/step - loss: 1.9857\n",
"313/313 [==============================] - 98s 312ms/step - loss: 2.2312\n",
"Epoch 3/50\n",
"313/313 [==============================] - 34s 110ms/step - loss: 1.7911\n",
"313/313 [==============================] - 96s 306ms/step - loss: 2.2992\n",
"Epoch 4/50\n",
"313/313 [==============================] - 34s 110ms/step - loss: 1.6789\n",
"313/313 [==============================] - 96s 308ms/step - loss: 2.4599\n",
"Epoch 5/50\n",
"313/313 [==============================] - 36s 114ms/step - loss: 1.6067\n",
"313/313 [==============================] - 97s 309ms/step - loss: 2.4062\n",
"Epoch 6/50\n",
"313/313 [==============================] - 39s 124ms/step - loss: 1.5517\n",
"313/313 [==============================] - 97s 310ms/step - loss: 2.0630\n",
"Epoch 7/50\n",
"313/313 [==============================] - 42s 133ms/step - loss: 1.5078\n",
"313/313 [==============================] - 97s 311ms/step - loss: 2.0933\n",
"Epoch 8/50\n",
"313/313 [==============================] - 42s 134ms/step - loss: 1.4756\n",
"313/313 [==============================] - 97s 309ms/step - loss: 2.0784\n",
"Epoch 9/50\n",
"313/313 [==============================] - 41s 132ms/step - loss: 1.4502\n",
"313/313 [==============================] - 95s 304ms/step - loss: 2.0101\n",
"Epoch 10/50\n",
"313/313 [==============================] - 41s 130ms/step - loss: 1.4291\n",
"313/313 [==============================] - 95s 302ms/step - loss: 1.9146\n",
"Epoch 11/50\n",
"313/313 [==============================] - 40s 129ms/step - loss: 1.4114\n",
"313/313 [==============================] - 97s 309ms/step - loss: 1.9204\n",
"Epoch 12/50\n",
"313/313 [==============================] - 40s 126ms/step - loss: 1.3961\n",
"313/313 [==============================] - 95s 305ms/step - loss: 1.9049\n",
"Epoch 13/50\n",
"313/313 [==============================] - 39s 126ms/step - loss: 1.3827\n",
"313/313 [==============================] - 92s 295ms/step - loss: 1.8894\n",
"Epoch 14/50\n",
"313/313 [==============================] - 39s 124ms/step - loss: 1.3708\n",
"313/313 [==============================] - 93s 296ms/step - loss: 1.8397\n",
"Epoch 15/50\n",
"313/313 [==============================] - 38s 123ms/step - loss: 1.3602\n",
"313/313 [==============================] - 93s 296ms/step - loss: 1.8147\n",
"Epoch 16/50\n",
"313/313 [==============================] - 39s 124ms/step - loss: 1.3505\n",
"313/313 [==============================] - 92s 293ms/step - loss: 1.8147\n",
"Epoch 17/50\n",
"313/313 [==============================] - 39s 123ms/step - loss: 1.3417\n",
"313/313 [==============================] - 92s 295ms/step - loss: 1.7741\n",
"Epoch 18/50\n",
"<<30 more lines>>\n",
"313/313 [==============================] - 37s 118ms/step - loss: 1.2591\n",
"313/313 [==============================] - 93s 298ms/step - loss: 1.6102\n",
"Epoch 34/50\n",
"313/313 [==============================] - 36s 116ms/step - loss: 1.2555\n",
"313/313 [==============================] - 93s 298ms/step - loss: 1.6063\n",
"Epoch 35/50\n",
"313/313 [==============================] - 37s 119ms/step - loss: 1.2521\n",
"313/313 [==============================] - 96s 306ms/step - loss: 1.6022\n",
"Epoch 36/50\n",
"313/313 [==============================] - 37s 119ms/step - loss: 1.2489\n",
"313/313 [==============================] - 91s 291ms/step - loss: 1.5984\n",
"Epoch 37/50\n",
"313/313 [==============================] - 38s 121ms/step - loss: 1.2457\n",
"313/313 [==============================] - 91s 291ms/step - loss: 1.5964\n",
"Epoch 38/50\n",
"313/313 [==============================] - 37s 119ms/step - loss: 1.2427\n",
"313/313 [==============================] - 92s 293ms/step - loss: 1.5924\n",
"Epoch 39/50\n",
"313/313 [==============================] - 39s 126ms/step - loss: 1.2398\n",
"313/313 [==============================] - 97s 310ms/step - loss: 1.5903\n",
"Epoch 40/50\n",
"313/313 [==============================] - 37s 117ms/step - loss: 1.2370\n",
"313/313 [==============================] - 93s 298ms/step - loss: 1.5882\n",
"Epoch 41/50\n",
"313/313 [==============================] - 36s 116ms/step - loss: 1.2343\n",
"313/313 [==============================] - 95s 303ms/step - loss: 1.5867\n",
"Epoch 42/50\n",
"313/313 [==============================] - 36s 115ms/step - loss: 1.2316\n",
"313/313 [==============================] - 92s 294ms/step - loss: 1.5826\n",
"Epoch 43/50\n",
"313/313 [==============================] - 36s 114ms/step - loss: 1.2291\n",
"313/313 [==============================] - 92s 294ms/step - loss: 1.5817\n",
"Epoch 44/50\n",
"313/313 [==============================] - 36s 115ms/step - loss: 1.2267\n",
"313/313 [==============================] - 92s 295ms/step - loss: 1.5796\n",
"Epoch 45/50\n",
"313/313 [==============================] - 36s 115ms/step - loss: 1.2243\n",
"313/313 [==============================] - 92s 295ms/step - loss: 1.5765\n",
"Epoch 46/50\n",
"313/313 [==============================] - 36s 116ms/step - loss: 1.2221\n",
"313/313 [==============================] - 92s 294ms/step - loss: 1.5741\n",
"Epoch 47/50\n",
"313/313 [==============================] - 36s 114ms/step - loss: 1.2199\n",
"313/313 [==============================] - 92s 295ms/step - loss: 1.5733\n",
"Epoch 48/50\n",
"313/313 [==============================] - 35s 113ms/step - loss: 1.2179\n",
"313/313 [==============================] - 92s 293ms/step - loss: 1.5706\n",
"Epoch 49/50\n",
"313/313 [==============================] - 37s 118ms/step - loss: 1.2159\n",
"313/313 [==============================] - 90s 289ms/step - loss: 1.5703\n",
"Epoch 50/50\n",
"313/313 [==============================] - 38s 120ms/step - loss: 1.2141\n"
"313/313 [==============================] - 90s 288ms/step - loss: 1.5666\n"
]
},
{
"data": {
"text/plain": [
"<tensorflow.python.keras.callbacks.History at 0x135f116a0>"
"<tensorflow.python.keras.callbacks.History at 0xd37b57908>"
]
},
"execution_count": 33,
Expand Down Expand Up @@ -820,11 +819,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"tham!\n",
"\n",
"coriolanus:\n",
"my loyald them done, live,\n",
"the c\n"
"torp:\n",
"unto most breathe blood him sight,\n",
"which rest\n"
]
}
],
Expand Down

0 comments on commit 1aa6650

Please sign in to comment.