From 386da3ca1e9438b15c8561f3120095e981aa5bf2 Mon Sep 17 00:00:00 2001 From: Siddharth Gururani Date: Tue, 13 Jun 2017 13:40:38 -0700 Subject: [PATCH 1/2] Fix tensorboard continue_from --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 0132c4ce..364e53e1 100644 --- a/train.py +++ b/train.py @@ -185,7 +185,7 @@ def main(): package['loss_results'] is not None and start_epoch > 0: # Previous scores to tensorboard logs loss_results, cer_results, wer_results = package['loss_results'], package['cer_results'], package[ 'wer_results'] - for i in range(len(loss_results)): + for i in range(epoch): info = { 'Avg Train Loss': loss_results[i], 'Avg WER': wer_results[i], From 528a60e40e49013c4043d03ca228115e79e46793 Mon Sep 17 00:00:00 2001 From: Siddharth Gururani Date: Fri, 16 Jun 2017 11:57:18 -0700 Subject: [PATCH 2/2] Fix bug in logging with continue_from flag --- train.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/train.py b/train.py index 364e53e1..de6bf60a 100644 --- a/train.py +++ b/train.py @@ -168,13 +168,12 @@ def main(): else: start_iter += 1 avg_loss = int(package.get('avg_loss', 0)) + loss_results, cer_results, wer_results = package['loss_results'], package[ + 'cer_results'], package['wer_results'] if args.visdom and \ package['loss_results'] is not None and start_epoch > 0: # Add previous scores to visdom graph - epoch = start_epoch - loss_results[0:epoch], cer_results[0:epoch], wer_results[0:epoch] = package['loss_results'], package[ - 'cer_results'], package['wer_results'] - x_axis = epochs[0:epoch] - y_axis = [loss_results[0:epoch], wer_results[0:epoch], cer_results[0:epoch]] + x_axis = epochs[0:start_epoch] + y_axis = [loss_results[0:start_epoch], wer_results[0:start_epoch], cer_results[0:start_epoch]] for x in range(len(viz_windows)): viz_windows[x] = viz.line( X=x_axis, @@ -183,9 +182,7 @@ def main(): ) if args.tensorboard and \ package['loss_results'] is not None and start_epoch > 0: # Previous scores to tensorboard logs - loss_results, cer_results, wer_results = package['loss_results'], package['cer_results'], package[ - 'wer_results'] - for i in range(epoch): + for i in range(start_epoch): info = { 'Avg Train Loss': loss_results[i], 'Avg WER': wer_results[i], @@ -327,7 +324,6 @@ def main(): epoch + 1, wer=wer, cer=cer)) if args.visdom: - # epoch += 1 x_axis = epochs[0:epoch + 1] y_axis = [loss_results[0:epoch + 1], wer_results[0:epoch + 1], cer_results[0:epoch + 1]]