video_captioning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-22T11:10:53.206543Z",
     "start_time": "2021-03-22T11:10:52.752293Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Import Path,Vocabulary, utility, evaluator and datahandler module\n",
    "from config import Path\n",
    "from dictionary import Vocabulary\n",
    "from utils import Utils\n",
    "from evaluate import Evaluator\n",
    "from data import DataHandler\n",
    "\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import copy\n",
    "\n",
    "#set seed for reproducibility\n",
    "utils = Utils()\n",
    "utils.set_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mean Pooling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-22T11:10:54.698930Z",
     "start_time": "2021-03-22T11:10:54.556381Z"
    }
   },
   "outputs": [],
   "source": [
    "#Import configuration and model \n",
    "from config import ConfigMP\n",
    "from models.mean_pooling.model import MeanPooling\n",
    "\n",
    "\n",
    "\n",
    "#create Mean pooling object\n",
    "cfg = ConfigMP()\n",
    "# specifying the dataset in configuration object from {'msvd','msrvtt'}\n",
    "cfg.dataset = 'msvd'\n",
    "#creation of path object\n",
    "path = Path(cfg,os.getcwd())\n",
    "\n",
    "#Changing the hyperparameters in configuration object\n",
    "#cfg.batch_size = 100 #training batch size\n",
    "cfg.n_layers = 1    # number of layers in decoder rnn\n",
    "cfg.decoder_type = 'lstm'  # from {'lstm','gru'}\n",
    "cfg.vocabulary_min_count = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Vocabulary creation or load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-22T11:10:56.692988Z",
     "start_time": "2021-03-22T11:10:56.670861Z"
    }
   },
   "outputs": [],
   "source": [
    "#Vocabulary object\n",
    "voc = Vocabulary(cfg)\n",
    "#If vocabulary is already saved or downloaded the saved file\n",
    "voc.load() #comment this if using vocabulary for the first time or with no saved file\n",
    "print('Vocabulary Size : ',voc.num_words) \n",
    "\n",
    "\n",
    "# # Uncomment this block if using vocabulary for the first time or if there is no saved file\n",
    "# text_dict = {}\n",
    "# voc = Vocabulary(cfg)\n",
    "# data_handler = DataHandler(cfg,path,voc)\n",
    "# import json\n",
    "# print(path.feature_file)\n",
    "# json.load(open(path.feature_file))\n",
    "# text_dict.update(data_handler.train_dict)\n",
    "# text_dict.update(data_handler.val_dict)\n",
    "# text_dict.update(data_handler.test_dict)\n",
    "# for k,v in text_dict.items():\n",
    "#     for anno in v:'\n",
    "#         voc.addSentence(anno)\n",
    "# voc.save()\n",
    "\n",
    "\n",
    "##Uncomment this block for filtering Rare Words from Dictionary\n",
    "min_count = cfg.vocabulary_min_count #remove all words below count min_count\n",
    "voc.trim(min_count=min_count)\n",
    "print('Vocabulary Size : ',voc.num_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataloaders model and evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-22T11:11:01.874306Z",
     "start_time": "2021-03-22T11:10:58.126859Z"
    }
   },
   "outputs": [],
   "source": [
    "# Datasets and dataloaders\n",
    "data_handler = DataHandler(cfg,path,voc)\n",
    "train_dset,val_dset,test_dset = data_handler.getDatasets()\n",
    "train_loader,val_loader,test_loader = data_handler.getDataloader(train_dset,val_dset,test_dset)\n",
    "\n",
    "#Model object\n",
    "model = MeanPooling(voc,cfg,path)\n",
    "#Evaluator object on test data\n",
    "test_evaluator_greedy = Evaluator(model,test_loader,path,cfg,data_handler.test_dict)\n",
    "test_evaluator_beam = Evaluator(model,test_loader,path,cfg,data_handler.test_dict,decoding_type='beam')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-22T10:51:10.412439Z",
     "start_time": "2021-03-22T10:51:10.044987Z"
    }
   },
   "outputs": [],
   "source": [
    "#Training Loop\n",
    "cfg.encoder_lr = 1e-4\n",
    "cfg.decoder_lr = 1e-3\n",
    "cfg.teacher_forcing_ratio = 1.0\n",
    "model.update_hyperparameters(cfg)\n",
    "val_loss = []\n",
    "for e in range(1,3001):\n",
    "    loss = model.train_epoch(train_loader,utils)\n",
    "    if e%50 == 0 :\n",
    "        print('Epoch -- >',e,'Loss -->',loss)\n",
    "        print('greedy :',test_evaluator_greedy.evaluate(utils,model,e,loss))\n",
    "        val_loss.append(model.loss_calculate(val_loader,utils))\n",
    "        #print('beam :',test_evaluator_beam.evaluate(utils,model,e,loss))\n",
    "        #print('semibeam :',test_evaluator_semibeam.evaluate(utils,model,e,loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features, targets, mask, max_length,_,_,_= dataiter.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsr,txt = model.GreedyDecoding(features.to(cfg.device))\n",
    "txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsr,txt,scores = model.BeamDecoding(features.to(cfg.device),return_single=False)\n",
    "txt,scores\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.target_tensor_to_caption(voc,targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SA-LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-17T11:02:32.210270Z",
     "start_time": "2021-03-17T11:02:32.062768Z"
    }
   },
   "outputs": [],
   "source": [
    "#Import configuration and model \n",
    "\n",
    "from config import ConfigSALSTM\n",
    "from models.SA_LSTM.model import SALSTM\n",
    "\n",
    "#create Mean pooling object\n",
    "cfg = ConfigSALSTM(opt_encoder=True)\n",
    "# specifying the dataset in configuration object from {'msvd','msrvtt'}\n",
    "cfg.dataset = 'msvd'\n",
    "\n",
    "#Changing the hyperparameters in configuration object\n",
    "cfg.batch_size = 100 #training batch size\n",
    "cfg.n_layers = 1    # number of layers in decoder rnn\n",
    "cfg.decoder_type = 'lstm'  # from {'lstm','gru'}\n",
    "cfg.dropout = 0.5\n",
    "cfg.opt_param_init = False\n",
    "\n",
    "\n",
    "\n",
    "#creation of path object\n",
    "path = Path(cfg,os.getcwd())\n",
    "#Vocabulary object, \n",
    "voc = Vocabulary(cfg)\n",
    "#If vocabulary is already saved or downloaded the saved file\n",
    "voc.load() #comment this if using vocabulary for the first time or with no saved file\n",
    "\n",
    "min_count = 5 #remove all words below count min_count\n",
    "voc.trim(min_count=min_count)\n",
    "print('Vocabulary Size : ',voc.num_words)\n",
    "#print('Vocabulary Size : ',voc.num_words) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-17T11:02:37.382514Z",
     "start_time": "2021-03-17T11:02:33.430116Z"
    }
   },
   "outputs": [],
   "source": [
    "# Datasets and dataloaders\n",
    "data_handler = DataHandler(cfg,path,voc)\n",
    "train_dset,val_dset,test_dset = data_handler.getDatasets()\n",
    "train_loader,val_loader,test_loader = data_handler.getDataloader(train_dset,val_dset,test_dset)\n",
    "\n",
    "#Model object\n",
    "model = SALSTM(voc,cfg,path)\n",
    "#Evaluator object on test data\n",
    "test_evaluator_greedy = Evaluator(model,test_loader,path,cfg,data_handler.test_dict)\n",
    "test_evaluator_beam = Evaluator(model,test_loader,path,cfg,data_handler.test_dict,decoding_type='beam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.load(os.path.join('Saved','sa_lstm_msvd.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-17T11:14:23.021770Z",
     "start_time": "2021-03-17T11:02:38.180710Z"
    }
   },
   "outputs": [],
   "source": [
    "#Training Loop\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "cfg.encoder_lr = 1e-4\n",
    "cfg.decoder_lr = 1e-4\n",
    "cfg.teacher_forcing_ratio = 1.0\n",
    "model.update_hyperparameters(cfg)\n",
    "# lr_scheduler = ReduceLROnPlateau(model.dec_optimizer, mode='min', factor=cfg.lr_decay_gamma,\n",
    "#                                      patience=cfg.lr_decay_patience, verbose=True)\n",
    "for e in range(1,1351):\n",
    "    loss_train = model.train_epoch(train_loader,utils)\n",
    "    #loss_val = model.train_epoch(val_loader,utils)\n",
    "    #lr_scheduler.step(loss_train)\n",
    "    if e%50 == 0 :\n",
    "        print('Epoch -- >',e,'Loss -->',loss_train)\n",
    "        print('greedy :',test_evaluator_greedy.evaluate(utils,model,e,loss_train))\n",
    "        print('beam :',test_evaluator_beam.evaluate(utils,model,e,loss_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataiter = iter(val_loader)\n",
    "features, targets, mask, max_length,_,motion_feat,object_feat= dataiter.next()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsr,txt,_ = model.GreedyDecoding(features.to(cfg.device))\n",
    "txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils.target_tensor_to_caption(voc,targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tsr,txt,scr = model.BeamDecoding(features.to(cfg.device))\n",
    "txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RecNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import configuration and model \n",
    "\n",
    "from config import ConfigRecNet\n",
    "from models.RecNet.model import RecNet\n",
    "\n",
    "#create Mean pooling object\n",
    "cfg = ConfigRecNet()\n",
    "# specifying the dataset in configuration object from {'msvd','msrvtt'}\n",
    "cfg.dataset = 'msvd'\n",
    "\n",
    "#Changing the hyperparameters in configuration object\n",
    "cfg.batch_size = 100 #training batch size\n",
    "cfg.n_layers = 1    # number of layers in decoder rnn\n",
    "cfg.decoder_type = 'lstm'  # from {'lstm','gru'}\n",
    "\n",
    "\n",
    "#creation of path object\n",
    "path = Path(cfg,os.getcwd())\n",
    "#Vocabulary object, \n",
    "voc = Vocabulary(cfg)\n",
    "#If vocabulary is already saved or downloaded the saved file\n",
    "voc.load() #comment this if using vocabulary for the first time or with no saved file\n",
    "min_count = cfg.vocabulary_min_count #remove all words below count min_count\n",
    "voc.trim(min_count=min_count)\n",
    "print('Vocabulary Size : ',voc.num_words)\n",
    "#print('Vocabulary Size : ',voc.num_words) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Datasets and dataloaders\n",
    "data_handler = DataHandler(cfg,path,voc)\n",
    "train_dset,val_dset,test_dset = data_handler.getDatasets()\n",
    "train_loader,val_loader,test_loader = data_handler.getDataloader(train_dset,val_dset,test_dset)\n",
    "\n",
    "#Model object\n",
    "model = RecNet(voc,cfg,path)\n",
    "#Evaluator object on test data\n",
    "test_evaluator_greedy = Evaluator(model,test_loader,path,cfg,data_handler.test_dict)\n",
    "test_evaluator_beam = Evaluator(model,test_loader,path,cfg,data_handler.test_dict,decoding_type='beam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stage-1 Training, Or load model after stage-1 Training.\n",
    "\n",
    "encoder_state_dict_file = os.path.join(path.saved_models_path,'sa_lstm_encoder_msvd.pt')\n",
    "decoder_state_dict_file = os.path.join(path.saved_models_path,'sa_lstm_decoder_msvd.pt')\n",
    "#print(encoder_state_dict_file)\n",
    "model.encoder.load_state_dict(torch.load(encoder_state_dict_file))\n",
    "model.decoder.load_state_dict(torch.load(decoder_state_dict_file))\n",
    "print('greedy :',test_evaluator_greedy.evaluate(utils,model,1350,1.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stage-2 Training\n",
    "cfg.encoder_lr = 1e-3\n",
    "cfg.decoder_lr = 1e-3\n",
    "cfg.global_lr = 1e-3\n",
    "cfg.local_lr = 1e-2\n",
    "cfg.teacher_forcing_ratio = 1.0\n",
    "cfg.training_stage = 2\n",
    "cfg.lmda = 0.1\n",
    "model.update_hyperparameters(cfg)\n",
    "# lr_scheduler = ReduceLROnPlateau(model.dec_optimizer, mode='min', factor=cfg.lr_decay_gamma,\n",
    "#                                      patience=cfg.lr_decay_patience, verbose=True)\n",
    "for e in range(1,2501):\n",
    "    lloss_train, recloss_train = model.train_epoch(train_loader,utils)\n",
    "    #loss_val = model.train_epoch(val_loader,utils)\n",
    "    #lr_scheduler.step(loss_train)\n",
    "    if e%10 == 0 :\n",
    "        model.encoder.eval()\n",
    "        model.decoder.eval()\n",
    "        model.local_reconstructor.eval()\n",
    "        print('Epoch -- >',e,'Likelihood Loss -->',lloss_train,'Reconstruction Loss -->',recloss_train)\n",
    "        print('greedy :',test_evaluator_greedy.evaluate(utils,model,e,lloss_train))\n",
    "        print('beam :',test_evaluator_beam.evaluate(utils,model,e,lloss_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataiter = iter(train_loader)\n",
    "features, targets, mask, max_length,_,motion_feat,object_feat= dataiter.next()\n",
    "features.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MARN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved/msvd_word2index_dic.p\n",
      "keep_words 3981 / 12596 = 0.3161\n",
      "Vocabulary Size :  3984\n"
     ]
    }
   ],
   "source": [
    "#Import configuration and model \n",
    "\n",
    "\n",
    "from config import ConfigMARN\n",
    "from models.MARN.model import MARN\n",
    "\n",
    "#create Mean pooling object\n",
    "cfg = ConfigMARN()\n",
    "# specifying the dataset in configuration object from {'msvd','msrvtt'}\n",
    "cfg.dataset = 'msvd'\n",
    "\n",
    "#Changing the hyperparameters in configuration object\n",
    "\n",
    "\n",
    "\n",
    "#creation of path object\n",
    "path = Path(cfg,os.getcwd())\n",
    "#Vocabulary object, \n",
    "voc = Vocabulary(cfg)\n",
    "#If vocabulary is already saved or downloaded the saved file\n",
    "voc.load() #comment this if using vocabulary for the first time or with no saved file\n",
    "min_count = cfg.vocabulary_min_count #remove all words below count min_count\n",
    "voc.trim(min_count=min_count)\n",
    "print('Vocabulary Size : ',voc.num_words)\n",
    "#print('Vocabulary Size : ',voc.num_words) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Datasets and dataloaders\n",
    "data_handler = DataHandler(cfg,path,voc)\n",
    "train_dset,val_dset,test_dset = data_handler.getDatasets()\n",
    "train_loader,val_loader,test_loader = data_handler.getDataloader(train_dset,val_dset,test_dset)\n",
    "\n",
    "#Model object\n",
    "model = MARN(voc,cfg,path)\n",
    "#Evaluator object on test data\n",
    "test_evaluator_greedy = Evaluator(model,test_loader,path,cfg,data_handler.test_dict)\n",
    "#test_evaluator_beam = Evaluator(model,test_loader,path,cfg,data_handler.test_dict,decoding_type='beam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch -- > 25 Loss --> 3.9958693362308884   AC loss ---> 0.03090188533067703\n",
      "{'testlen': 3890, 'reflen': 3886, 'guess': [3890, 3220, 2550, 1880], 'correct': [2759, 1181, 518, 106]}\n",
      "ratio: 1.0010293360779718\n",
      "greedy : {'Bleu_1': 0.7092544987144707, 'Bleu_2': 0.5100327486080173, 'Bleu_3': 0.3752567976872323, 'Bleu_4': 0.23363259354817334, 'METEOR': 0.24120785959776894, 'ROUGE_L': 0.610236566114215, 'CIDEr': 0.15639253169251544}\n",
      "Epoch -- > 50 Loss --> 3.59046994521975   AC loss ---> 0.02184500128030777\n",
      "{'testlen': 3727, 'reflen': 3719, 'guess': [3727, 3057, 2387, 1717], 'correct': [2890, 1430, 711, 249]}\n",
      "ratio: 1.0021511158910992\n",
      "greedy : {'Bleu_1': 0.77542259189676, 'Bleu_2': 0.6022676335250706, 'Bleu_3': 0.47628336154073075, 'Bleu_4': 0.3537987715241685, 'METEOR': 0.277348359131785, 'ROUGE_L': 0.6552164314236341, 'CIDEr': 0.390500539843794}\n",
      "Epoch -- > 75 Loss --> 3.3389997142232097   AC loss ---> 0.018926209211349486\n",
      "{'testlen': 3721, 'reflen': 3712, 'guess': [3721, 3051, 2381, 1711], 'correct': [2924, 1527, 766, 283]}\n",
      "ratio: 1.0024245689652471\n",
      "greedy : {'Bleu_1': 0.7858102660573002, 'Bleu_2': 0.6271295483596641, 'Bleu_3': 0.5020280314835279, 'Bleu_4': 0.38034707608806123, 'METEOR': 0.2885635233307096, 'ROUGE_L': 0.6677921646372974, 'CIDEr': 0.46530693547810265}\n",
      "Epoch -- > 100 Loss --> 3.2335948057674493   AC loss ---> 0.015862594991922378\n",
      "{'testlen': 3786, 'reflen': 3780, 'guess': [3786, 3116, 2446, 1776], 'correct': [3013, 1681, 876, 366]}\n",
      "ratio: 1.0015873015870367\n",
      "greedy : {'Bleu_1': 0.7958267300578986, 'Bleu_2': 0.6552309349057115, 'Bleu_3': 0.5357293994428838, 'Bleu_4': 0.4219088555338067, 'METEOR': 0.30312048986285467, 'ROUGE_L': 0.6774881782494551, 'CIDEr': 0.5560915068260947}\n",
      "Epoch -- > 125 Loss --> 3.075437706838148   AC loss ---> 0.014236636608839035\n",
      "{'testlen': 3920, 'reflen': 3917, 'guess': [3920, 3250, 2580, 1910], 'correct': [3084, 1711, 920, 398]}\n",
      "ratio: 1.0007658922642326\n",
      "greedy : {'Bleu_1': 0.7867346938773503, 'Bleu_2': 0.643572495760664, 'Bleu_3': 0.5285925332305239, 'Bleu_4': 0.41884486392170145, 'METEOR': 0.3085079267589841, 'ROUGE_L': 0.6788769439302236, 'CIDEr': 0.6082798450014398}\n",
      "Epoch -- > 150 Loss --> 2.97519330942332   AC loss ---> 0.017049746066331865\n",
      "{'testlen': 3758, 'reflen': 3763, 'guess': [3758, 3088, 2418, 1748], 'correct': [3018, 1733, 945, 410]}\n",
      "ratio: 0.9986712729202767\n",
      "greedy : {'Bleu_1': 0.8020189559153501, 'Bleu_2': 0.6704463623321849, 'Bleu_3': 0.5598116355849188, 'Bleu_4': 0.4502436445018508, 'METEOR': 0.31596816810557277, 'ROUGE_L': 0.6845317280491038, 'CIDEr': 0.6434386994466349}\n",
      "Epoch -- > 175 Loss --> 2.7853921287763628   AC loss ---> 0.017188223302364348\n",
      "{'testlen': 3957, 'reflen': 3952, 'guess': [3957, 3287, 2617, 1947], 'correct': [3158, 1823, 1018, 446]}\n",
      "ratio: 1.0012651821859815\n",
      "greedy : {'Bleu_1': 0.7980793530450346, 'Bleu_2': 0.6652984627970656, 'Bleu_3': 0.5563213131377474, 'Bleu_4': 0.4456423960829763, 'METEOR': 0.3195923589050506, 'ROUGE_L': 0.6913701565256254, 'CIDEr': 0.6707115762657333}\n",
      "Epoch -- > 200 Loss --> 2.7066300502667193   AC loss ---> 0.016030927002429963\n",
      "{'testlen': 3985, 'reflen': 3990, 'guess': [3985, 3315, 2645, 1975], 'correct': [3174, 1841, 1040, 478]}\n",
      "ratio: 0.9987468671676695\n",
      "greedy : {'Bleu_1': 0.7954880961646515, 'Bleu_2': 0.6642468670290692, 'Bleu_3': 0.5574945082482713, 'Bleu_4': 0.4523862709829777, 'METEOR': 0.321368718237567, 'ROUGE_L': 0.6943729445352252, 'CIDEr': 0.7162367955837502}\n",
      "Epoch -- > 225 Loss --> 2.6754710204190575   AC loss ---> 0.018900204598903656\n",
      "{'testlen': 4052, 'reflen': 4050, 'guess': [4052, 3382, 2712, 2042], 'correct': [3232, 1887, 1066, 484]}\n",
      "ratio: 1.0004938271602468\n",
      "greedy : {'Bleu_1': 0.7976307996049364, 'Bleu_2': 0.6671140788655268, 'Bleu_3': 0.5592713566147934, 'Bleu_4': 0.4512470117397467, 'METEOR': 0.3274234864398704, 'ROUGE_L': 0.6965507018547925, 'CIDEr': 0.7412796947044867}\n",
      "Epoch -- > 250 Loss --> 2.589164914172032   AC loss ---> 0.02262635961174965\n",
      "{'testlen': 3944, 'reflen': 3944, 'guess': [3944, 3274, 2604, 1934], 'correct': [3184, 1872, 1064, 489]}\n",
      "ratio: 0.9999999999997464\n",
      "greedy : {'Bleu_1': 0.8073022312369131, 'Bleu_2': 0.6794095717462393, 'Bleu_3': 0.5734840233073716, 'Bleu_4': 0.4673089404087852, 'METEOR': 0.329799387661286, 'ROUGE_L': 0.704644314453793, 'CIDEr': 0.7549530892568287}\n",
      "Epoch -- > 275 Loss --> 2.5337486573563193   AC loss ---> 0.01901746854186058\n",
      "{'testlen': 4072, 'reflen': 4070, 'guess': [4072, 3402, 2732, 2062], 'correct': [3267, 1902, 1084, 515]}\n",
      "ratio: 1.0004914004911547\n",
      "greedy : {'Bleu_1': 0.8023084479369346, 'Bleu_2': 0.6697439269470253, 'Bleu_3': 0.5624993849138858, 'Bleu_4': 0.45916754223029776, 'METEOR': 0.3315906703441964, 'ROUGE_L': 0.7052194413006333, 'CIDEr': 0.7925864719039174}\n",
      "Epoch -- > 300 Loss --> 2.41988559159911   AC loss ---> 0.015305523127317429\n",
      "{'testlen': 3968, 'reflen': 3964, 'guess': [3968, 3298, 2628, 1958], 'correct': [3219, 1927, 1094, 511]}\n",
      "ratio: 1.001009081735368\n",
      "greedy : {'Bleu_1': 0.8112399193546342, 'Bleu_2': 0.6884781920442574, 'Bleu_3': 0.5821802200089644, 'Bleu_4': 0.47637080527822495, 'METEOR': 0.3405265110989553, 'ROUGE_L': 0.7109566917946866, 'CIDEr': 0.8291895303190925}\n",
      "Epoch -- > 325 Loss --> 2.3829907736384834   AC loss ---> 0.016008519232273102\n",
      "{'testlen': 4048, 'reflen': 4041, 'guess': [4048, 3378, 2708, 2038], 'correct': [3261, 1919, 1109, 527]}\n",
      "ratio: 1.0017322444936894\n",
      "greedy : {'Bleu_1': 0.8055830039523701, 'Bleu_2': 0.6764922291581406, 'Bleu_3': 0.5722724473714277, 'Bleu_4': 0.4691953901735516, 'METEOR': 0.33583694873902453, 'ROUGE_L': 0.7082736000570841, 'CIDEr': 0.8322870443515807}\n",
      "Epoch -- > 350 Loss --> 2.2750166412235346   AC loss ---> 0.02653258338570595\n",
      "{'testlen': 4079, 'reflen': 4073, 'guess': [4079, 3409, 2739, 2069], 'correct': [3283, 1960, 1131, 533]}\n",
      "ratio: 1.0014731156393317\n",
      "greedy : {'Bleu_1': 0.8048541309142425, 'Bleu_2': 0.6802571633789545, 'Bleu_3': 0.5759776170713647, 'Bleu_4': 0.47102708933198584, 'METEOR': 0.3387897438211377, 'ROUGE_L': 0.7106565434186339, 'CIDEr': 0.8417295153790754}\n",
      "Epoch -- > 375 Loss --> 2.268754849296054   AC loss ---> 0.028440549075603484\n",
      "{'testlen': 4004, 'reflen': 3997, 'guess': [4004, 3334, 2664, 1994], 'correct': [3257, 1943, 1116, 531]}\n",
      "ratio: 1.0017513134848632\n",
      "greedy : {'Bleu_1': 0.8134365634363603, 'Bleu_2': 0.6885182360363749, 'Bleu_3': 0.5834275812434945, 'Bleu_4': 0.47954852444373264, 'METEOR': 0.3445412864881122, 'ROUGE_L': 0.713277499707953, 'CIDEr': 0.8484618164899964}\n",
      "Epoch -- > 400 Loss --> 2.2041979912460223   AC loss ---> 0.021966117918491363\n",
      "{'testlen': 4137, 'reflen': 4131, 'guess': [4137, 3467, 2797, 2127], 'correct': [3332, 1981, 1145, 550]}\n",
      "ratio: 1.0014524328247394\n",
      "greedy : {'Bleu_1': 0.8054145516072503, 'Bleu_2': 0.6783831510758082, 'Bleu_3': 0.5732636114777573, 'Bleu_4': 0.46980169723536713, 'METEOR': 0.3403749191461473, 'ROUGE_L': 0.7115417873680908, 'CIDEr': 0.8589295958296881}\n",
      "Epoch -- > 425 Loss --> 2.2090038911578267   AC loss ---> 0.01886691093444824\n",
      "{'testlen': 4078, 'reflen': 4072, 'guess': [4078, 3408, 2738, 2068], 'correct': [3300, 1970, 1139, 548]}\n",
      "ratio: 1.0014734774064338\n",
      "greedy : {'Bleu_1': 0.8092202059831267, 'Bleu_2': 0.6839379136829402, 'Bleu_3': 0.5794836846598146, 'Bleu_4': 0.4765279907384327, 'METEOR': 0.34152488569471867, 'ROUGE_L': 0.7123702540772384, 'CIDEr': 0.8389883935461858}\n",
      "Epoch -- > 450 Loss --> 2.1653517934436173   AC loss ---> 0.025127877444028855\n",
      "{'testlen': 4009, 'reflen': 4004, 'guess': [4009, 3339, 2669, 1999], 'correct': [3260, 1956, 1128, 534]}\n",
      "ratio: 1.001248751248501\n",
      "greedy : {'Bleu_1': 0.8131703666747785, 'Bleu_2': 0.6901873380538651, 'Bleu_3': 0.5860907048248948, 'Bleu_4': 0.48156591589769737, 'METEOR': 0.3408323450291638, 'ROUGE_L': 0.7151395685777704, 'CIDEr': 0.86274228617226}\n",
      "Epoch -- > 475 Loss --> 2.1396833926419303   AC loss ---> 0.021066931188106538\n",
      "{'testlen': 4085, 'reflen': 4078, 'guess': [4085, 3415, 2745, 2075], 'correct': [3294, 1952, 1121, 529]}\n",
      "ratio: 1.001716527709416\n",
      "greedy : {'Bleu_1': 0.80636474908181, 'Bleu_2': 0.6789070516895099, 'Bleu_3': 0.5730967791512497, 'Bleu_4': 0.468037195659896, 'METEOR': 0.3372661482690198, 'ROUGE_L': 0.7126271287029997, 'CIDEr': 0.8408304052446127}\n",
      "Epoch -- > 500 Loss --> 2.075977917742943   AC loss ---> 0.018512180149555205\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'testlen': 4096, 'reflen': 4089, 'guess': [4096, 3426, 2756, 2086], 'correct': [3335, 1998, 1156, 552]}\n",
      "ratio: 1.0017119100022005\n",
      "greedy : {'Bleu_1': 0.8142089843748012, 'Bleu_2': 0.6890837488680113, 'Bleu_3': 0.5839928708663213, 'Bleu_4': 0.47913936448993966, 'METEOR': 0.3411564377362805, 'ROUGE_L': 0.7124298204181696, 'CIDEr': 0.87774070599289}\n"
     ]
    }
   ],
   "source": [
    "#Training Loop (Stage-1)(Without memory decoder)\n",
    "cfg.encoder_lr = 1e-4\n",
    "cfg.decoder_lr = 1e-4\n",
    "cfg.teacher_forcing_ratio = 1.0\n",
    "#model.generate_memory(data_handler)\n",
    "#print('Memory Generated')\n",
    "model.opt_memory_decoder = False\n",
    "model.update_hyperparameters(cfg)\n",
    "\n",
    "#lr_scheduler = StepLR(model.dec_optimizer,300,gamma=0.1,verbose=False)\n",
    "# lr_scheduler = ReduceLROnPlateau(model.dec_optimizer, mode='min', factor=cfg.lr_decay_gamma,\n",
    "#                                      patience=cfg.lr_decay_patience, verbose=True)\n",
    "for e in range(1,501):\n",
    "    loss_train,ac_loss = model.train_epoch(train_loader,utils)\n",
    "    #loss_val = model.train_epoch(val_loader,utils)\n",
    "    #lr_scheduler.step()\n",
    "    if e%25 == 0 :\n",
    "        print('Epoch -- >',e,'Loss -->',loss_train,'  AC loss --->',ac_loss)\n",
    "        print('greedy :',test_evaluator_greedy.evaluate(utils,model,e,loss_train))\n",
    "        #print('beam :',test_evaluator_beam.evaluate(utils,model,e,loss_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory Generated\n"
     ]
    }
   ],
   "source": [
    "#Training Loop (Stage-2)(With Attended Memory decoder)\n",
    "cfg.encoder_lr = 1e-4\n",
    "cfg.decoder_lr = 1e-4\n",
    "cfg.teacher_forcing_ratio = 1.0\n",
    "model.generate_memory(data_handler)\n",
    "print('Memory Generated')\n",
    "model.opt_memory_decoder = True\n",
    "model.update_hyperparameters(cfg)\n",
    "\n",
    "#lr_scheduler = StepLR(model.dec_optimizer,300,gamma=0.1,verbose=False)\n",
    "# lr_scheduler = ReduceLROnPlateau(model.dec_optimizer, mode='min', factor=cfg.lr_decay_gamma,\n",
    "#                                      patience=cfg.lr_decay_patience, verbose=True)\n",
    "for e in range(501,1501):\n",
    "    loss_train,ac_loss = model.train_epoch(train_loader,utils)\n",
    "    #loss_val = model.train_epoch(val_loader,utils)\n",
    "    #lr_scheduler.step()\n",
    "    if e%25 == 0 :\n",
    "        print('Epoch -- >',e,'Loss -->',loss_train,'  AC loss --->',ac_loss)\n",
    "        print('greedy :',test_evaluator_greedy.evaluate(utils,model,e,loss_train))\n",
    "        #print('beam :',test_evaluator_beam.evaluate(utils,model,e,loss_train))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}