15.nn-lm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Language Models\n",
    "Status of Notebook: Work in Progress\n",
    "\n",
    "Reference: https://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf\n",
    "\n",
    "Dynet Version: https://github.com/neubig/nn4nlp-code/blob/master/02-lm/nn-lm.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch#安装torch库，支持在图形处理单元上计算张量\n",
    "import random#安装random库，用于产生各种分布的伪随机数序列\n",
    "import torch#安装torch库，支持在图形处理单元上计算张量\n",
    "import torch.nn as nn#加载神经网络常用模块\n",
    "import math#加载函数库\n",
    "import time#加载时间元组\n",
    "import numpy as np#加载numpy科学计算库"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#取消注释就可以下载数据集\n",
    "#!wget https://raw.githubusercontent.com/neubig/nn4nlp-code/master/data/ptb/test.txt\n",
    "#!wget https://raw.githubusercontent.com/neubig/nn4nlp-code/master/data/ptb/train.txt\n",
    "#!wget https://raw.githubusercontent.com/neubig/nn4nlp-code/master/data/ptb/valid.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#函数读取数据，处理每一行并按“|||”分割列\n",
    "def read_data(filename):#函数读取数据，处理每一行\n",
    "    data = []#初始化数据列表\n",
    "    with open(filename, \"r\") as f:#读取filename文件到f中\n",
    "        for line in f:#逐行打印f文件\n",
    "            line = line.strip().split(\" \")#将读取数据的所有字符小写，并去除两端的空格或者制表符、换行符等无效字符\n",
    "            data.append(line)#将读取的数据添加到数据列表中\n",
    "    return data#返回读取后的数据列表\n",
    "\n",
    "#读取数据\n",
    "train_data = read_data('data/ptb/train.txt')#保存到训练数据集\n",
    "val_data = read_data('data/ptb/valid.txt')#保存到测试数据集\n",
    "\n",
    "#创建单词和标签索引以及特殊标记\n",
    "word_to_index = {}#创建单词索引列表\n",
    "index_to_word = {}#创建标签索引列表\n",
    "word_to_index[\"<s>\"] = len(word_to_index)\n",
    "index_to_word[len(word_to_index)-1] = \"<s>\"\n",
    "word_to_index[\"<unk>\"] = len(word_to_index)#添加<UNK>到字典\n",
    "index_to_word[len(word_to_index)-1] = \"<unk>\"\n",
    "\n",
    "def create_dict(data, check_unk=False):#根据数据创建单词到索引字典和标记到索引字典\n",
    "    for line in data:#逐行迭代\n",
    "        for word in line:\n",
    "            if check_unk == False:\n",
    "                if word not in word_to_index:#如果word不在字典里\n",
    "                    word_to_index[word] = len(word_to_index)#添加word到字典中\n",
    "                    index_to_word[len(word_to_index)-1] = word#添加索引\n",
    "            \n",
    "            #无效，因为<unk>已附带数据\n",
    "            #应在未处理＜unk＞的情况下处理数据\n",
    "            else: \n",
    "                if word not in word_to_index:#如果word不在字典里\n",
    "                    word_to_index[word] = word_to_index[\"<unk>\"]#添加<UNK>到字典\n",
    "                    index_to_word[len(word_to_index)-1] = word#添加索引\n",
    "\n",
    "create_dict(train_data)#创建训练数据字典\n",
    "create_dict(val_data, check_unk=True)#创建测试数据字典\n",
    "\n",
    "def create_tensor(data):#根据数据创建单词和标记张量\n",
    "    for line in data:\n",
    "        yield([word_to_index[word] for word in line])\n",
    "\n",
    "train_data = list(create_tensor(train_data))#创建训练数据张量\n",
    "val_data = list(create_tensor(val_data))#创建测试数据张量\n",
    "\n",
    "number_of_words = len(word_to_index)#单词个数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In our implementation we are using batched training. There are a few differences from the original implementation found [here](https://github.com/neubig/nn4nlp-code/blob/master/02-lm/loglin-lm.py). "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义模型\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'#检测当前计算机是否支持使用cuda，如果支持则将device变量设置为cuda，否则设置为cpu\n",
    "\n",
    "N = 2#n-gram的长度\n",
    "EMB_SIZE = 128#嵌入的大小\n",
    "HID_SIZE = 128#隐藏层的大小\n",
    "\n",
    "class NeuralLM(nn.Module):#构建神经语言模型模型\n",
    "    def __init__(self, number_of_words, ngram_length, EMB_SIZE, HID_SIZE):\n",
    "        super(NeuralLM, self).__init__()#初始化\n",
    "\n",
    "        self.embedding = nn.Embedding(number_of_words, EMB_SIZE)#嵌入层\n",
    "        self.hidden = nn.Linear(EMB_SIZE * ngram_length, HID_SIZE)#隐藏层\n",
    "        self.output = nn.Linear(HID_SIZE, number_of_words)#输出层\n",
    "\n",
    "    def forward(self, x):#计算分数\n",
    "        embs = self.embedding(x)#Size:[batch_size x num_hist x emb_size]\n",
    "        embs = embs.view(embs.size(0), -1)#Size:[batch_size x (num_hist*emb_size)]\n",
    "        h = torch.nn.functional.tanh(self.hidden(embs))#Size:[batch_size x hid_size]\n",
    "        scores = self.output(h)#Size:batch_size x num_words\n",
    "        return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Settings and Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NeuralLM(number_of_words, N, EMB_SIZE, HID_SIZE)#加载神经语言模型模型\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.1)#加载神经网络训练优化器，学习率为0.1\n",
    "criterion = torch.nn.CrossEntropyLoss()#加载损失函数\n",
    "\n",
    "if torch.cuda.is_available():#如果可以使用cuda\n",
    "    model.to(device)#模型加载到相应设备中\n",
    "\n",
    "def calc_sent_loss(sent):#计算句子损失的函数\n",
    "    S = word_to_index[\"<s>\"]\n",
    "\n",
    "    hist = [S] * N#起始历史等于句末符号\n",
    "    \n",
    "    #收集所有目标和历史记录\n",
    "    all_targets = []#初始化目标队列\n",
    "    all_histories = []#初始化历史记录\n",
    "    \n",
    "    #逐步完成句子，包括句尾标记\n",
    "    for next_word in sent + [S]:\n",
    "        all_histories.append(list(hist))#读取的数据加入历史记录队列\n",
    "        all_targets.append(next_word)#下一个数据加入目标队列\n",
    "        hist = hist[1:] + [next_word]\n",
    "\n",
    "    logits = model(torch.LongTensor(all_histories).to(device))#logits是转换成概率之前的值，是下一步通常被投给softmax的向量\n",
    "    loss = criterion(logits, torch.LongTensor(all_targets).to(device))#计算损失\n",
    "\n",
    "    return loss\n",
    "\n",
    "MAX_LEN = 100#定义句子最大长度100\n",
    "def generate_sent():#生成句子的函数\n",
    "    S = word_to_index[\"<s>\"]#<s>添加到字典中\n",
    "    hist = [S] * N\n",
    "    sent = []#初始化列表\n",
    "    while True:\n",
    "        logits = model(torch.LongTensor([hist]).to(device))#将模型加载到相应的设备中，64位整型\n",
    "        p = torch.nn.functional.softmax(logits) # 1 x number_of_words\n",
    "        next_word = p.multinomial(num_samples=1).item()#抽取样本，为每行切片绘制的独立样本数为1\n",
    "        if next_word == S or len(sent) == MAX_LEN:\n",
    "            break\n",
    "        sent.append(next_word)#添加到样本中\n",
    "        hist = hist[1:] + [next_word]\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 0: train loss/word=4.1802, ppl=65.3775\n",
      "iter 0: dev loss/word=4.4128, ppl=82.4961, time=1.26s\n",
      "in constitution physics which could counting suspect include be on\n",
      "dealers manufacturers plans commissions\n",
      "in constitution physics which could counting suspect include be on behalf he declares\n",
      "in constitution physics which could counting suspect include be and which and an for was designed on themes of weakness jobs n't be and which <unk> developed the sale such from about other objectives\n",
      "N have in prolonged damage\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/ipykernel_launcher.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 1: train loss/word=4.4307, ppl=83.9873\n",
      "iter 1: dev loss/word=4.5315, ppl=92.8970, time=1.27s\n",
      "two <unk> hours <unk> relations clark new an index big were medicine more 'm bank N in october this fall\n",
      "two this said its consumer puts for democratic futures the ringers out N note a day to get this affairs\n",
      "this time\n",
      "two this said its consumer puts for democratic futures the ringers out N note a top outstanding bank for $ candidates savings <unk> relationship in shopping for declared futures the ringers out N note a day to get this said its consumer puts to highlight this fall\n",
      "this time the\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 2: train loss/word=4.4670, ppl=87.0953\n",
      "iter 2: dev loss/word=4.5699, ppl=96.5306, time=1.28s\n",
      "there is by being it on the first was estimated for possible the experiment of those after the key\n",
      "there is by\n",
      "<unk> intensity excess $ co. spot N N N N N the tanker of those after in <unk> forced around who participated in <unk> forced around to $ N million wednesday\n",
      "there is by\n",
      "to one why N gallons iii bush from $ N million wednesday\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 3: train loss/word=4.4909, ppl=89.1985\n",
      "iter 3: dev loss/word=4.5530, ppl=94.9163, time=1.31s\n",
      "in of western actions it does about service pilots a the company costs there with chief executive retailing <unk> under and are will for which the department showed N N of stock funds profit as well a buildup and an interest has expects <unk> up in the friday-the-13th third\n",
      "in of <unk> and the products costs has about shareholders fidelity with agreed was it to a less and stock will okla. say the former economic to make\n",
      "in it does about service pilots a the company costs there with a five-year and then <unk> more\n",
      "in of western actions it does about service pilots at profit common runs has\n",
      "thus declined to comment\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 4: train loss/word=4.4966, ppl=89.7113\n",
      "iter 4: dev loss/word=4.6409, ppl=103.6412, time=1.28s\n",
      "the apparent centers groups by to reform\n",
      "are consumers too deep over that we do n't want to continue owning stocks we oct. in <unk>\n",
      "the apparent centers groups by the <unk> missile n't available five former corp to slow owning u.k. <unk>\n",
      "the apparent centers groups by the and centers but to limit owning investment\n",
      "the apparent centers groups by the to share mr. lehman attributed n't available five former corp to slow owning it was who i was coast to round owning revenue to specific another dramatic worked and financial announcements <unk> wo n't end anytime soon\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 5: train loss/word=4.5213, ppl=91.9530\n",
      "iter 5: dev loss/word=4.7837, ppl=119.5463, time=1.28s\n",
      "the other involving plant the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits in many other say central\n",
      "the other involving plant the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits is for filled <unk> senior the commission value to consolidate several lawsuits in many other say central\n",
      "the other involving plant the commission groups\n",
      "the other involving plant the commission value of leading funds code growth channel grows aide market against her for clearance 's <unk> of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era to mr. a ratio process some air fares about rumors <unk> <unk> process the <unk> era of the <unk> era the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era to mr. the <unk> era to mr.\n",
      "the other involving plant the commission value bank leading some continental a senior the commission value to consolidate several lawsuits on fetch for violations <unk> senior the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits <unk> senior the commission value in draw other time strict fire american express for two operations n't if debt only on behalf n't increase he also questioned for two operations n't if debt\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 6: train loss/word=4.5284, ppl=92.6074\n",
      "iter 6: dev loss/word=4.8860, ppl=132.4199, time=1.27s\n",
      "toyota have expressed recent durable attempt <unk> chief stocks spend notes\n",
      "toyota have expressed recent durable attempt <unk> development\n",
      "toyota have expressed recent durable attempt to\n",
      "toyota have expressed recent durable attempt <unk> development 's also <unk> concedes stocks could back average resolve by\n",
      "toyota of N <unk> occurred stocks less today <unk>\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 7: train loss/word=4.5339, ppl=93.1220\n",
      "iter 7: dev loss/word=4.9127, ppl=136.0103, time=1.28s\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place the partnership more than year <unk> <unk> new york $ N to N a share a year earlier assets at fairly insurance will open and assets <unk> subsidiaries by and mae <unk> and his wife by fannie mae N N from turnover citing commitment place the partnership more than a partial on <unk> changes by of companies and assets <unk> subsidiaries in a trading range N N from turnover citing commitment place the partnership more than year <unk> <unk> new york $ N to N to N\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place only will open to represent this week\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place only will open\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place the partnership more than year <unk> <unk> new york $ N to N N from turnover citing commitment place only will open and assets <unk> subsidiaries that not officials monday N N from turnover citing commitment place the partnership more other <unk> match by by mae <unk> and <unk> and his wife by fannie mae N N from turnover citing commitment place the partnership more than a partial of refusing and assets <unk> subsidiaries by and mae <unk> and his wife by fannie mae N N from\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place only will open to represent this week\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 8: train loss/word=4.5402, ppl=93.7127\n",
      "iter 8: dev loss/word=4.9000, ppl=134.2900, time=1.28s\n",
      "barney any projections case for purchase\n",
      "barney any projections case for purchase about liability says fast financial profit would increase close sassy cie are into imports the will concern the work industry did doing to record other cash and the but section N <unk> N gorbachev games the continued mr. sohmer says <unk> for N N aided executive into attacking and increased risks <unk> where with ratings pitch democratic\n",
      "barney any projections case for purchase\n",
      "barney any projections case for purchase\n",
      "barney any projections case for purchase\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_14352/404430955.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0msent_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msent\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# CHANGE to all train_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mmy_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalc_sent_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mtrain_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mmy_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_14352/2289298869.py\u001b[0m in \u001b[0;36mcalc_sent_loss\u001b[0;34m(sent)\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mhist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhist\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnext_word\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m     \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_histories\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_targets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/nlp/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1108\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1109\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1110\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1111\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1112\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_14352/2217491764.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0membs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# [batch_size x num_hist x emb_size]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0membs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# [batch_size x (num_hist*emb_size)]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# batch_size x hid_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# batch_size x num_words\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#开始训练\n",
    "for ITER in range (10):#10轮次\n",
    "    random.shuffle(train_data)#训练\n",
    "\n",
    "    model.train()#模型训练\n",
    "    train_words, train_loss = 0, 0.0#训练单词总数，训练损失\n",
    "    for sent_id, sent in enumerate(train_data):#更改为所有的train_data\n",
    "        \n",
    "        my_loss = calc_sent_loss(sent)#计算句子损失\n",
    "        \n",
    "        train_loss += my_loss.item()#将得到的句子损失加到训练损失中\n",
    "        train_words += len(sent)#训练单词总数+1\n",
    "\n",
    "        optimizer.zero_grad()#把梯度置零\n",
    "        my_loss.backward()#反向传播，计算分数\n",
    "        optimizer.step()#更新模型参数\n",
    "        #这三个函数的作用是将梯度归零，然后反向传播计算得到每个参数的梯度值，最后通过梯度下降执行一步参数更新。\n",
    "        if (sent_id+1) % 5000 == 0:\n",
    "            print(\"--finished %r sentences\" % (sent_id+1))\n",
    "    print(\"iter %r: train loss/word=%.4f, ppl=%.4f\" % (ITER, train_loss/train_words, math.exp(train_loss/train_words)))\n",
    "\n",
    "    #评价\n",
    "    model.eval()#切换评估模式\n",
    "    dev_words, dev_loss = 0, 0.0#评估单词总数，评估损失\n",
    "    start = time.time()#记录开始时间\n",
    "    for sent_id, sent in enumerate(val_data):\n",
    "        my_loss = calc_sent_loss(sent)#记录句子损失\n",
    "        dev_loss += my_loss.item()#将得到的句子损失加到评估损失中\n",
    "        dev_words += len(sent)#评估单词总数+1\n",
    "    print(\"iter %r: dev loss/word=%.4f, ppl=%.4f, time=%.2fs\" % (ITER, dev_loss/dev_words, math.exp(dev_loss/dev_words), time.time()-start))\n",
    "\n",
    "    #生成几个句子\n",
    "    for _ in range(5):\n",
    "        sent = generate_sent()#生成句子\n",
    "        print(\" \".join([index_to_word[x] for x in sent]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "154abf72fb8cc0db1aa0e7366557ff891bff86d6d75b7e5f2e68a066d591bfd7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}