STS-B.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "793b5b43",
   "metadata": {},
   "source": [
    "# 导包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "488dfbc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 15:15:19 - INFO - <module> - 54 : Loading faiss with AVX2 support.\n",
      "2021-10-19 15:15:19 - INFO - <module> - 58 : Could not load library with AVX2 support due to:\n",
      "ModuleNotFoundError(\"No module named 'faiss.swigfaiss_avx2'\")\n",
      "2021-10-19 15:15:19 - INFO - <module> - 64 : Loading faiss.\n",
      "2021-10-19 15:15:19 - INFO - <module> - 66 : Successfully loaded faiss.\n",
      "2021-10-19 15:15:20 - INFO - from_pretrained - 125 : loading vocabulary file /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/vocab.txt\n"
     ]
    }
   ],
   "source": [
    "import nlp_basictasks\n",
    "import os,json\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import random\n",
    "from tqdm.autonotebook import tqdm, trange\n",
    "from torch.utils.data import DataLoader\n",
    "from nlp_basictasks.modules import SBERT\n",
    "from nlp_basictasks.modules.transformers import BertTokenizer,BertModel,BertConfig\n",
    "from nlp_basictasks.readers.sts import InputExample,convert_examples_to_features,getExamples,convert_sentences_to_features\n",
    "from nlp_basictasks.modules.utils import get_optimizer,get_scheduler\n",
    "from nlp_basictasks.Trainer import Trainer\n",
    "from nlp_basictasks.evaluation import stsEvaluator\n",
    "from sentence_transformers import SentenceTransformer,models\n",
    "model_path1='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distill-simcse/'\n",
    "# model_path2=\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distiluse-base-multilingual-cased-v1/\"\n",
    "model_path3='/data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/'\n",
    "# data_folder='/data/nfs14/nfs/aisearch/asr/xhsun/datasets/lcqmc/'\n",
    "# train_file=os.path.join(data_folder,'lcqmc_train.tsv')\n",
    "# dev_file=os.path.join(data_folder,'lcqmc_dev.tsv')\n",
    "#tokenizer=BertTokenizer.from_pretrained(os.path.join(model_path1,'0_Transformer'))\n",
    "tokenizer=BertTokenizer.from_pretrained(model_path3)\n",
    "max_seq_len=64\n",
    "batch_size=128"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f151cd7",
   "metadata": {},
   "source": [
    "# 获取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7525086a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-train.txt'\n",
    "dev_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-dev.txt'\n",
    "test_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-test.txt'\n",
    "def read_data(file_path):\n",
    "    sentences=[]\n",
    "    labels=[]\n",
    "    with open(file_path) as f:\n",
    "        lines=f.readlines()\n",
    "    for line in lines:\n",
    "        line_split=line.strip().split('||')\n",
    "        sentences.append([line_split[1],line_split[2]])\n",
    "        labels.append(line_split[3])\n",
    "    return sentences,labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c8ae5fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sentences,train_labels=read_data(train_file)\n",
    "dev_sentences,dev_labels=read_data(dev_file)\n",
    "test_sentences,test_labels=read_data(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7628163f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['一架飞机要起飞了。', '一架飞机正在起飞。'], ['一个男人在吹一支大笛子。', '一个人在吹长笛。']] ['5', '3']\n",
      "[['一个戴着安全帽的男人在跳舞。', '一个戴着安全帽的男人在跳舞。'], ['一个小孩在骑马。', '孩子在骑马。']] ['5', '4']\n",
      "[['一个女孩在给她的头发做发型。', '一个女孩在梳头。'], ['一群男人在海滩上踢足球。', '一群男孩在海滩上踢足球。']] ['2', '3']\n"
     ]
    }
   ],
   "source": [
    "print(train_sentences[:2],train_labels[:2])\n",
    "print(dev_sentences[:2],dev_labels[:2])\n",
    "print(test_sentences[:2],test_labels[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "94e1756d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['一个女孩在给她的头发做发型。', '一个女孩在梳头。']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sentences[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cee6dea",
   "metadata": {},
   "source": [
    "## create unsupervised train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7aba043f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5231\n",
      "['一架飞机要起飞了。', '一个男人在吹一支大笛子。', '一个人正把切碎的奶酪撒在比萨饼上。']\n",
      "<InputExample> label: 1, text pairs : 一架飞机要起飞了。; 一架飞机要起飞了。\n"
     ]
    }
   ],
   "source": [
    "train_sentences=[sentence[0] for sentence in train_sentences]#只取一般数据作为训练集\n",
    "print(len(train_sentences))\n",
    "print(train_sentences[:3])\n",
    "train_examples=[InputExample(text_list=[sentence,sentence],label=1) for sentence in train_sentences]\n",
    "train_dataloader=DataLoader(train_examples,shuffle=True,batch_size=batch_size)\n",
    "def smart_batching_collate(batch):\n",
    "    features_of_a,features_of_b,labels=convert_examples_to_features(examples=batch,tokenizer=tokenizer,max_seq_len=max_seq_len)\n",
    "    return features_of_a,features_of_b,labels\n",
    "train_dataloader.collate_fn=smart_batching_collate\n",
    "print(train_examples[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dde833d",
   "metadata": {},
   "source": [
    "## create supervised train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4d8011b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(train_sentences))\n",
    "print(train_sentences[:3])\n",
    "#train_examples=[InputExample(text_list=sentence,label=1) for sentence in train_sentences]\n",
    "train_dataloader=DataLoader(train_examples,shuffle=True,batch_size=batch_size)\n",
    "def smart_batching_collate(batch):\n",
    "    features_of_a,features_of_b,labels=convert_examples_to_features(examples=batch,tokenizer=tokenizer,max_seq_len=max_seq_len)\n",
    "    return features_of_a,features_of_b,labels\n",
    "train_dataloader.collate_fn=smart_batching_collate\n",
    "print(train_examples[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9826c977",
   "metadata": {},
   "source": [
    "# SimCSE模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "66876608",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimCSE(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert_model_path,\n",
    "                 is_sbert_model=True,\n",
    "                temperature=0.05,\n",
    "                is_distilbert=False,\n",
    "                device='cpu'):\n",
    "        super(SimCSE,self).__init__()\n",
    "        if is_sbert_model:\n",
    "            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)\n",
    "        else:\n",
    "            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)\n",
    "            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())\n",
    "            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)\n",
    "        self.temperature=temperature\n",
    "        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids\n",
    "    def cal_cos_sim(self,embeddings1,embeddings2):\n",
    "        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)\n",
    "        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)\n",
    "        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)\n",
    "        \n",
    "    def forward(self,batch_inputs):\n",
    "        '''\n",
    "        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None\n",
    "        get token_embeddings,cls_token_embeddings,sentence_embeddings\n",
    "        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略\n",
    "        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768\n",
    "        '''\n",
    "        batch1_features,batch2_features,_=batch_inputs\n",
    "        if self.is_distilbert:\n",
    "            del batch1_features['token_type_ids']\n",
    "            del batch2_features['token_type_ids']\n",
    "        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']\n",
    "        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']\n",
    "        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)\n",
    "        batch_size=cos_sim.size(0)\n",
    "        assert cos_sim.size()==(batch_size,batch_size)\n",
    "        labels=torch.arange(batch_size).to(cos_sim.device)\n",
    "        return nn.CrossEntropyLoss()(cos_sim,labels)\n",
    "    \n",
    "    def encode(self, sentences,\n",
    "               batch_size: int = 32,\n",
    "               show_progress_bar: bool = None,\n",
    "               output_value: str = 'sentence_embedding',\n",
    "               convert_to_numpy: bool = True,\n",
    "               convert_to_tensor: bool = False,\n",
    "               device: str = None,\n",
    "               normalize_embeddings: bool = False):\n",
    "        '''\n",
    "        传进来的sentences只能是single_batch\n",
    "        '''\n",
    "        return self.encoder.encode(sentences=sentences,\n",
    "                                         batch_size=batch_size,\n",
    "                                         show_progress_bar=show_progress_bar,\n",
    "                                         output_value=output_value,\n",
    "                                         convert_to_numpy=convert_to_numpy,\n",
    "                                         convert_to_tensor=convert_to_tensor,\n",
    "                                         device=device,\n",
    "                                         normalize_embeddings=normalize_embeddings)\n",
    "    \n",
    "    def save(self,output_path):\n",
    "        os.makedirs(output_path,exist_ok=True)\n",
    "        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:\n",
    "            json.dump(self.get_config_dict(output_path), fOut)\n",
    "        self.encoder.save(output_path)\n",
    "        \n",
    "    def get_config_dict(self,output_path):\n",
    "        '''\n",
    "        一定要有dict，这样才能初始化Model\n",
    "        '''\n",
    "        return {'output_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert}\n",
    "    @staticmethod\n",
    "    def load(input_path):\n",
    "        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:\n",
    "            config = json.load(fIn)\n",
    "        return SimCSE(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ccda487f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-19 08:23:00 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\n",
      "2021-10-19 08:23:00 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\n"
     ]
    }
   ],
   "source": [
    "device='cpu'\n",
    "#simcse=SimCSE(bert_model_path=model_path3,is_distilbert=False,device=device,is_sbert_model=False)\n",
    "simcse=SimCSE(bert_model_path=\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/\",is_distilbert=False,device=device,is_sbert_model=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1145a69e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "255f23da",
   "metadata": {},
   "source": [
    "# 构造evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6d953b47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['一个戴着安全帽的男人在跳舞。', '一个戴着安全帽的男人在跳舞。'] 5\n"
     ]
    }
   ],
   "source": [
    "#dev_sentences=[example.text_list for example in dev_examples]\n",
    "#dev_labels=[example.label for example in dev_examples]\n",
    "print(dev_sentences[0],dev_labels[0])\n",
    "sentences1_list=[sen[0] for sen in dev_sentences]\n",
    "sentences2_list=[sen[1] for sen in dev_sentences]\n",
    "dev_labels=[int(score) for score in dev_labels]\n",
    "evaluator=stsEvaluator(sentences1=sentences1_list,sentences2=sentences2_list,batch_size=64,write_csv=True,scores=dev_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "5612d631",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-18 19:53:18 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c454eefa261e45e29042899878e1de77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c4ce7150f214839a2e9fc6de14a5f3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-18 19:53:32 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7585\tSpearman: 0.7632\n",
      "2021-10-18 19:53:32 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7228\tSpearman: 0.7396\n",
      "2021-10-18 19:53:32 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7195\tSpearman: 0.7360\n",
      "2021-10-18 19:53:32 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7377\tSpearman: 0.7477\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7632353820991463"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator(simcse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "dfb28a86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'一个戴着安全帽的男人在跳舞。'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences1_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "28671317",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'一个戴着安全帽的男人在跳舞。'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences2_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "2bf39873",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "654ecae8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52a8ae932bf44356af3876fbd7ae6685",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f12aee4fadea4cbd847896233f7d9cb3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "3c9e7c14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1458])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)\n",
    "(sentences1_embeddings-sentences2_embeddings).norm(dim=1,or).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "cfa24518",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.7389)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)\n",
    "(sentences1_embeddings-sentences2_embeddings).norm(dim=1,or).size()\n",
    "s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "(s1+s2)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc53a664",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b6f8b86",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "cb15d4ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence=\"数据转换方式决定了最终学习的向量表示的不变性\"\n",
    "v1=simcse.encode(sentence,normalize_embeddings=True,show_progress_bar=False)\n",
    "v2=simcse.encode(sentence[::-1],normalize_embeddings=True,show_progress_bar=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ddccda67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7955214"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(v1*v2).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb67a405",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9dc2a6e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "32f65eb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'性变不的示表量向的习学终最了定决式方换转据数'"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed6aecaa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a82d4014",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/ were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "bert_encoder=SimCSE(model_path3,is_sbert_model=False,is_distilbert=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdb65d7b",
   "metadata": {},
   "source": [
    "# 展示contrastive loss的计算过程"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d6aa0b2",
   "metadata": {},
   "source": [
    "## 首先将每一个句子编码为向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "649df7e1",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "too many values to unpack (expected 3)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-57cde251768b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0ms2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"今天组会的内容是对比学习在文本相似度计算任务的应用\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0ms2_pie\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0ms2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mz1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbert_encoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mz1_pie\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbert_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms1_pie\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mz2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbert_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1049\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1050\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1052\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1053\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-21-d03eb3bff30a>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, batch_inputs)\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0m一般来讲\u001b[0m\u001b[0;31m，\u001b[0m\u001b[0m只会取一种pooling策略\u001b[0m\u001b[0;31m，\u001b[0m\u001b[0m要么直接cls要么mean\u001b[0m \u001b[0mlast\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mmean\u001b[0m \u001b[0mlast2\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mmean\u001b[0m \u001b[0mfirst\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mlast\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;31m，\u001b[0m\u001b[0m所以sentence_embeddings的维度也是768\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m         '''\n\u001b[0;32m---> 29\u001b[0;31m         \u001b[0mbatch1_features\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch2_features\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_distilbert\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0mbatch1_features\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'token_type_ids'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: too many values to unpack (expected 3)"
     ]
    }
   ],
   "source": [
    "s1=\"这几行代码为了展示对比损失函数的计算\"\n",
    "s1_pie=s1\n",
    "s2=\"今天组会的内容是对比学习在文本相似度计算任务的应用\"\n",
    "s2_pie=s2\n",
    "z1=bert_encoder.encode(s1)\n",
    "z1_pie=bert_encoder.encode(s1_pie)\n",
    "z2=bert_encoder.encode(s2)\n",
    "z2_pie=bert_encoder.encode(s2_pie)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2509696",
   "metadata": {},
   "source": [
    "## normalize每一个向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4c9841ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "v1=z1/np.linalg.norm(z1,ord=2)\n",
    "v1_pie=z1_pie/np.linalg.norm(z1_pie,ord=2)\n",
    "v2=z2/np.linalg.norm(z2,ord=2)\n",
    "v2_pie=z2_pie/np.linalg.norm(z2_pie,ord=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ffde6315",
   "metadata": {},
   "source": [
    "## 得到logits和labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "73c4748e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.0000, 0.8564],\n",
      "        [0.8564, 1.0000]])\n",
      "tensor([[1, 0],\n",
      "        [0, 1]])\n"
     ]
    }
   ],
   "source": [
    "logits=torch.tensor([[np.sum(v1*v1_pie),np.sum(v1*v2_pie)],\n",
    "                     [np.sum(v2*v1_pie),np.sum(v2*v2_pie)]])\n",
    "labels=torch.LongTensor([[1,0],[0,1]])\n",
    "print(logits)\n",
    "print(labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ab20062",
   "metadata": {},
   "source": [
    "## 对logits进行softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "c6926139",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.6537, -0.7342],\n",
       "        [-0.7410, -0.6475]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_probs=torch.nn.functional.log_softmax(logits,dim=1)\n",
    "log_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d36c0537",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.6506)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contrastive_loss=-(log_probs[0][0] + log_probs[1][1])/2\n",
    "contrastive_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3330f48",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8773e4ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "877e3681",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.5488)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contrastive_loss=-torch.sum(torch.nn.functional.log_softmax(logits,dim=1)*labels)/2\n",
    "contrastive_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "04bfca41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "3+-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3b620bb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr,spearmanr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fff433de",
   "metadata": {},
   "outputs": [],
   "source": [
    "X=[0.9,0.7,0.3,0.8,0.6]\n",
    "Y=[4,5,1,3,2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb1fb431",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spearman rank correlation coefficient:  0.7\n"
     ]
    }
   ],
   "source": [
    "rank_X=[5,3,1,4,2]\n",
    "d_i=[-1,2,0,1,0]\n",
    "d_i2=[1,4,0,1,0]\n",
    "print(\"spearman rank correlation coefficient: \",1-(6*sum(d_i2))/(5*(5**2-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "88f3ce8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spearman rank correlation coefficient:  0.7\n"
     ]
    }
   ],
   "source": [
    "print(\"spearman rank correlation coefficient: \",spearmanr(a=rank_X,b=Y)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dda56768",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pearson correlation coefficient:  0.7554831017177897\n"
     ]
    }
   ],
   "source": [
    "print(\"pearson correlation coefficient: \",pearsonr(X,Y)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89ab8ba6",
   "metadata": {},
   "source": [
    "# ConSERT模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c223d841",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConSERT(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert_model_path,\n",
    "                 is_sbert_model=True,\n",
    "                temperature=0.05,\n",
    "                is_distilbert=False,\n",
    "                 cutoff_rate=0.15,\n",
    "                device='cpu',\n",
    "                close_dropout=True):\n",
    "        super(ConSERT,self).__init__()\n",
    "        if is_sbert_model:\n",
    "            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)\n",
    "        else:\n",
    "            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)\n",
    "            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())\n",
    "            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)\n",
    "        self.temperature=temperature\n",
    "        self.cutoff_rate=cutoff_rate\n",
    "        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids\n",
    "        self.close_dropout=close_dropout\n",
    "        \n",
    "    def cal_cos_sim(self,embeddings1,embeddings2):\n",
    "        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)\n",
    "        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)\n",
    "        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)\n",
    "\n",
    "    def shuffle_and_cutoff(self,sentence_feature):\n",
    "        input_ids, attention_mask=sentence_feature['input_ids'],sentence_feature['attention_mask']\n",
    "        bsz, seq_len = input_ids.shape\n",
    "        shuffled_input_ids=[]\n",
    "        cutoff_attention_mask=[]\n",
    "        for bsz_id in range(bsz):\n",
    "            sample_mask = attention_mask[bsz_id]\n",
    "            num_tokens = sample_mask.sum().int().item()\n",
    "            cur_input_ids=input_ids[bsz_id]\n",
    "            if 102 not in cur_input_ids:\n",
    "                indexes = list(range(num_tokens))[1:]\n",
    "                random.shuffle(indexes)\n",
    "                indexes=[0]+indexes#保证第一个位置是0\n",
    "            else:\n",
    "                indexes = list(range(num_tokens))[1:-1]\n",
    "                random.shuffle(indexes)\n",
    "                indexes=[0]+indexes+[num_tokens-1]#保证第一个位置是0，最后一个位置是SEP不变\n",
    "            rest_indexes = list(range(num_tokens, seq_len))\n",
    "            total_indexes = indexes + rest_indexes\n",
    "            shuffled_input_id=input_ids[bsz_id][total_indexes]\n",
    "            #print(shuffled_input_id,indexes)\n",
    "            if self.cutoff_rate>0.0:\n",
    "                sample_len=max(int(num_tokens*(1-self.cutoff_rate)),1)#if true_len is 32, cutoff_rate is 0.15 then sample_len is 27\n",
    "                start_id = np.random.randint(1, high=num_tokens-sample_len+1)# start_id random select from (0,6)，避免删除CLS\n",
    "                cutoff_mask=[1]*seq_len\n",
    "                for idx in range(start_id, start_id+sample_len):\n",
    "                    cutoff_mask[idx]=0#这些位置是0，bool之后就变成了False，而masked_fill是选择True的位置替换为value的\n",
    "                cutoff_mask[0]=0#避免CLS被替换\n",
    "                cutoff_mask[num_tokens-1]=0#避免SEP被替换\n",
    "                cutoff_mask=torch.ByteTensor(cutoff_mask).bool().to(input_ids.device)\n",
    "                shuffled_input_id=shuffled_input_id.masked_fill(cutoff_mask,value=0).to(input_ids.device)\n",
    "                sample_mask=sample_mask.masked_fill(cutoff_mask,value=0).to(input_ids.device)\n",
    "\n",
    "            shuffled_input_ids.append(shuffled_input_id)\n",
    "            cutoff_attention_mask.append(sample_mask)\n",
    "        shuffled_input_ids=torch.vstack(shuffled_input_ids)\n",
    "        cutoff_attention_mask=torch.vstack(cutoff_attention_mask)\n",
    "        return shuffled_input_ids,cutoff_attention_mask\n",
    "        \n",
    "    def forward(self,batch_inputs):\n",
    "        '''\n",
    "        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None\n",
    "        get token_embeddings,cls_token_embeddings,sentence_embeddings\n",
    "        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略\n",
    "        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768\n",
    "        '''\n",
    "        batch1_features,batch2_features,_=batch_inputs\n",
    "        if self.is_distilbert:\n",
    "            del batch1_features['token_type_ids']\n",
    "            del batch2_features['token_type_ids']\n",
    "        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']\n",
    "        shuffled_input_ids,cutoff_attention_mask=self.shuffle_and_cutoff(sentence_feature=batch1_features)\n",
    "        #new_features{'input_ids'}=shuffled_input_ids\n",
    "        batch2_features['input_ids']=shuffled_input_ids\n",
    "        batch2_features['attention_mask']=cutoff_attention_mask\n",
    "        orig_attention_probs_dropout_prob=self.encoder[0].auto_model.encoder.config.attention_probs_dropout_prob\n",
    "        orig_hidden_dropout_prob=self.encoder[0].auto_model.encoder.config.hidden_dropout_prob\n",
    "        if self.close_dropout:\n",
    "            self.encoder[0].auto_model.encoder.config.attention_probs_dropout_prob=0.0\n",
    "            self.encoder[0].auto_model.encoder.config.hidden_dropout_prob=0.0\n",
    "        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']\n",
    "        if self.close_dropout:\n",
    "            self.encoder[0].auto_model.encoder.config.attention_probs_dropout_prob=orig_attention_probs_dropout_prob\n",
    "            self.encoder[0].auto_model.encoder.config.hidden_dropout_prob=orig_hidden_dropout_prob\n",
    "            \n",
    "        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)\n",
    "        batch_size=cos_sim.size(0)\n",
    "        assert cos_sim.size()==(batch_size,batch_size)\n",
    "        labels=torch.arange(batch_size).to(cos_sim.device)\n",
    "        return nn.CrossEntropyLoss()(cos_sim,labels)\n",
    "    \n",
    "    def encode(self, sentences,\n",
    "               batch_size: int = 32,\n",
    "               show_progress_bar: bool = None,\n",
    "               output_value: str = 'sentence_embedding',\n",
    "               convert_to_numpy: bool = True,\n",
    "               convert_to_tensor: bool = False,\n",
    "               device: str = None,\n",
    "               normalize_embeddings: bool = False):\n",
    "        '''\n",
    "        传进来的sentences只能是single_batch\n",
    "        '''\n",
    "        return self.encoder.encode(sentences=sentences,\n",
    "                                         batch_size=batch_size,\n",
    "                                         show_progress_bar=show_progress_bar,\n",
    "                                         output_value=output_value,\n",
    "                                         convert_to_numpy=convert_to_numpy,\n",
    "                                         convert_to_tensor=convert_to_tensor,\n",
    "                                         device=device,\n",
    "                                         normalize_embeddings=normalize_embeddings)\n",
    "    \n",
    "    def save(self,output_path):\n",
    "        os.makedirs(output_path,exist_ok=True)\n",
    "        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:\n",
    "            json.dump(self.get_config_dict(output_path), fOut)\n",
    "        self.encoder.save(output_path)\n",
    "        \n",
    "    def get_config_dict(self,output_path):\n",
    "        '''\n",
    "        一定要有dict，这样才能初始化Model\n",
    "        '''\n",
    "        return {'bert_model_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert,\n",
    "               'close_dropout':self.close_dropout,'cutoff_rate':self.cutoff_rate}\n",
    "    @staticmethod\n",
    "    def load(input_path):\n",
    "        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:\n",
    "            config = json.load(fIn)\n",
    "            if 'bert_model_path' not in config:\n",
    "                config['bert_model_path']=config['output_path']\n",
    "                del config['output_path']\n",
    "        return ConSERT(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "562a2143",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:38:36 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unConSERT_STS-B_closedropout/\n",
      "2021-10-13 16:38:36 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unConSERT_STS-B_closedropout/\n"
     ]
    }
   ],
   "source": [
    "consert_closedropout=ConSERT.load(\"/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unConSERT_STS-B_closedropout/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "95146ce5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:40 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b29f19d901f04b0b9fc92ea9a3b26ae6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e44f249900947538094cd01583f735c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:53 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7792\tSpearman: 0.7808\n",
      "2021-10-13 16:39:53 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7560\tSpearman: 0.7741\n",
      "2021-10-13 16:39:53 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7564\tSpearman: 0.7744\n",
      "2021-10-13 16:39:53 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7515\tSpearman: 0.7558\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7807855257752795"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator(consert_closedropout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c871b756",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/ were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "consert_donotclosedropout=ConSERT(bert_model_path=model_path3,is_distilbert=False,is_sbert_model=False,close_dropout=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "92bd9150",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:40:02 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "deaf63cd78f9489eb8e6f6e823e8a376",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc2aa446176a4e95ab7230960f1fa010",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:40:15 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6641\tSpearman: 0.6808\n",
      "2021-10-13 16:40:15 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6740\tSpearman: 0.6882\n",
      "2021-10-13 16:40:15 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6680\tSpearman: 0.6824\n",
      "2021-10-13 16:40:15 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.4705\tSpearman: 0.4646\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6807509857943427"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator(consert_donotclosedropout)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72f43c7d",
   "metadata": {},
   "source": [
    "# EsimCSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "d50f4c37",
   "metadata": {},
   "outputs": [],
   "source": [
    "from queue import Queue\n",
    "class ESimCSE(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert_model_path,\n",
    "                 q_size=256,\n",
    "                 dup_rate=0.32,\n",
    "                 is_sbert_model=True,\n",
    "                temperature=0.05,\n",
    "                is_distilbert=False,\n",
    "                 gamma=0.99,\n",
    "                device='cpu'):\n",
    "        super(ESimCSE,self).__init__()\n",
    "        if is_sbert_model:\n",
    "            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)\n",
    "            self.moco_encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)\n",
    "        else:\n",
    "            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)\n",
    "            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())\n",
    "            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)\n",
    "            self.moco_encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)\n",
    "        self.gamma=gamma\n",
    "        self.q=[]\n",
    "        self.q_size=q_size\n",
    "        self.dup_rate=dup_rate\n",
    "        self.temperature=temperature\n",
    "        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids\n",
    "    def cal_cos_sim(self,embeddings1,embeddings2):\n",
    "        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)\n",
    "        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)\n",
    "        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)\n",
    "\n",
    "    def word_repetition(self,sentence_feature):\n",
    "        input_ids, attention_mask, token_type_ids=sentence_feature['input_ids'].cpu().tolist(),sentence_feature['attention_mask'].cpu().tolist(),sentence_feature['token_type_ids'].cpu().tolist()\n",
    "        bsz, seq_len = len(input_ids),len(input_ids[0])\n",
    "        #print(bsz,seq_len)\n",
    "        repetitied_input_ids=[]\n",
    "        repetitied_attention_mask=[]\n",
    "        repetitied_token_type_ids=[]\n",
    "        rep_seq_len=seq_len\n",
    "        for bsz_id in range(bsz):\n",
    "            sample_mask = attention_mask[bsz_id]\n",
    "            actual_len = sum(sample_mask)\n",
    "\n",
    "            cur_input_id=input_ids[bsz_id]\n",
    "            dup_len=random.randint(a=0,b=max(2,int(self.dup_rate*actual_len)))\n",
    "            dup_word_index=random.sample(list(range(1,actual_len)),k=dup_len)\n",
    "            \n",
    "            r_input_id=[]\n",
    "            r_attention_mask=[]\n",
    "            r_token_type_ids=[]\n",
    "            for index,word_id in enumerate(cur_input_id):\n",
    "                if index in dup_word_index:\n",
    "                    r_input_id.append(word_id)\n",
    "                    r_attention_mask.append(sample_mask[index])\n",
    "                    r_token_type_ids.append(token_type_ids[bsz_id][index])\n",
    "\n",
    "                r_input_id.append(word_id)\n",
    "                r_attention_mask.append(sample_mask[index])\n",
    "                r_token_type_ids.append(token_type_ids[bsz_id][index])\n",
    "\n",
    "            after_dup_len=len(r_input_id)\n",
    "            #assert after_dup_len==actual_len+dup_len\n",
    "            repetitied_input_ids.append(r_input_id)#+rest_input_ids)\n",
    "            repetitied_attention_mask.append(r_attention_mask)#+rest_attention_mask)\n",
    "            repetitied_token_type_ids.append(r_token_type_ids)#+rest_token_type_ids)\n",
    "\n",
    "            assert after_dup_len==dup_len+seq_len\n",
    "            if after_dup_len>rep_seq_len:\n",
    "                rep_seq_len=after_dup_len\n",
    "\n",
    "        for i in range(bsz):\n",
    "            after_dup_len=len(repetitied_input_ids[i])\n",
    "            pad_len=rep_seq_len-after_dup_len\n",
    "            repetitied_input_ids[i]+=[0]*pad_len\n",
    "            repetitied_attention_mask[i]+=[0]*pad_len\n",
    "            repetitied_token_type_ids[i]+=[0]*pad_len\n",
    "\n",
    "        repetitied_input_ids=torch.LongTensor(repetitied_input_ids)\n",
    "        repetitied_attention_mask=torch.LongTensor(repetitied_attention_mask)\n",
    "        repetitied_token_type_ids=torch.LongTensor(repetitied_token_type_ids)\n",
    "        return {\"input_ids\":repetitied_input_ids,'attention_mask':repetitied_attention_mask,'token_type_ids':repetitied_token_type_ids}\n",
    "\n",
    "    def forward(self,batch_inputs):\n",
    "        '''\n",
    "        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None\n",
    "        get token_embeddings,cls_token_embeddings,sentence_embeddings\n",
    "        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略\n",
    "        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768\n",
    "        '''\n",
    "        batch1_features,batch2_features,_=batch_inputs\n",
    "        if self.is_distilbert:\n",
    "            del batch1_features['token_type_ids']\n",
    "            del batch2_features['token_type_ids']\n",
    "        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']\n",
    "        batch2_features=self.word_repetition(sentence_feature=batch2_features)\n",
    "        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']\n",
    "        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)\n",
    "        batch_size=cos_sim.size(0)\n",
    "        assert cos_sim.size()==(batch_size,batch_size)\n",
    "        labels=torch.arange(batch_size).to(cos_sim.device)\n",
    "        negative_samples=None\n",
    "        if len(self.q)>0:\n",
    "            negative_samples=torch.vstack(self.q[:self.q_size])#(q_size,768)\n",
    "        if len(self.q)+batch_size>=self.q_size:\n",
    "            del self.q[:batch_size]\n",
    "            \n",
    "        with torch.no_grad():\n",
    "            self.moco_encoder[0].auto_model.encoder.config.attention_probs_dropout_prob=0.0\n",
    "            self.moco_encoder[0].auto_model.encoder.config.hidden_dropout_prob=0.0\n",
    "            self.q.extend(self.moco_encoder(batch1_features)['sentence_embedding'])\n",
    "            \n",
    "        if negative_samples is not None:\n",
    "            batch_size+=negative_samples.size(0)#(N+M)\n",
    "            cos_sim_with_neg=self.cal_cos_sim(batch1_embeddings,negative_samples)/self.temperature#(N,M) not (N,N) N is bsz\n",
    "            cos_sim=torch.cat([cos_sim,cos_sim_with_neg],dim=1)#(N,N+M)\n",
    "            #labels=\n",
    "        for encoder_param,moco_encoder_param in zip(self.encoder.parameters(),self.moco_encoder.parameters()):\n",
    "            moco_encoder_param.data=self.gamma*moco_encoder_param.data+(1.-self.gamma)*encoder_param.data\n",
    "            \n",
    "        return nn.CrossEntropyLoss()(cos_sim,labels)\n",
    "    \n",
    "    def encode(self, sentences,\n",
    "               batch_size: int = 32,\n",
    "               show_progress_bar: bool = None,\n",
    "               output_value: str = 'sentence_embedding',\n",
    "               convert_to_numpy: bool = True,\n",
    "               convert_to_tensor: bool = False,\n",
    "               device: str = None,\n",
    "               normalize_embeddings: bool = False):\n",
    "        '''\n",
    "        传进来的sentences只能是single_batch\n",
    "        '''\n",
    "        return self.encoder.encode(sentences=sentences,\n",
    "                                         batch_size=batch_size,\n",
    "                                         show_progress_bar=show_progress_bar,\n",
    "                                         output_value=output_value,\n",
    "                                         convert_to_numpy=convert_to_numpy,\n",
    "                                         convert_to_tensor=convert_to_tensor,\n",
    "                                         device=device,\n",
    "                                         normalize_embeddings=normalize_embeddings)\n",
    "    \n",
    "    def save(self,output_path):\n",
    "        os.makedirs(output_path,exist_ok=True)\n",
    "        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:\n",
    "            json.dump(self.get_config_dict(output_path), fOut)\n",
    "        self.encoder.save(output_path)\n",
    "        \n",
    "    def get_config_dict(self,output_path):\n",
    "        '''\n",
    "        一定要有dict，这样才能初始化Model\n",
    "        '''\n",
    "        return {'bert_model_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert,\n",
    "                'q_size':self.q_size,'dup_rate':self.dup_rate,'gamma':self.gamma}\n",
    "    @staticmethod\n",
    "    def load(input_path):\n",
    "        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:\n",
    "            config = json.load(fIn)\n",
    "        return ESimCSE(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "fba8f113",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-18 20:04:03 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-18 20:04:03 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-18 20:04:06 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-18 20:04:06 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    }
   ],
   "source": [
    "device='cpu'\n",
    "esimcse=ESimCSE(bert_model_path='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE',\n",
    "                is_distilbert=False,\n",
    "                is_sbert_model=True,\n",
    "                dup_rate=0.32,gamma=0.99,\n",
    "                device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a69b19c4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "12f3fafd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-18 20:04:10 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "726473a79dca4c56a700ad38f45d7da1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5184ffb332134f8387bb9c4021c027b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-18 20:04:24 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7900\tSpearman: 0.7929\n",
      "2021-10-18 20:04:24 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7555\tSpearman: 0.7709\n",
      "2021-10-18 20:04:24 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7549\tSpearman: 0.7702\n",
      "2021-10-18 20:04:24 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7507\tSpearman: 0.7531\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7929133672005957"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator(esimcse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "e3fb6740",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4050000f29b44e46b8e07189cc4e0574",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=43.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3c61d86e88c4f878ca26243fa57b34c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=43.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tensor(183.4928)\n",
      "tensor(-8.0145)\n",
      "tensor(-9.4073)\n",
      "tensor(-8.7109)\n",
      "tensor(0.0132)\n"
     ]
    }
   ],
   "source": [
    "sentences1_list=[sen[0] for sen in test_sentences]\n",
    "sentences2_list=[sen[1] for sen in test_sentences]\n",
    "sentences1_embeddings=esimcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=esimcse.encode(sentences2_list,convert_to_tensor=True)\n",
    "print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mean())\n",
    "s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "print(s1)\n",
    "s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "print(s2)\n",
    "print((s1+s2)/2)\n",
    "print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mul(-2).exp().mean())#.log())\n",
    "mengbi_esimcse=(sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "0a582ee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def uniform_loss(x, t=2):\n",
    "    return torch.pdist(x, p=2).pow(2).mul(-t).exp().mean().log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "cecda5af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-8.0145)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(sentences1_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "2469a37d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-9.4073)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniform_loss(sentences2_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "8faa167e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_loss(x, y, alpha=2):\n",
    "    return (x - y).norm(p=2, dim=1).pow(alpha).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "b7e733a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(183.4928)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "align_loss(sentences1_embeddings,sentences2_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eed059a5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "9dde349a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e45550e76ff4ebdb3dac293fd116143",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "040cbfe2e0874af8af5b97cc1158bd49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tensor(194.2286)\n",
      "tensor(-9.8505)\n",
      "tensor(-9.6273)\n",
      "tensor(-9.7389)\n",
      "tensor(-5.3389)\n"
     ]
    }
   ],
   "source": [
    "sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)\n",
    "print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mean())\n",
    "s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "print(s1)\n",
    "s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "print(s2)\n",
    "print((s1+s2)/2)\n",
    "print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mul(-2).exp().mean().log())\n",
    "mengbi_simcse=(sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "cd5a7e94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(7)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum((mengbi_esimcse==mengbi_simcse).bool())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "48662910",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0048)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mengbi_esimcse.mul(-2).exp().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "fcfb01f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0048)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mengbi_simcse.mul(-2).exp().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ae1b181",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "34e1a980",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/ were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "bert_model=SimCSE(model_path3,is_sbert_model=False,is_distilbert=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "d386ea96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcd8e4cbb74c4db49100851f9a7c2611",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9d7d8a927e34377bf4249bc942bafbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=46.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "tensor(58.9933)\n",
      "tensor(-9.8504)\n",
      "tensor(-9.6273)\n",
      "tensor(-9.7389)\n"
     ]
    }
   ],
   "source": [
    "sentences1_embeddings=bert_model.encode(sentences1_list,convert_to_tensor=True)\n",
    "sentences2_embeddings=bert_model.encode(sentences2_list,convert_to_tensor=True)\n",
    "print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mean())\n",
    "s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "print(s1)\n",
    "s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()\n",
    "print(s2)\n",
    "print((s1+s2)/2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb350e40",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8a5f736c",
   "metadata": {},
   "source": [
    "# train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "cb010100",
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs=5\n",
    "output_path='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE'\n",
    "tensorboard_logdir=os.path.join(output_path,'log')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6c0d9a5",
   "metadata": {},
   "source": [
    "## get optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "561741a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_type='AdamW'\n",
    "scheduler='WarmupLinear'\n",
    "warmup_proportion=0.1\n",
    "optimizer_params={'lr': 2e-5}\n",
    "weight_decay=0.01\n",
    "num_train_steps = int(len(train_dataloader) * epochs)\n",
    "warmup_steps = num_train_steps*warmup_proportion\n",
    "optimizer = get_optimizer(model=esimcse,optimizer_type=optimizer_type,weight_decay=weight_decay,optimizer_params=optimizer_params)\n",
    "scheduler = get_scheduler(optimizer, scheduler=scheduler, warmup_steps=warmup_steps, t_total=num_train_steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b025841",
   "metadata": {},
   "source": [
    "## get Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "6ea83e74",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 20:54:28 - INFO - train - 56 : 一个epoch 下，每隔8个step会输出一次loss，每隔20个step会评估一次模型\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "662c8c18845c4276bb65c744ef04618d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 20:54:38 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 0 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39e5983dc9904b3592d69adcca720e7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "277559b0149a4d1f9170a58850962cbc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 20:54:51 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.6641\tSpearman: 0.6808\n",
      "2021-10-13 20:54:51 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.6740\tSpearman: 0.6882\n",
      "2021-10-13 20:54:51 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.6680\tSpearman: 0.6824\n",
      "2021-10-13 20:54:51 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.4705\tSpearman: 0.4646\n",
      "2021-10-13 20:54:51 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 20:54:52 - INFO - train - 98 : In epoch 0, training_step 0, the eval score is 0.6807509857943427, previous eval score is -9999999, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-13 20:56:12 - INFO - train - 75 : Epoch : 0, train_step : 8/205, loss_value : 1.1475277915596962 \n",
      "2021-10-13 20:57:38 - INFO - train - 75 : Epoch : 0, train_step : 16/205, loss_value : 0.3286867868155241 \n",
      "2021-10-13 20:58:28 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 0 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e1622f109044bea9e35553e64aa8328",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a3cca79d47994e9680e0239c97271bc7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 20:58:40 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7551\tSpearman: 0.7585\n",
      "2021-10-13 20:58:40 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7346\tSpearman: 0.7449\n",
      "2021-10-13 20:58:40 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7343\tSpearman: 0.7447\n",
      "2021-10-13 20:58:40 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7145\tSpearman: 0.7155\n",
      "2021-10-13 20:58:40 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 20:58:41 - INFO - train - 98 : In epoch 0, training_step 20, the eval score is 0.7585202474064063, previous eval score is 0.6807509857943427, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-13 20:59:20 - INFO - train - 75 : Epoch : 0, train_step : 24/205, loss_value : 0.055874085519462824 \n",
      "2021-10-13 21:00:42 - INFO - train - 75 : Epoch : 0, train_step : 32/205, loss_value : 0.023807268124073744 \n",
      "2021-10-13 21:02:09 - INFO - train - 75 : Epoch : 0, train_step : 40/205, loss_value : 0.020951081591192633 \n",
      "2021-10-13 21:02:15 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 0 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41a80c117852488c91e6b9d5e4cb4b41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c477f63cd4104823999e63ed6fa9b695",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:02:28 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7679\tSpearman: 0.7693\n",
      "2021-10-13 21:02:28 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7419\tSpearman: 0.7564\n",
      "2021-10-13 21:02:28 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7411\tSpearman: 0.7552\n",
      "2021-10-13 21:02:28 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7385\tSpearman: 0.7403\n",
      "2021-10-13 21:02:28 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:02:29 - INFO - train - 98 : In epoch 0, training_step 40, the eval score is 0.7693038887787182, previous eval score is 0.7585202474064063, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed96b4235ebb4900bc255a835b1627ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:02:39 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 1 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79a647d40ba8477fa808db9090ae56de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "feefd7c556d544d890e73d918d8b0cb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:02:52 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7694\tSpearman: 0.7710\n",
      "2021-10-13 21:02:52 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7425\tSpearman: 0.7570\n",
      "2021-10-13 21:02:52 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7418\tSpearman: 0.7560\n",
      "2021-10-13 21:02:52 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7405\tSpearman: 0.7427\n",
      "2021-10-13 21:02:52 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:02:53 - INFO - train - 98 : In epoch 1, training_step 0, the eval score is 0.771035587975385, previous eval score is 0.7693038887787182, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-13 21:04:09 - INFO - train - 75 : Epoch : 1, train_step : 16/205, loss_value : 0.035574153531342745 \n",
      "2021-10-13 21:05:37 - INFO - train - 75 : Epoch : 1, train_step : 32/205, loss_value : 0.019113536807708442 \n",
      "2021-10-13 21:06:29 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 1 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75a8e2d137b8451ba3b20dc821b44e23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "719469ce59ee4d2d9645b3b2a8aef0b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:06:41 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7860\tSpearman: 0.7889\n",
      "2021-10-13 21:06:41 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7570\tSpearman: 0.7710\n",
      "2021-10-13 21:06:41 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7565\tSpearman: 0.7705\n",
      "2021-10-13 21:06:41 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7384\tSpearman: 0.7409\n",
      "2021-10-13 21:06:41 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:06:42 - INFO - train - 98 : In epoch 1, training_step 40, the eval score is 0.7888571436807816, previous eval score is 0.771035587975385, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-13 21:07:19 - INFO - train - 75 : Epoch : 1, train_step : 48/205, loss_value : 0.017784759518690407 \n",
      "2021-10-13 21:08:46 - INFO - train - 75 : Epoch : 1, train_step : 64/205, loss_value : 0.029313169419765472 \n",
      "2021-10-13 21:10:11 - INFO - train - 75 : Epoch : 1, train_step : 80/205, loss_value : 0.019131976325297728 \n",
      "2021-10-13 21:10:17 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 1 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f152a39f7329410d802709a525d0ad39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a285cc899e949d9a7f075b5dd22389e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:10:30 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7843\tSpearman: 0.7871\n",
      "2021-10-13 21:10:30 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7514\tSpearman: 0.7662\n",
      "2021-10-13 21:10:30 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7506\tSpearman: 0.7653\n",
      "2021-10-13 21:10:30 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7453\tSpearman: 0.7484\n",
      "2021-10-13 21:10:30 - INFO - train - 102 : No improvement over previous best eval score (0.787076 vs 0.788857), patience = 19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a28bbecce47c47798f1c3cd0f583e48d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:10:39 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 2 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60ff26da33a04291905d62906440eaed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "181c535836b24504a40034d43b7450a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:10:52 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7838\tSpearman: 0.7864\n",
      "2021-10-13 21:10:52 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7508\tSpearman: 0.7655\n",
      "2021-10-13 21:10:52 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7499\tSpearman: 0.7646\n",
      "2021-10-13 21:10:52 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7452\tSpearman: 0.7485\n",
      "2021-10-13 21:10:52 - INFO - train - 102 : No improvement over previous best eval score (0.786445 vs 0.788857), patience = 18\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:12:13 - INFO - train - 75 : Epoch : 2, train_step : 24/205, loss_value : 0.0334612459409982 \n",
      "2021-10-13 21:13:36 - INFO - train - 75 : Epoch : 2, train_step : 48/205, loss_value : 0.021292690536938608 \n",
      "2021-10-13 21:14:28 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 2 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94ac9c05cacf4137b1d65c379f990878",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47ef891aad4048ca97a69a180d81637d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:14:41 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7797\tSpearman: 0.7828\n",
      "2021-10-13 21:14:41 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7471\tSpearman: 0.7617\n",
      "2021-10-13 21:14:41 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7460\tSpearman: 0.7604\n",
      "2021-10-13 21:14:41 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7468\tSpearman: 0.7492\n",
      "2021-10-13 21:14:41 - INFO - train - 102 : No improvement over previous best eval score (0.782786 vs 0.788857), patience = 17\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:15:18 - INFO - train - 75 : Epoch : 2, train_step : 72/205, loss_value : 0.027970600873231888 \n",
      "2021-10-13 21:16:42 - INFO - train - 75 : Epoch : 2, train_step : 96/205, loss_value : 0.020671935402788222 \n",
      "2021-10-13 21:18:07 - INFO - train - 75 : Epoch : 2, train_step : 120/205, loss_value : 0.015347252017818391 \n",
      "2021-10-13 21:18:13 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 2 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecd3e3e62a314ed28557db5b18f15b6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0da31f5f4c9d455285e0f8d57a015af5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:18:25 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7886\tSpearman: 0.7917\n",
      "2021-10-13 21:18:25 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7539\tSpearman: 0.7690\n",
      "2021-10-13 21:18:25 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7531\tSpearman: 0.7681\n",
      "2021-10-13 21:18:25 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7532\tSpearman: 0.7554\n",
      "2021-10-13 21:18:25 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:18:26 - INFO - train - 98 : In epoch 2, training_step 120, the eval score is 0.7916804790959123, previous eval score is 0.7888571436807816, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c296cc265ea04bbc8ec3bf397fc9eb1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:18:38 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 3 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01660bbe907b477eb4b4d612d2d5a6c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4539ef4649164ee1ac63d6a4271aafd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:18:51 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7887\tSpearman: 0.7918\n",
      "2021-10-13 21:18:51 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7540\tSpearman: 0.7692\n",
      "2021-10-13 21:18:51 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7532\tSpearman: 0.7683\n",
      "2021-10-13 21:18:51 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7530\tSpearman: 0.7552\n",
      "2021-10-13 21:18:51 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:18:52 - INFO - train - 98 : In epoch 3, training_step 0, the eval score is 0.7917551052847813, previous eval score is 0.7916804790959123, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-13 21:20:11 - INFO - train - 75 : Epoch : 3, train_step : 32/205, loss_value : 0.029419887636322528 \n",
      "2021-10-13 21:21:32 - INFO - train - 75 : Epoch : 3, train_step : 64/205, loss_value : 0.022672737133689225 \n",
      "2021-10-13 21:22:22 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 3 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4164d86a62ae438788e962bcbb59c75a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1d0b845750943fea68bc217a358f54f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:22:34 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7900\tSpearman: 0.7929\n",
      "2021-10-13 21:22:34 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7555\tSpearman: 0.7709\n",
      "2021-10-13 21:22:34 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7549\tSpearman: 0.7702\n",
      "2021-10-13 21:22:34 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7507\tSpearman: 0.7531\n",
      "2021-10-13 21:22:34 - INFO - save - 371 : Save model to /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:22:35 - INFO - train - 98 : In epoch 3, training_step 80, the eval score is 0.7929133672005957, previous eval score is 0.7917551052847813, model has been saved in /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE\n",
      "2021-10-13 21:23:11 - INFO - train - 75 : Epoch : 3, train_step : 96/205, loss_value : 0.01519197560264729 \n",
      "2021-10-13 21:24:40 - INFO - train - 75 : Epoch : 3, train_step : 128/205, loss_value : 0.019646525790449232 \n",
      "2021-10-13 21:26:10 - INFO - train - 75 : Epoch : 3, train_step : 160/205, loss_value : 0.026167668955167755 \n",
      "2021-10-13 21:26:15 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 3 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "516d8eba9f624a2385b4ce6f94b7fd69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e762303acaf64f679142b459e0cdf39c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:26:28 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7885\tSpearman: 0.7917\n",
      "2021-10-13 21:26:28 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7542\tSpearman: 0.7697\n",
      "2021-10-13 21:26:28 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7534\tSpearman: 0.7692\n",
      "2021-10-13 21:26:28 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7501\tSpearman: 0.7528\n",
      "2021-10-13 21:26:28 - INFO - train - 102 : No improvement over previous best eval score (0.791657 vs 0.792913), patience = 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68ff1c3a379b463eb9b71842199212a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:26:38 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 4 after 1 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8ce9e3b93c1444f86c8b89dadffedda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3c00cc406bf43818f2f16cf3936e2ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:26:51 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7885\tSpearman: 0.7915\n",
      "2021-10-13 21:26:51 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7542\tSpearman: 0.7697\n",
      "2021-10-13 21:26:51 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7534\tSpearman: 0.7691\n",
      "2021-10-13 21:26:51 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7502\tSpearman: 0.7529\n",
      "2021-10-13 21:26:51 - INFO - train - 102 : No improvement over previous best eval score (0.791529 vs 0.792913), patience = 15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:28:08 - INFO - train - 75 : Epoch : 4, train_step : 40/205, loss_value : 0.030378026654943824 \n",
      "2021-10-13 21:29:32 - INFO - train - 75 : Epoch : 4, train_step : 80/205, loss_value : 0.017732904729200527 \n",
      "2021-10-13 21:30:22 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 4 after 21 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66cba4f3057f47adad2ba4dc2129a1e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0932c51fc6a44befad2b3dc8c124cc7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:30:35 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7872\tSpearman: 0.7902\n",
      "2021-10-13 21:30:35 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7532\tSpearman: 0.7689\n",
      "2021-10-13 21:30:35 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7524\tSpearman: 0.7680\n",
      "2021-10-13 21:30:35 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7497\tSpearman: 0.7522\n",
      "2021-10-13 21:30:35 - INFO - train - 102 : No improvement over previous best eval score (0.790173 vs 0.792913), patience = 14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:31:11 - INFO - train - 75 : Epoch : 4, train_step : 120/205, loss_value : 0.019710425898665562 \n",
      "2021-10-13 21:32:32 - INFO - train - 75 : Epoch : 4, train_step : 160/205, loss_value : 0.01952880504541099 \n",
      "2021-10-13 21:33:58 - INFO - train - 75 : Epoch : 4, train_step : 200/205, loss_value : 0.015049356035888195 \n",
      "2021-10-13 21:34:03 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset in epoch 4 after 41 steps:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2823ae94c4834c92909810f1eee37d16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ef1ee22d52a43c387080b223f88e757",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 21:34:16 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7869\tSpearman: 0.7899\n",
      "2021-10-13 21:34:16 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7528\tSpearman: 0.7684\n",
      "2021-10-13 21:34:16 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7519\tSpearman: 0.7676\n",
      "2021-10-13 21:34:16 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7489\tSpearman: 0.7517\n",
      "2021-10-13 21:34:16 - INFO - train - 102 : No improvement over previous best eval score (0.789881 vs 0.792913), patience = 13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trainer=Trainer(epochs=epochs,output_path=output_path,tensorboard_logdir=tensorboard_logdir,early_stop_patience=20)\n",
    "trainer.train(train_dataloader=train_dataloader,\n",
    "             model=esimcse,\n",
    "             optimizer=optimizer,\n",
    "             scheduler=scheduler,\n",
    "             evaluator=evaluator,\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17b66304",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "448718e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 18:23:51 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f18f94c0cea4d7b996cb5ea8e05c5af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ef3b77593d744c88f6edd2b25c9a4e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Batches'), FloatProgress(value=0.0, max=23.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-13 18:24:04 - INFO - __call__ - 103 : Cosine-Similarity :\tPearson: 0.7764\tSpearman: 0.7783\n",
      "2021-10-13 18:24:04 - INFO - __call__ - 105 : Manhattan-Distance:\tPearson: 0.7496\tSpearman: 0.7682\n",
      "2021-10-13 18:24:04 - INFO - __call__ - 107 : Euclidean-Distance:\tPearson: 0.7499\tSpearman: 0.7688\n",
      "2021-10-13 18:24:04 - INFO - __call__ - 109 : Dot-Product-Similarity:\tPearson: 0.7536\tSpearman: 0.7592\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7783171236503472"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator(consert_donotclosedropout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58138e0e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "010401c5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}