sandbox-deeprank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from importlib import reload "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deeprank.dataset import DataLoader, PairGenerator, ListGenerator\n",
    "from deeprank import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fd6fc1c6ad0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed = 1234\n",
    "torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[./data/letor/r5w/word_dict.txt]\n",
      "\tWord dict size: 193367\n",
      "[./data/letor/r5w/qid_query.txt]\n",
      "\tData size: 1692\n",
      "[./data/letor/r5w/docid_doc.txt]\n",
      "\tData size: 65323\n",
      "[./data/letor/r5w/embed_wiki-pdc_d50_norm]\n",
      "\tEmbedding size: 109282\n",
      "[./data/letor/r5w/embed.idf]\n",
      "\tEmbedding size: 193367\n",
      "Generate numpy embed: (193368, 50)\n",
      "Generate numpy embed: (193368, 1)\n"
     ]
    }
   ],
   "source": [
    "loader = DataLoader('./config/letor07_mp_fold1.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "letor_config = json.loads(open('./config/letor07_mp_fold1.model').read())\n",
    "#device = torch.device(\"cuda\")\n",
    "#device = torch.device(\"cpu\")\n",
    "select_device = torch.device(\"cpu\")\n",
    "rank_device = torch.device(\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[./data/letor/r5w/relation.train.fold1.txt]\n",
      "\tInstance size: 47828\n",
      "Pair Instance Count: 325439\n"
     ]
    }
   ],
   "source": [
    "Letor07Path = letor_config['data_dir']\n",
    "\n",
    "letor_config['fill_word'] = loader._PAD_\n",
    "letor_config['embedding'] = loader.embedding\n",
    "letor_config['feat_size'] = loader.feat_size\n",
    "letor_config['vocab_size'] = loader.embedding.shape[0]\n",
    "letor_config['embed_dim'] = loader.embedding.shape[1]\n",
    "letor_config['pad_value'] = loader._PAD_\n",
    "\n",
    "pair_gen = PairGenerator(rel_file=Letor07Path + '/relation.train.fold%d.txt'%(letor_config['fold']), \n",
    "                         config=letor_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deeprank import select_module\n",
    "from deeprank import rank_module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "QueryCentricNet()"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "letor_config['max_match'] = 20\n",
    "letor_config['win_size'] = 5\n",
    "select_net = select_module.QueryCentricNet(config=letor_config, out_device=rank_device)\n",
    "select_net = select_net.to(select_device)\n",
    "select_net.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nletor_config['q_limit'] = 20\\nletor_config['d_limit'] = 2000\\nletor_config['max_match'] = 20\\nletor_config['win_size'] = 5\\nletor_config['finetune_embed'] = True\\nletor_config['lr'] = 0.0001\\nselect_net = select_module.PointerNet(config=letor_config)\\nselect_net = select_net.to(device)\\nselect_net.embedding.weight.data.copy_(torch.from_numpy(loader.embedding))\\nselect_net.train()\\nselect_optimizer = optim.RMSprop(select_net.parameters(), lr=letor_config['lr'])\\n\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "letor_config['q_limit'] = 20\n",
    "letor_config['d_limit'] = 2000\n",
    "letor_config['max_match'] = 20\n",
    "letor_config['win_size'] = 5\n",
    "letor_config['finetune_embed'] = True\n",
    "letor_config['lr'] = 0.0001\n",
    "select_net = select_module.PointerNet(config=letor_config)\n",
    "select_net = select_net.to(device)\n",
    "select_net.embedding.weight.data.copy_(torch.from_numpy(loader.embedding))\n",
    "select_net.train()\n",
    "select_optimizer = optim.RMSprop(select_net.parameters(), lr=letor_config['lr'])\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "letor_config[\"dim_q\"] = 1\n",
    "letor_config[\"dim_d\"] = 1\n",
    "letor_config[\"dim_weight\"] = 1\n",
    "letor_config[\"c_reduce\"] = [1, 1]\n",
    "letor_config[\"k_reduce\"] = [1, 50]\n",
    "letor_config[\"s_reduce\"] = 1\n",
    "letor_config[\"p_reduce\"] = [0, 0]\n",
    "\n",
    "letor_config[\"c_en_conv_out\"] = 4\n",
    "letor_config[\"k_en_conv\"] = 3\n",
    "letor_config[\"s_en_conv\"] = 1\n",
    "letor_config[\"p_en_conv\"] = 1\n",
    "\n",
    "letor_config[\"en_pool_out\"] = [1, 1]\n",
    "letor_config[\"en_leaky\"] = 0.2\n",
    "\n",
    "letor_config[\"dim_gru_hidden\"] = 3\n",
    "\n",
    "letor_config['lr'] = 0.005\n",
    "letor_config['finetune_embed'] = False\n",
    "\n",
    "rank_net = rank_module.DeepRankNet(config=letor_config)\n",
    "rank_net = rank_net.to(rank_device)\n",
    "rank_net.embedding.weight.data.copy_(torch.from_numpy(loader.embedding))\n",
    "rank_net.qw_embedding.weight.data.copy_(torch.from_numpy(loader.idf_embedding))\n",
    "rank_net.train()\n",
    "rank_optimizer = optim.Adam(rank_net.parameters(), lr=letor_config['lr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_device(*variables, device):\n",
    "    return (torch.from_numpy(variable).to(device) for variable in variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_text(x):\n",
    "    print(' '.join([loader.word_dict[w.item()] for w in x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "britain west indy consular information sheet britain west indy consular information sheet september country description anguilla britain virgin island cayman island montserrat turk caico britain overseas territory comprise britain west indy three embassy caribbean area share responsibility citizen visiting island britain west indy area name refer specific embassy consular jurisdiction appropriate consular information sheet necessary additional information anguilla britain virgin island montserrat consular district embassy located bridgetown barbados anguilla britain virgin island moderately develop economy tourist facilities widely available montserrat economy recover series eruption soufriere hill volcano access southern part island restricted resident concentrated northern designate safe zone accommodations limited airport remains closed most visitor island travel daily ferry antigua traveler check carib world travel antigua telephone sailing times cayman island consular district embassy located kingston jamaica turk caico consular district embassy located nassau bahamas turk caico archipelago eight major island numerous uninhabited cay most tourist facilities located providenciale provo grand turk island dollar unit currency large hotel shop accept credit cards entry requirement tourist stays three month anguilla britain virgin island montserrat stays days turk caico citizen passport naturalize certificate original certify birth certificate photo identification onward return ticket sufficient funds stay cayman island turk caico require payment departure tax person years age old cayman island tax airfare arrange long stays arrange work permit cayman island turk caico traveler contact department immigrate cayman island turk caico immigrate department information regarding entry customs requirement britain west indy citizen contact britain embassy massachusetts avenue nw washington dc telephone nearest britain consulate atlanta boston chicago dallas los angeles new york san francisco internet http nsi org travel britain west indy txt http www britain info org customs regulation anguilla britain virgin island montserrat customs authority enforce strict regulation concerning temporary importation export territory item firearm agriculture product currency advisable contact turk caico customs department specific information regarding customs requirement territory importation firearm turk caico strict forbidden prior approval writing commissioner police crime information petty street crime occur visitor leave valuable unattended hotel rooms beach turk caico visitor dial emergency police fire medical assistance loss theft abroad passport report immediately local police nearest embassy consulate citizen refer department states pamphlet safe trip abroad ways promote trouble free journey publication tip traveler caribbean available mail superintendent document government printing office washington dc internet http www gpoaccess gov index html bureau consular affair home page http travel state gov criminal penalty foreign country citizen subject country law regulation differ significant united states afford protection available individual law penalty break law severe united states similar offense person violate law anguilla britain virgin island montserrat cayman island turk caico unknowing expel arrest imprison penalty possession use traffick illegal drug territory strict convict offender expect jail sentence heavy fine medical facilities medical facilities available limited anguilla britain virgin island turk caico small public hospital grand turk private clinic provo clinic hyperbaric chamber most serious case require medical evacuate air turk caico united states expensive traveler ensure medical insurance valid cover expense air evacuate medical facilities limited montserrat quality medical care cayman island comparable procedure critical care require medical evacuate united states cayman island highly develop dive sector hyperbaric chamber available doctor hospital expect immediate cash payment health service medical insurance medical insurance valid united states medicare medicaid program payment medical service united states uninsured traveler require medical care overseas face extreme difficulty check insurance company confirm policy apply overseas provision medical evacuate ascertain payment made overseas hospital doctor reimburse later expenses incur insurance policy coverage psychiatric treatment disposition remains event death useful information medical emergency abroad overseas insurance program provided department state bureau consular affair brochure medical information america travel abroad available bureau consular affair home page autofax health information information vaccination health precaution obtain center disease control prevention hotline international traveler fyi trip fax cdc faxx internet site http www cdc gov traffic safety road conditions foreign country citizen encounter road conditions differ significant united states information concerning anguilla britain virgin island montserrat turk caico provided general reference total accurate particular location circumstance four territory driving left hand side road anguilla britain virgin island montserrat safety public transportation fair urban road conditions maintenance good rural road conditions maintenance fair available roadside assistance fair night driving mountain driving britain virgin island montserrat done great caution steep narrow winding nature road turk caico safety public transportation good urban road conditions maintenance fair rural road conditions maintenance fair available roadside assistance poor specific information concerning turk caico driver permit vehicle inspection road tax mandatory insurance contact turk caico tourist board cayman island safety public transportation good urban road conditions maintenance good rural road conditions maintenance good available roadside assistance good specific information concerning britain driver permit vehicle inspection road tax mandatory insurance contact britain national tourist organization offices new york telephone internet http www com aviation safety oversight federal aviation administration faa assess anguilla montserrat civil aviation authority category compliance international aviation safety standard oversight anguilla montserrat air carrier operation consultation correct deficiency ongoing air carrier territory permit conduct limited operation subject heighten faa surveillance federal aviation administration faa assess turk caico britain virgin island cayman island civil aviation authority category compliance international aviation safety standard oversight great britain air carrier operation information traveler contact department transportation visit faa internet website http www faa gov avr iasa department defense dod separate assess foreign air carrier suitability official provider air service information regarding dod policy specific carrier traveler contact dod disaster preparedness montserrat volcano prone territory volcanic eruption abate volcano dangerous caribbean country affected hurricane hurricane season runs june november general information natural disaster preparedness available internet federal emergency management agency fema http www fema gov children issue information international adoption children international parental child abduct international child support enforce issue please refer internet site http travel state gov children issue html telephone registration embassy consulate location citizen living visiting anguilla britain virgin island montserrat encourage register consular section embassy bridgetown barbados jurisdiction territory obtain update information travel security region consular section located america life insurance company alico building cheapside bridgetown barbados telephone fax hour operation monday friday local holiday citizen register consular agent antigua address bluff house pigeon point english harbour telephone fax e mail address ag consular agent hour operation monday friday local holiday please call appointments embassy consulate turk caico citizen living visiting turk caico encourage register consular section embassy nassau bahamas jurisdiction territory embassy physically located mcdonald restaurant queen street downtown nassau reach monday friday bahamian holiday noon telephone hour fax additional information travel security territory citizen contact embassy turk caico tourism office embassy consulate cayman island citizen living visiting cayman island encourage register consular section embassy kingston jamaica jurisdiction territory obtain update information travel security consular section located floor life jamaica building oxford road kingston telephone fax office hour monday friday jamaica holiday window service hour duty officer contact embassy main switchboard telephone chancery located three block away consular section jamaica mutual life center oxford road kingston citizen register consular agency george town grand cayman largest three cayman island consular agency located office adventure travel seven mile beach telephone fax internet office hour noon monday friday jamaica holiday replace consular information sheet dated march update section country description entry requirement customs regulation crime information criminal penalty medical information traffic safety road conditions aviation safety oversight disaster preparedness children issue registration embassy consulate location consular information sheet travel warning travel state gov britain windy html $ [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "living in cayman island $$ [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]\n",
      "0 registration embassy consulate location citizen living visiting anguilla britain virgin island\n",
      "1 embassy consulate turk caico citizen living visiting turk caico encourage register\n",
      "2 embassy consulate cayman island citizen living visiting cayman island encourage register\n",
      "3 description anguilla britain virgin island cayman island montserrat turk caico britain\n",
      "4 travel antigua telephone sailing times cayman island consular district embassy located\n"
     ]
    }
   ],
   "source": [
    "X1, X1_len, X1_id, X2, X2_len, X2_id, Y, F = \\\n",
    "        pair_gen.get_batch(data1=loader.query_data, data2=loader.doc_data)\n",
    "X1, X1_len, X2, X2_len, Y, F = \\\n",
    "        to_device(X1, X1_len, X2, X2_len, Y, F, device=rank_device)\n",
    "\n",
    "show_text(X2[0])\n",
    "\n",
    "X1, X2_new, X1_len, X2_len_new, X2_pos = select_net(X1, X2, X1_len, X2_len, X1_id, X2_id)\n",
    "\n",
    "show_text(X1[0])\n",
    "for i in range(5):\n",
    "    print(i, end=' ')\n",
    "    show_text(X2_new[0][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([26])\n",
      "200\n",
      "200\n",
      "tensor([ 967., 1043., 1091.,   19.,  119.,  193.,  205.,  215.,  223.,  428.,\n",
      "         502.,  513.,  750.,  845., 1088., 1093., 1158., 1161.,   18.,   20.,\n",
      "          39.,   60.,   71.,   90.,  105.,  120.,  140.,  152.,  171.,  194.,\n",
      "         206.,  216.,  224.,  279.,  426.,  429.,  460.,  503.],\n",
      "       device='cuda:0')\n",
      "tensor([ 0., 10., 20., 37.,  1., 11., 21., 30., 38.], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "print(X2_pos[20].shape)\n",
    "print(len(X2_pos))\n",
    "print(len(X2))\n",
    "print(X2_pos[0])\n",
    "print(X2_pos[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X1 = X1[:1]\n",
    "# X1_len = X1_len[:1]\n",
    "# X2 = X2[:1]\n",
    "# X2_len = X2_len[:1]\n",
    "# X1_id = X1_id[:1]\n",
    "# X2_id = X2_id[:1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show_text(X2[0])\n",
    "# X1, X2_new, X1_len, X2_len_new = select_net(X1, X2, X1_len, X2_len, X1_id, X2_id)\n",
    "# show_text(X1[0])\n",
    "# for i in range(5):\n",
    "#     print(i, end=' ')\n",
    "#     show_text(X2_new[0][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rank loss: 0.8253192901611328\n",
      "rank loss: 0.9061598777770996\n",
      "rank loss: 0.7371701002120972\n",
      "rank loss: 0.6861473917961121\n",
      "rank loss: 0.8359648585319519\n",
      "rank loss: 0.9824948906898499\n",
      "rank loss: 1.086227297782898\n",
      "rank loss: 0.9257540702819824\n",
      "rank loss: 0.8410568237304688\n",
      "rank loss: 0.8319268822669983\n",
      "rank loss: 0.8617979288101196\n",
      "rank loss: 0.8329433798789978\n",
      "rank loss: 0.6752440333366394\n",
      "rank loss: 0.9370722770690918\n",
      "rank loss: 0.8927839398384094\n",
      "rank loss: 0.8497340083122253\n",
      "rank loss: 0.7453449964523315\n",
      "rank loss: 0.7862136960029602\n",
      "rank loss: 0.7250550389289856\n",
      "rank loss: 0.7466641068458557\n",
      "rank loss: 0.8688445091247559\n",
      "rank loss: 0.8354814052581787\n",
      "rank loss: 0.7522762417793274\n",
      "rank loss: 0.6215545535087585\n",
      "rank loss: 0.862816572189331\n",
      "rank loss: 0.7009199857711792\n",
      "rank loss: 0.6947179436683655\n",
      "rank loss: 0.7155901193618774\n",
      "rank loss: 0.739418625831604\n",
      "rank loss: 0.7291609048843384\n",
      "rank loss: 0.7221815586090088\n",
      "rank loss: 0.7942367196083069\n",
      "rank loss: 0.645375669002533\n",
      "rank loss: 0.8599945902824402\n",
      "rank loss: 0.7585619688034058\n",
      "rank loss: 0.9197361469268799\n",
      "rank loss: 0.808783769607544\n",
      "rank loss: 0.7190381288528442\n",
      "rank loss: 0.7260241508483887\n",
      "rank loss: 0.6644251346588135\n",
      "rank loss: 0.7117756009101868\n",
      "rank loss: 0.7056992650032043\n",
      "rank loss: 0.6694395542144775\n",
      "rank loss: 0.6533949375152588\n",
      "rank loss: 0.8180677890777588\n",
      "rank loss: 0.6237444281578064\n",
      "rank loss: 0.6857447028160095\n",
      "rank loss: 0.5818533301353455\n",
      "rank loss: 0.6223163604736328\n",
      "rank loss: 0.5993679165840149\n",
      "rank loss: 0.6882268190383911\n",
      "rank loss: 0.7574049830436707\n",
      "rank loss: 0.7883985638618469\n",
      "rank loss: 0.6909044981002808\n",
      "rank loss: 0.5793841481208801\n",
      "rank loss: 0.6546987295150757\n",
      "rank loss: 0.8344674706459045\n",
      "rank loss: 0.8723913431167603\n",
      "rank loss: 0.7280072569847107\n",
      "rank loss: 0.7079762816429138\n",
      "rank loss: 0.6346868276596069\n",
      "rank loss: 0.5469557642936707\n",
      "rank loss: 0.6577250361442566\n",
      "rank loss: 0.7035403251647949\n",
      "rank loss: 0.6854127645492554\n",
      "rank loss: 0.8054794073104858\n",
      "rank loss: 0.7555392980575562\n",
      "rank loss: 0.8174128532409668\n",
      "rank loss: 0.7170220613479614\n",
      "rank loss: 0.6334143877029419\n",
      "rank loss: 0.7377594113349915\n",
      "rank loss: 0.727661669254303\n",
      "rank loss: 0.654536247253418\n",
      "rank loss: 0.6697186231613159\n",
      "rank loss: 0.6948603391647339\n",
      "rank loss: 0.6571195125579834\n",
      "rank loss: 0.6675547361373901\n",
      "rank loss: 0.6355634927749634\n",
      "rank loss: 0.6263956427574158\n",
      "rank loss: 0.7449415326118469\n",
      "rank loss: 0.5782480239868164\n",
      "rank loss: 0.7611068487167358\n",
      "rank loss: 0.6447526216506958\n",
      "rank loss: 0.6705676317214966\n",
      "rank loss: 0.6753314137458801\n",
      "rank loss: 0.7123861312866211\n",
      "rank loss: 0.750924825668335\n",
      "rank loss: 0.7021487355232239\n",
      "rank loss: 0.5994362235069275\n",
      "rank loss: 0.49022066593170166\n",
      "rank loss: 0.6561969518661499\n",
      "rank loss: 0.6952511668205261\n",
      "rank loss: 0.6888825297355652\n",
      "rank loss: 0.5608211159706116\n",
      "rank loss: 0.654406726360321\n",
      "rank loss: 0.8109426498413086\n",
      "rank loss: 0.7869074940681458\n",
      "rank loss: 0.6555036306381226\n",
      "rank loss: 0.5492518544197083\n",
      "rank loss: 0.6668916344642639\n",
      "rank loss: 0.7097367644309998\n",
      "rank loss: 0.8545863032341003\n",
      "rank loss: 0.5428604483604431\n",
      "rank loss: 0.7902082800865173\n",
      "rank loss: 0.5704790949821472\n",
      "rank loss: 0.6370575428009033\n",
      "rank loss: 0.5423408150672913\n",
      "rank loss: 0.867685079574585\n",
      "rank loss: 0.6105930805206299\n",
      "rank loss: 0.475509911775589\n",
      "rank loss: 0.7611523270606995\n",
      "rank loss: 0.7158710956573486\n",
      "rank loss: 0.6647457480430603\n",
      "rank loss: 0.6767051815986633\n",
      "rank loss: 0.5616432428359985\n",
      "rank loss: 0.7292777299880981\n",
      "rank loss: 0.588063657283783\n",
      "rank loss: 0.751769483089447\n",
      "rank loss: 0.8119333982467651\n",
      "rank loss: 0.6637603640556335\n",
      "rank loss: 0.5659765005111694\n",
      "rank loss: 0.6056384444236755\n",
      "rank loss: 0.7444398999214172\n",
      "rank loss: 0.7454335689544678\n",
      "rank loss: 0.7483544945716858\n",
      "rank loss: 0.5820091366767883\n",
      "rank loss: 0.5047313570976257\n",
      "rank loss: 0.6184120178222656\n",
      "rank loss: 0.7028547525405884\n",
      "rank loss: 0.4972821772098541\n",
      "rank loss: 0.7774192094802856\n",
      "rank loss: 0.767616868019104\n",
      "rank loss: 0.5514692068099976\n",
      "rank loss: 0.643605649471283\n",
      "rank loss: 0.6625069379806519\n",
      "rank loss: 0.7233222723007202\n",
      "rank loss: 0.5310849547386169\n",
      "rank loss: 0.6904588341712952\n",
      "rank loss: 0.7665449380874634\n",
      "rank loss: 0.7415623068809509\n",
      "rank loss: 0.6009477972984314\n",
      "rank loss: 0.5934821367263794\n",
      "rank loss: 0.4787769913673401\n",
      "rank loss: 0.8038679361343384\n",
      "rank loss: 0.7596362233161926\n",
      "rank loss: 0.6418794989585876\n",
      "rank loss: 0.7452442646026611\n",
      "rank loss: 0.7411455512046814\n",
      "rank loss: 0.7134140729904175\n",
      "rank loss: 0.7329197525978088\n",
      "rank loss: 0.6332678198814392\n",
      "rank loss: 0.6853761672973633\n",
      "rank loss: 0.6416361927986145\n",
      "rank loss: 0.6676276922225952\n",
      "rank loss: 0.6256608963012695\n",
      "rank loss: 0.6293230056762695\n",
      "rank loss: 0.6936831474304199\n",
      "rank loss: 0.7669994235038757\n",
      "rank loss: 0.621979832649231\n",
      "rank loss: 0.6652216911315918\n",
      "rank loss: 0.5823826193809509\n",
      "rank loss: 0.5484753847122192\n",
      "rank loss: 0.6745011806488037\n",
      "rank loss: 0.7030272483825684\n",
      "rank loss: 0.589518666267395\n",
      "rank loss: 0.7250195145606995\n",
      "rank loss: 0.643869161605835\n",
      "rank loss: 0.5818212032318115\n",
      "rank loss: 0.49639928340911865\n",
      "rank loss: 0.6948899030685425\n",
      "rank loss: 0.5694196820259094\n",
      "rank loss: 0.6521367430686951\n",
      "rank loss: 0.655613362789154\n",
      "rank loss: 0.6057036519050598\n",
      "rank loss: 0.5698220133781433\n",
      "rank loss: 0.7007342576980591\n",
      "rank loss: 0.7440113425254822\n",
      "rank loss: 0.6381808519363403\n",
      "rank loss: 0.7143141031265259\n",
      "rank loss: 0.5456219911575317\n",
      "rank loss: 0.563964307308197\n",
      "rank loss: 0.6509751081466675\n",
      "rank loss: 0.6574314832687378\n",
      "rank loss: 0.6345028281211853\n",
      "rank loss: 0.7750428318977356\n",
      "rank loss: 0.7941579818725586\n",
      "rank loss: 0.6345378756523132\n",
      "rank loss: 0.7104957699775696\n",
      "rank loss: 0.7166115641593933\n",
      "rank loss: 0.5856930613517761\n",
      "rank loss: 0.5825032591819763\n",
      "rank loss: 0.6563040018081665\n",
      "rank loss: 0.7580392360687256\n",
      "rank loss: 0.6157196164131165\n",
      "rank loss: 0.7710130214691162\n",
      "rank loss: 0.6196264624595642\n",
      "rank loss: 0.6800066828727722\n",
      "rank loss: 0.5978471040725708\n",
      "rank loss: 0.5798062682151794\n",
      "rank loss: 0.647723913192749\n",
      "rank loss: 0.6615773439407349\n",
      "rank loss: 0.6547691226005554\n",
      "rank loss: 0.6185985803604126\n",
      "rank loss: 0.6838380098342896\n",
      "rank loss: 0.6108643412590027\n",
      "rank loss: 0.4659000337123871\n",
      "rank loss: 0.5920618772506714\n",
      "rank loss: 0.61629319190979\n",
      "rank loss: 0.7056947946548462\n",
      "rank loss: 0.6112760305404663\n",
      "rank loss: 0.6107545495033264\n",
      "rank loss: 0.5221754312515259\n",
      "rank loss: 0.7269941568374634\n",
      "rank loss: 0.7442633509635925\n",
      "rank loss: 0.675088107585907\n",
      "rank loss: 0.7163035273551941\n",
      "rank loss: 0.5927048921585083\n",
      "rank loss: 0.7673153281211853\n",
      "rank loss: 0.621222734451294\n",
      "rank loss: 0.7173669338226318\n",
      "rank loss: 0.7313451766967773\n",
      "rank loss: 0.5910603404045105\n",
      "rank loss: 0.640478253364563\n",
      "rank loss: 0.5678544640541077\n",
      "rank loss: 0.5833525061607361\n",
      "rank loss: 0.6422555446624756\n",
      "rank loss: 0.6571154594421387\n",
      "rank loss: 0.5806067585945129\n",
      "rank loss: 0.7988702058792114\n",
      "rank loss: 0.7815283536911011\n",
      "rank loss: 0.6340240240097046\n",
      "rank loss: 0.7146464586257935\n",
      "rank loss: 0.7401726245880127\n",
      "rank loss: 0.623992383480072\n",
      "rank loss: 0.6378418803215027\n",
      "rank loss: 0.6037620306015015\n",
      "rank loss: 0.6515266299247742\n",
      "rank loss: 0.658591628074646\n",
      "rank loss: 0.5866996049880981\n",
      "rank loss: 0.6697197556495667\n",
      "rank loss: 0.5930948257446289\n",
      "rank loss: 0.6309228539466858\n",
      "rank loss: 0.5535991787910461\n",
      "rank loss: 0.6649977564811707\n",
      "rank loss: 0.6433483958244324\n",
      "rank loss: 0.49392613768577576\n",
      "rank loss: 0.4813811779022217\n",
      "rank loss: 0.5638495087623596\n",
      "rank loss: 0.557433009147644\n",
      "rank loss: 0.5753852128982544\n",
      "rank loss: 0.5896557569503784\n",
      "rank loss: 0.6928886771202087\n",
      "rank loss: 0.4539565145969391\n",
      "rank loss: 0.5155171751976013\n",
      "rank loss: 0.6018991470336914\n",
      "rank loss: 0.6476000547409058\n",
      "rank loss: 0.6208788156509399\n",
      "rank loss: 0.6157430410385132\n",
      "rank loss: 0.6680580973625183\n",
      "rank loss: 0.6645251512527466\n",
      "rank loss: 0.7243916392326355\n",
      "rank loss: 0.7339990735054016\n",
      "rank loss: 0.53104168176651\n",
      "rank loss: 0.6530911326408386\n",
      "rank loss: 0.6900074481964111\n",
      "rank loss: 0.6268577575683594\n",
      "rank loss: 0.6640608906745911\n",
      "rank loss: 0.6300787925720215\n",
      "rank loss: 0.633104681968689\n",
      "rank loss: 0.6400140523910522\n",
      "rank loss: 0.5904444456100464\n",
      "rank loss: 0.5831116437911987\n",
      "rank loss: 0.6452039480209351\n",
      "rank loss: 0.6984691619873047\n",
      "rank loss: 0.7015999555587769\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rank loss: 0.5133711695671082\n",
      "rank loss: 0.6093894243240356\n",
      "rank loss: 0.6936553716659546\n",
      "rank loss: 0.5612749457359314\n",
      "rank loss: 0.6028836369514465\n",
      "rank loss: 0.6241759657859802\n",
      "rank loss: 0.5825188159942627\n",
      "rank loss: 0.592739999294281\n",
      "rank loss: 0.5665154457092285\n",
      "rank loss: 0.6135952472686768\n",
      "rank loss: 0.6288796663284302\n",
      "rank loss: 0.4932781755924225\n",
      "rank loss: 0.5185708999633789\n",
      "rank loss: 0.5801147222518921\n",
      "rank loss: 0.531109631061554\n",
      "rank loss: 0.6218476891517639\n",
      "rank loss: 0.5993818640708923\n",
      "rank loss: 0.5731241703033447\n",
      "rank loss: 0.7112193703651428\n",
      "rank loss: 0.4649350643157959\n",
      "rank loss: 0.5803585648536682\n",
      "rank loss: 0.6520506143569946\n",
      "rank loss: 0.6149023175239563\n",
      "rank loss: 0.5500425696372986\n",
      "rank loss: 0.6771690249443054\n",
      "rank loss: 0.6709114909172058\n",
      "rank loss: 0.5567389726638794\n",
      "rank loss: 0.5823621153831482\n",
      "rank loss: 0.6645567417144775\n",
      "rank loss: 0.621821939945221\n",
      "rank loss: 0.601882815361023\n",
      "rank loss: 0.5645096302032471\n",
      "rank loss: 0.5257892608642578\n",
      "rank loss: 0.6914139986038208\n",
      "rank loss: 0.744803249835968\n",
      "rank loss: 0.6775184273719788\n",
      "rank loss: 0.4683395028114319\n",
      "rank loss: 0.5107113718986511\n",
      "rank loss: 0.6382501721382141\n",
      "rank loss: 0.7100539207458496\n",
      "rank loss: 0.6234136819839478\n",
      "rank loss: 0.6835761666297913\n",
      "rank loss: 0.4858143627643585\n",
      "rank loss: 0.5329545140266418\n",
      "rank loss: 0.48997196555137634\n",
      "rank loss: 0.6524157524108887\n",
      "rank loss: 0.524574875831604\n",
      "rank loss: 0.6328427195549011\n",
      "rank loss: 0.5360468029975891\n",
      "rank loss: 0.5431642532348633\n",
      "rank loss: 0.5694161653518677\n",
      "rank loss: 0.632455587387085\n",
      "rank loss: 0.6302438974380493\n",
      "rank loss: 0.6566185355186462\n",
      "rank loss: 0.4697479009628296\n",
      "rank loss: 0.5842887759208679\n",
      "rank loss: 0.5525298714637756\n",
      "rank loss: 0.7080880403518677\n",
      "rank loss: 0.6430352926254272\n",
      "rank loss: 0.5879358053207397\n",
      "rank loss: 0.5920752286911011\n",
      "rank loss: 0.5825058221817017\n",
      "rank loss: 0.649506688117981\n",
      "rank loss: 0.700621485710144\n",
      "rank loss: 0.5957927107810974\n",
      "rank loss: 0.5853244662284851\n",
      "rank loss: 0.5099084973335266\n",
      "rank loss: 0.6348800659179688\n",
      "rank loss: 0.6921374201774597\n",
      "rank loss: 0.5615172982215881\n",
      "rank loss: 0.6553217768669128\n",
      "rank loss: 0.6082420945167542\n",
      "rank loss: 0.6166708469390869\n",
      "rank loss: 0.67294842004776\n",
      "rank loss: 0.6450339555740356\n",
      "rank loss: 0.6379097104072571\n",
      "rank loss: 0.6777207851409912\n",
      "rank loss: 0.5248715877532959\n",
      "rank loss: 0.5974276065826416\n",
      "rank loss: 0.5538598895072937\n",
      "rank loss: 0.5527054667472839\n",
      "rank loss: 0.49821630120277405\n",
      "rank loss: 0.6855984330177307\n",
      "rank loss: 0.7373784184455872\n",
      "rank loss: 0.5248663425445557\n",
      "rank loss: 0.6098999381065369\n",
      "rank loss: 0.5772374272346497\n",
      "rank loss: 0.7994548678398132\n",
      "rank loss: 0.5106441974639893\n",
      "rank loss: 0.5459521412849426\n",
      "rank loss: 0.7638841867446899\n",
      "rank loss: 0.6447718739509583\n",
      "rank loss: 0.7000759840011597\n",
      "rank loss: 0.5539072155952454\n",
      "rank loss: 0.6441413760185242\n",
      "rank loss: 0.6609251499176025\n",
      "rank loss: 0.7082569599151611\n",
      "rank loss: 0.5513425469398499\n",
      "rank loss: 0.5752824544906616\n",
      "rank loss: 0.7349660992622375\n",
      "rank loss: 0.7014517784118652\n",
      "rank loss: 0.6771320104598999\n",
      "rank loss: 0.5942216515541077\n",
      "rank loss: 0.6211420893669128\n",
      "rank loss: 0.6408814787864685\n",
      "rank loss: 0.4657655656337738\n",
      "rank loss: 0.6669146418571472\n",
      "rank loss: 0.45995351672172546\n",
      "rank loss: 0.6363571882247925\n",
      "rank loss: 0.5623579025268555\n",
      "rank loss: 0.6132044792175293\n",
      "rank loss: 0.5062593221664429\n",
      "rank loss: 0.6171649694442749\n",
      "rank loss: 0.668350338935852\n",
      "rank loss: 0.5898702144622803\n",
      "rank loss: 0.6838979721069336\n",
      "rank loss: 0.6309739351272583\n",
      "rank loss: 0.6375854015350342\n",
      "rank loss: 0.5493778586387634\n",
      "rank loss: 0.5471910834312439\n",
      "rank loss: 0.6286852955818176\n",
      "rank loss: 0.5602430701255798\n",
      "rank loss: 0.6547220945358276\n",
      "rank loss: 0.511917233467102\n",
      "rank loss: 0.72225022315979\n",
      "rank loss: 0.8138279318809509\n",
      "rank loss: 0.6462910175323486\n",
      "rank loss: 0.6229498982429504\n",
      "rank loss: 0.5974141359329224\n",
      "rank loss: 0.5886201858520508\n",
      "rank loss: 0.5726275444030762\n",
      "rank loss: 0.5690548419952393\n",
      "rank loss: 0.5904621481895447\n",
      "rank loss: 0.6232371926307678\n",
      "rank loss: 0.695364236831665\n",
      "rank loss: 0.6393831968307495\n",
      "rank loss: 0.5794901847839355\n",
      "rank loss: 0.6744499206542969\n",
      "rank loss: 0.5961605906486511\n",
      "rank loss: 0.46713751554489136\n",
      "rank loss: 0.6336479187011719\n",
      "rank loss: 0.5003744959831238\n",
      "rank loss: 0.5976220965385437\n",
      "rank loss: 0.7056907415390015\n",
      "rank loss: 0.6319918036460876\n",
      "rank loss: 0.6885788440704346\n",
      "rank loss: 0.5705408453941345\n",
      "rank loss: 0.5946744680404663\n",
      "rank loss: 0.5208673477172852\n",
      "rank loss: 0.647142767906189\n",
      "rank loss: 0.5795824527740479\n",
      "rank loss: 0.5697316527366638\n",
      "rank loss: 0.5241800546646118\n",
      "rank loss: 0.6675418019294739\n",
      "rank loss: 0.6036832332611084\n",
      "rank loss: 0.6253550052642822\n",
      "rank loss: 0.5374031066894531\n",
      "rank loss: 0.547782301902771\n",
      "rank loss: 0.5389584898948669\n",
      "rank loss: 0.5232020616531372\n",
      "rank loss: 0.6761074662208557\n",
      "rank loss: 0.4285225570201874\n",
      "rank loss: 0.722886323928833\n",
      "rank loss: 0.5139389634132385\n",
      "rank loss: 0.4044558107852936\n",
      "rank loss: 0.4741155803203583\n",
      "rank loss: 0.6421760320663452\n",
      "rank loss: 0.5879985690116882\n",
      "rank loss: 0.5746995210647583\n",
      "rank loss: 0.6290029883384705\n",
      "rank loss: 0.5238049030303955\n",
      "rank loss: 0.5915272831916809\n",
      "rank loss: 0.5953396558761597\n",
      "rank loss: 0.608862042427063\n",
      "rank loss: 0.4833449423313141\n",
      "rank loss: 0.579334020614624\n",
      "rank loss: 0.6458004713058472\n",
      "rank loss: 0.6158807873725891\n",
      "rank loss: 0.6636658310890198\n",
      "rank loss: 0.64467453956604\n",
      "rank loss: 0.6981528997421265\n",
      "rank loss: 0.5824010968208313\n",
      "rank loss: 0.5844647288322449\n",
      "rank loss: 0.35877394676208496\n",
      "rank loss: 0.5156503319740295\n",
      "rank loss: 0.655929684638977\n",
      "rank loss: 0.6576936841011047\n",
      "rank loss: 0.5813105702400208\n",
      "rank loss: 0.5818280577659607\n",
      "rank loss: 0.7810451984405518\n",
      "rank loss: 0.5543988943099976\n",
      "rank loss: 0.543402910232544\n",
      "rank loss: 0.6471899747848511\n",
      "rank loss: 0.5553327798843384\n",
      "rank loss: 0.43229156732559204\n",
      "rank loss: 0.6128778457641602\n",
      "rank loss: 0.3698439598083496\n",
      "rank loss: 0.641465425491333\n",
      "rank loss: 0.6127437949180603\n",
      "rank loss: 0.680927574634552\n",
      "rank loss: 0.6502053141593933\n",
      "rank loss: 0.615877628326416\n",
      "rank loss: 0.5190286040306091\n",
      "rank loss: 0.7879676818847656\n",
      "rank loss: 0.6273728013038635\n",
      "rank loss: 0.6981329917907715\n",
      "rank loss: 0.5590851902961731\n",
      "rank loss: 0.6176275610923767\n",
      "rank loss: 0.5896059274673462\n",
      "rank loss: 0.5429995059967041\n",
      "rank loss: 0.6336995363235474\n",
      "rank loss: 0.6050646901130676\n",
      "rank loss: 0.5544248819351196\n",
      "rank loss: 0.5246632099151611\n",
      "rank loss: 0.6547473669052124\n",
      "rank loss: 0.7398380637168884\n",
      "rank loss: 0.5007464289665222\n",
      "rank loss: 0.5303335189819336\n",
      "rank loss: 0.5356393456459045\n",
      "rank loss: 0.6333893537521362\n",
      "rank loss: 0.7600122094154358\n",
      "rank loss: 0.6899213790893555\n",
      "rank loss: 0.5697799324989319\n",
      "rank loss: 0.4957682192325592\n",
      "rank loss: 0.5388685464859009\n",
      "rank loss: 0.7361215949058533\n",
      "rank loss: 0.4685225486755371\n",
      "rank loss: 0.6226116418838501\n",
      "rank loss: 0.5476768612861633\n",
      "rank loss: 0.5968860983848572\n",
      "rank loss: 0.8028150200843811\n",
      "rank loss: 0.5993894338607788\n",
      "rank loss: 0.6875239610671997\n",
      "rank loss: 0.5729924440383911\n",
      "rank loss: 0.7071168422698975\n",
      "rank loss: 0.5324933528900146\n",
      "rank loss: 0.4885619878768921\n",
      "rank loss: 0.56663578748703\n",
      "rank loss: 0.45354971289634705\n",
      "rank loss: 0.6879916787147522\n",
      "rank loss: 0.5527989268302917\n",
      "rank loss: 0.6227177977561951\n",
      "rank loss: 0.69500333070755\n",
      "rank loss: 0.6626946926116943\n",
      "rank loss: 0.5508628487586975\n",
      "rank loss: 0.6669080853462219\n",
      "rank loss: 0.6292222738265991\n",
      "rank loss: 0.5759738683700562\n",
      "rank loss: 0.5574280619621277\n",
      "rank loss: 0.4836215078830719\n",
      "rank loss: 0.5756626129150391\n",
      "rank loss: 0.5291585922241211\n",
      "rank loss: 0.616489827632904\n",
      "rank loss: 0.6023639440536499\n",
      "rank loss: 0.614048957824707\n",
      "rank loss: 0.46234703063964844\n",
      "rank loss: 0.6352792978286743\n",
      "rank loss: 0.48141932487487793\n",
      "rank loss: 0.6159082651138306\n",
      "rank loss: 0.6045082211494446\n",
      "rank loss: 0.5968570709228516\n",
      "rank loss: 0.5791180729866028\n",
      "rank loss: 0.6221844553947449\n",
      "rank loss: 0.48178809881210327\n",
      "rank loss: 0.596959114074707\n",
      "rank loss: 0.5859464406967163\n",
      "rank loss: 0.44934067130088806\n",
      "rank loss: 0.8341890573501587\n",
      "rank loss: 0.5355315804481506\n",
      "rank loss: 0.8695276975631714\n",
      "rank loss: 0.5308929681777954\n",
      "rank loss: 0.5392665266990662\n",
      "rank loss: 0.5666631460189819\n",
      "rank loss: 0.5604323148727417\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rank loss: 0.4959826171398163\n",
      "rank loss: 0.5804392099380493\n",
      "rank loss: 0.4816310703754425\n",
      "rank loss: 0.7640804648399353\n",
      "rank loss: 0.7401973009109497\n",
      "rank loss: 0.5766350626945496\n",
      "rank loss: 0.6007418036460876\n",
      "rank loss: 0.646103024482727\n",
      "rank loss: 0.574966549873352\n",
      "rank loss: 0.6843050718307495\n",
      "rank loss: 0.5124703645706177\n",
      "rank loss: 0.5367817282676697\n",
      "rank loss: 0.6134668588638306\n",
      "rank loss: 0.5180090665817261\n",
      "rank loss: 0.7356290817260742\n",
      "rank loss: 0.8127268552780151\n",
      "rank loss: 0.5558343529701233\n",
      "rank loss: 0.6449012756347656\n",
      "rank loss: 0.550355076789856\n",
      "rank loss: 0.649695873260498\n",
      "rank loss: 0.5372965931892395\n",
      "rank loss: 0.5424820184707642\n",
      "rank loss: 0.6034793853759766\n",
      "rank loss: 0.6027082204818726\n",
      "rank loss: 0.637467622756958\n",
      "rank loss: 0.6520441770553589\n",
      "rank loss: 0.6662905216217041\n",
      "rank loss: 0.4601402282714844\n",
      "rank loss: 0.6279621720314026\n",
      "rank loss: 0.6649020910263062\n",
      "rank loss: 0.6187517046928406\n",
      "rank loss: 0.597774088382721\n",
      "rank loss: 0.48795974254608154\n",
      "rank loss: 0.5230541825294495\n",
      "rank loss: 0.5392048358917236\n",
      "rank loss: 0.6306567192077637\n",
      "rank loss: 0.6831409931182861\n",
      "rank loss: 0.6177390217781067\n",
      "rank loss: 0.5845361948013306\n",
      "rank loss: 0.6045563220977783\n",
      "rank loss: 0.4795595407485962\n",
      "rank loss: 0.5760976076126099\n",
      "rank loss: 0.7316612005233765\n",
      "rank loss: 0.5375339984893799\n",
      "rank loss: 0.6300511956214905\n",
      "rank loss: 0.7494047284126282\n",
      "rank loss: 0.6382573246955872\n",
      "rank loss: 0.6018816232681274\n",
      "rank loss: 0.6653091907501221\n",
      "rank loss: 0.6264314651489258\n",
      "rank loss: 0.594818115234375\n",
      "rank loss: 0.6157723665237427\n",
      "rank loss: 0.6277817487716675\n",
      "rank loss: 0.7726138234138489\n",
      "rank loss: 0.5517677664756775\n",
      "rank loss: 0.6950049996376038\n",
      "rank loss: 0.7346392869949341\n",
      "rank loss: 0.601835310459137\n",
      "rank loss: 0.586587131023407\n",
      "rank loss: 0.5724804401397705\n",
      "rank loss: 0.5145185589790344\n",
      "rank loss: 0.5340775847434998\n",
      "rank loss: 0.600912868976593\n",
      "rank loss: 0.5955340266227722\n",
      "rank loss: 0.6120234727859497\n",
      "rank loss: 0.5874027609825134\n",
      "rank loss: 0.5175620913505554\n",
      "rank loss: 0.6114723682403564\n",
      "rank loss: 0.5862346291542053\n",
      "rank loss: 0.7411625385284424\n",
      "rank loss: 0.6652158498764038\n",
      "rank loss: 0.5995690226554871\n",
      "rank loss: 0.668172299861908\n",
      "rank loss: 0.5162229537963867\n",
      "rank loss: 0.6367594599723816\n",
      "rank loss: 0.672239363193512\n",
      "rank loss: 0.5617910027503967\n",
      "rank loss: 0.6091117858886719\n",
      "rank loss: 0.7323104739189148\n",
      "rank loss: 0.8181512355804443\n",
      "rank loss: 0.6967644095420837\n",
      "rank loss: 0.5532618165016174\n",
      "rank loss: 0.5230858325958252\n",
      "rank loss: 0.5678370594978333\n",
      "rank loss: 0.4649471640586853\n",
      "rank loss: 0.6539223194122314\n",
      "rank loss: 0.4857584536075592\n",
      "rank loss: 0.6391837000846863\n",
      "rank loss: 0.5573137402534485\n",
      "rank loss: 0.5632376074790955\n",
      "rank loss: 0.6440931558609009\n",
      "rank loss: 0.6508482098579407\n",
      "rank loss: 0.6550161838531494\n",
      "rank loss: 0.5891088247299194\n",
      "rank loss: 0.5410140752792358\n",
      "rank loss: 0.5150939226150513\n",
      "rank loss: 0.5661767721176147\n",
      "rank loss: 0.6439443826675415\n",
      "rank loss: 0.7646253705024719\n",
      "rank loss: 0.52385014295578\n",
      "rank loss: 0.6061913371086121\n",
      "rank loss: 0.454039603471756\n",
      "rank loss: 0.5128725171089172\n",
      "rank loss: 0.6645280122756958\n",
      "rank loss: 0.5428637862205505\n",
      "rank loss: 0.5248359441757202\n",
      "rank loss: 0.6090695858001709\n",
      "rank loss: 0.5361109972000122\n",
      "rank loss: 0.6031575798988342\n",
      "rank loss: 0.5292489528656006\n",
      "rank loss: 0.6500598192214966\n",
      "rank loss: 0.6069156527519226\n",
      "rank loss: 0.5974709987640381\n",
      "rank loss: 0.702275812625885\n",
      "rank loss: 0.40749311447143555\n",
      "rank loss: 0.6497770547866821\n",
      "rank loss: 0.517363429069519\n",
      "rank loss: 0.5880185961723328\n",
      "rank loss: 0.6810697913169861\n",
      "rank loss: 0.5039733648300171\n",
      "rank loss: 0.5740867257118225\n",
      "rank loss: 0.6732761859893799\n",
      "rank loss: 0.6237808465957642\n",
      "rank loss: 0.5840416550636292\n",
      "rank loss: 0.5167364478111267\n",
      "rank loss: 0.45426633954048157\n",
      "rank loss: 0.6134015321731567\n",
      "rank loss: 0.6156768798828125\n",
      "rank loss: 0.5297285318374634\n",
      "rank loss: 0.6374748349189758\n",
      "rank loss: 0.6346221566200256\n",
      "rank loss: 0.6512209177017212\n",
      "rank loss: 0.609465479850769\n",
      "rank loss: 0.6019415259361267\n",
      "rank loss: 0.4699207544326782\n",
      "rank loss: 0.6337809562683105\n",
      "rank loss: 0.5916303396224976\n",
      "rank loss: 0.5794947147369385\n",
      "rank loss: 0.5992717146873474\n",
      "rank loss: 0.6035544276237488\n",
      "rank loss: 0.7631650567054749\n",
      "rank loss: 0.5215895771980286\n",
      "rank loss: 0.6502174139022827\n",
      "rank loss: 0.5306412577629089\n",
      "rank loss: 0.6788583397865295\n",
      "rank loss: 0.5072223544120789\n",
      "rank loss: 0.5738585591316223\n",
      "rank loss: 0.5629715323448181\n",
      "rank loss: 0.5194010734558105\n",
      "rank loss: 0.6363349556922913\n",
      "rank loss: 0.44282862544059753\n",
      "rank loss: 0.6271577477455139\n",
      "rank loss: 0.5780470967292786\n",
      "rank loss: 0.5999751687049866\n",
      "rank loss: 0.5192636251449585\n",
      "rank loss: 0.7151976823806763\n",
      "rank loss: 0.5697516202926636\n",
      "rank loss: 0.46866774559020996\n",
      "rank loss: 0.6010950803756714\n",
      "rank loss: 0.5137875080108643\n",
      "rank loss: 0.4419160783290863\n",
      "rank loss: 0.6567449569702148\n",
      "rank loss: 0.6893525719642639\n",
      "rank loss: 0.618077278137207\n",
      "rank loss: 0.5222010016441345\n",
      "rank loss: 0.6508082151412964\n",
      "rank loss: 0.5737641453742981\n",
      "rank loss: 0.6435055136680603\n",
      "rank loss: 0.5867516994476318\n",
      "rank loss: 0.49894076585769653\n",
      "rank loss: 0.5746033191680908\n",
      "rank loss: 0.540411114692688\n",
      "rank loss: 0.5562062859535217\n",
      "rank loss: 0.6638458371162415\n",
      "rank loss: 0.6023404002189636\n",
      "rank loss: 0.6110756993293762\n",
      "rank loss: 0.5827265977859497\n",
      "rank loss: 0.5484525561332703\n",
      "rank loss: 0.5289906859397888\n",
      "rank loss: 0.4283185601234436\n",
      "rank loss: 0.5353800654411316\n",
      "rank loss: 0.6395533084869385\n",
      "rank loss: 0.5717236399650574\n",
      "rank loss: 0.599930465221405\n",
      "rank loss: 0.7662286162376404\n",
      "rank loss: 0.3729392886161804\n",
      "rank loss: 0.5008604526519775\n",
      "rank loss: 0.4210158884525299\n",
      "rank loss: 0.4585442543029785\n",
      "rank loss: 0.6354654431343079\n",
      "rank loss: 0.4361203610897064\n",
      "rank loss: 0.46124762296676636\n",
      "rank loss: 0.4450981914997101\n",
      "rank loss: 0.6048001050949097\n",
      "rank loss: 0.5586264133453369\n",
      "rank loss: 0.6514484882354736\n",
      "rank loss: 0.5338215231895447\n",
      "rank loss: 0.5620989203453064\n",
      "rank loss: 0.5962615609169006\n",
      "rank loss: 0.645932674407959\n",
      "rank loss: 0.5732113718986511\n",
      "rank loss: 0.5602632761001587\n",
      "rank loss: 0.7383816242218018\n",
      "rank loss: 0.5731799006462097\n",
      "rank loss: 0.5435423254966736\n",
      "rank loss: 0.6765855550765991\n",
      "rank loss: 0.6492910385131836\n",
      "rank loss: 0.7136900424957275\n",
      "rank loss: 0.5662294030189514\n",
      "rank loss: 0.5511871576309204\n",
      "rank loss: 0.6477751135826111\n",
      "rank loss: 0.7031962275505066\n",
      "rank loss: 0.6175748109817505\n",
      "rank loss: 0.4691884517669678\n",
      "rank loss: 0.5837807059288025\n",
      "rank loss: 0.5836053490638733\n",
      "rank loss: 0.7854065299034119\n",
      "rank loss: 0.43475034832954407\n",
      "rank loss: 0.5189374685287476\n",
      "rank loss: 0.5975436568260193\n",
      "rank loss: 0.5337499380111694\n",
      "rank loss: 0.5565330982208252\n",
      "rank loss: 0.5317055583000183\n",
      "rank loss: 0.5530390739440918\n",
      "rank loss: 0.5787030458450317\n",
      "rank loss: 0.5538532733917236\n",
      "rank loss: 0.5987327694892883\n",
      "rank loss: 0.7397071719169617\n",
      "rank loss: 0.5205843448638916\n",
      "rank loss: 0.7209644913673401\n",
      "rank loss: 0.4199492633342743\n",
      "rank loss: 0.6275160312652588\n",
      "rank loss: 0.5293305516242981\n",
      "rank loss: 0.6103397011756897\n",
      "rank loss: 0.5077642202377319\n",
      "rank loss: 0.5085546970367432\n",
      "rank loss: 0.6818061470985413\n",
      "rank loss: 0.42594459652900696\n",
      "rank loss: 0.5196923613548279\n",
      "rank loss: 0.4858259856700897\n",
      "rank loss: 0.4223059117794037\n",
      "rank loss: 0.682047426700592\n",
      "rank loss: 0.4966898560523987\n",
      "rank loss: 0.631591796875\n",
      "rank loss: 0.48890167474746704\n",
      "rank loss: 0.46881556510925293\n",
      "rank loss: 0.4064020812511444\n",
      "rank loss: 0.6481003165245056\n",
      "rank loss: 0.5708467960357666\n",
      "rank loss: 0.5811027884483337\n",
      "rank loss: 0.6569998264312744\n",
      "rank loss: 0.5245026350021362\n",
      "rank loss: 0.739422619342804\n",
      "rank loss: 0.5919159054756165\n",
      "rank loss: 0.6002605557441711\n",
      "rank loss: 0.5305280089378357\n",
      "rank loss: 0.603735625743866\n",
      "rank loss: 0.4968346357345581\n",
      "rank loss: 0.5244911909103394\n",
      "rank loss: 0.5562028288841248\n",
      "rank loss: 0.6551820039749146\n",
      "rank loss: 0.6021937727928162\n",
      "rank loss: 0.5946544408798218\n",
      "rank loss: 0.5466862320899963\n",
      "rank loss: 0.6610601544380188\n",
      "rank loss: 0.6010615229606628\n",
      "rank loss: 0.653766393661499\n",
      "rank loss: 0.6367529630661011\n",
      "rank loss: 0.46342214941978455\n",
      "rank loss: 0.6628741025924683\n",
      "rank loss: 0.5507064461708069\n",
      "rank loss: 0.5205163359642029\n",
      "rank loss: 0.7260319590568542\n",
      "rank loss: 0.6037302017211914\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rank loss: 0.48663732409477234\n",
      "rank loss: 0.5374844670295715\n",
      "rank loss: 0.46874046325683594\n",
      "rank loss: 0.5891987681388855\n",
      "rank loss: 0.6559246778488159\n",
      "rank loss: 0.7458073496818542\n",
      "rank loss: 0.6179835200309753\n",
      "rank loss: 0.48474937677383423\n",
      "rank loss: 0.611075222492218\n",
      "rank loss: 0.5645518898963928\n",
      "rank loss: 0.5106174945831299\n",
      "rank loss: 0.6290766596794128\n",
      "rank loss: 0.5679289102554321\n",
      "rank loss: 0.48271214962005615\n",
      "rank loss: 0.7024364471435547\n",
      "rank loss: 0.6389115452766418\n",
      "rank loss: 0.5404137969017029\n",
      "rank loss: 0.702846348285675\n",
      "rank loss: 0.7886378169059753\n",
      "rank loss: 0.4048720896244049\n",
      "rank loss: 0.5624221563339233\n",
      "rank loss: 0.5382150411605835\n",
      "rank loss: 0.6934891939163208\n",
      "rank loss: 0.636634111404419\n",
      "rank loss: 0.5546224117279053\n",
      "rank loss: 0.6348323822021484\n",
      "rank loss: 0.4190647006034851\n",
      "rank loss: 0.7016768455505371\n",
      "rank loss: 0.5422465801239014\n",
      "rank loss: 0.4568864703178406\n",
      "rank loss: 0.5150576829910278\n",
      "rank loss: 0.6098272800445557\n",
      "rank loss: 0.6064140796661377\n",
      "rank loss: 0.526688277721405\n",
      "rank loss: 0.6716177463531494\n",
      "rank loss: 0.5258026123046875\n",
      "rank loss: 0.5182700753211975\n",
      "rank loss: 0.6585767865180969\n",
      "rank loss: 0.6419798731803894\n",
      "rank loss: 0.5229746103286743\n",
      "rank loss: 0.46765920519828796\n",
      "rank loss: 0.4586448669433594\n",
      "rank loss: 0.5140469074249268\n",
      "rank loss: 0.5737773180007935\n",
      "rank loss: 0.7220437526702881\n",
      "rank loss: 0.6313402652740479\n",
      "rank loss: 0.6825506687164307\n",
      "rank loss: 0.5732608437538147\n",
      "rank loss: 0.6208743453025818\n",
      "rank loss: 0.5165846347808838\n",
      "rank loss: 0.5467522144317627\n",
      "rank loss: 0.5075498223304749\n",
      "rank loss: 0.5577554702758789\n",
      "rank loss: 0.5301543474197388\n",
      "rank loss: 0.478442907333374\n",
      "rank loss: 0.7070451378822327\n",
      "rank loss: 0.543590784072876\n",
      "rank loss: 0.5139694809913635\n",
      "rank loss: 0.5966993570327759\n",
      "rank loss: 0.5988609194755554\n",
      "rank loss: 0.4897705614566803\n",
      "rank loss: 0.4654216766357422\n",
      "rank loss: 0.6072706580162048\n",
      "rank loss: 0.6772657632827759\n",
      "rank loss: 0.6068309545516968\n",
      "rank loss: 0.5735353231430054\n",
      "rank loss: 0.49366018176078796\n",
      "rank loss: 0.5883808732032776\n",
      "rank loss: 0.6868321299552917\n",
      "rank loss: 0.4612933099269867\n",
      "rank loss: 0.4549793601036072\n",
      "rank loss: 0.5044448971748352\n",
      "rank loss: 0.5653022527694702\n",
      "rank loss: 0.4743686616420746\n",
      "rank loss: 0.5103377103805542\n",
      "rank loss: 0.5034747123718262\n",
      "rank loss: 0.748773455619812\n",
      "rank loss: 0.5700854659080505\n",
      "rank loss: 0.5461919903755188\n",
      "rank loss: 0.5528277158737183\n",
      "rank loss: 0.5718920826911926\n",
      "rank loss: 0.6875066757202148\n",
      "rank loss: 0.5404905080795288\n",
      "rank loss: 0.6071205139160156\n",
      "rank loss: 0.6005838513374329\n",
      "rank loss: 0.6610015034675598\n",
      "rank loss: 0.46457257866859436\n",
      "rank loss: 0.45899420976638794\n",
      "rank loss: 0.5892502665519714\n",
      "rank loss: 0.47835397720336914\n",
      "rank loss: 0.5152376294136047\n",
      "rank loss: 0.6330558657646179\n",
      "rank loss: 0.5529945492744446\n",
      "rank loss: 0.5412518978118896\n",
      "rank loss: 0.5181496143341064\n",
      "rank loss: 0.5445736646652222\n",
      "rank loss: 0.5159862041473389\n",
      "rank loss: 0.737484335899353\n",
      "rank loss: 0.5857850313186646\n",
      "rank loss: 0.5662296414375305\n",
      "rank loss: 0.6875936985015869\n",
      "rank loss: 0.4536193609237671\n",
      "rank loss: 0.5354499220848083\n",
      "rank loss: 0.5903902053833008\n",
      "rank loss: 0.523009717464447\n",
      "rank loss: 0.5018133521080017\n",
      "rank loss: 0.5720149278640747\n",
      "rank loss: 0.5681282877922058\n",
      "rank loss: 0.5608694553375244\n",
      "rank loss: 0.5481735467910767\n",
      "rank loss: 0.5545434355735779\n",
      "rank loss: 0.4342561662197113\n",
      "rank loss: 0.5673940777778625\n",
      "rank loss: 0.684893012046814\n",
      "rank loss: 0.4933570623397827\n",
      "rank loss: 0.6568296551704407\n",
      "rank loss: 0.6799030900001526\n",
      "rank loss: 0.4809870719909668\n",
      "rank loss: 0.7884386777877808\n",
      "rank loss: 0.4304344952106476\n",
      "rank loss: 0.5597058534622192\n",
      "rank loss: 0.4798685610294342\n",
      "rank loss: 0.7502918839454651\n",
      "rank loss: 0.5653550028800964\n",
      "rank loss: 0.5614262819290161\n",
      "rank loss: 0.5456466674804688\n",
      "rank loss: 0.46029624342918396\n",
      "rank loss: 0.5326287150382996\n",
      "rank loss: 0.6218786835670471\n",
      "rank loss: 0.509010374546051\n",
      "rank loss: 0.573014497756958\n",
      "rank loss: 0.5181798338890076\n",
      "rank loss: 0.5893905758857727\n",
      "rank loss: 0.5071518421173096\n",
      "rank loss: 0.5916610360145569\n",
      "rank loss: 0.49577078223228455\n",
      "rank loss: 0.6667149066925049\n",
      "rank loss: 0.55787193775177\n",
      "rank loss: 0.7543686032295227\n",
      "rank loss: 0.5657207369804382\n",
      "rank loss: 0.4933196008205414\n",
      "rank loss: 0.5938815474510193\n",
      "rank loss: 0.5578587055206299\n",
      "rank loss: 0.49163538217544556\n",
      "rank loss: 0.5314846038818359\n",
      "rank loss: 0.5894911289215088\n",
      "rank loss: 0.6046978831291199\n",
      "rank loss: 0.523015558719635\n",
      "rank loss: 0.7298634648323059\n",
      "rank loss: 0.6604974269866943\n",
      "rank loss: 0.5069760084152222\n",
      "rank loss: 0.5119900107383728\n",
      "rank loss: 0.5504332780838013\n",
      "rank loss: 0.5355781316757202\n",
      "rank loss: 0.5441694259643555\n",
      "rank loss: 0.5605000853538513\n",
      "rank loss: 0.47836539149284363\n",
      "rank loss: 0.5666305422782898\n",
      "rank loss: 0.5526399612426758\n",
      "rank loss: 0.6593759059906006\n",
      "rank loss: 0.6474283933639526\n",
      "rank loss: 0.5728915929794312\n",
      "rank loss: 0.5174154043197632\n",
      "rank loss: 0.5307165384292603\n",
      "rank loss: 0.5334379076957703\n",
      "rank loss: 0.5467674732208252\n",
      "rank loss: 0.5010791420936584\n",
      "rank loss: 0.761025071144104\n",
      "rank loss: 0.5646198987960815\n",
      "rank loss: 0.5473676323890686\n",
      "rank loss: 0.39601632952690125\n",
      "rank loss: 0.4865350127220154\n",
      "rank loss: 0.6478384733200073\n",
      "rank loss: 0.5860742330551147\n",
      "rank loss: 0.46642184257507324\n",
      "rank loss: 0.7153288125991821\n",
      "rank loss: 0.5341423749923706\n",
      "Time Cost: 1625.1807734966278 s\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "rank_loss_list = []\n",
    "\n",
    "start_t = time.time()\n",
    "for i in range(1000):\n",
    "    # One Step Forward\n",
    "    X1, X1_len, X1_id, X2, X2_len, X2_id, Y, F = \\\n",
    "        pair_gen.get_batch(data1=loader.query_data, data2=loader.doc_data)\n",
    "    X1, X1_len, X2, X2_len, Y, F = \\\n",
    "        to_device(X1, X1_len, X2, X2_len, Y, F, device=select_device)\n",
    "    X1, X2, X1_len, X2_len, X2_pos = select_net(X1, X2, X1_len, X2_len, X1_id, X2_id)\n",
    "    X2, X2_len = utils.data_adaptor(X2, X2_len, select_net, rank_net, letor_config)\n",
    "    output = rank_net(X1, X2, X1_len, X2_len, X2_pos)\n",
    "    \n",
    "    # Update Rank Net\n",
    "    rank_loss = rank_net.pair_loss(output, Y)\n",
    "    print('rank loss:', rank_loss.item())\n",
    "    rank_loss_list.append(rank_loss.item())\n",
    "    rank_optimizer.zero_grad()\n",
    "    rank_loss.backward()\n",
    "    rank_optimizer.step()\n",
    "    \n",
    "end_t = time.time()\n",
    "print('Time Cost: %s s' % (end_t-start_t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXmcFMX5/z/PzO5yn4KIXAuKIh5BXPGOoBgBzxhjRPM1GBNzaDT+EhOMUROjxiTGIwY1xmg8okaNBwpeKKiIICCHHKILohwCy32zx9Tvj+mjuvqq7umZ2Rme9+vFi53u6urq66mnnnrqeUgIAYZhGKa8SBW7AQzDMEzysHBnGIYpQ1i4MwzDlCEs3BmGYcoQFu4MwzBlCAt3hmGYMoSFO8MwTBnCwp1hGKYMYeHOMAxThlQU68RdunQR1dXVxTo9wzBMSTJ79uz1QoiuYeWKJtyrq6sxa9asYp2eYRimJCGiL3TKsVmGYRimDAkV7kT0MBGtI6IFPvsHENEHRLSHiH6ZfBMZhmGYqOho7v8GMCJg/0YAVwG4I4kGMQzDMLkTKtyFEO8iK8D99q8TQswE0JBkwxiGYZj4sM2dYRimDCmocCeiy4loFhHNqqurK+SpGYZh9ioKKtyFEA8KIWqEEDVdu4a6aTIMwzAxKVmzzNqtuzFp0dpiN4NhGKZZErqIiYieAjAUQBciWgngJgCVACCEeICI9gMwC0B7ABki+jmAgUKIrXlrNYDzH5iGFRt3YfntZ+TzNAzDMCVJqHAXQowO2b8GQM/EWqTJio27zPODiAp9eoZhmGZNyZplTDKi2C1gGIZpfpS8cBeCpTvDMIxKyQt31twZhmHclIFwZ+nOMAyjUvLCnWU7wzCMm5IX7qy5MwzDuCl54c6inWEYxk3JC3fW3BmGYdyUvHAXmWK3gGEYpvlR8sKdNXeGYRg3LNwZhmHKkJIX7izaGYZh3JS8cGfNnWEYxk3JC3eW7QzDMG5KXriz5s4wDOOm5IU7y3aGYRg3JS/cWXNnGIZxU/LCnWU7wzCMm1DhTkQPE9E6Ilrgs5+I6G9EVEtE84locPLN9Ic1d4ZhGDc6mvu/AYwI2D8SQH/j3+UA7s+9Wfpwsg6GYRg3ocJdCPEugI0BRc4B8JjIMh1ARyLqnlQDw+A0ewzDMG6SsLn3ALBC+r3S2FYQWHNnGIZxU9AJVSK6nIhmEdGsurq6ROpkzZ1hGMZNEsJ9FYBe0u+exjYXQogHhRA1Qoiarl27JnBq1twZhmG8SEK4jwdwieE1cyyALUKIrxKoVwv2lmEYhnFTEVaAiJ4CMBRAFyJaCeAmAJUAIIR4AMBEAKMA1ALYCeDSfDXWC5btDMMwbkKFuxBidMh+AeCKxFoUEdbcGYZh3PAKVYZhmDKk5IU7a+4MwzBuWLgzDMOUISUv3Fm0MwzDuCl94c6aO8MwjIuSF+68iIlhGMZN6Qt3lu4MwzAuSl64s2hnGIZxU/LCnb1lGIZh3JS8cGfZzjAM46bkhTtr7gzDMG5KXrhP/qQO1WMnYOOO+mI3hWEYptlQ8sL94fc/BwAsXL2lyC1hGIZpPpS8cDdpaMoUuwkMwzDNhpIX7inK/t/QxLZ3hmEYk5IX7hWp7CV4ae5bdzdg/fY9hW4SwzBM0Sl54U6G5t7oobkf/8e3UXPLpAK3iGEYpviUvHBPGdK93kNz376nsdDNYRiGaRZoCXciGkFES4iolojGeuzvQ0RvEdF8IppCRD2Tb6o3ts2dJ1QZhmFMQoU7EaUBjAMwEsBAAKOJaKBS7A4AjwkhjgBwM4A/Jt1QmWdmrZDbB8DbLMMwDLO3oqO5DwFQK4RYJoSoB/A0gHOUMgMBvG38Pdljf2Js39OIXz033/pNrLkzDMO40BHuPQCskH6vNLbJzANwnvH3NwG0I6J9cm+emyZFQ08bdplGDv3LMAxjkdSE6i8BnExEcwCcDGAVgCa1EBFdTkSziGhWXV1drBM1KbFkzAnVhkbW3BmGYUx0hPsqAL2k3z2NbRZCiNVCiPOEEEcCuN7YtlmtSAjxoBCiRghR07Vr11gNVgOFWROqrLkzDMNY6Aj3mQD6E1FfIqoCcCGA8XIBIupCRGZd1wF4ONlm2qjC3ZxQZZs7wzCMTahwF0I0ArgSwOsAFgN4RgixkIhuJqKzjWJDASwhok8BdANwa57ai4wiwy3Nnc0yDMMwFhU6hYQQEwFMVLbdKP39HIDnkm2aN6rNncATqgzDMColt0JV9ZYxhbrXClWGYZi9ldIT7kIV7lmh3hhRuM9bsZkTfDAMU7aUnnBXjO6mJl/fmMHrC9cgo2meOWfc+7jgHx/EakMmI9DEZiBGYsGqLXhhzspiN4NhLEpOuKu29QZD2L84dzV+9PhsPD79i9A6zA6gdt32WG044U9vY9DNb8Q6lilPzrx3Kq7577xiN4NhLEpOuKsa8+4GpyZ/0/iF+PDzjY5tQjHl5Gqf/2rLbmzbzREnGYZpvpS8cPfiew9/iNlfbLJ+q4fsYbdJhmHKHC1XyOaEjnAXEJj48VeOY8wYNEDWPg8AlWlyHRvEsrrtWLJmW6RjGIZhikF5CncByGJbXdW6pzEb9sZM0afLKX99J1J5hmGYYlGWZhkBICVp6qpwj6u5MwzDlAplKdwBO8671zF7LOFecpfPMAyjRclJN90wAyQZZtR4NCzcGYYpd0pOuqkrVD0RiubuY5apYLMMwzBlSukJd41cqQICqQCzTD1r7kyCLKuLtxiOYfJJyUk3Hc096y3jP6Fq1kGsuDMJ8P7SDcVuAsO4KD3hLmnht5x7mG+5IM3dEvYcHoZJANYRmOZIyQn3KsmUIi9MUiHy19xZqDMMU+6UnHAfPrCbpZX7yXYBp8klkwGembkCC1Ztyf42hL0p4z/8fCNWb96VnwYzTII89N4yfLaWV0kz4ZSccJchH6O5EMJhc28SAr/633ycee9UY7+z/AX/+ADD7piSr2YyMdi4ox6TFq0tdjNyprEpg627GxKpK5MRuGXCYpw77v1E6mPKGy3hTkQjiGgJEdUS0ViP/b2JaDIRzSGi+UQ0Kvmm2piyORUwI6plc5cICia2avMu3Dphkee+txavRfXYCdiyM5kPmMly6SMf4gePzcL2Pc0/+mbQxPyv/jcfR/wumfDQpiPAzoamROpjyptQ4U5EaQDjAIwEMBDAaCIaqBT7LbKJs48EcCGA+5JuqIwpm7XNMkIV7mY9wnKLDOLXz83HP9/73HPfuMm1AIDaOh4qJ8myuh0AvDvi5kZQE5//aJVRJvfr4AQxTBR0NPchAGqFEMuEEPUAngZwjlJGAGhv/N0BwOrkmuiP7oSq+6Owf//0Px+FnidIM8tYHY13occ/WI7/zAhPIMI4MYV6sTxR1m/fgzcTNAsl0UeVQD/HNCN0hHsPACuk3yuNbTK/A/BdIloJYCKAnyXSuhD8be5OYes2yxjlAExaHP4Bt6pM++4zhZCfcL/hpYW4/oUFoefIlQWrtmBVM58UfvrDL/HVFr02FluOXfKvD/HDx2ZhZ30yZqEkrqepyB0eU1okNaE6GsC/hRA9AYwC8DgRueomosuJaBYRzaqrq8v5pAGKuxXWF3BrPFE1oFZV/sLd7DiCRhFR2d3QpJ0L1uTMe6fihNvfTqwNSbNxRz3GPv8xxjw8U6u8+YyKZYn4YoNpFgovq7MYLkmzjJ9SwzAyOsJ9FYBe0u+exjaZywA8AwBCiA8AtATQRa1ICPGgEKJGCFHTtWvXeC2WCJpQvXvSZ9bf6qrWqHbcYM09+3+S39uAG17D2OfnJ1dhM6DRiN62YUe9VnnLXbVMbBFJXIXZ4bNoZ3TQEe4zAfQnor5EVIXshOl4pcyXAE4FACI6BFnhnrtqHoKusiybZVZs3IlnZ2ez1OvKjZZBwj0TbJaJijnieGbWykTqK1WsRcR5ku2167Zb6x4KQRITw6UwuZxvhBC4963PsG7r7oKf++OVW/DGwjUFP29cQoW7EKIRwJUAXgewGFmvmIVEdDMRnW0U+wWAHxLRPABPARgjCqBy6Q5P5Y/i/Aem4d1Po/U7QWYZa+IvIXXKTLzdoqKklyDkjDB03XwJtOF3vmOteygESVyGVkTUZsDmnfW46qk5ifn3y8xfuQV/ffNT/Py/cxOvO4yz/j4Vlz8+u+DnjYtWmj0hxERkJ0rlbTdKfy8CcEKyTQtHV1uWNfe1W/dYfwvNwXLQCMEUPkm5qZnCvW2LksuAGEzE25O0zf2luatwRM+O6NulTcR2hDeACmQoMfMSNHeT+wPvLMP4eatx8H7tcMWwAxOtu6EpexN2s69/KCWtHupG7I06OakS9PFmFPPBB0s34J2IIwOZbYa207ZlmQl3A13BZIeISEa6X/30XJx+97uRj0uqc0lC6bbdQ5u5dDfIRydkPo8kHRjKlZIW7rpmGd3sTXEwNXbzwxv9z+n43sMfxq5vu6G5t6kqT+Gui/nEkrRE6CxY821Ijuh2Uqs37/JdlVsqi5iS6pC9YI8hfUpauMcxy8joCo7gRUzJmmXqm+zk3eu27cbpd72LlZt2JlJ3MYl6d2yzTHEFWlKCSvcyjr/9bXzrvmme+6x7USJyLR8jDPMepFm4h1Liwl2vnGmni0vQaTKW5u7cXrdtD/76xhJX+TlfbvKNUwM4hcALH63CkrXb8NgH7hWu67fvwYi738WKjaUl+KN+ksVWVhPzc49wziU+UR+DFIjGHN9xXV6auyo8gmoen5kl3NksE0qJC/fcNHc/Gpoyzgkbn/PsaWxyxKmR+esbS3Dv27XWb3P/N++bhn++97nvRJ2upvrinFX4ZM02PPy+d8yb5sq6bXswY5l+5qJi+7lHPf/67XtQPXYCptWuz6keLzI+ivuCVVtw4PWvYvKSddp1LV+/A7e8sihSu+obM7j66bn4zoMfBJYza8yHcm2bZZKvO580NmVyVjKjUtLCXfcBN0QU7mfdOxUDbngttNwpd7zja5ZpUHK91isP1q9J8vagcAKlbHOUF5iFUUjZPmH+V7j4oemObVFHDh8bvvMPvLssp3q88Ov4Zy7fCAB4Z4n+RP6Pn5iNh6Z+js/W6ed/Nc8ve5wFkY83tFQ195P/MgUH//bVgp6zpGftdO1uUYesn6zRi/C4avMudGlbBcD98VYpfuqq8M9qTO7223ZV8jTHlCqyXIpix07C5q6rnV7xpDuIXFSbe8uK7JqIXWpMmiT83H201jgdRy5OBmFfXT5HW6Y7aHOyuW/b3YB12/bggK5tfcsUI+5TSWvuKc3eu7HJ+2UTAjimb2cAwD5tqlz7P1i6wRGjxgvzg1Nf6Kq0s23qt+T3bUX9MEpkXYtDSEZpczGiKTpcZ3Vs7tLf5oK3nfXO9yaJiVlLuCvi1Xxnosg765gI55f0Di3yYpaxrrV4wr1um3PkcsE/puPUv75TpNb4U9rCXfP5Bmkp+xia934dWrr2jf7ndPxx4ieB0sHcM/3zjbhvim1jr0wHa+5+GqmuQuV36cW2UfshoslLi2Is25dXgkZVcCuNTn2XKtwD6nnn0zpUj52ADduDzR1hlxHFOyWOXVzXzz6fr6CwzDK51TN3xWZc/tisWF5uR986yfGsFn+1NbfG5ImSFu66vXdTxt8sYz5c8//NO52BrWrXbQ/8wM0W/O2tz/Dn12zvmErFLPPwVOfEp/oBLFi1BbO/2KQliNZu3Y0nfGLE5+vD2t3QhOuenx8qgPyQmxWlA0rCVh21CvmDj6pxm5fm1tz9eei9rH1+wepgIeEXfiCqRu0kfx1CEq6Qu+qbUD12Ap6ZlY06blpYc43ldOWTH+GNRWtj507eVAKZ10pauOs+YHVyU8b8jk2h+sA7zokwonhD6kplWHHPW85JRFWIn3nvVHzr/mmewkwVhmMemWllKlKZ/eWmyG3V4eV5q/HUhyscHVgU4q4STsbLJLyOuSs2W3/LI72ozbaFe6OyPf713/LKIixZs83X5m6+n/meY9RNoKJ2AkvrtruUJqusEPh8vfe7DGQVGcDOeGZ2cLom2TCa6UA3EUpcuOuVawzQ3E2h8+na7Zj62XrXfiIK1tx9OpgwoeAncHRsoV9u8P8Yvv1AsJtaXHIJB7BtdwN++6KdsCSSzT3y2TzqCKlky64GR9LpeSs2Y4eheWvFlvFI6ajm5I17HXXb9+ChqZ/j4odm+ApXO+x0FKN79LYIn/PcN6UWB0meIOotO/Wv72DE3e951vnsrJUYdscUTFvq/vYAdzIcyyyTo+bud7gQArdNXIxP1oSZWpp/r1Diwj238ANCCIeQ/e6/ZriEV4qCNT+/FjR4dChfSELZ3xUy/KXZUR8/aNJ9U2rxwpzo4YTtxZHRP6r7pyx1xNvJCIEX5qxE9dgJWB9i5knGWyZ4/x4lCNXFD83QPlbF7gT127DDI9zAE9OzZjfzHc8IYcdzV977OAtX49jc/RSPP7+2JDS0wxqfEL3zVmZHTEt9XDKtdhr/28I+tLlaqN/75p0NePDdZbjwwek+R5QOe4dw1zDLWCi/CcEfpl/yiYZG90HD7phin8ZvQjXiOod/T1uOZXX6vsp/fm0JrvnvvGgnQW5JSdQPXwB46sOsDfWztcFtj3o/vAgbbQSNsqIL92htePuTtfjoy83GuewyN7+SXcWcloS7r80d0aV7HG+ZjCpp/epOUKsVyjktm3uO0t1UUvyeb9hz99v/6dptePDdpTm0LDlKW7hrtv4tnzypTYrmDrg1rhRRLHuxlylIriYjspOoaujSOJqqrl9+EkvU4wh3deQkhB2vPszVNAlBEfqhBpwj+oSqMA/UasO0Wnu1rvzs1TmbTEbYIX9d5zS3R384UUw5UTsEnbrNIurtaWzK4K43P7WCqFkjGDOtZZ5cIa32hLw0fnvPHfc+bpv4Sc6RaJOgJIW7uTpNV3Oft9I7486u+qZQYUoUz2MjbAHS+u17cOa9U/HLZ51atFd7wj6Sjq0qtdq0frteijsvhGuArI/a0QkALYzFPrsbgjscnb5uZ30jfvqf2dbkm0rYMw7arRVbRronwvrf3aF5nls+l3QrTG8rO91geLKO/E+oZv8Pex+jmIn8OqQX567GPW99hjteX+KoKywhfSYj8PgHy2PHe8/Fw0cIgV3GefMZiVaXEhfu9rbPbh2Jwb07RqpnZ32Tz8pRGyLKSwhT084658vNju1xzB8tAzJFyYRpyUHkYpZx+RILgZaVepq7zkjm5XmrMfHjNZYgUAmrIegcUb1cTI1NPUznHZJLVKRM4W630S8qZMbHi0b3XNrHhCyWcn87zv1mmATvY52/TVOeqrnb3jLe9byxaA1ueGkh/uLzLvi1zdWe4N2enbUQ9oiiOYRnLknhXmFIdVmDqEynrA9Cl8aMcNnj1YeWJsqLu5RZpSpYvIR+6BBRs31BLqG6xNFrXPcYtuauepWoJOLnHmKNylVz1ymvY9uV3wXzHTeffZM0oeqqw/g/0iImj6qembXC02PMbp95Hm9WbNyF3Q1Nvu9rrcekqZ8ZRN1u/g7LWbxjT1ZZ2KiZiN338cZ47wTsuQA/D72tuxvwmxc+di1yywclKdzN3lF9wF7P+8QDuwTW5ZcYQa4zqZjiprYqo1Z9y4TFkevV1S6DXEJVHp22HGu2SGYO4xxbdzdGTk7sHh0BLYx7EeZlESXNnV/JMK05+BTxbO7qUTq1yNdakXZeU0bI4QfU47L/R9PczRGGfc5fPTcf3/3XDL9DrGP8zDJf/8tk/OypOdZvtZTXUX5NNrer5o2wTEy6NnPzaLVcLqP0jBChmvvf367FkzO+xH98FiEmiZZwJ6IRRLSEiGqJaKzH/ruIaK7x71Mi2uxVT1KYvaM6qeL1zn22Lniyccsu50ozzwnVhIT7qMO62+eRhttR8HppdbXLIK8hmVWbd+Gm8Qtx2aMzXed4ed5qDLntLUf53Q1N+OWz87Bum7fQV6Nyrt68C0/O+BJAdM1dCOG+ByFCLez+BH3Q//toVaTJMb8Q0DqdlHyaqrRqc7fNMi5XSHgLfR2ivH1hmjsATP5EP+xwUDvMSzTf2ZQiNP00d3WCdu6KzZ7eZHHXpwQhhN3p+Nnc7fwP+TfbhAp3IkoDGAdgJICBAEYT0UC5jBDiGiHEICHEIAD3Ang+H401sc0y4WXbhCSa/mqLUyC57nnMCVUv5Akxc9lz1Lr7XjfRtU33RdGNJ91kfFByxxcknCbM/wrPzV6ZjcPjVZ8yYpDdR8MmvtTz9r1uIm4avzDwmLA6VIKewf1TluLJD7/UPldUP3e5Y8l4aO62zV3D5CN9EPWNmcChv+XUE+H907HtN2YEPt/gnUDG6zhTyLr76+x2c71IKpV9juakuZ9wt9cFZH+fO+59nBIhqJffyMtVzqNERgjrGv0090LGO9PR3IcAqBVCLBNC1AN4GsA5AeVHA3gqicb5YWruqq+rp80xovBUH1pTk4emGBP5gZvDVxFgS41Tb9Ryw+98B/co8dXt4be9LegUprayevMu3PDiApfLZdCIIUxz9zrSzxPJ7zHFXS1somu/1akr+Fj7b2tCVdL0fNNFGv/Ln8PZf5+KQ24Mz0kQ5QOxLy1YQr1rLFjLKXKjIiQJhP/OXIF/GHHyUwS8uWgtqsdOcEVpzLY13nMwj4p7fJjmbp0n/4q7lnDvAWCF9Hulsc0FEfUB0BfA2z77LyeiWUQ0q65OP7GASkWIvU0m6semarfb9jRoJycIw6stuxuaLPepuOi6XZkTquZ9WrV5F2rXbcddkz51lPPymw6y15ud7IzPN+Lx6V9gqpKFKKjzCfWWyQhs2L4H67bt9v3g/ETIt+6fhhP/9Hag2eW/M78M7VwJ2bkZHfc6Wzg4t/uHm5DKSO0wo0vKGrZtllHrsAWgSdjaB9ssGFjMeYxHpx8Ekb6Q9BgwA7AVAyJgupTBK5UiPDptOQBnVEZrJKDXxEgricPKOWzuPgpNIUMVJz2heiGA54QQnl+BEOJBIUSNEKKma9eusU+iLmhQtzvOGbHuJ6Y7h+Dv127A2zHtiCpe8nFHfRNG3uMdd8PE/sCD7Xh+vDJ/NT76cpMloM3Ocdtu2+wyYf5Xdn2WL7FdR5CnjTr38enabY7crkGdz54QP/eMAI66ZRKG3PpW6AhFFeKzv9iElZt2Bb4Ej077QkvAHXbT6zj5L5O9d0qX7/eMdISGrFiYGqDcKSQ5oRqlXSY6NncZUur3Gln7e8sYZhnjnhCRY6T+4LvLLCVCPtIqEXJd9oSqc7u2WcajgBC2ouMVfsRRNqT+JNAR7qsA9JJ+9zS2eXEh8mySAWx7pCo0zBflZ6ccaG0rxPBHF79FKF9qJrn2E0JhQu/KJ+fgvPumWVpQQ5PAc7NXOswlchYi24Qrae5Bwl15i26b+AlO+rMtCIM1d31vGb/7F6YNBd2eVCr8/m00IhrqjOD8vmmd11BuB5FbuPstIoozoWoLMXfLJi9Zh4kff+Xa7jdyCCJ0kaBPq82tcoemsyrVnlDVk+676pscZkTzqJ31/i6dfuh4yxTQ5K4l3GcC6E9EfYmoClkBPl4tREQDAHQCkJ+whBJmKjO/F6emurP1dyFmpXV5c5F3GIQwiLIvi6xpy+ibZeyX+JfPzvOdYPVaKxMYPC3kowuayJWDdjU2ZVzXKJ9VbsKCVd6rjr0I+tB1vKEeeX+59rnkuj5ba5tGdASFV65feYuvn3vCmvulj8zET//jkXLQw1wXhqPJEdpnXov57qTIVurc7bJPYo/qves99a9T8DtpQv6sv0/FmEdsrzD5fjz03ufW368vXOM8p8c7JWCPdnU90/JJqHAXQjQCuBLA6wAWA3hGCLGQiG4morOlohcCeFoUIBXQg5cchR+d3A/9urTxa7P0d75bk38aMwI3jV+AQTe/6blftwNTtYnQTiHA3HD/FDs4kt8ciH2s/z5Zc7/2ufk4/HdvOPZ7mSUA70iKfjJcvUw5nCsl6Oqqnuu0u971bdrW3Q2oHjsBb0hCwyv2j9fIxWWWMf736mTfr10fOFcQ5dIjp/NT7q0QAuMm1+L1hWvwrJF8wzbLuA4FYL+jKSJfDxn5UPNVfE0RxiZL63bg34at3kSeI5LbO8mISTWtdj1+9Phsz/oc7cjYZplQE2IB5JJWgmwhxEQAE5VtNyq/f5dcs4Lps08bXDfyENf2qJMpJv26tMGygIQBxeaR95dbE2xeuFL4ZYRn1DxVM/TXqO1ys5ZvxFF9OrkE5JMffoGfDD0AgH+Evm27GzB5SfDEuTyh+sKcVVb7TeTzymYZ+ZxhskbtmOTY4oTcl4rL59e1uZvJVlZLrrhe7XBcv5/vdIDQvfihGRg9pBf+eN4RzvYY/1/00HRMuOok9OjYyrNur7bEtblPWrzOMXr9dk2vgEVMTtMrkf/CJecH711md0OT1mjPMdAwqtrokWjE0+YOIXnL+HxbzcwVsmSwnr1043W0MjXfaamhXqPfNeu6KJqHL6vbgfMf+ACPTlvuqlO2f/rZQsc8MhNXPTUncCGZl81dnoySheX0pZK3hHTOXzwbHMI47BVIUnP3r8m5x6sTkCetzf2q5gv4a85+5pJXF6zxWFSV/X/zzgbcq2QJ88NvEZUf6uruoNXIqpnDXsRkTKgiSHOX5yq86//9ywtxvpTIxu8K5PsUFBbY21vGvdjKj3zEq1IpbakG4InLjsE9Fw4CYD8w5wcRXkdVRfO/DUHXoQrpbbsbPeN4qOX8tAv1VJMWr3Ov3JW0KL+PbvYXmzzPK+P1ETiFnL39cmlo7KXE+Z0lXLgH74+Cjssj4N1Wr1y/8qYbXvJZvBVic9+8s8GKn28fEv2i4/SBOlFXg+qWA9b5ae7rt9XjzHvfw+rNu3yF9kIlP61fB+Xw7rEmZz3KednchbBt7r4TqqXrCllwTuzfBecMyrrdez0wnZc4yORRCqheJJc8/CGG3/mOq9x9U2odv/3cG9UPbf32PYGae5gi5zcRBmQ194feW+ZIvC2PMPyEQzpFGDe5FjMk32c/wt6BKGaZT9cG+4/7J+vIMm3petQ3ZjyFmZpUO1ufV4XkWSboMXy8KveIIFG9ZQjBKSoDj/XIEqphAAAgAElEQVQILeIn3J+dvQILVm3Fox8sz9mP3Eu4e+EVZkLW3MMmVJvLIqaSQ75xHVtVhZYvBc09aPJT9aL42LAtVo+dgFrJJLJcWRbum7RYEYafrNmGFz5yer/KH1qYdpYOiNY5+4tNuGXCYox9/mNrW71DuHsflyLCX15fgu9I6dDkD22TtKo0dIVqBAn0DWmS1EQWKL5rEYTAzOUbcdE/Z+Cg376K+Svdwva+Ke4MPl7VuRcxZf//46uf4CdPeE/8xZ1X+GrLLvz2xY/R2GR3SJEClMWUYpMUz7IUka9wj3MKX7OM9O6reVud58xumyaZCnVs7s0t/EDJ4GFyx8OXHh16XFWFXjz05kqQ4H/nU9sToFdn56TZf2euUIv7sk5Z4u0U7sHH6iSRkF0gGz1sz1G54w07nrcs2LwEeaHMMnIAKy8/cs/jNEaecvtfXeDtJeIRUt8iSOD85vmP8cT0LzFt6YbIcxM6iW5UJ4ibXlqA6rETMH7ealddfnM75rGfrtkWexHZxQ9N1ypnYl6XHFVWCCn8gBG25JlZK7CzPjjybL4oL+Hu0dP26NgKl53YN/C4qhI3y7ysfAgyu6QXq3ObFo59umaZIJbWbcevn5sfWEZHeMpl5Im3uAu3ZGQTlddCqLAMR1HwXcQkgE077Q5Mt/lh5RZ/tRW7GsKFh9qp6V6xWS4j7G5GNwOajifSg0asGACoXbcNj/rEDSKQr1eW+b1PXlKH6cv8E4IE8b6R7tCrA/OeUHWbwoSwO8qMEJizYjN+9dx8/PaFBbHalCtarpClgn1jndu7tW/hLizh62JVIsz43P+Flu24DYq3gp8rpI6GtnD1VsxcvhG/fWGBb2Z7Ex3tWy4ja91BZg5XHdLfLSu9R2OqLZQomlkmDF/NHQKbJeEe7k3hX1/dtj2oHjsBfzzvcFwnmbN02rVkzTb037etVge+u6FJMk24bfvrtu7GjpCkE441J4HlgOF3uk1eJkGau8zC1ba7Y9B751eVfERQJ2aWk01y6mpic/2HHOenua1QLRn8btyY44M1d1O4/yBEwy9F5KBkX2xw+vL7CnfNnB7ffuADraBnOp2FLOvekXzj/Y7cEJIP1isxCuDtjperK6RDe/Mpc8bfpmKJtHhKd+QRJKD8BPvdSiA4AGgSwLwVm3H63e/in+8tU/Z6fznH3/62ZVLLCDs6qinQhtz2FobdMcW3fWFmGdnvPMz8lCLyX6Eq/S2HqY7TZ3t7y3iY8kJWEwPCmsvbsMM2aZrBzwqw1rO8hLuN88aFTZiaPfTB+7VzbC/k5Ee+kGN6q1qWn1kmirDTiZSoZ5axC8kTqn5C8AePzQqsb9xk9+QkAPzuZbc7YZL5LoM+WnlBl27IiDhNu3uS2289kxFYatj8s1EUwyveuKNeinFjC74on0XQu3TmvVO16yEKSIotnUIW7s5YPUp9Plfh9HM3t3mUM/6XB/2ZjLDqzQj7/PKI7aMv85rHyEFZCXfZZ/ahS2rwo6/30zrOzyxzbN99kmpa0fByrzPxC1gWRbjnQ3OXhXtYYDGZl+auxmsLgicqzVWwJqTZPl10hbGXT7tnfQl1PE0ZYT2rVpoJ1QE7HlJGCHvUE8kVUneEElaXO0Cd18GbHfMa0e+dfIRfIhG5brnTUG3u5nHFim9VVsI9Jc28Dx/YDdeNcoco8CKtzNibdG0XbKsvBeJopVEOCQvZC+hN0K6XvHHk8lE9DX78hDvgVRDZ2DKRDvGow/5b90POp+buXY/AW4uzoatbVVY47vGOPY2hCUmEELjooWx+VW3NnZLz59aNLSOHs4gl3GWzjEf9nie1Njlt7ua35/UM2c89InKvGe0475cmqYnWS47rk0g9cYj3gusfU6+Ruk+ng1llpB1UMbPZ54vl63e4Iv7p8Pe3P0PNLZNc27W9YHRt7gktU88IYeUlmLJknSPV4fh5qzH4D95B6ezj7b+X1u1A9dgJ2udNgiCzjPN89t9B751XVUvrtjuyOnl536nncU6oyqYcYXlhFUtzLy9vmZhz0ecN7oH/fbQSQ6RQwUByNncCcOqAffFWQkk/ohDlxapvzGDLroZE/b6B3ARUHB9h3VyxQDaf60tz/V1J/bjjDXvSUr7Fuh1jUPITuc6k5EJ29WT2/zhB8uIIKIJ+Z6dz35atd4fUCCLqe3yqkms16Pv3iqEvJLuMPAEdZLPPJ2WluZtEfQ9POLALlt9+BqqVEMI6rlc6EBH+NeZofGtwT63yz/74uETOC0R7wa96ag6OvnVSohOMUdugst0rtG8IN/rFYCkAuu+ebmrFpLS+pozAYT06xD4+7jOU22/mVvUi7DInLV7nypIWdmyu8xVBZhmvquVtmQwQQcfIC+Ul3AMC/ehy5TA7i1PS/u9H9emkVU53kYgOUUwsZgzs2oAojvlug8q23dGFe5AQSZrl63dYXiiAvjDeFeIfbtcXq1ke9fgn2NYhzjMUMY+Lfh7vc8iL09Rm6MSgoQDpvnlnPb7YsEOpRzj+CrrfbHOPyNGG8OzTubVr350XfE2rjisk4Z5UMluzmtFDeuF/PwnXyuN2Kt+p6eXa5veC/eK0g3zr8Y0+GJNcBJRf/JvmwtA7pjhiwuhe6+6QxOB2fdFvntf7kxEip+cQdwShbZaJVbtxrM/Bt7yyKIdapZC/Hq27+um5OPkvU5RsZfLfIrBjK0TI37KyuX/v+GoMG7Av+uzjztA0pG9njyPcyPI8qTDv5ktCRDiqT2f07NQqm7jZh7DMRn60bel+nH4fl98KznyQi2lhQ4gXhxf51hb/9NonOZ9bt4lxriWdIlenvnLTLnyxQS9Xrxe6C9scCP1nn49H9mKMuRQZHd3O5Qpp/S0SDWsRh7LS3InIU7AD+tqwbBJJzubu/N0qRLB6mWVaaESu9M7k4/2CtfBZwZkPcnnHN8UR7vFPp8X9HtEbTZL0jJi/cjN+/3J07dNLOchFsANwmJ50ERDxOoWo58nTAw+LNQ84nTgyQjhCoCQ9dxUVrS+ciEYQ0RIiqiWisT5lLiCiRUS0kIieTLaZuaMrqOVifoGKciXMpi53RD8+OZvKTkebDwt6dEFNT7RrkdXudTqL5kCY/7UXXxmp6zq3CQ/3nDRJfc8CwNl/fz+WUA5auBYXr3DEYYgomnsOXfKir7aGF1LQ+bJts0xAGUVzN8mawZq5cCeiNIBxAEYCGAhgNBENVMr0B3AdgBOEEIcC+Hke2poTuvZzWfAmlTVFrSWsKbJwN4WwTkfj9TLJ2kNVRcp6UVsUIMzx4N4dc65ja4wJVZNiJGFJ7IMusmBIguyEql7ZGTGjOeriCj8QweQSdA2qQJ9jhBfIiGBTVnOZUB0CoFYIsUwIUQ/gaQDnKGV+CGCcEGITAAghCu/QHYK+Wcb771yIat2R22rGxdFpf0YA1fu0VrbZb1FFKmXZcAuhuV9yXHXezxFEMXLjJvXRhvnBlwq6nd0HGhm1ciGOk4Ida97/GuTrm7nc7qBKxebeA4Cc1WGlsU3mIAAHEdH7RDSdiEZ4VURElxPRLCKaVVdXOHc1IIpZRtLcExPuFPhbRW6rpbnrrM7LCEy5dphjm/x6VabJ+l2ICdVih1KuKopwT+aDjmNqyCdxHuVNLy3UWsFcCObECNgVFDjMRN4lmxAzGRHoZ19Ki5gqAPQHMBTAaAD/JCLXmFwI8aAQokYIUdO1a9eETq1HQKa3vKNGTgz7UOS2tjVs5I0aH4m3zd3elk6lrBe1EJp7sXPTFkNzL/IcWt6oiHEv65symLR4bXjBZoqOcicHgLv3bTtHsUDx3wWdJ7YKgOxA3dPYJrMSwHghRIMQ4nMAnyIr7JsNcRYGJbWYSI09HppQWpLunYxJwd0aAbq85L9sc0+n7CFmIbxliiFcHeevKC2b+5M/PCbBliSLVxx8HXQCyxUDnU/7pbmrsX77nkAt+/v/9g49nRHJZviKg87XNxNAfyLqS0RVAC4EMF4p8yKyWjuIqAuyZho1I0BRiWUiSEg2RPUEkDX3Tq2zwl0d3g7u3RHLbz8DPTraeVG9TAJyp5AmsjT3jq3z70kSR9tLkqJo7jmoa8cf0CXBljQPooRsLiS6zhL3vvVZrIkUIYLNMoWYUQ19+4UQjQCuBPA6gMUAnhFCLCSim4nobKPY6wA2ENEiAJMBXCuEyO8MSURMLTyKjE/KW0Z9jlZykG7tPEo7be6dWld6ljHfm1GH7ydty2687ZuHW9s2SSs801KvUQh7dGWMDlXurHI166z2iTSZT4o9FG9u7NFciducifNIS8IVEgCEEBOFEAcJIQ4QQtxqbLtRCDHe+FsIIf6fEGKgEOJwIcTT+Wx0HEw/8WuG+y+7zxfqQzaFu1/asIpUCo9cejROPqgr9peEnYyppcsBqPp2aQsAuOiY3tY2eZIna5Yx2pAi3PUdvZAMugzq5ZxmyVVz79Aqt9HFsf0Kn2zl75NrwwvtRby2IHo45UIQxeIaR0aHLWIqpQnVZk8qRVh++xn42an9cdNZA/HEZcH2zR+ffEBirpDqMza1Zr+FSakUMOzgffHo94egTYsK3PbNw/HUD491lDGr3G74gX/7qJ64YtgBrrrkYXEqRVankCbCN4/smdg1AsBjlw1xfDR+nZcuHVrlFh3j6Gq9kBNM/sjHgqpCkp0YjS6KwzT35uLnXnZcekJfnNjf3765/PYzMHbkgMRcIVVbuDnR5zcPoG6/6Jje6KuEIzZfHDNq4mkDu4VqynJnYv6Z5DuWIqchqzJHFyV5odXYkQMiH19sb53mzDmD9i92E0qCxz74Apuk1H26ZFfn5qFBEdgrhXuhUXtpc6LP79l7CX11k+mBZQr3di29bfPOOuwJ1aD8kHFJKdlyknQ/jSOm00X0f33yB83X8wVIzFegWXPKgH1990WJ+Pq3t9wJx8PIZIQVBqNYsHAPIKkJVXV4Zgp3czZ9SN/OmHDVidZ+rwVX6sto1nnDmQNR06cTjtRY6p9O2YuY8rHAiODMcxnnHPIoRwB49eqT8Oj3h8RqT66a+/4dWsY+Vkd48MgiP1SlUxjYvX2gzXveiuiLmqIgEK9TSBIW7gHodu7v/WoYHh5T47tffcVMm7uZJPnIXh1x6P52lhwdzd2UgYf37IDnfnK81orTCsnm7iV3r9dMKO4HERwqYa7rBIQQOKR7e5x8UNdYJrKKHDX3dA7CV6djK+Y6gGKbDPJJRZo8wx4XEjkXqxeFiOfOwj2AsM/zkO7t8ciYo9Grc2v09kgQYuKaUDVWhzYa8UNUzd5L61MFZZxJnpSkuavnGDtyALrloKkCTpv7T4YmNyEdl1wndHPpnHTkdpy4/T/6er8YrXFTbDe9fFKRIqQImFq7vmhteHz6F0U7twkL9yCMj/vqU/ujTZVbM+7XtQ2GGXa9IEHgmlA1hE6jYTjXUTCSEO7yIiZVszxvcA9fYTz8kG5a9cs2928e2SOxTFZAPBNZmNkjTLjmItz1zDLRPz81X8GY46sj1wEkO5EelNWrGFSmU3kL150U7C1TZKzAQX4FpB1BJgC/CVVTqOsMH0mpPuwIrwm9lIe3jEmayFeY/eP/jgptX7ZOsswn6uSqzMWGH/4QD1dF+bpy/QDCJlS9MlfJ5CIfdALVeY0s+u/bNtIxneKuNE5QuLRvFT6ZX0gq0pRYop18wX7uRcZOkCvw5/PdC35k7TnIPqtq2acNzGrCg3t3MqoPf9SqoAw75PgDu1iJOUycrpDO+tLGUFblN6MGaNmP9+/QEqmUrV+rbpEy155+MN645ut45NKjXfvMe6IS51uVV8h29FjpG3YPa/rYnU/3Di3xt9FHap9bvmd+SUO8BNDLPzsRc288zbdeNYtXXNNTkmYZOQjd947rk1i9calIpTBvZX4nTEuBvVq4v/DT4/H8T4/33S9nYjnjiO745Tecw0/5+5A/1H2Uj1n9kIYevC9qbx2Jr/XqYOzPbh89pBd+dLK3TdXlCqnxcaol0gHCPZUiT1PC5V93L4zywjw2pfzvRTpFOKhbO7Rp4dac/3rB1/CToQcY7c9NAMl+/0f0dHsThd3Dm8891Po7nSKc/TV933D58g/p7h1mwst00LIyHRj3R504j+txk6RZwJxDalOVxpC+ua0KviSBzqEyTWUTDz8X9mrhfmTvTr6aIuDOxPKDk/rhimEHWJqKLHxkwam+Vl4fUkU6ZQlEU8j88bwjcN1Ib4+VODZ3v7AH2b+dZYPMMlGwzTLkGxUvyLOnZWUa5w5S0wU40Z2IlLXaP553uGu/EMB9Fw/2PV5eRBX13jjeB59HFcdVVNXc48YIiqK5qwlgVKoqUnj2x8fhzf93cs6T6KOH9EafkPOFoRv2YmD39ujV2Tu8RzmwVwv3MGybe/ZDaFmZxrWnD8A9Fw7KbnfY3IMmVL23m4fofGiqbNFJPKxWK7dRFSx+ZhldzPaZHRaRv7lJVzjnKhTlFbJeE+IZITDq8O5adUW9N/JIzny+Zx7hPFfrKnvk8n/H6mmsLZVQza08rkuHKHrtEyELsqrSKRxd3Rn7d2yV86ruVpXpnFeX6L5fRMDEq07CBTU9czxjdHhCtcj451C0zTUmss1dFWp+5gVTAOgI6jhatUtzl1561QSTFe7RzzHuosFGfXD8n06RrxdQmCeJdd99jpE/3v3a+7tvypq71zmjaK9RvS/IIdyBRTefjnsudNrs5dg5fzj3MK16Va00rq98lIxRYROmh3RvL/3KTTQnkSFM956kiNCuZaVvcL5Sh4V7AHYORXV79n8/mzuQdS00NQI/IZdSzDJBqIJXx8PGZXMnwj/+7yiceKA7rk6aKJb7mOnfb85PyDb3uNqJVyvkbemQiVITr1g6MlHWuET1vnCMLkRWS1dHHB1ieJmo7Ygv3PXLBmnCM68fjl6ONR7Oiq8+NVrOHtXsFAfdSWZLISlCMIZfnX5w3s/Bwl0D9UPwsq6rH+6dFwzCd47uZRzv/SWZL5een7vzt9aEqlImnSacfuh+nsPslGSW6RFBk1E1dttbJvfl9f73za63rcekrIms5XqNSqJorzqyXY406mWWUYkj3FXvTq973K9LG88OvFv7FtJktT5BZrAwE9k1EX3gW1blLpJ0A9bJ72qhKYQfPgv3AGzzgPfEpJ/N3d5saube9UfR3FWzwtCDw3PQqtWGaZ+W1u3xVsy8frhLkHRqXekS6iRt6Ne1Lf5y/hGh7VTxMsuo5wWAMw7vjruN+Q8TOaZ8kOsnYN+fF684IbRNOv2A0wxkb/d7vl7eQmGoK6HV9RUtK7OTm17ulx1bVVn3JKhTVAl6b/zCYuhyUv8uuPMC2804iSQy2u6h0vxQOcLCPQBruKZq7h4ToUHxYPw0RPOYqD7HU389DLec6/b+UFHrDdOyLHu5x9vetV0LtFciT2bjwZsfiGmOye4zT/3tml6Ijvv8cpMev+wY/O6sgRh38WD07OQUdlcOO9D6WzZZeH3AlxxXDcCdZMQLnSckn0++1wfv5+0KqSvI2kuLrdTon5VKovPzj+qJfdq28KynMZOx7uwBXdvi8cv8A7LJcwBB743OStyg9QFH9OyA8wbbE5pE3i65UYiaJCbJldTNCRbuAfhpkGETfqZgszXz4PqjBjjq2am15VschFptmHA326s7qdWYyXho7tn/E/EGkOqQW96rc2uMOaGv5yHydxrk119760jccKZ+oDQdE47fSOGmsw71Kq5lL3/okhpMvPoknHdkD88sYv2MOP+mRm8206u16iWc1N85+jumr71oq1cn2zQXJPzUXV7nPbavf9KUpOzdU389DIf1yE7sVqZIL/6/cUPkazjhwGSzd/Xo2Ao/PMn7Xc03WsKdiEYQ0RIiqiWisR77xxBRHRHNNf79IPmmFh7zmasftrW4KeR7t8w3Pnqfl3knn3jZZydcdSL+cE5W+Jh7/YS72syGJmELMeO/zm2yWmMuJsX9jABml54Y/aOQhWqlYiaR2ySvM9BB5xHJ1VVVpHBQt7b4+0VH+t7PIOE+9dfDMOM3p2L4wG7o2ak17vzOIFw93D052atza8y54TRcZtyroEvyW3fghe690Vo5HVBVUkpzl7YtcEDXbOiGirTegrN5K7cAcF7D90/oi+W3n5FMo5BVMH6YULC3qIQa3ogoDWAcgNMArAQwk4jGCyEWKUX/K4S4Mg9tLBq+WmiITVg93s/VMa5ZJi5esVYO3b+DFW7YHEGovtQmDU3OC2lssof55v+PjDkaby5ag309XBTfuXao1uRx2xYVsT+wIM09RRT7Xusc5ohlT4Q3rjk5sHyQbVg1NwXRqU2VdV1B9nF5hBi2+ldX5rps7h71BmnnSRlEKtMpa/6hIp1CO8mU1boqHZjuT26DvHAtCQREQZLRe6Fz1iEAaoUQy4QQ9QCeBnBOfpvVPJDDDzi3Z/EbqgtluOf3GZkfRqHiToct7jCTbftpmo3Kku7GjP0pm5refh1a4v8MW7ZKn33auNIFygTltY2jScqTjUGBzHSIvNBM41RJJusw3yHTC8Pr3cxkhNVIc/e719rmDBnde6VTLqhIUvbudIqs+1mZyvqv/9wY6YQ9O/kaWvgoNl6MHtLbtS1F2dHwkz+03+VcE8XHReesPQCskH6vNLapfIuI5hPRc0QUZxat2eGnufu9kPNu/IazHPw/NLmeQuUUCLPxmsm0W/poL6rmLv9O4hMNymuri/xo5IFKdqLO/7h+Xduge0A8+6iau47MSjJZh5/mLidXbxLC9Zx679PacsmUD9WVuS6bu8d9CqrK6zxx3yVzJGQKU3Nxkt+z+4GHKauFMpdluo760V6JLJpOEQ7dvwO6d7DnLJLw3Y9DUm/XywCqhRBHAHgTwKNehYjociKaRUSz6urqEjp1/lGHmrbm7izXukX2IV5lLNwwhYvfy2VOgh3Vxz++TZKETajuNjR3vyXtjUov1NgkpJys/vX+4rSDtMMG+2HWf96RwXFn/DR3uQ4v3v7FUHxw3am+cwU6Qcxk90IdAZWkRmf2s+ozbtuywjJRhCkRsvmEALxxzdetFcg6x/iWCZqQhTmJn/KNnqmL+bxNDb7CGsV4l+9txLCR26c6Kvx6RPDE7IDuzlGP6jUGZJ/J538cFdL65NFxdl0FQNbEexrbLIQQG6SfDwH4s1dFQogHATwIADU1NQXSV+Pjl0Tab6K0Mp1y2IrNF9dvWHhI9/aY8suhgVmckiTMDGDa3kcd3h0vzFkVWBbIesuY9yDoI/9ZxFWKQZgdqB8OzV1pko4gmvrrU3D87W+7tuto7vu293ZB9KMqQbOM+Y5ZZhlpn6nRZhw292CIspE7D+rm7cZp4ra5e9QVeJ7s//NvOj2kReGYglm2vWfb5BfjyN25RrW5//OSGixYtQUXPzQDgJ13V3W6KIa7pY7qMBNAfyLqS0RVAC4EMF4uQERyRKSzASxOronF4xtG3PULFF/tsIlSE53AYNVd2uRttZo6ZAzTFA/erx1qbx1pxZsP46Jjemtp7kmgW71D+1QapdNGvzgjOsKdQFYMfR3NRV03oIvX62La3FWzDMEWYk1CxDa3+LclR5u78X9VRUrLvTcIU1OvkGzvMsf07Wx903I52WyqmmXC6NCqEicc2AXLbhuFv377a3jSMIE1B0KvRAjRCOBKAK8jK7SfEUIsJKKbiehso9hVRLSQiOYBuArAmHw1uJD06tway28/AwP3V4Zexv+hHgcFdnVUUYe5OtHydEwFlWlC+5YVOGVAt4JfW7j7qf++OP3Pcz8+zjiv3oSqNU8T0vHfdNZAnH7ofjFaZL9X/brak9PqhKqM+dyjTNzrehW5be7RvGWCFJtRh0e7P+a7a85lmCYq01tld2PGcV1eI9m48yCpFOFbR/VsVkHItNYgCyEmApiobLtR+vs6ANcl27RmjM9Eq4q1WjO/rfHlxP5dsHzDl9bvOAmZvfj4d6dbH45llsm76q7pvZEiPDymBq/M/8qjiuht7G5Oyhm/H/juYGzYUY/rX1gAILty18x0TwCGDdgXL81djcqK4HNd6rMISwez5hd+YodNCHKF9DLLhL28up221j2N+Wr8dOiBmPjxGgDZtJETF3yF92s34PP1OzzLm5q6eTrzultUprGjvgl7Gpoc80ama7B8rVHcFoO9gLSryRu8QjUGfi6SKqavd9wkxrly01mHOuKmJDWB17IybWk4llkmkZrDCbvnKQJOGdANd14wyLUvThvVcAojDuuOi4/pY+3/YOwpjvJ/Pv8IvHvtMEes9iAuqOnpmUgkCFNwyB2IZZbxeMSmWSYj9N/dJNdeRBaCxjbZ5HP8gV1wy7mHY/Ivh/rWZdvYs/TfNztfcM6g7IKm3Q1NDnde1WwDBMfH7xfgxqvSwYh/FCV7V9JEj1zEYOD+7VGRIvzslAMDy+WyGCcJKtMp30BaSdNcbO7ByyHjnDd4UrzCEb+G0CKdsrwwdPDKzRvaJiIAwmHuaFImVB3J2w0NNmyFqvwMk1x7katro/Z5lOK9OrfG4ptHYMOOPXjk/eXY1dCERmmirNoQ1uboc/SQ3oF2/6OrO2PZ+h04rEd7LFi1NbAt7VtWYsHvT0frIrlBAizcY9GhVSVqbyu8a1OuxE2mHIS5mlX26y0mQR1YTpq7RtlCjV68zpPxm1AlOwRuJiMsj56u7YI9e+Iq7l7utkGmm90BK0ej6iJexVtVpdGmISvmdtU3WU4Q1486xEoyYl5ru5bB4tDySNLUZKJE3swHLNz3InTjXPvxh3MOxasL1ji2HbhvO9xz4SAMPXjfnOoOw0wyfXR18JqAIF9+Xa+k60YOwDrDjq47vwIUzs5qr3y2G+Xn5w7YWcKahMB3anqhXcsKjDrMdnAzr230kN6YtjTr1RzXLDPi0P3wk6EH4P4pS+32BpTf0+SeeZbLX3JcH4w8TC8Voo9vDe4AAAsNSURBVHntexqcHYZpaiEiS3Ovlkws1irrkPp1yzUXWLiXEA+PqclpNj6Ky+W1px/sEhT/d1y1Z2iBc0ISWgfx5jVfx2l3vRta7qg+nTHjN6eiW0BaPSB4dKJ79T862V6VaJs+ms+yjEfGDMHj05c7Vj4GaZXyYp5UinDmEd524M5tqjD8kH0xafG62KumK9Ip/HrEAKdwD7jxGY8TnXpINyytW4aOratw8zl66QcBe47L6pgNsrmPD8bwQ7rhV8/NM9ppN8rUsDfsqA+sP6P4/cYV8i9feWLMI6PBwr2EOGWAnv95ElwxLHg+ISn6G4tkdHzrwwQ74L0wxSSOt4y9VsG5/ckfHOMybejW/zWN+PFBHHfAPjjuAGdoWntC1V+462CHxEjS5u5/fnXVM5BdFfqDE/uii09cej/6GIsBvXzVzffZTJDSJE2sHtk7+zz8AuZZGIcc27cz5q3YHNuV9fCeHWIdFxUW7kzRWfj70yMvHvEjaZu7n7A73iONnQ5LbxuVl2G9a0JVIizshIzVmSU5oRpRc0+nyDOqqB/HGx3dET074I5vfw3DD/E3EQ7u3QnTlm7ATsl0c+j+HfDY94dgcEgYELOlB+/XrqiOErqwcN8LeHhMDd7+ZF2xm+FLnHRzfgRnDYpeX/uWFejZqRWuG6mf2COIKII2Cj8f3h+bd9bjm0bsHdkeT0S4ZvhBOGVA+LxIWIKZpPHS3KNirgolIpx/VM/Asled2h/7dWiJUYc5te6vH+RMXHLuoP1dYZfNDr45+LDrwMJ9L+CUAd0KatIpJsEeQdG/yop0ClN/fUpgmSuGHYBxk5cGlsk3+7ZrifsudgdnM00iXok+vDA1/0L5uRcq3LVJVUUK3z22T2i5uy90pwZUM6w1d3gRE1NWJK2563Dt6QNKYpiuQ5Sk7boE2dwLLdxzoVBJdZKChTtTVgROqBawHaXETWcdimP7dcZRfTrhHGNF5RE9wyd9jwnIjapLlNR/xUZNTNPcYbMMU1YUQ3MvdQ7erx2evjwbIG34wG7ao5A+mitxo06oNlsss0xxm6ELC3emrAjylvFLH1iOFEIhvv6MgYH7zSxGQbIwiQnVfJNOETq2qoy8QrXYsHBnSp7KNKHB8FsOmlB99NIhGHrHlAK1qnmQLznUr2sbKz2fF46kNZpJu5sri28eAQB4bvZKvLpgDQ7ct22RW6QH29yZkuelK+wVf0E29+oIUf0YJ09ffix+9PV+1u8ooSyC+pdSEO5mIpHRQ3rhoxtOC81O1Vxg4c6UPHIylVKxh5Yax/bbB9eNsn39o/jry4r7xKtOcuwrJQ8UIso5z2shYeHOlBWl4slQ6oy7ODhxtoz5TK4+tT8G7t8e4688AX85/wgAuWnuSa1qLlfY5s4wZUi+FeK+EU1csg3+iJ4dsdEI0pXLhOqHvxmOPU3+IYP3drS6PiIaQURLiKiWiMYGlPsWEQkiqkmuiQzDxKW5jmPs7FDxhXuH1pXYt51+DJq9jVDhTkRpAOMAjAQwEMBoInL5QBFROwBXA5iRdCMZholGny5ZH/ROzdRGnJbCEDP5QUdzHwKgVgixTAhRD+BpAOd4lPsDgD8B2J1g+xhGi8N6tA8vtBfxi9MOxiNjjsax/fYJL1wEhvTtjDHHV+Mv346eZhDQX0C1N6Njc+8BYIX0eyWAY+QCRDQYQC8hxAQiujbB9jGMFs/+6Hhs29NQ7GY0G6oqUhimEQWyWKRThN+dfWisYz+7dWSzNTc1J3KeUCWiFIA7AYzRKHs5gMsBoHfv3rmemmEsWlWlAzPXM+VDZZq9ZHTQEe6rAPSSfvc0tpm0A3AYgCmGy9N+AMYT0dlCiFlyRUKIBwE8CAA1NTVsbWMYZq/hqR8ei9YFVEB0hPtMAP2JqC+yQv1CABeZO4UQWwBYaWmIaAqAX6qCnWEYZm9GTY2Yb0LHN0KIRgBXAngdwGIAzwghFhLRzUR0dr4byDAMw0RHy+YuhJgIYKKy7UafskNzbxbDMAyTCzwzwTAMU4awcGcYhilDOLYMwzDa3HTWQGzeyesJSgEW7sxexT0XDkLXdi2K3YyS5dIT+ha7CYwmLNyZvYpzBvUodhMYpiCwzZ1hGKYMYc2dKWvGXTQYbVpwWAJm74OFO1PWnHFE92I3gWGKAptlGIZhyhAW7gzDMGUIC3eGYZgyhIU7wzBMGcLCnWEYpgxh4c4wDFOGsHBnGIYpQ1i4MwzDlCEkRHFSmRJRHYAvYh7eBcD6BJtTCvA17x3wNe8d5HLNfYQQXcMKFU245wIRzRJC1BS7HYWEr3nvgK9576AQ18xmGYZhmDKEhTvDMEwZUqrC/cFiN6AI8DXvHfA17x3k/ZpL0ubOMAzDBFOqmjvDMAwTQMkJdyIaQURLiKiWiMYWuz1JQUS9iGgyES0iooVEdLWxvTMRvUlEnxn/dzK2ExH9zbgP84locHGvIB5ElCaiOUT0ivG7LxHNMK7rv0RUZWxvYfyuNfZXF7PduUBEHYnoOSL6hIgWE9Fx5fyciega451eQERPEVHLcnzORPQwEa0jogXStsjPlYi+Z5T/jIi+F7c9JSXciSgNYByAkQAGAhhNRAOL26rEaATwCyHEQADHArjCuLaxAN4SQvQH8JbxG8jeg/7Gv8sB3F/4JifC1QAWS7//BOAuIcSBADYBuMzYfhmATcb2u4xypco9AF4TQgwA8DVkr78snzMR9QBwFYAaIcRhANIALkR5Pud/AxihbIv0XImoM4CbABwDYAiAm8wOITJCiJL5B+A4AK9Lv68DcF2x25Wna30JwGkAlgDobmzrDmCJ8fc/AIyWylvlSuUfgJ7GC38KgFcAELILOyrU5w3gdQDHGX9XGOWo2NcQ45o7APhcbXu5PmcAPQCsANDZeG6vADi9XJ8zgGoAC+I+VwCjAfxD2u4oF+VfSWnusF8Uk5XGtrLCGIoeCWAGgG5CiK+MXWsAdDP+Lod7cTeAXwHIGL/3AbBZCNFo/JavybpeY/8Wo3yp0RdAHYBHDHPUQ0TUBmX6nIUQqwDcAeBLAF8h+9xmo/yfs0nU55rY8y414V72EFFbAP8D8HMhxFZ5n8h25WXh3kREZwJYJ4SYXey2FJgKAIMB3C+EOBLADthDdQBl95w7ATgH2U5tfwBt4DZd7BUU+rmWmnBfBaCX9Lunsa0sIKJKZAX7f4QQzxub1xJRd2N/dwDrjO2lfi9OAHA2ES0H8DSyppl7AHQkIjNxu3xN1vUa+zsA2FDIBifESgArhRAzjN/PISvsy/U5DwfwuRCiTgjRAOB5ZJ99uT9nk6jPNbHnXWrCfSaA/sZMexWyEzPji9ymRCAiAvAvAIuFEHdKu8YDMGfMv4esLd7cfokx634sgC3S8K/ZI4S4TgjRUwhRjexzfFsIcTGAyQDON4qp12veh/ON8iWn3Qoh1gBYQUQHG5tOBbAIZfqckTXHHEtErY133Lzesn7OElGf6+sAvkFEnYxRzzeMbdEp9gREjAmLUQA+BbAUwPXFbk+C13UiskO2+QDmGv9GIWtvfAvAZwAmAehslCdkPYeWAvgYWW+Eol9HzGsfCuAV4+9+AD4EUAvgWQAtjO0tjd+1xv5+xW53Dtc7CMAs41m/CKBTOT9nAL8H8AmABQAeB9CiHJ8zgKeQnVdoQHaEdlmc5wrg+8b11wK4NG57eIUqwzBMGVJqZhmGYRhGAxbuDMMwZQgLd4ZhmDKEhTvDMEwZwsKdYRimDGHhzjAMU4awcGcYhilDWLgzDMOUIf8fGA2dsWWjXikAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline\n",
    "\n",
    "plt.figure()\n",
    "plt.plot(rank_loss_list)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(select_net, \"qcentric.model\")\n",
    "torch.save(rank_net, \"deeprank.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[./data/letor/r5w/relation.test.fold1.txt]\n",
      "\tInstance size: 13652\n",
      "List Instance Count: 336\n",
      "[Test] 0.5042477865174272\n"
     ]
    }
   ],
   "source": [
    "select_net_e = torch.load(f='qcentric.model')\n",
    "rank_net_e = torch.load(f='deeprank.model')\n",
    "\n",
    "list_gen = ListGenerator(rel_file=Letor07Path+'/relation.test.fold%d.txt'%(letor_config['fold']),\n",
    "                         config=letor_config)\n",
    "map_v = 0.0\n",
    "map_c = 0.0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for X1, X1_len, X1_id, X2, X2_len, X2_id, Y, F in \\\n",
    "        list_gen.get_batch(data1=loader.query_data, data2=loader.doc_data):\n",
    "        #print(X1.shape, X2.shape, Y.shape)\n",
    "        X1, X1_len, X2, X2_len, Y, F = to_device(X1, X1_len, X2, X2_len, Y, F, device=select_device)\n",
    "        X1, X2, X1_len, X2_len, X2_pos = select_net_e(X1, X2, X1_len, X2_len, X1_id, X2_id)\n",
    "        X2, X2_len = utils.data_adaptor(X2, X2_len, select_net, rank_net, letor_config)\n",
    "        #print(X1.shape, X2.shape, Y.shape)\n",
    "        pred = rank_net_e(X1, X2, X1_len, X2_len, X2_pos)\n",
    "        map_o = utils.eval_MAP(pred.tolist(), Y.tolist())\n",
    "        #print(pred.shape, Y.shape)\n",
    "        map_v += map_o\n",
    "        map_c += 1.0\n",
    "    map_v /= map_c\n",
    "\n",
    "print('[Test]', map_v)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Reward v0: 0.4359386539000405\n",
    "Reward v1: 0.42572616969349864\n",
    "Reward v2: 0.4245778777643799"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nips",
   "language": "python",
   "name": "nips"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}