chess.com_exporter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chess.com Exporter\n",
    "\n",
    "### Download Data from User Stats and Games Played Results and Export to CSV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "\n",
    "API Docs:\n",
    "\n",
    "* https://www.chess.com/club/chess-com-developer-community\n",
    "* https://www.chess.com/news/view/published-data-api"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configure User Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_user = \"markwk\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries and Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import time\n",
    "import json\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get User Info and Current Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getChessUser(username):\n",
    "    r = requests.get(\"https://api.chess.com/pub/player/\"+username+\"/\")\n",
    "    user_info = r.json()\n",
    "    return user_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_info = getChessUser(my_user)\n",
    "# print(json.dumps(user_info, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018-08-30 09:43:58\n"
     ]
    }
   ],
   "source": [
    "joined_date = datetime.utcfromtimestamp(user_info['joined'])\n",
    "\n",
    "print(joined_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getChessRatings(username):\n",
    "    r = requests.get(\"https://api.chess.com/pub/player/\"+username+\"/stats\")\n",
    "    user_stats = r.json()\n",
    "    return user_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_stats = getChessRatings(my_user)\n",
    "# print(json.dumps(user_stats, indent=4, sort_keys=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rating': 980, 'date': 1537067051, 'rd': 45}\n"
     ]
    }
   ],
   "source": [
    "# latest blitz rating\n",
    "print(user_stats['chess_blitz']['last'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Archives List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getChessUserArchives(username):\n",
    "    r = requests.get(\"https://api.chess.com/pub/player/\"+username+\"/games/archives\")\n",
    "    a = r.json()\n",
    "    archives_list = a['archives']\n",
    "    \n",
    "    return archives_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "archives = getChessUserArchives(my_user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# total archives / months with games\n",
    "len(archives)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "monthlist = []\n",
    "\n",
    "for i in archives:\n",
    "    full_archive_link = i\n",
    "    month = full_archive_link.replace('https://api.chess.com/pub/player/' + my_user + '/games/',\"\")\n",
    "    monthlist.append(month)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function\n",
    "def months(start_month, start_year, end_month, end_year):\n",
    "\n",
    "    monthlist = []\n",
    "    month, year = end_month, end_year\n",
    "\n",
    "    while (year, month) >= (start_year, start_month):\n",
    "\n",
    "        str = format(year) + '/' + '{:02d}'.format(month)\n",
    "        monthlist.append(str)\n",
    "\n",
    "        month -= 1\n",
    "        if month < 1:\n",
    "            month = 12\n",
    "            year -= 1\n",
    "    \n",
    "    return monthlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Alternative Option 2: Get all Months Since Joined\n",
    "start_year = joined_date.year\n",
    "start_month = joined_date.month\n",
    "\n",
    "end_year = datetime.now().year\n",
    "end_month = datetime.now().month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Alternative Option 3: manually set dates\n",
    "\n",
    "# start_year = 2018\n",
    "# start_month = 1\n",
    "\n",
    "# end_year = 2018\n",
    "# end_month = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncoment to run option 2 or 3\n",
    "# monthlist = months(start_month, start_year, end_month, end_year)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download Data on Chess.com Games Played"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getChessGamesPlayed(username, monthlist):\n",
    "    \n",
    "    games_list = []\n",
    "    \n",
    "    for i in range(0,len(monthlist)):\n",
    "        print(\"Getting Game Data for \" + monthlist[i])\n",
    "        \n",
    "        r = requests.get(\"https://api.chess.com/pub/player/\" + \n",
    "                         username + \"/games/\" + monthlist[i] +\"/\")\n",
    "        \n",
    "        games_obj = r.json()\n",
    "        games_list.extend(games_obj['games'])\n",
    "        \n",
    "        # Wait for 10 seconds\n",
    "        time.sleep(10)\n",
    "                  \n",
    "    return games_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Getting Game Data for 2018/09\n"
     ]
    }
   ],
   "source": [
    "# get all games stats\n",
    "games_list = getChessGamesPlayed(my_user, monthlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to unpack nested data\n",
    "def unpack(df, column, fillna=None):\n",
    "    ret = None\n",
    "    if fillna is None:\n",
    "        ret = pd.concat([df, pd.DataFrame((d for idx, d in df[column].iteritems()))], axis=1)\n",
    "        del ret[column]\n",
    "    else:\n",
    "        ret = pd.concat([df, pd.DataFrame((d for idx, d in df[column].iteritems())).fillna(fillna)], axis=1)\n",
    "        del ret[column]\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Games Played Data Cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# unpack the nested data \n",
    "games_played = pd.DataFrame(games_list)\n",
    "\n",
    "games_played = unpack(games_played, 'black', 0)\n",
    "games_played = unpack(games_played, 'white', 0)\n",
    "\n",
    "# if don't play daily then not all games will have a start time\n",
    "if 'start_time' in games_played.columns:\n",
    "    games_played.columns = ['end_time', 'fen', 'pgn', 'rated', 'rules', 'start_time', 'time_class',\n",
    "       'time_control', 'url', 'black_id', 'black_rating', 'black_result',\n",
    "       'black_username', 'white_id', 'white_rating', 'white_result', 'white_username']\n",
    "    games_played['start_time'] = pd.to_datetime(games_played['start_time'], unit='s')\n",
    "    games_played['end_time'] = pd.to_datetime(games_played['end_time'], unit='s')\n",
    "else: \n",
    "    games_played.columns = ['end_time', 'fen', 'pgn', 'rated', 'rules', 'time_class',\n",
    "       'time_control', 'url', 'black_id', 'black_rating', 'black_result',\n",
    "       'black_username', 'white_id', 'white_rating', 'white_result', 'white_username']\n",
    "    games_played['end_time'] = pd.to_datetime(games_played['end_time'], unit='s')\n",
    "    \n",
    "# set which side I played\n",
    "games_played['my_side'] = np.where(games_played['white_username'] == my_user, 'white', 'black')\n",
    "\n",
    "# get and assign my result\n",
    "game_results = []\n",
    "\n",
    "for index, row in games_played.iterrows():\n",
    "    if row['white_username'] == my_user:\n",
    "        # print(\"I was white\")\n",
    "        game_results.append(row['white_result'])\n",
    "    else:\n",
    "        # print(\"I was black\")\n",
    "        game_results.append(row['black_result'])\n",
    "        \n",
    "games_played['my_result'] = game_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export to csv\n",
    "games_played.to_csv(\"data/\" + my_user + \"_games_played\" + \".csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# total games played\n",
    "total_games_played = len(games_played)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_games_played"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advanced PGN Parsing: Extract Additional Game Info from PGN Game Files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE:** To use this section, you need to install the Chess-Python Library, which you can do on the command line using the following command: \n",
    "\n",
    "> pip install python-chess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import chess.pgn\n",
    "import io\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Advanced Parsing of PGN\n",
    "\n",
    "terminations = []\n",
    "total_moves = []\n",
    "start_times = []\n",
    "end_times  = []\n",
    "moves_times_white = []\n",
    "moves_times_black = []\n",
    "\n",
    "# TODO: Additional Data to Exract\n",
    "# Result\n",
    "# ECO\n",
    "# ECOUrl\n",
    "# WhiteElo\n",
    "# BlackElo\n",
    "\n",
    "for index, row in games_played.iterrows():\n",
    "    raw_pgn = io.StringIO(row['pgn'])\n",
    "    game = chess.pgn.read_game(raw_pgn)\n",
    "    \n",
    "    terminations.append(game.headers['Termination'] )\n",
    "\n",
    "    start_time = game.headers['Date'] + \" \" + game.headers['StartTime']\n",
    "    start_times.append(start_time)  \n",
    "\n",
    "    end_time = game.headers['Date'] + \" \" + game.headers['EndTime']\n",
    "    end_times.append(end_time) \n",
    "        \n",
    "    move_times = re.findall('\\{\\[%clk (.*?)\\]\\}', row['pgn'])\n",
    "    \n",
    "    total_moves.append(len(move_times))\n",
    "    \n",
    "    # last move time: \n",
    "    # print(move_times[-1])\n",
    "    \n",
    "    moves_times_white.append(', '.join(move_times[0:][::2]))\n",
    "    moves_times_black.append(', '.join(move_times[1:][::2]))\n",
    "\n",
    "games_played['termination'] = terminations \n",
    "games_played['total_moves'] = total_moves                       \n",
    "                       \n",
    "games_played['start_time'] = start_times   \n",
    "games_played['start_time'] = pd.to_datetime(games_played['start_time'])\n",
    "\n",
    "games_played['pgn_end_time'] = end_times   \n",
    "games_played['pgn_end_time'] = pd.to_datetime(games_played['pgn_end_time'])\n",
    "\n",
    "games_played['moves_times_white'] = moves_times_white\n",
    "games_played['moves_times_black'] = moves_times_black"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: \n",
    "# * Get the total play time of each player from last timestamp minus total seconds of your side game\n",
    "# * Calculate the average move time "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# games_played.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export detailed stats to csv\n",
    "games_played.to_csv(\"data/\" + my_user + \"_games_played_detailed\" + \".csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}