notebooks/Colab_RN152V2+GeM+ArcFace.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GLC2020 Keetar ResNet152V2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pklc4PIcj3tw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "cf2ba937-9687-431c-a86b-d71d280e3bd8"
      },
      "source": [
        "!git clone https://hav4ik:3bbdb6bd02341ef0ebdf165ff819fc4f45d64ef9@github.com/hav4ik/google-landmarks-2020.git\n",
        "!cd google-landmarks-2020; git checkout base_train_loop\n",
        "!pip install google-landmarks-2020/python/"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'google-landmarks-2020'...\n",
            "remote: Enumerating objects: 252, done.\u001b[K\n",
            "remote: Counting objects: 100% (252/252), done.\u001b[K\n",
            "remote: Compressing objects: 100% (139/139), done.\u001b[K\n",
            "remote: Total 252 (delta 102), reused 229 (delta 98), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (252/252), 78.58 KiB | 3.14 MiB/s, done.\n",
            "Resolving deltas: 100% (102/102), done.\n",
            "Branch 'base_train_loop' set up to track remote branch 'base_train_loop' from 'origin'.\n",
            "Switched to a new branch 'base_train_loop'\n",
            "Processing ./google-landmarks-2020/python\n",
            "Requirement already satisfied: tensorflow>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from glrec==0.1.0) (2.3.0)\n",
            "Collecting colorlog>=4.1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/2a/81/12d77537c82c5d46aa2721dfee25a0e873ef5920ebd0827152f411effb57/colorlog-4.2.1-py2.py3-none-any.whl\n",
            "Collecting efficientnet>=1.1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/28/91/67848a143b54c331605bfba5fd31cf4e9db13d2e429d103fe807acc3bcf4/efficientnet-1.1.0-py3-none-any.whl\n",
            "Collecting gcloud\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/11/ab/d0cee58db2d8445c26e6f5db25d9b1f1aa14a3ab30eea8ce77ae808d10ef/gcloud-0.18.3.tar.gz (454kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 6.7MB/s \n",
            "\u001b[?25hCollecting keras-tqdm\n",
            "  Downloading https://files.pythonhosted.org/packages/16/5c/ac63c65b79a895b8994474de2ad4d5b66ac0796b8903d60cfea3f8308d5c/keras_tqdm-2.0.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.1.0)\n",
            "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.1.2)\n",
            "Requirement already satisfied: numpy<1.19.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.18.5)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.12.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (3.3.0)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (2.10.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.31.0)\n",
            "Requirement already satisfied: tensorboard<3,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (2.3.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (0.8.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (0.35.1)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (3.12.4)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (0.2.0)\n",
            "Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (2.3.0)\n",
            "Requirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.4.1)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.6.3)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (0.3.3)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow>=2.2.0->glrec==0.1.0) (1.15.0)\n",
            "Requirement already satisfied: scikit-image in /usr/local/lib/python3.6/dist-packages (from efficientnet>=1.1.0->glrec==0.1.0) (0.16.2)\n",
            "Collecting keras-applications<=1.0.8,>=1.0.7\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 3.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: httplib2>=0.9.1 in /usr/local/lib/python3.6/dist-packages (from gcloud->glrec==0.1.0) (0.17.4)\n",
            "Requirement already satisfied: googleapis-common-protos in /usr/local/lib/python3.6/dist-packages (from gcloud->glrec==0.1.0) (1.52.0)\n",
            "Requirement already satisfied: oauth2client>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from gcloud->glrec==0.1.0) (4.1.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from keras-tqdm->glrec==0.1.0) (4.41.1)\n",
            "Requirement already satisfied: Keras in /usr/local/lib/python3.6/dist-packages (from keras-tqdm->glrec==0.1.0) (2.4.3)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (2.23.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (49.6.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (1.7.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (3.2.2)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (0.4.1)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (1.17.2)\n",
            "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (3.2.2)\n",
            "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (1.1.1)\n",
            "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (2.5)\n",
            "Requirement already satisfied: pillow>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (7.0.0)\n",
            "Requirement already satisfied: imageio>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (2.4.1)\n",
            "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=2.0.1->gcloud->glrec==0.1.0) (4.6)\n",
            "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=2.0.1->gcloud->glrec==0.1.0) (0.2.8)\n",
            "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=2.0.1->gcloud->glrec==0.1.0) (0.4.8)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from Keras->keras-tqdm->glrec==0.1.0) (3.13)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (1.24.3)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (1.7.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (1.3.0)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (1.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (2.8.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (0.10.0)\n",
            "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx>=2.0->scikit-image->efficientnet>=1.1.0->glrec==0.1.0) (4.4.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (3.1.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow>=2.2.0->glrec==0.1.0) (3.1.0)\n",
            "Building wheels for collected packages: glrec, gcloud\n",
            "  Building wheel for glrec (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for glrec: filename=glrec-0.1.0-cp36-none-any.whl size=18201 sha256=d72fc53f9b6c7a1c0c06c9f225fb4ac68a6719335c96b015244577ec18c8463f\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-goihl_hv/wheels/52/73/92/4f680bcf94f91663f493c6d74605b0d7ec59ccc72f549749c7\n",
            "  Building wheel for gcloud (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gcloud: filename=gcloud-0.18.3-cp36-none-any.whl size=602938 sha256=1e00e5a3c0a19b1884ec061eff925db230ba440c431fa5af573eaadc62f4d7fc\n",
            "  Stored in directory: /root/.cache/pip/wheels/b9/9b/9c/a01be401658fea33b93a35d03921b0c638266821b264dc8662\n",
            "Successfully built glrec gcloud\n",
            "Installing collected packages: colorlog, keras-applications, efficientnet, gcloud, keras-tqdm, glrec\n",
            "Successfully installed colorlog-4.2.1 efficientnet-1.1.0 gcloud-0.18.3 glrec-0.1.0 keras-applications-1.0.8 keras-tqdm-2.0.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tnZeHT8Mj5Vg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "02f76fbf-4e0c-4a95-ea41-23d2005eb571"
      },
      "source": [
        "!cd google-landmarks-2020; git branch -a; git log --oneline -n 5"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "* \u001b[32mbase_train_loop\u001b[m\n",
            "  master\u001b[m\n",
            "  \u001b[31mremotes/origin/HEAD\u001b[m -> origin/master\n",
            "  \u001b[31mremotes/origin/base_train_loop\u001b[m\n",
            "  \u001b[31mremotes/origin/master\u001b[m\n",
            "  \u001b[31mremotes/origin/notes\u001b[m\n",
            "  \u001b[31mremotes/origin/refactor_prediction_pipeline\u001b[m\n",
            "\u001b[33m8fd2d6f\u001b[m\u001b[33m (\u001b[m\u001b[1;36mHEAD -> \u001b[m\u001b[1;32mbase_train_loop\u001b[m\u001b[33m, \u001b[m\u001b[1;31morigin/base_train_loop\u001b[m\u001b[33m)\u001b[m Fixed bugs with CosFace\n",
            "\u001b[33m810d406\u001b[m Added full model weights loading\n",
            "\u001b[33m6ea24b9\u001b[m Fixed potential bug when batch_size=1 (on a single TPU replica)\n",
            "\u001b[33mfa9cb20\u001b[m Removed TQDM due to memory leak in Colab; Set verbose=1;\n",
            "\u001b[33mfa88f9b\u001b[m Fixed label shape expansion bug; Fixed TPU dataset shape issues\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KCCrTa9vkBMN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import auth\n",
        "auth.authenticate_user()"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fBKo8EykCOS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "os.environ[\"GCLOUD_PROJECT\"] = \"daring-runway-287322\"\n",
        "!export GCLOUD_PROJECT=daring-runway-287322"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4CkAAJ6kIk9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trn_cng = \"\"\"\n",
        "# Information about the experiment\n",
        "experiment:\n",
        "  name: RN152V2-512x512-512-GeM+ArcFace_001\n",
        "  storage: gs:https://khch-glc2020-models/global/RN152V2\n",
        "  description: Global model. Copy the strategy of Keetar, but for ResNet152V2\n",
        "\n",
        "\n",
        "\n",
        "# Competition-specific configuration\n",
        "glc_config:\n",
        "  gld_version: gld_v2_clean\n",
        "  gld_id_mapping: gs:https://gld-v2-clean/gld-v2-clean-id-mapping.csv\n",
        "\n",
        "\n",
        "\n",
        "# Configure the dataset feeding pipeline\n",
        "dataset_config:\n",
        "  image_size: [512, 512, 3]\n",
        "  imagenet_crop: false\n",
        "\n",
        "  train_tfrecords:\n",
        "    tfrecord_dir: gs:https://khch-gldv2-clean-keetar-split/tfrecord/\n",
        "    basename: train\n",
        "    shards: 512\n",
        "\n",
        "  validation_tfrecords:\n",
        "    tfrecord_dir: gs:https://khch-gldv2-clean-keetar-split/tfrecord/\n",
        "    basename: validation\n",
        "    shards: 64\n",
        "\n",
        "  train_shuffle:\n",
        "    buffer_size: 2048\n",
        "    seed: 17061998\n",
        "\n",
        "  train_augmentations:\n",
        "    # Using only horizontal flips, as in Keetar's solution\n",
        "    - class: TFImageTransform\n",
        "      kwargs:\n",
        "        horizontal_flip: true\n",
        "        vertical_flip: false\n",
        "        brightness_adjustments: false\n",
        "\n",
        "\n",
        "\n",
        "# Model configurations\n",
        "model_config:\n",
        "\n",
        "  # Backbone CNN configuration\n",
        "  backbone_config:\n",
        "    architecture: ResNet152V2\n",
        "    weights: imagenet\n",
        "    trainable: true\n",
        "\n",
        "  # Global branch: [backbone]->[pooling]->[dense]->[head]\n",
        "  global_branch_config:\n",
        "    # Currently, the following pooling methods are supported:\n",
        "    #   - GAP()\n",
        "    #   - GeM(p=3, train_p=False)\n",
        "    pooling_config:\n",
        "      method: GeM\n",
        "      kwargs:\n",
        "        p: 3.0\n",
        "        train_p: false\n",
        "\n",
        "    # Embedding after pooling (output units of a Dense layer\n",
        "    # without activation function because the backbone already\n",
        "    # have it, and without bias.\n",
        "    embedding_dim: 512\n",
        "\n",
        "    # Currently, the following heads are supported:\n",
        "    #   - ArcFace(s=30, m=0.5)\n",
        "    #   - ArcMarginProduct(s=30, m=0.5, easy_margin=False)\n",
        "    #   - AdaCos(m=0.5, is_dynamic=True)\n",
        "    #   - CosFace(s=30, m=0.35)\n",
        "    head_config:\n",
        "      layer: ArcFace\n",
        "      kwargs:\n",
        "        s: 15.98916767 # = sqrt(2) * log(C-1), where C=81313\n",
        "        m: 0.1 # Google used this margin\n",
        "\n",
        "  # Local features extractor\n",
        "  local_branch_config: null\n",
        "\n",
        "  # Places classifier\n",
        "  places_branch_config: null\n",
        "\n",
        "\n",
        "\n",
        "# Configuration of the training process\n",
        "training_config:\n",
        "\n",
        "  # Previously trained weights (continue training)\n",
        "  previous_weights: gs:https://khch-glc2020-models/global/RN152V2/RN152V2-512x512-512-GeM+ArcFace_001/checkpoints/005_val_loss=0.20117.hdf5\n",
        "\n",
        "  # single TPU core batch size for 512x512\n",
        "  batch_size:\n",
        "    v: 12\n",
        "    tpu: lin\n",
        "\n",
        "  epochs: 10\n",
        "  samples_per_epoch: 1508148 # 1580470 - 72322 items in validation\n",
        "  initial_epoch: 5\n",
        "\n",
        "  # Using SDG, with settings as suggested by Keetar:\n",
        "  # lr 1e-3 (adjusted for GPU), 0.9 momentum, and 1e-5 decay\n",
        "  optimizer:\n",
        "    algorithm: SGD\n",
        "    kwargs:\n",
        "      learning_rate:\n",
        "        v: 0.003 # 1e-3\n",
        "        tpu: sqrt\n",
        "      momentum: 0.9\n",
        "      decay: 0.00001 # 1e-5\n",
        "\n",
        "  # Keetar used weights proportional to 1/log(class cnt + 1)\n",
        "  class_weights: inv_log\n",
        "\n",
        "  # Not supported yet\n",
        "  learning_rate_scheduler: null\n",
        "\n",
        "  # We train until convergence, i.e. till val_loss not increasing\n",
        "  additional_callbacks:\n",
        "    - callback: EarlyStopping\n",
        "      kwargs:\n",
        "        monitor: val_loss\n",
        "        patience: 2\n",
        "\"\"\""
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HMdEAH82lWMe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c26ffe88-8430-43d1-b4ed-c36b62e97e63"
      },
      "source": [
        "debug_mode = False\n",
        "\n",
        "\n",
        "import os\n",
        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
        "import sys\n",
        "import yaml\n",
        "sys.path.append('google-landmarks-2020/training')\n",
        "from train import train_delg\n",
        "\n",
        "\n",
        "train_delg(debug_mode=debug_mode, **yaml.safe_load(trn_cng))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[0m2020-09-03 19:37:17 | INFO     | Started Experiment: RN152V2-512x512-512-GeM+ArcFace_001 (train.py:131)\u001b[0m\n",
            "INFO:eyesight:Started Experiment: RN152V2-512x512-512-GeM+ArcFace_001\n",
            "\u001b[0m2020-09-03 19:37:17 | INFO     | All data will be saved to: gs:https://khch-glc2020-models/global/RN152V2 (train.py:132)\u001b[0m\n",
            "INFO:eyesight:All data will be saved to: gs:https://khch-glc2020-models/global/RN152V2\n",
            "\u001b[0m2020-09-03 19:37:17 | INFO     | Experiment description: Global model. Copy the strategy of Keetar, but for ResNet152V2 (train.py:133)\u001b[0m\n",
            "INFO:eyesight:Experiment description: Global model. Copy the strategy of Keetar, but for ResNet152V2\n",
            "\u001b[0m2020-09-03 19:37:17 | INFO     | Running on  TPU grpc:https://10.4.49.58:8470 (utils.py:13)\u001b[0m\n",
            "INFO:eyesight:Running on  TPU grpc:https://10.4.49.58:8470\n",
            "\u001b[0m2020-09-03 19:37:17 | INFO     | Initializing TPU... (utils.py:20)\u001b[0m\n",
            "INFO:eyesight:Initializing TPU...\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc:https://10.4.49.58:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc:https://10.4.49.58:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n",
            "\u001b[0m2020-09-03 19:38:18 | INFO     | TPU initialized. (utils.py:24)\u001b[0m\n",
            "INFO:eyesight:TPU initialized.\n",
            "\u001b[0m2020-09-03 19:38:18 | INFO     | Replicas: 8 (utils.py:31)\u001b[0m\n",
            "INFO:eyesight:Replicas: 8\n",
            "\u001b[36m2020-09-03 19:38:18 | DEBUG    | Resolved <{'v': 12, 'tpu': 'lin'}> to <96> (strat=lin) (utils.py:98)\u001b[0m\n",
            "DEBUG:eyesight:Resolved <{'v': 12, 'tpu': 'lin'}> to <96> (strat=lin)\n",
            "\u001b[36m2020-09-03 19:38:18 | DEBUG    | Resolved <{'v': 0.003, 'tpu': 'sqrt'}> to <0.008485281374238571> (strat=sqrt) (utils.py:108)\u001b[0m\n",
            "DEBUG:eyesight:Resolved <{'v': 0.003, 'tpu': 'sqrt'}> to <0.008485281374238571> (strat=sqrt)\n",
            "\u001b[36m2020-09-03 19:38:18 | DEBUG    | Using GCS to store models. Employ the `rsync` strategy (train.py:91)\u001b[0m\n",
            "DEBUG:eyesight:Using GCS to store models. Employ the `rsync` strategy\n",
            "\u001b[36m2020-09-03 19:38:19 | DEBUG    | Found `gld_id_mapping` file (train.py:167)\u001b[0m\n",
            "DEBUG:eyesight:Found `gld_id_mapping` file\n",
            "INFO:oauth2client.transport:Attempting refresh to obtain initial access_token\n",
            "INFO:oauth2client.client:Refreshing access_token\n",
            "\u001b[0m2020-09-03 19:38:19 | INFO     | Blob gs:https://gld-v2-clean/gld-v2-clean-id-mapping.csv downloaded to /tmp/glrec/gld-v2-clean/gld-v2-clean-id-mapping.csv in: 0:00:00.731854 (utils.py:83)\u001b[0m\n",
            "INFO:eyesight:Blob gs:https://gld-v2-clean/gld-v2-clean-id-mapping.csv downloaded to /tmp/glrec/gld-v2-clean/gld-v2-clean-id-mapping.csv in: 0:00:00.731854\n",
            "\u001b[0m2020-09-03 19:38:20 | INFO     | Batch size (adjusted for number of replicas): 96 (train.py:208)\u001b[0m\n",
            "INFO:eyesight:Batch size (adjusted for number of replicas): 96\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet152v2_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
            "234553344/234545216 [==============================] - 2s 0us/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[36m2020-09-03 19:39:04 | DEBUG    | Class weights is set to 1/log(class count) (train.py:111)\u001b[0m\n",
            "DEBUG:eyesight:Class weights is set to 1/log(class count)\n",
            "INFO:oauth2client.transport:Attempting refresh to obtain initial access_token\n",
            "INFO:oauth2client.client:Refreshing access_token\n",
            "\u001b[0m2020-09-03 19:39:52 | INFO     | Blob gs:https://khch-glc2020-models/global/RN152V2/RN152V2-512x512-512-GeM+ArcFace_001/checkpoints/005_val_loss=0.20117.hdf5 downloaded to /tmp/glrec/khch-glc2020-models/global/RN152V2/RN152V2-512x512-512-GeM+ArcFace_001/checkpoints/005_val_loss=0.20117.hdf5 in: 0:00:35.020458 (utils.py:83)\u001b[0m\n",
            "INFO:eyesight:Blob gs:https://khch-glc2020-models/global/RN152V2/RN152V2-512x512-512-GeM+ArcFace_001/checkpoints/005_val_loss=0.20117.hdf5 downloaded to /tmp/glrec/khch-glc2020-models/global/RN152V2/RN152V2-512x512-512-GeM+ArcFace_001/checkpoints/005_val_loss=0.20117.hdf5 in: 0:00:35.020458\n",
            "\u001b[0m2020-09-03 19:39:58 | INFO     | Model compiled in: 0:01:37.528249 (utils.py:83)\u001b[0m\n",
            "INFO:eyesight:Model compiled in: 0:01:37.528249\n",
            "\u001b[36m2020-09-03 19:39:58 | DEBUG    | Setting TensorBoard to write to GCS bucket. (train.py:306)\u001b[0m\n",
            "DEBUG:eyesight:Setting TensorBoard to write to GCS bucket.\n",
            "\u001b[0m2020-09-03 19:39:58 | INFO     | Loading `EarlyStopping` callback from tf.keras with parameters {'monitor': 'val_loss', 'patience': 2} (callbacks.py:26)\u001b[0m\n",
            "INFO:eyesight:Loading `EarlyStopping` callback from tf.keras with parameters {'monitor': 'val_loss', 'patience': 2}\n",
            "\u001b[36m2020-09-03 19:39:58 | DEBUG    | Initial epoch is set to 5 (train.py:337)\u001b[0m\n",
            "DEBUG:eyesight:Initial epoch is set to 5\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 6/10\n",
            "    1/15709 [..............................] - ETA: 41s - loss: 0.1644 - sparse_categorical_accuracy: 0.6875WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/summary_ops_v2.py:1277: stop (from tensorflow.python.eager.profiler) is deprecated and will be removed after 2020-07-01.\n",
            "Instructions for updating:\n",
            "use `tf.profiler.experimental.stop` instead.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/summary_ops_v2.py:1277: stop (from tensorflow.python.eager.profiler) is deprecated and will be removed after 2020-07-01.\n",
            "Instructions for updating:\n",
            "use `tf.profiler.experimental.stop` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r    2/15709 [..............................] - ETA: 8:02:01 - loss: 0.1791 - sparse_categorical_accuracy: 0.6979WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0276s vs `on_train_batch_end` time: 3.6519s). Check your callbacks.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0276s vs `on_train_batch_end` time: 3.6519s). Check your callbacks.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "15709/15709 [==============================] - 9194s 585ms/step - loss: 0.1643 - sparse_categorical_accuracy: 0.7255 - val_loss: 0.1948 - val_sparse_categorical_accuracy: 0.8226\n",
            "Epoch 7/10\n",
            "15709/15709 [==============================] - 9195s 585ms/step - loss: 0.1571 - sparse_categorical_accuracy: 0.7402 - val_loss: 0.1919 - val_sparse_categorical_accuracy: 0.8251\n",
            "Epoch 8/10\n",
            "15709/15709 [==============================] - 9207s 586ms/step - loss: 0.1518 - sparse_categorical_accuracy: 0.7514 - val_loss: 0.1900 - val_sparse_categorical_accuracy: 0.8268\n",
            "Epoch 9/10\n",
            "15709/15709 [==============================] - 9242s 588ms/step - loss: 0.1474 - sparse_categorical_accuracy: 0.7608 - val_loss: 0.1887 - val_sparse_categorical_accuracy: 0.8278\n",
            "Epoch 10/10\n",
            " 3913/15709 [======>.......................] - ETA: 1:55:41 - loss: 0.1471 - sparse_categorical_accuracy: 0.7609"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWKhitUdlZBN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}