notebooks/4_Evaluate_Presidio_Analyzer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "847acd88",
   "metadata": {},
   "source": [
    "# Evaluate Presidio Analyzer using the Presidio Evaluator framework"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b946feda",
   "metadata": {},
   "outputs": [],
   "source": [
    "# install presidio via pip if not yet installed\n",
    "\n",
    "#!pip install presidio-evaluator\n",
    "#!pip install \"presidio-analyzer[transformers]\"\n",
    "#!pip install presidio-evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae85cae9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from copy import deepcopy\n",
    "from pprint import pprint\n",
    "from collections import Counter\n",
    "from typing import List\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from presidio_evaluator import InputSample\n",
    "from presidio_evaluator.evaluation import Evaluator, ModelError\n",
    "from presidio_evaluator.models import PresidioAnalyzerWrapper\n",
    "from presidio_evaluator.experiment_tracking import get_experiment_tracker\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "736fdd23",
   "metadata": {},
   "source": [
    "Select data for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4cbd55c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"synth_dataset_v2.json\"\n",
    "dataset = InputSample.read_dataset_json(Path(Path.cwd().parent, \"data\", dataset_name))\n",
    "\n",
    "dataset = dataset[:300] # top 300 samples\n",
    "\n",
    "print(len(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c164ea07",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_entity_counts(dataset:List[InputSample]):\n",
    "    entity_counter = Counter()\n",
    "    for sample in dataset:\n",
    "        for tag in sample.tags:\n",
    "            entity_counter[tag] += 1\n",
    "    return entity_counter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77aedae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Count per entity:\")\n",
    "pprint(get_entity_counts(dataset).most_common())\n",
    "\n",
    "print(\"\\nExample sentence:\")\n",
    "print(dataset[1])\n",
    "\n",
    "print(\"\\nMin and max number of tokens in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.tokens) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.tokens) for sample in dataset])}\"\n",
    ")\n",
    "\n",
    "print(\"\\nMin and max sentence length in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.full_text) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.full_text) for sample in dataset])}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c5e16cb-bee8-4f0a-a543-4879daa35b9e",
   "metadata": {},
   "source": [
    "### Define the AnalyzerEngine object \n",
    "In this case, using a huggingface model: obi/deid_roberta_i2b2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "313b508f-e901-40b9-b575-c7fb8a794652",
   "metadata": {},
   "outputs": [],
   "source": [
    "from presidio_analyzer import AnalyzerEngine\n",
    "from presidio_analyzer.nlp_engine import TransformersNlpEngine, NerModelConfiguration\n",
    "\n",
    "\n",
    "# Here we define a transformers based NLP engine, \n",
    "# but you can use this cell to customize your Presidio Analyzer instance\n",
    "\n",
    "# Define which model to use\n",
    "model_config = [{\"lang_code\": \"en\", \"model_name\": {\n",
    "    \"spacy\": \"en_core_web_sm\",  # use a small spaCy model for lemmas, tokens etc.\n",
    "    \"transformers\": \"obi/deid_roberta_i2b2\"\n",
    "    }\n",
    "}]\n",
    "\n",
    "# Map transformers model labels to Presidio's\n",
    "model_to_presidio_entity_mapping = dict(\n",
    "    PER=\"PERSON\",\n",
    "    PERSON=\"PERSON\",\n",
    "    LOC= \"LOCATION\",\n",
    "    LOCATION= \"LOCATION\",\n",
    "    GPE=\"LOCATION\",\n",
    "    ORG=\"ORGANIZATION\",\n",
    "    ORGANIZATION=\"ORGANIZATION\",\n",
    "    NORP=\"NRP\",\n",
    "    AGE=\"AGE\",\n",
    "    ID=\"ID\",\n",
    "    EMAIL=\"EMAIL\",\n",
    "    PATIENT=\"PERSON\",\n",
    "    STAFF=\"PERSON\",\n",
    "    HOSP=\"ORGANIZATION\",\n",
    "    PATORG=\"ORGANIZATION\",\n",
    "    DATE=\"DATE_TIME\",\n",
    "    TIME=\"DATE_TIME\",\n",
    "    PHONE=\"PHONE_NUMBER\",\n",
    "    HCW=\"PERSON\",\n",
    "    HOSPITAL=\"ORGANIZATION\",\n",
    "    FACILITY=\"LOCATION\",\n",
    ")\n",
    "\n",
    "ner_model_configuration = NerModelConfiguration(labels_to_ignore = [\"O\"], \n",
    "                                                model_to_presidio_entity_mapping=model_to_presidio_entity_mapping)\n",
    "\n",
    "nlp_engine = TransformersNlpEngine(models=model_config,\n",
    "                                   ner_model_configuration=ner_model_configuration)\n",
    "\n",
    "# Set up the engine, loads the NLP module (spaCy model by default) \n",
    "# and other PII recognizers\n",
    "analyzer_engine = AnalyzerEngine(nlp_engine=nlp_engine)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aae4c379",
   "metadata": {},
   "source": [
    "### Run evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16dbf6d6-a554-4602-8907-589786d47a12",
   "metadata": {},
   "source": [
    "#### Define experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29d39ff1-4f14-4e32-ae84-ecc6c739f829",
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment = get_experiment_tracker()\n",
    "model = PresidioAnalyzerWrapper(analyzer_engine)\n",
    "\n",
    "# Define evaluator and experiment tracking\n",
    "\n",
    "evaluator = Evaluator(model=model)\n",
    "dataset = Evaluator.align_entity_types(\n",
    "    deepcopy(dataset), entities_mapping=PresidioAnalyzerWrapper.presidio_entities_map\n",
    ")\n",
    "\n",
    "print(\"Count per entity after alignment:\")\n",
    "pprint(get_entity_counts(dataset).most_common())\n",
    "\n",
    "# Track model and dataset params\n",
    "params = {\"dataset_name\": dataset_name, \"model_name\": model.name}\n",
    "params.update(model.to_log())\n",
    "experiment.log_parameters(params)\n",
    "experiment.log_dataset_hash(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a7d6626-d094-4dfd-8f37-c0443edf00dc",
   "metadata": {},
   "source": [
    "#### Run experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf65af8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run experiment\n",
    "evaluation_results = evaluator.evaluate_all(dataset)\n",
    "results = evaluator.calculate_score(evaluation_results)\n",
    "\n",
    "# Track experiment results\n",
    "experiment.log_metrics(results.to_log())\n",
    "entities, confmatrix = results.to_confusion_matrix()\n",
    "experiment.log_confusion_matrix(matrix=confmatrix, \n",
    "                                labels=entities)\n",
    "\n",
    "# Plot output\n",
    "plotter = evaluator.Plotter(model=model, \n",
    "                            results=results, \n",
    "                            output_folder = \".\", \n",
    "                            model_name = model.name, \n",
    "                            beta = 2)\n",
    "\n",
    "# end experiment\n",
    "experiment.end()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b4d662d-596c-4a69-b3c9-1edcda20cc5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "plotter.plot_scores()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "070f8287",
   "metadata": {},
   "source": [
    "### Results analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd04db3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sent = \"I am taiwanese but I live in Cambodia.\"\n",
    "# sent = input(\"Enter sentence: \")\n",
    "model.predict(InputSample(full_text=sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08ae9bda",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = results.model_errors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "819eb905",
   "metadata": {},
   "source": [
    "#### False positives"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98f4802e",
   "metadata": {},
   "source": [
    "1. Most common false positive tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "640037af",
   "metadata": {},
   "outputs": [],
   "source": [
    "ModelError.most_common_fp_tokens(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03432506",
   "metadata": {},
   "outputs": [],
   "source": [
    "fps_df = ModelError.get_fps_dataframe(errors, entity=[\"LOCATION\"])\n",
    "fps_df[[\"full_text\", \"token\", \"annotation\", \"prediction\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0852513",
   "metadata": {},
   "source": [
    "2. Most common false negative examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afae40fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "ModelError.most_common_fn_tokens(errors, n=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44ed6416",
   "metadata": {},
   "source": [
    "More FN analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7abfcbe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fns_df = ModelError.get_fns_dataframe(errors, entity=[\"IP_ADDRESS\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ae73b2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fns_df[[\"full_text\", \"token\", \"annotation\", \"prediction\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24c8be1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"All errors:\\n\")\n",
    "[print(error, \"\\n\") for error in errors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a67ff38d-0817-4864-9991-b3eb1f80eecc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "presidio-evaluator",
   "language": "python",
   "name": "presidio-evaluator"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}