updated notebooks

Former-commit-id: 59d6a2e

.gitignore

@@ -12,3 +12,6 @@ voxel51-import/notebooks/.ipynb_checkpoints
 *-checkpoint.ipynb
 voxel51-import/*.json
 *.env
+voxel51-import/notebooks/.Trash-0/*
+voxel51-import/notebooks/*.json
+voxel51-import/models/*

voxel51-import/Dockerfile

@@ -1,3 +1,6 @@
 FROM tensorflow/tensorflow:2.4.1-gpu-jupyter
 
-RUN pip3 install numpy scipy fiftyone
+RUN apt install -y libgl1-mesa-glx
+
+RUN pip3 install opencv-contrib-python numpy scipy fiftyone imutils 
+

voxel51-import/notebooks/Add FAA Data to Dataset.ipynb

@@ -0,0 +1,272 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Add FAA Data to a Voxel 51 Dataset\n",
+ "adapted from: https://voxel51.com/docs/fiftyone/recipes/model_inference.html"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you haven’t already, install FiftyOne:"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Build FiftyOne Dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#config\n",
+ "import fiftyone as fo\n",
+ "import os\n",
+ "import pandas as pd\n",
+ "\n",
+ "planes = pd.read_csv(\"./aircraftDatabase.csv\",index_col='icao24')\n",
+ "#print(planes)\n",
+ "batch_size = 1000\n",
+ "# Create the dataset\n",
+ "dataset = fo.load_dataset(\"plane-dataset\")\n",
+ "\n",
+ " \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "\n",
+ "def addPlaneData(sample):\n",
+ "\n",
+ " plane_id = sample[\"icao24\"].label.lower()\n",
+ " try:\n",
+ " plane = planes.loc[ plane_id.lower()]\n",
+ " if plane.size == 26:\n",
+ "\n",
+ " #print(\"Adding metadata for plane {} onto image {}\".format(plane[\"icao24\"].values[0], image[\"external_id\"]))\n",
+ " if isinstance(plane[\"model\"], str):\n",
+ " sample[\"model\"] = fo.Classification(label=plane[\"model\"] )\n",
+ " if isinstance(plane[\"manufacturername\"], str):\n",
+ " sample[\"manufacturer\"] = fo.Classification(label=plane[\"manufacturername\"])\n",
+ " #print(sample)\n",
+ " sample.save()\n",
+ " else:\n",
+ " print(plane.size)\n",
+ " except KeyError:\n",
+ " print(\"FAA Data entry not found for: {}\".format(plane_id)) \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#sample=dataset.shuffle().first()\n",
+ "for sample in dataset:\n",
+ " addPlaneData(sample)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Find unique models\n",
+ "Go through the models that were attached to samples and create a list of all of the unique model types"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "models = []\n",
+ "view = dataset.exists(\"model\")\n",
+ "\n",
+ "for s in view.select_fields(\"model\"):\n",
+ " if s.model.label not in models:\n",
+ " models.append(s.model.label)\n",
+ "for model in models:\n",
+ " print(model)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Normalize the Model\n",
+ "There are a lot of different types of models and different ways of writing the same models. We need to try and group together very similar aircraft types. This creates a new field with the normalized model call: `norm_model`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def normalizeModel(model):\n",
+ " norm_model=None\n",
+ " if model==\"A319-112\" or model==\"A319-115\" or model==\"A319-132\":\n",
+ " norm_model=\"A319\"\n",
+ "\n",
+ " if model==\"A320 232\" or model==\"A320 232SL\" or model==\"A320-211\" or model==\"A320-212\" or model==\"A320-214\" or model==\"A320-232\" or model==\"A320-251N\" or model==\"A320-271N\":\n",
+ " norm_model=\"A320\"\n",
+ "\n",
+ " if model==\"A321-211\" or model==\"A321-231\" or model==\"A321-271NX\" or model==\"Airbus A321-231(SL)\":\n",
+ " norm_model=\"A321\"\n",
+ "\n",
+ " if model==\"A330 243\" or model==\"A330-243F\":\n",
+ " norm_model=\"A330\"\n",
+ "\n",
+ " if model==\"737-71B\" or model==\"737-724\" or model==\"737-73V\" or model==\"737-752\" or model==\"737-7H4\" or model==\"737-7Q8\":\n",
+ " norm_model=\"737-700\"\n",
+ "\n",
+ " if model==\"737-800\" or model==\"Boeing 737-852\" or model==\"737-823\" or model==\"737-824\" or model==\"737-832\" or model==\"737-83N\" or model==\"737-84P\" or model==\"737-890\" or model==\"737-8EH\" or model==\"737-8H4\" or model==\"737NG 823/W\" or model==\"737NG 852/W\" or model==\"737NG 85P/W\" or model==\"737NG 86N/W\" or model==\"737NG 8V3/W\":\n",
+ " norm_model=\"737-800\"\n",
+ "\n",
+ " if model==\"737-900ER\" or model==\"737-924ER\" or model==\"737-932ER\":\n",
+ " norm_model=\"737-900\"\n",
+ "\n",
+ "\n",
+ " if model==\"747-48EF\":\n",
+ " norm_model=\"747-400\"\n",
+ "\n",
+ " if model==\"757-231\" or model==\"757-232\" or model==\"757-251\":\n",
+ " norm_model=\"757-200\"\n",
+ "\n",
+ " if model==\"767 330ER/W\":\n",
+ " norm_model=\"767-300\"\n",
+ "\n",
+ " if model==\"777-223\":\n",
+ " norm_model=\"777-200\"\n",
+ "\n",
+ " if model==\"787-8\":\n",
+ " norm_model=\"787-800\"\n",
+ "\n",
+ " if model==\"787-9 (Boeing)\" or model==\"BOEING 787-9 Dreamliner\":\n",
+ " norm_model=\"787-800\"\n",
+ "\n",
+ " if model==\"45\" or model==\"60\":\n",
+ " norm_model=\"Learjet 45/60\"\n",
+ "\n",
+ " if model==\"510\" or model==\"Citation Excel\" or model==\"Citation Sovereign+\" or model==\"525\" or model==\"550\" or model==\"560\" or model==\"680\" or model==\"750\" or model==\"525A\" or model==\"525B\" or model==\"525C\" or model==\"560XL\" or model==\"680A\":\n",
+ " norm_model=\"Cessna Jet\"\n",
+ "\n",
+ " if model==\"CL-600-2B16\" or model==\"BD-100-1A10\" or model==\"BD-700-1A11\":\n",
+ " norm_model=\"Bombardier Challanger\"\n",
+ "\n",
+ "\n",
+ " if model==\"CL-600-2C10\":\n",
+ " norm_model=\"CRJ700\"\n",
+ "\n",
+ " if model==\"CL-600-2C11\":\n",
+ " norm_model=\"CRJ550\"\n",
+ "\n",
+ " if model==\"CL-600-2D24\" or model==\"CRJ 900 LR NG\" or model==\"CRJ-900\":\n",
+ " norm_model=\"CRJ900\"\n",
+ "\n",
+ " if model==\"ERJ 170-100 SE\" or model==\"ERJ 170-100SU\" or model==\"ERJ 170-200 LR\" or model==\"ERJ 190-100 IGW\" or model==\"EMB-190 AR\":\n",
+ " norm_model=\"ERJ-170\"\n",
+ "\n",
+ " if model==\"EMB-135BJ\" or model==\"EMB-145LR\":\n",
+ " norm_model=\"EMB-135\"\n",
+ "\n",
+ " if model==\"EMB-505\" or model==\"EMB-545\":\n",
+ " norm_model=\"EMB-505\"\n",
+ "\n",
+ " if model==\"PA-23-250\":\n",
+ " norm_model=\"Piper PA-23\"\n",
+ "\n",
+ " if model==\"PC-12/47E\":\n",
+ " norm_model=\"Pilatus PC-12\"\n",
+ " \n",
+ " if model==\"FALCON 10\" or model==\"FALCON 2000\" or model==\"FALCON 50\" or model==\"FALCON 7X\" or model==\"FALCON 900 EX\" or model==\"FALCON 900EX\": \n",
+ " norm_model=\"Falcon\"\n",
+ " \n",
+ " if model==\"G-IV\" or model==\"G-V\" or model==\"GALAXY\" or model==\"GIV-X (G450)\" or model==\"GULFSTREAM 200\" or model==\"GV-SP (G550)\" or model==\"GVI(G650ER)\":\n",
+ " norm_model=\"Gulfstream\"\n",
+ " \n",
+ " if model==\"HAWKER 800XP\" or model==\"HAWKER 900XP\":\n",
+ " norm_model=\"Hawker\"\n",
+ " \n",
+ " if model==\"SF50\":\n",
+ " norm_model=\"Cirrus\"\n",
+ " \n",
+ " if model==\"PRESSURIZED LANCR IV\":\n",
+ " norm_model=\"Lancair IV\"\n",
+ " \n",
+ " if model==\"B300\":\n",
+ " norm_model=\"King Air\"\n",
+ " return norm_model\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Add the normalized label for the model\n",
+ "This groups to together similar models and different names for the same model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for sample in dataset.exists(\"model\"):\n",
+ " norm_model = normalizeModel(sample[\"model\"].label)\n",
+ " if norm_model != None:\n",
+ " sample[\"norm_model\"] = fo.Classification(label=norm_model)\n",
+ " sample.save()\n",
+ " else:\n",
+ " print(\"Match not found for: {}\".format(sample[\"model\"].label))\n",
+ "\n",
+ " #addPlaneData(sample)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}