methods/01_collecting_and_processing_twitter_data/collecting_and_processing_twitter_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calling and Processing Data\n",
    "\n",
    "This markdown file contains code for the following:\n",
    "1) Call the number of tweets that meet our search criteria by hour in each country;\n",
    "\n",
    "2) Call a sample of tweets at the period of maximum activity for each day during the study period;\n",
    "\n",
    "3) Remove unwanted tweets and clean the tweet text for subsequent analysis;\n",
    "\n",
    "4) Identify the number of tweets related to COVID-19;\n",
    "\n",
    "5) Save tweet_ids."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Packages & Define Key Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import yaml\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from searchtweets import ResultStream, gen_rule_payload, load_credentials, collect_results\n",
    "from datetime import datetime, date, timedelta\n",
    "import matplotlib.pyplot as plt\n",
    "import json_lines\n",
    "import re\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('punkt')\n",
    "import csv\n",
    "import demoji\n",
    "demoji.download_codes()\n",
    "\n",
    "# Define rootpath\n",
    "rp = 'C:\\\\Users\\\\sgmmahon\\\\Documents\\\\GitHub\\\\iom_project\\\\'\n",
    "mp = 'methods\\\\accessing_tweets\\\\'\n",
    "dp = 'data\\\\tweet_data\\\\'     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Premium Twitter Key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Define YAML with key details for accessing Twitter API\n",
    "config = dict(\n",
    "    search_tweets_api=dict(\n",
    "        account_type='premium',\n",
    "        endpoint=f\"https://api.twitter.com/1.1/tweets/search/fullarchive/datacollection.json\",\n",
    "        consumer_key=' ',\n",
    "        consumer_secret=' '\n",
    "    )\n",
    ")\n",
    "\n",
    "\n",
    "# Save YAML file\n",
    "with open(rp + mp + 'twitter_keys.yaml', 'w') as config_file:\n",
    "    yaml.dump(config, config_file, default_flow_style=False)\n",
    "\n",
    "# Define rules for premium search for streaming tweets\n",
    "premium_search_args = load_credentials(rp + mp + 'twitter_keys.yaml',\n",
    "                                       yaml_key=\"search_tweets_api\",\n",
    "                                       env_overwrite=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Search Terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create functions which concatenates vectors\n",
    "def cnct (x): return(\" OR \".join(x))\n",
    "def cnctwb (x): return(\"(\" + \" OR \".join(x) + \")\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# UK Search Terms\n",
    "\n",
    "# Migrant terms\n",
    "uk_neutral_migrant_terms  = [\"immigrant\", \"immigration\", \"migrant\", \"migration\", \"\\\"asylum seeker\\\"\", \"refugee\", \"\\\"undocumented worker\\\"\", \"\\\"guest worker\\\"\", \n",
    "                             \"\\\"EU worker\\\"\", \"\\\"non-UK workers\\\"\", \"\\\"foreign worker\\\"\", \"(human smuggling)\", \"(human trafficking)\"]\n",
    "uk_negative_migrant_terms = [\"illegals\", \"foreigner\", \"\\\"illegal alien\\\"\", \"\\\"illegal worker\\\"\"]\n",
    " \n",
    "# Racial terms\n",
    "uk_negative_racial_terms  = [\"islamophob\", \"sinophob\", \"\\\"china flu\\\"\", \"\\\"kung flu\\\"\", \"\\\"china virus\\\"\", \"\\\"chinese virus\\\"\", \"shangainese\"]\n",
    "\n",
    "# Twitter accounts\n",
    "uk_pro_migrant_account_1  = [\"@UNmigration\", \"@IOM_UN\", \"@IOMatUN\", \"@IOMatEU\", \"@IOM_UK\", \"@IOMResearch\", \"@IOM_GMDAC\", \"@hrw\", \"@Right_to_Remain\",\n",
    "                             \"@CommonsHomeAffs\", \"@fcukba\", \"@Mark_George_QC\", \"@MigrantVoiceUK\", \"@MigrantChildren\", \"@MigrantHelp\", \"@thevoiceofdws\"]\n",
    "uk_pro_migrant_account_2  = [\"@WORCrights\", \"@UbuntuGlasgow\", \"@MigrantsUnionUK\", \"@migrants_rights\", \"@MigrantsMRC\", \"@Consenant_UK\", \"@RomaSupport\",\n",
    "                             \"@MigrantsLawProj\", \"@MigRightsScot\", \"@IRMOLondon\", \"@HighlySkilledUK\", \"@WeBelong19\", \"@Project17UK\"]\n",
    "uk_neutral_account        = [\"@ukhomeoffice\", \"@pritipatel\", \"@UKHomeSecretary\", \"@EUHomeAffairs\", \"@MigrMatters\", \"@MigObs\"]\n",
    "uk_anti_migrant_account   = [\"@Nigel_Farage\", \"@MigrationWatch\"]\n",
    "\n",
    "# Hashtags\n",
    "uk_positive_hashtags      = [\"#RefugeesWelcome\", \"#MigrantsWelcome\", \"#LeaveNoOneBehind\", \"#FreedomForImmigrants\", \"#illegalmigantsUK\", \"#LondonIsOpen\",\n",
    "                             \"#EndHostileEnvironment\", \"#FamiliesBelongTogether\"]\n",
    "uk_neutral_hashtags       = [\"#Pritiuseless\", \"#migrationEU\", \"#immigration\", \"#migration\", \"#immigrant\", \"#migrant\", \"#immigrate\", \"#migrate\", \"#refugees\",\n",
    "                             \"#NigelFarage\", \"#ImmigrationReform\"]\n",
    "uk_negative_hashtags      = [\"#illegals\", \"#foreigner\", \"#foreigners\", \"#illegalalien\", \"#illegalaliens\", \"#illegalworker\", \"#illegalworkers\", \"#KeepThemOut\",\n",
    "                             \"#OurCountry\", \"#SendThemBack\", \"#migrantsnotwelcome\", \"#refugeesnotwelcome\", \"#illegals\", \"#ChinaVirus\", \"#chinaflu\", \"#kungflu\",\n",
    "                             \"#chinesevirus\", \"#TheyHaveToGoBack\", \"#DeportThemAll\"]\n",
    "uk_event_hashtags         = [\"#Moria\", \"#CampFire\", \"#closethecamps\"]\n",
    "\n",
    "# Define final search queries\n",
    "uk_terms    = cnctwb([cnct(uk_neutral_migrant_terms), cnct(uk_negative_migrant_terms), cnct(uk_negative_racial_terms)])\n",
    "uk_accounts = cnctwb([cnct(uk_pro_migrant_account_1), cnct(uk_pro_migrant_account_2), cnct(uk_neutral_account), cnct(uk_anti_migrant_account)])\n",
    "uk_hashtags = cnctwb([cnct(uk_positive_hashtags), cnct(uk_neutral_hashtags), cnct(uk_negative_hashtags), cnct(uk_event_hashtags)])\n",
    "\n",
    "# Append all search term into single list\n",
    "uk_search_terms = uk_neutral_migrant_terms + uk_negative_migrant_terms + uk_negative_racial_terms + uk_pro_migrant_account_1 + \\\n",
    "                  uk_pro_migrant_account_2 + uk_neutral_account + uk_positive_hashtags + uk_neutral_hashtags + uk_negative_hashtags + uk_event_hashtags\n",
    "\n",
    "\n",
    "# USA Search Terms\n",
    "\n",
    "# Migrant terms\n",
    "usa_neutral_migrant_terms  = [\"immigrant\", \"immigration\", \"migrant\", \"migration\", \"\\\"asylum seeker\\\"\", \"refugee\", \"\\\"undocumented worker\\\"\", \"\\\"guest worker\\\"\", \n",
    "                              \"\\\"foreign worker\\\"\", \"(human smuggling)\", \"(human trafficking)\"]\n",
    "usa_negative_migrant_terms = [\"illegals\", \"foreigner\", \"\\\"illegal alien\\\"\", \"\\\"illegal worker\\\"\"]\n",
    " \n",
    "# Racial terms\n",
    "usa_negative_racial_terms  = [\"islamophob\", \"sinophob\", \"\\\"china flu\\\"\", \"\\\"kung flu\\\"\", \"\\\"china virus\\\"\", \"\\\"chinese virus\\\"\", \"shangainese\"]\n",
    "\n",
    "# Twitter accounts\n",
    "usa_pro_migrant_account_1  = [\"@UNmigration\", \"@IOM_UN\", \"@IOMatUN\", \"@IOMatEU\", \"@IOM_UK\", \"@IOMResearch\", \"@IOM_GMDAC\", \"@hrw\", \"@NIJC\", \"@CIYJA\", \n",
    "                              \"@ImmAdvocates\", \"@NWIRP\", \"@RAICESTEXAS\", \"@ImmJusticeNOW\", \"@icirr\", \"@IAmAnImmigrant\", \"@Am4ImmJustice\", \"@NILCJusticeFund\"]\n",
    "usa_pro_migrant_account_2  = [\"@CIRCimmigrant\", \"@FLImmigrant\", \"@ImmFamTogether\", \"@ImmJustice\", \"@NICE4Workers\", \"@CA4ImmiJustice\", \"@immigrantarc\", \"@Join_SIM\",\n",
    "                              \"@SDIRC\", \"@RMIAN_org\", \"@NJAIJ\", \"@NVImmigrants\", \"@VA_Immigrants\"]\n",
    "usa_neutral_account        = [\"@ICEgov\", \"@PhillyOIA\", \"@iandraffairs\", \"@LAC4Immigrants\", \"@CoreCivic\"]\n",
    "usa_anti_migrant_account   = [\"@EuropidWhites\"]\n",
    "\n",
    "# Hashtags\n",
    "usa_positive_hashtags      = [\"#RefugeesWelcome\", \"#MigrantsWelcome\", \"#LeaveNoOneBehind\", \"#FreedomForImmigrants\", \"#illegalmigantsUSA\", \"#KillTheImmigrationBill\", \n",
    "                              \"#ImmigrantsMakeAmericaGreat\", \"#NoWall\", \"#NoWallEver\", \"#NoBan\", \"#FamiliesBelongTogether\", \"#stopICEcold\", \"#EndRemainInMexico\"]\n",
    "usa_neutral_hashtags       = [\"#ICE\", \"#immigration\", \"#migration\", \"#immigrant\", \"#migrant\", \"#immigrate\", \"#migrate\", \"#refugees\", \"#ImmigrationReform\"]\n",
    "usa_negative_hashtags      = [\"#illegals\", \"#foreigner\", \"#foreigners\", \"#illegalalien\", \"#illegalaliens\", \"#illegalworker\", \"#illegalworkers\", \"#KeepThemOut\",\n",
    "                              \"#OurCountry\", \"#SendThemBack\", \"#migrantsnotwelcome\", \"#refugeesnotwelcome\", \"#illegals\", \"#ChinaVirus\", \"#chinaflu\", \"#kungflu\",\n",
    "                              \"#chinesevirus\", \"#TheyHaveToGoBack\", \"#DeportThemAll\"]\n",
    "usa_event_hashtags         = [\"#Moria\", \"#closethecamps\", \"#divestfromdetention\"]\n",
    "\n",
    "# Define final search queries\n",
    "usa_terms    = cnctwb([cnct(usa_neutral_migrant_terms), cnct(usa_negative_migrant_terms), cnct(usa_negative_racial_terms)])\n",
    "usa_accounts = cnctwb([cnct(usa_pro_migrant_account_1), cnct(usa_pro_migrant_account_2), cnct(usa_neutral_account), cnct(usa_anti_migrant_account)])\n",
    "usa_hashtags = cnctwb([cnct(usa_positive_hashtags), cnct(usa_neutral_hashtags), cnct(usa_negative_hashtags), cnct(usa_event_hashtags)])\n",
    "\n",
    "# Append all search term into single list\n",
    "usa_search_terms = usa_neutral_migrant_terms + usa_negative_migrant_terms + usa_negative_racial_terms + usa_pro_migrant_account_1 + \\\n",
    "                   usa_pro_migrant_account_2 + usa_neutral_account + usa_positive_hashtags + usa_neutral_hashtags + usa_negative_hashtags + usa_event_hashtags\n",
    "\n",
    "\n",
    "\n",
    "# Spain Search Terms\n",
    "\n",
    "# Migrant terms\n",
    "spain_neutral_migrant_terms  = [\"migraci\", \"migrante\", \"refugiad\", \"migratorio\", \"refugiada\", \"discrimin\", \"extranjer\", \"xenofo\"]\n",
    "spain_negative_migrant_terms = [\"racista\", \"deporta\", \"legal\", \"patriota\", \"racismo\", \"invasores\"]\n",
    " \n",
    "# Racial terms\n",
    "spain_negative_racial_terms  = [\"musulman\", \"islamofobia\", \"marroquí\"]\n",
    "\n",
    "# Twitter accounts\n",
    "spain_pro_migrant_account_1  = [\"@UNmigration\", \"@IOM_UN\", \"@IOMatUN\", \"@IOMatEU\", \"@IOMPOS\", \"@IOMResearch\", \"@IOM_GMDAC\", \"@hrw\", \"@IOMspain\", \"@FundacionMigrar\",\n",
    "                                \"@inclusiongob\", \"@CooperacionAND\", \"@migrantes_sj\"]\n",
    "spain_pro_migrant_account_2  = [\"@PMigraciones\", \"@walkingborders\", \"@CanalMigrantes\", \"@RegularizacionY\", \"@RSAcogida\", \"@campsoscar\", \"@openarms_fund\", \"@APDHA\",\n",
    "                                \"@TrasLaManta\", \"@EspacioInmigran\", \"@CIEsNoMadrid\"]\n",
    "spain_neutral_account        = [\"@SpainMFA\", \"@MAECgob\", \"@EUHomeAffairs\", \"@redinmigracion\", \"@inmigrantes\", \"@m_migracion\", \"@desalambre\", \"@PoderMigrante_N\"]\n",
    "spain_anti_migrant_account   = [\"@FUERAINMIGRANTE\", \"@rubnpulido\", \"@ArturoVilla_\", \"@angladarj\"]\n",
    "\n",
    "# Hashtags\n",
    "spain_positive_hashtags      = [\"#InmigracionNoInvasion\", \"#nomasdiscriminacion\", \"#conlosrefugiados\", \"#nohayserhumanoilegal\",\n",
    "                                \"#multicultural\", \"#bienvenidosaespana\", \"#enddeportaciones\"]\n",
    "spain_neutral_hashtags       = [\"#Extranjería\", \"#Inmigración\", \"#Nacionalidad\", \"#inmigrantes\", \"#migrantes\", \"#migración\", \"#todossomosmigrante\", \"#redmigrante\",\n",
    "                                \"#migraciónlaboral\", \"#leydemigración\", \"#leydemigraciones\", \"#migracionysalud\", \"#migracionyeducacion\", \"#migracionytrabajo\",\n",
    "                                \"#migracionyeconomia\", \"#soymigrante\", \"#soymigranteandalucia\"]\n",
    "spain_negative_hashtags      = [\"#leymigratoriaya\", \"#Quelosdeportenatodos\", \"#DeportacionYa\", \"#fueramigrantes\", \"#cierrenfronteras\", \"#NoMasMigrantes\",\n",
    "                                \"#DeportacionInmediata\", \"#inmigrantesilegales\", \"#nomasinmigrantes\", \"#TrataDePersonas\", \"#TráficoIlícitodeMigrantes\"]\n",
    "spain_event_hashtags         = [\"#siria\", \"#MoriaCamp\", \"#lesbos\"]\n",
    "\n",
    "# Define final search queries\n",
    "spain_terms    = cnctwb([cnct(spain_neutral_migrant_terms), cnct(spain_negative_migrant_terms), cnct(spain_negative_racial_terms)])\n",
    "spain_accounts = cnctwb([cnct(spain_pro_migrant_account_1), cnct(spain_pro_migrant_account_2), cnct(spain_neutral_account), cnct(spain_anti_migrant_account)])\n",
    "spain_hashtags = cnctwb([cnct(spain_positive_hashtags), cnct(spain_neutral_hashtags), cnct(spain_negative_hashtags), cnct(spain_event_hashtags)])\n",
    "\n",
    "# Append all search term into single list\n",
    "spain_search_terms = spain_neutral_migrant_terms + spain_negative_migrant_terms + spain_negative_racial_terms + spain_pro_migrant_account_1 + \\\n",
    "                     spain_pro_migrant_account_2 + spain_neutral_account + spain_positive_hashtags + spain_neutral_hashtags + spain_negative_hashtags + spain_event_hashtags\n",
    "\n",
    "\n",
    "\n",
    "# Italy Search Terms\n",
    "\n",
    "# Migrant terms\n",
    "italy_neutral_migrant_terms  = [\"immigrato\", \"immigrati\", \"\\\"richiedente asilo\\\"\", \"\\\"richiedenti asilo\\\"\", \"\\\"persona irregolare\\\"\", \"\\\"persone irregolari\\\"\", \n",
    "                                \"migrante\", \"migranti\", \"straniero\", \"stranieri\", \"rifugiato\", \"rifugiati\", \"\\\"vittima di tratta\\\"\", \"\\\"vittime di tratta\\\"\", \n",
    "                                \"\\\"minore non accompagnato\\\"\", \"\\\"minori non accompagnati\\\"\", \"\\\"traffico di migranti\\\"\", \"sbarco\", \"sbarchi\"]\n",
    "italy_negative_migrant_terms = [\"clandestino\", \"clandestini\", \"illegale\", \"illegali\", \"profugo\", \"profughi\", \n",
    "                                \"\\\"immigrato abusivo\\\"\", \"\\\"immigrati abusivi\\\"\", \"invasione\"]\n",
    " \n",
    "# Racial terms\n",
    "italy_negative_racial_terms  = [\"zingaro\", \"zingari\", \"negro\", \"negri\", \"negre\", \"sporco\", \"sporchi\", \"scimmia\", \"scimmie\", \"extracomunitario\", \"extracomunitari\" ]\n",
    "\n",
    "# Twitter accounts\n",
    "italy_pro_migrant_account_1  = [\"@UNmigration\", \"@IOM_UN\", \"@IOMatUN\", \"@IOMatEU\", \"@IOMResearch\", \"@OIMItalia\", \"@IOM_GMDAC\", \"@LHartIOM\", \"@hrw\", \"@Refugees\", \"@MSF_Sea\", \"@RefugeesIntl\",\n",
    "                                \"@DetentionForum\", \"@OpenSociety\", \"@Amnesty\", \"@FilippoGrandi\", \"@IOMchief\", \"@UNSR_Migration\", \"@PICUM_post\", \"@seawatch_intl\", \"@CentroAstalli\"]\n",
    "italy_pro_migrant_account_2  = [\"@emergency_ong\", \"@amnestyitalia\", \"@SeaWatchItaly\", \"@BaobabExp\", \"@aboubakar_soum\", \"@CaritasItaliana\", \"@cartadiroma\", \"@caritas_milano\", \n",
    "                                \"@open_migration\", \"@Medhope_FCEI\", \"@MSF_ITALIA\", \"@OIMItalia\", \"@UNHCRItalia\", \"@SOSMedItalia\", \"@openarms_it\", \"@RescueMed\", \"@UNICEF_Italia\", \n",
    "                                \"@valigiablu\", \"@OxfamItalia\", \"@CIRRIFUGIATI\", \"@SaveChildrenIT\", \"@InMigrazione\", \"@Pontifex\"]\n",
    "italy_neutral_account        = [\"@Viminale\", \"@EUHomeAffairs\", \"@GiuseppeConteIT\", \"@ItalyMFA\"]\n",
    "italy_anti_migrant_account   = [\"@matteosalvinimi\", \"@giorgiameloni\", \"@CasaPoundItalia\", \"@ForzaNuova\", \"@FratellidItalia\", \"@LegaSalvini\", \"@Dsantanche\", \n",
    "                                \"@ilgiornale\", \"@Libero_official\", \"@lumorisi\", \"@RobertoFioreFN\"]\n",
    "\n",
    "# Hashtags\n",
    "italy_positive_hashtags      = [\"#portiaperti\", \"#apriamoiporti\", \"#aprireiporti\", \"#accoglienza\", \"#bastarazzismo\", \"#fatelientrare\", \"#SeaWatch\", \"#SeaEye\", \"#Openarms\", \"#TuttiFratelli\",\n",
    "                                \"#3ottobre\", \"#restiamoumani\", \"#regolarizzazione\", \"#corridoiumanitari\", \"#solidarieta\", \"#dirittodiasilo\", \"#ioaccolgo\"]\n",
    "italy_neutral_hashtags       = [\"#DecretiSicurezza\", \"#decretoimmigrazione\", \"#migranti\", \"#migrazioni\", \"#immigrati\", \"#immigrazione\", \"#SAR\", \"#searchandrescue\", \"#stranieri\",\n",
    "                                \"#richiedentiasilo\", \"#asilo\", \"#rifugiati\", \"#integrazione\", \"#ONG\", \"#iussoli\", \"#decretiSalvini\", \"#razzismo\", \"#cittadinanza\", \"#MareNostrum\", \n",
    "                                \"#reinsediamento\", \"#rimpatri\", \"#Mediterraneo\", \"#RegolamentoDublino\", \"#hotspot\", \"#tratta\"]\n",
    "italy_negative_hashtags      = [\"#portichiusi\", \"#tolleranzazero\", \"#descretisicurezza\", \"#decretosalvini\", \"#decretoimmigrazione\", \"#BloccoNavale\", \"#blocconavalesubito\", \n",
    "                                \"#chiudiamoiporti\", \"#invasione\", \"#lamorgesedimettiti\", \"#extracomunitari\", \"#clandestini\", \"#profughi\", \"#irregolari\", \"#ItaliaAgliItaliani\", \n",
    "                                \"#aiutiamoliacasaloro\", \"#primagliitaliani\", \"#Decretoclandestini\", \"#DifendiamolItalia\", \"#iostoconSalvini\"]\n",
    "italy_event_hashtags         = [\"#Moria\", \"#Etienne\", \"#Abou\", \"#Lesbo\", \"#WillyMonteroDuarte\", \"#WillyMonteiro\"]\n",
    "\n",
    "# Define final search queries\n",
    "# In order to meet API character limits 'italy_positive_hashtags' have been added to 'italy_terms'\n",
    "italy_terms    = cnctwb([cnct(italy_neutral_migrant_terms), cnct(italy_negative_migrant_terms), cnct(italy_negative_racial_terms), cnct(italy_positive_hashtags)])\n",
    "italy_accounts = cnctwb([cnct(italy_pro_migrant_account_1), cnct(italy_pro_migrant_account_2), cnct(italy_neutral_account), cnct(italy_anti_migrant_account)])\n",
    "italy_hashtags = cnctwb([cnct(italy_neutral_hashtags), cnct(italy_negative_hashtags), cnct(italy_event_hashtags)])\n",
    "\n",
    "# Append all search term into single list\n",
    "italy_search_terms = italy_neutral_migrant_terms + italy_negative_migrant_terms + italy_negative_racial_terms + italy_pro_migrant_account_1 + \\\n",
    "                     italy_pro_migrant_account_2 + italy_neutral_account + italy_positive_hashtags + italy_neutral_hashtags + italy_negative_hashtags + italy_event_hashtags\n",
    "\n",
    "\n",
    "# German Search Terms\n",
    "\n",
    "# Migrant terms\n",
    "grman_neutral_migrant_terms  = [\"migrant\", \"migranten\", \"migrantin\", \"migrantinnen\", \"migrierende\", \"migration\", \"asylbewerber\", \"asylbewerberin\", \"asylbewerbende\", \"flüchtling\", \n",
    "                                \"flüchtende\", \"flüchtlinge\", \"\\\"ausländische arbeiter\\\"\", \"\\\"ausländische arbeiterin\\\"\", \"\\\"ausländischer arbeiterinnen\\\"\", \"gastarbeiter\", \"gastarbeiterin\", \n",
    "                                \"gastarbeiterinnen\", \"\\\"ausländische arbeitskräfte\\\"\", \"\\\"ausländische arbeitskraft\\\"\"]\n",
    "grman_negative_migrant_terms = [\"\\\"illegale arbeiter\\\"\", \"\\\"illegale arbeiterin\\\"\", \"\\\"illegale arbeiterinnen\\\"\", \"\\\"illegal arbeitende\\\"\", \"ausländer\", \"ausländerin\", \"ausländerinnen\"]\n",
    " \n",
    "# Racial terms\n",
    "grman_negative_racial_terms  = [\"zigeuner\", \"zigeunerinnen\", \"\\\"kung flu\\\"\", \"kanacken\", \"kanacke\", \"kanackin\", \"kanackinnen\"]\n",
    "\n",
    "# Twitter accounts\n",
    "grman_pro_migrant_account_1  = [\"@UNmigration\", \"@IOM_UN\", \"@IOMatUN\", \"@IOMatEU\", \"@IOMGermany\", \"@IOMResearch\", \"@IOM_GMDAC\", \"@hrw\", \"@missingmigrants\", \"@ECRE\", \"@Refugees\", \n",
    "                                \"@unoflucht\", \"@MoriaMediaTeam\", \"@ProAsyl\", \"@Balkanbrücke\", \"@CLAIM_Allianz\", \"@Project_SAS\", \"@NoBorder_Berlin\", \"@RLC_Berlin\", \"@hamburgasyl\",\n",
    "                                \"@seebruecke_intl\", \"@refugeesresist\", \"@grenzenlwaerme\", \"@lwob\", \"@seawatch_intl\", \"@_seebruecke_\", \"@M_RSection\", \"@Flchtlngshlfr_n\", \"@noborderkitchen\"]\n",
    "grman_pro_migrant_account_2  = [\"@BBgegenRechts\", \"@refugee_supp\", \"@AmnestyAntira\", \"@ProjektSeehilfe\", \"@areyousyrious\", \"@berlinfueralle\", \"@AmnestyEU\", \"@SOSMedGermany\", \"@seawatchcrew\",\n",
    "                                \"@buegera\", \"@ENStatelessness\", \"@derbraunemob\", \"@Amnesty\", \"@IL_berlin\", \"@nothilfe\", \"@ErikMarquardt\", \"@MVLouiseMIchel\", \"@SEENOTRETTUNG\", \"@civilfleet\",\n",
    "                                \"@f_grillmeier\", \"@isabelschayani\", \"@chrjkb\"]\n",
    "grman_neutral_account        = [\"@BMI_Bund\", \"@BAMF_Dialog\", \"@EUHomeAffairs\", \"@BK_Amt\"]\n",
    "grman_anti_migrant_account   = [\"@BjoernHoecke\", \"@afd\", \"@compactmagazin\", \"@Hauptstadt_NPD\", \"@aktionsblogb\", \"@refugeecrimemap\", \"@DS_redaktion\", \"@ER_MV\", \"@JNDeutschland\"]\n",
    "\n",
    "# Hashtags\n",
    "grman_positive_hashtags      = [\"#RefugeesWelcome\", \"#MigrantsWelcome\", \"#LeaveNoOneBehind\", \"#FreedomForImmigrants\", \"#BerlinIsOpen\", \"#familyreunion\", \"#wirschaffendas\", \n",
    "                                \"#TogetherForRescue\", \"#blacklivesmatter\", \"#RescueFamily\", \"#FightForSolidarity\", \"#FreeTheShips\", \"#WirHabenPlatz\", \"#SafePassage\", \"#FlattenTheBorders\",\n",
    "                                \"#NotMyEU\", \"#ShameOnYouEurope\"]\n",
    "grman_neutral_hashtags       = [\"#migration\", \"#immigration\", \"#immigrant\", \"#migrant\", \"#flüchtling\", \"#flüchtlinge\", \"#Asylrecht\", \"#EUmigration\", \"#Seenotrettung\", \"#Migrationspakt\",\n",
    "                                \"#MIgrationPact\", \"#Migrationspaket\", \"#Rückführungspatenschaft\", \"#Asylreform\"]\n",
    "grman_negative_hashtags      = [\"#illegale\", \"#ausländer\", \"#Zigeuner\", \"#KungFlu\", \"#whitelivesmatter\", \"#Kanacken\", \"#MGGA\", \"#Europistan\", \"#Germanystan\", \"#PEGIDA\", \"#Zukunftheimat\",\n",
    "                                \"#EinProzent\", \"#ausländerraus\", \"#keepthemout\", \"#unserland\", \"#migrantsnotwelcome\", \"#refugeesunwelcome\", \"#refugeesnotwelcome\", \"#chinaflu\",\n",
    "                                \"#chinavirus\", \"#alleabschieben\", \"#abschieben\", \"#gegenmigration\", \"#Befreiungvomislam\"]\n",
    "grman_event_hashtags         = [\"#Moria\", \"#flüchtlingscamps\", \"#campschliessung\", \"#MoriaStory\", \"#dreizehntausend\", \"#Iuventa\", \"#SeaWatch3\", \"#Libyen\", \"#RefugeesGR\"]\n",
    "\n",
    "# Define final search queries\n",
    "# In order to meet API character limits 'grman_neutral_account', 'grman_anti_migrant_account' and 'grman_event_hashtags' have been added to 'grman_terms'\n",
    "grman_terms    = cnctwb([cnct(grman_neutral_migrant_terms), cnct(grman_negative_migrant_terms), cnct(grman_negative_racial_terms), \n",
    "                         cnct(grman_neutral_account), cnct(grman_anti_migrant_account), cnct(grman_event_hashtags)])\n",
    "grman_accounts = cnctwb([cnct(grman_pro_migrant_account_1), cnct(grman_pro_migrant_account_2)])\n",
    "grman_hashtags = cnctwb([cnct(grman_positive_hashtags), cnct(grman_neutral_hashtags), cnct(grman_negative_hashtags)])\n",
    "\n",
    "# Append all search term into single list\n",
    "grman_search_terms = grman_neutral_migrant_terms + grman_negative_migrant_terms + grman_negative_racial_terms + grman_pro_migrant_account_1 + \\\n",
    "                     grman_pro_migrant_account_2 + grman_neutral_account + grman_positive_hashtags + grman_neutral_hashtags + grman_negative_hashtags + grman_event_hashtags\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Country Search Terms\n",
    "uk_add_terms     = ' lang:en (place_country:GB OR profile_country:GB)'\n",
    "usa_add_terms    = ' lang:en (place_country:US OR profile_country:US)'\n",
    "spain_add_terms  = ' -is:retweet lang:es (place_country:ES OR profile_country:ES)'\n",
    "italy_add_terms  = ' lang:it (place_country:IT OR profile_country:IT)'\n",
    "grman_add_terms  = ' -is:retweet lang:de (place_country:DE OR profile_country:DE)'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Access Tweet Counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function which calls daily counts for a given search query\n",
    "def call_counts (x,y,z,a,b):\n",
    "    # Put together search terms and rules\n",
    "    count_rule = gen_rule_payload(x, from_date=y, to_date=z,\n",
    "                                  results_per_call=a,\n",
    "                                  count_bucket=b)\n",
    "    # Collect counts\n",
    "    counts = collect_results(count_rule, result_stream_args=premium_search_args)\n",
    "    return(counts)\n",
    "\n",
    "# Function which selects a substring from a string column\n",
    "def date_sub (x,y,z):\n",
    "    return(x.date.str.slice(y,z))\n",
    "\n",
    "# Function which converts string to datetime\n",
    "def str2dt (x):\n",
    "    return(datetime.strptime(x, \"%Y-%m-%d %H:%M\"))\n",
    "\n",
    "# Function which converts string to datetime\n",
    "def dt2str (x):\n",
    "    return(datetime.strftime(x, \"%Y-%m-%d %H:%M\"))\n",
    "\n",
    "# Function which converts counts call into a dataframe\n",
    "def format_counts (x,y,z):\n",
    "    tc = {'date':            [dc['timePeriod'] for dc in x],\n",
    "                    'terms_tweets':    [dc['count'] for dc in x], \n",
    "                    'accounts_tweets': [dc['count'] for dc in y], \n",
    "                    'hashtags_tweets': [dc['count'] for dc in z]}\n",
    "    tc = pd.DataFrame(tc, columns = ['date', 'terms_tweets', 'accounts_tweets', 'hashtags_tweets'])\n",
    "    tc['total_tweets'] = tc.sum(axis = 1)\n",
    "    tc['date'] = date_sub(tc,0,4) + '-' + date_sub(tc,4,6) + '-' + date_sub(tc,6,8) + ' ' + date_sub(tc,8,10) + ':' + date_sub(tc,10,12)\n",
    "    return(tc)\n",
    "\n",
    "# Function which calls all tweet counts for a desired location, time interval and time bucke\n",
    "def call_tweet_counts(x,y,z,a,b,c,d):\n",
    "    \n",
    "    # Convert start and end dates to datetime objects\n",
    "    st  = str2dt(x)\n",
    "    et  = str2dt(y)\n",
    "    # Obtain time delta between start and end time\n",
    "    tdf = et-st\n",
    "\n",
    "    # Identify how many calls of 500 are needed to obtain counts for every hour between start and end date\n",
    "    loops = int((tdf.total_seconds() / 3600) // 500)\n",
    "    \n",
    "    # Create list to append twitter count infromation to\n",
    "    counts = []\n",
    "    \n",
    "    # for loop with calls tweets and appends them to counts list\n",
    "    # Counts can only be called in groups of 500, to the loop must be run \n",
    "    for i in range(loops + 1):\n",
    "        # In all but the last loop, counts are called in groups of 500 hours at a time\n",
    "        if i < loops:\n",
    "            start = str(et - timedelta(hours=(i+1)*500))[0:16]\n",
    "            end   = str(et - timedelta(hours=i*500))[0:16]\n",
    "            calls = 500\n",
    "        # In the final loop, the remaining hours between the previous call and the desired start date are called\n",
    "        else:\n",
    "            end   = str(et - timedelta(hours=loops*500))[0:16]\n",
    "            start = x\n",
    "            calls = int((str2dt(end) - str2dt(x)).total_seconds() / 3600 )\n",
    "        # Counts are called for search terms\n",
    "        terms_tweets    = call_counts((z + c), start, end, calls, d)\n",
    "        # Counts are called for accounts\n",
    "        accounts_tweets = call_counts((a + c), start, end, calls, d)\n",
    "        # Counts are called for hashtags\n",
    "        hashtags_tweets = call_counts((b + c), start, end, calls, d)\n",
    "        # All counts are formatted into a dataframe\n",
    "        tweet_counts    = format_counts(terms_tweets, accounts_tweets, hashtags_tweets)\n",
    "        # Dataframe is appended to the counts list\n",
    "        counts.append(tweet_counts)\n",
    "    \n",
    "    # All counts dataframes are concatenated into a single dataframe\n",
    "    all_counts = pd.concat(counts)\n",
    "    # The date variable is converted from str to datetime\n",
    "    all_counts['date'] = pd.to_datetime(all_counts['date'])\n",
    "    # Dataframe is ordered by datetime and the index is reset\n",
    "    all_counts = all_counts.sort_values(by='date', ascending=False).reset_index().drop(['index'], axis=1)    \n",
    "    \n",
    "    # Return output\n",
    "    return(all_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define count search parameters\n",
    "start_date  = \"2020-05-01 00:00\"\n",
    "end_date    = \"2020-11-01 00:00\"\n",
    "time_bucket = \"hour\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call UK tweet counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#uk_tweet_counts = call_tweet_counts(start_date, end_date, uk_terms, uk_accounts, uk_hashtags, uk_add_terms, time_bucket)\n",
    "uk_tweet_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save uk_tweet_counts as csv\n",
    "#uk_tweet_counts.to_csv(rp + dp + 'tweet_counts\\\\uk_tweet_counts.csv')\n",
    "#uk_tweet_counts.to_csv(rp + dp + 'tweet_counts\\\\uk_tweet_counts_2.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call USA tweet counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#usa_tweet_counts = call_tweet_counts(start_date, end_date, usa_terms, usa_accounts, usa_hashtags, usa_add_terms, time_bucket)\n",
    "usa_tweet_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save usa_tweet_counts as csv\n",
    "#usa_tweet_counts.to_csv(rp + dp + 'tweet_counts\\\\usa_tweet_counts.csv')\n",
    "#usa_tweet_counts.to_csv(rp + dp + 'tweet_counts\\\\usa_tweet_counts2.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call Spain tweet counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#spain_tweet_counts = call_tweet_counts(start_date, end_date, spain_terms, spain_accounts, spain_hashtags, spain_add_terms, time_bucket)\n",
    "spain_tweet_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save spain_tweet_counts as csv\n",
    "#spain_tweet_counts.to_csv(rp + dp + 'tweet_counts\\\\spain_tweet_counts.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call Italy tweet counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#italy_tweet_counts = call_tweet_counts(start_date, end_date, italy_terms, italy_accounts, italy_hashtags, italy_add_terms, time_bucket)\n",
    "italy_tweet_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save uk_tweet_counts as csv\n",
    "#italy_tweet_counts.to_csv(rp + dp + 'tweet_counts\\\\italy_tweet_counts_01122019_01052020.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call German tweet counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#grman_tweet_counts = call_tweet_counts(start_date, end_date, grman_terms, grman_accounts, grman_hashtags, grman_add_terms, time_bucket)\n",
    "grman_tweet_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save grman_tweet_counts as csv\n",
    "#grman_tweet_counts.to_csv(rp + dp + 'tweet_counts\\\\grman_tweet_counts.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot Houry Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Country tweet count to read in\n",
    "country = 'italy'\n",
    "\n",
    "# Read in tweet_counts as dataframe\n",
    "country_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\' + country + '_tweet_counts_01122019_01052020.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "country_tweet_counts['date'] = pd.to_datetime(country_tweet_counts['date'])\n",
    "\n",
    "# Plot hourly tweet counts\n",
    "date   = country_tweet_counts['date'] \n",
    "tweets = country_tweet_counts['total_tweets']\n",
    "\n",
    "plt.figure(figsize=(20,6))\n",
    "plt.plot(date, tweets)\n",
    "plt.title(country + ' hourly tweets')\n",
    "plt.xlabel('Hour')\n",
    "plt.ylabel('tweet Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Access Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function which subsets tweet counts df to only include N day with the most activity\n",
    "def max_act_days(a,N):\n",
    "    # Create day variable\n",
    "    a['day'] = a['date'].apply(lambda x: datetime.strftime(x, '%Y-%m-%d'))\n",
    "    # Create new df to be subsetted later\n",
    "    x = a\n",
    "    # Get total tweets for each day and order days from most to least activity\n",
    "    a = a.groupby('day').sum().sort_values(by='total_tweets',ascending=False)\n",
    "    # Subset to only include N days with most activity\n",
    "    a = a.loc[a.index.values.tolist()[0:N],]\n",
    "    # Subset x to only include max activity days\n",
    "    x = x[x['day'].astype(str).isin(a.index.values)]\n",
    "    # Remove day variable\n",
    "    del x['day']\n",
    "    # Return results\n",
    "    return(x)\n",
    "\n",
    "# Function which calls daily counts for a given search query\n",
    "def call_tweets (x,y,z):\n",
    "    # Put together search terms and rules\n",
    "    rule = gen_rule_payload(x, from_date=y, to_date=z, results_per_call=500)\n",
    "    # Collect counts\n",
    "    tweets = collect_results(rule, result_stream_args=premium_search_args)\n",
    "    return(tweets)\n",
    "\n",
    "# Function to call tweets as json files\n",
    "def get_tweets (a,b,c,d,e):\n",
    "    \n",
    "    # Obtain the time to call on each day of the study period\n",
    "    ## This is the hour after the one with the most tweets that meet the search criteria within a 24 hours period.\n",
    "    \n",
    "    # Create new 'day' datetime variable which does not include hours of minutes\n",
    "    a['day'] = a['date'].apply(lambda x: datetime.strftime(x, '%Y-%m-%d'))\n",
    "    \n",
    "    # Subset the dataframe to only include the hour with most tweets on each day\n",
    "    max_hour = a.loc[a.groupby('day')['total_tweets'].agg(pd.Series.idxmax)]['date']\n",
    "    \n",
    "    # Add an hour to all of these times (Max Hour Plus One) and convert to a list of string variables\n",
    "    #mhpo = max_hour.apply(lambda x: x + timedelta(hours = 1) ).apply(lambda x: dt2str(x)).tolist()\n",
    "    \n",
    "    # Add 30 minutes to all of these times (for the USA's second call) and convert to a list of string variables\n",
    "    #mhpo = max_hour.apply(lambda x: x + timedelta(minutes = 30) ).apply(lambda x: dt2str(x)).tolist()\n",
    "    \n",
    "    # Don't add an hour. Convert to a list of string variables\n",
    "    mhpo = max_hour.apply(lambda x: dt2str(x)).tolist()\n",
    "    \n",
    "    # Identify how many days there are between the start and end date\n",
    "    #days = (str2dt(end_date) - str2dt(start_date)).days\n",
    "    days = len(a['day'].unique())\n",
    "    \n",
    "    # Create empty list to put tweets in\n",
    "    tweets = []\n",
    "    \n",
    "    for i in range(days):\n",
    "        # Define the end datetime as the time in the mhpo (Max Hour Plus One) list for a given date\n",
    "        end   = mhpo[i]\n",
    "        # Define start time as 72 hours before the end time (arbitrary time period considered sufficient to ensure 500 tweets are collected)\n",
    "        start = dt2str(str2dt(mhpo[i]) - timedelta(days = 3))\n",
    "        \n",
    "        # Creates list of tweets for a given day\n",
    "        tweets_temp = []\n",
    "        # Counts are called for search terms\n",
    "        terms_tweets    = call_tweets((b + e), start, end)\n",
    "        # Counts are called for accounts\n",
    "        accounts_tweets = call_tweets((c + e), start, end)\n",
    "        # Counts are called for hashtags\n",
    "        hashtags_tweets = call_tweets((d + e), start, end)\n",
    "        # Compile tweets into 'tweets_temp' lists\n",
    "        tweets_temp.extend((terms_tweets, accounts_tweets, hashtags_tweets))\n",
    "        # Append 'tweets_temp' to 'tweets' list\n",
    "        tweets.append(tweets_temp)\n",
    "        # Print to report completion\n",
    "        print(str(i + 1) + ' of ' + str(days) + ' days called.', end=\"\\r\")\n",
    "    \n",
    "    return(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define tweet search parameters\n",
    "start_date  = \"2019-12-01 00:00\"\n",
    "end_date    = \"2020-05-01 00:00\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call UK Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in uk_tweet_counts.csv as dataframe\n",
    "uk_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\uk_tweet_counts_01122019_01052020.csv').drop(['Unnamed: 0'], axis=1)\n",
    "#uk_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\uk_tweet_counts_01052020_01112020.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "uk_tweet_counts['date'] = pd.to_datetime(uk_tweet_counts['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Subset tweet count to only include the 75 days with the most activity\n",
    "uk_tweet_counts_max_days = max_act_days(uk_tweet_counts,75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call tweets from API\n",
    "#uk_tweets = get_tweets(uk_tweet_counts_max_days, uk_terms, uk_accounts, uk_hashtags, uk_add_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as json file\n",
    "#with open(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020.json', 'w') as f:\n",
    "#    json.dump(uk_tweets, f)\n",
    "#with open(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_max.json', 'w') as f:\n",
    "#    json.dump(uk_tweets, f)\n",
    "#with open(rp + dp + 'tweets\\\\uk_tweets_01052020_01112020.json', 'w') as f:\n",
    "#    json.dump(uk_tweets, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call USA Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in usa_tweet_counts.csv as dataframe\n",
    "#usa_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\usa_tweet_counts_01122019_01052020.csv').drop(['Unnamed: 0'], axis=1)\n",
    "usa_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\usa_tweet_counts_01052020_01112020.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "usa_tweet_counts['date'] = pd.to_datetime(usa_tweet_counts['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call tweets from API\n",
    "#usa_tweets = get_tweets(usa_tweet_counts, usa_terms, usa_accounts, usa_hashtags, usa_add_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as json file\n",
    "#with open(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_with_retweets.json', 'w') as f:\n",
    "#    json.dump(usa_tweets, f)\n",
    "with open(rp + dp + 'tweets\\\\usa_tweets_01052020_01112020.json', 'w') as f:\n",
    "    json.dump(usa_tweets, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call Spain Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in usa_tweet_counts.csv as dataframe\n",
    "spain_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\spain_tweet_counts.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "spain_tweet_counts['date'] = pd.to_datetime(spain_tweet_counts['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call tweets from API\n",
    "#spain_tweets = get_tweets(spain_tweet_counts, spain_terms, spain_accounts, spain_hashtags, spain_add_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as json file\n",
    "#with open(rp + dp + 'tweets\\\\spain_tweets.json', 'w') as f:\n",
    "#    json.dump(spain_tweets, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call Italy Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in italy_tweet_counts.csv as dataframe\n",
    "italy_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\italy_tweet_counts_01122019_01052020.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "italy_tweet_counts['date'] = pd.to_datetime(italy_tweet_counts['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call tweets from API\n",
    "#italy_tweets = get_tweets(italy_tweet_counts, italy_terms, italy_accounts, italy_hashtags, italy_add_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as json file\n",
    "#with open(rp + dp + 'tweets\\\\italy_tweets.json', 'w') as f:\n",
    "#    json.dump(italy_tweets, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Call German Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in usa_tweet_counts.csv as dataframe\n",
    "grman_tweet_counts = pd.read_csv(rp + dp + 'tweet_counts\\\\grman_tweet_counts.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "grman_tweet_counts['date'] = pd.to_datetime(grman_tweet_counts['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Call tweets from API\n",
    "#grman_tweets = get_tweets(grman_tweet_counts, grman_terms, grman_accounts, grman_hashtags, grman_add_terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as json file\n",
    "#with open(rp + dp + 'tweets\\\\grman_tweets.json', 'w') as f:\n",
    "#    json.dump(grman_tweets, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reformat Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function which extracts only essential information from tweets\n",
    "def reduce_tweets(tw_list):\n",
    "    tweets = [] # Create balnk file to read tweets into \n",
    "    for tweet in tw_list: # For each tweet\n",
    "        \n",
    "        reduced_tweet = { # Store key details\n",
    "            'created_at': tweet['created_at'], # Time and date of tweet\n",
    "            'status_id': tweet['id_str'], # Unique ID of Tweet\n",
    "            'username': tweet['user']['screen_name'], # Username of Twitter profile\n",
    "            'user_id': tweet['user']['id_str'], # Unique ID for Twtter profile\n",
    "            'text': tweet['text'] # Store text of tweet (140 characters max)\n",
    "        }\n",
    "        \n",
    "        if 'extended_tweet' in tweet: # If tweet is more than 140 characters (Twitter seperates out old and current tweet lengths)\n",
    "            reduced_tweet.update({'text':tweet['extended_tweet']['full_text']}) # Store full text (else cut off)\n",
    "        elif 'retweeted_status' in tweet and 'extended_tweet' in tweet['retweeted_status']: # If a retweet and tweet more than 140 characters\n",
    "            reduced_tweet.update({'text':tweet['retweeted_status']['extended_tweet']['full_text']}) # Store full text\n",
    "        else: # Else if neither of previous two options, keep 140 characters text\n",
    "            reduced_tweet.update({'text':tweet['text']})\n",
    "            \n",
    "        if 'derived' in tweet['user']: # If present in the users information\n",
    "            if 'locations' in tweet['user']['derived']: # Store country\n",
    "                reduced_tweet.update({'country':tweet['user']['derived']['locations'][0]['country']})\n",
    "#            else:\n",
    "#                reduced_tweet.update({}'country':''}) # If not present then store as missing\n",
    "            if 'region' in tweet['user']['derived']['locations'][0]: # If present in the users information\n",
    "                reduced_tweet.update({'region':tweet['user']['derived']['locations'][0]['region']}) # Store region\n",
    "#            else:\n",
    "#                reduced_tweet.update({'region':''}) # If not present then store as missing\n",
    "            \n",
    "        if 'retweeted_status' in tweet: # If a retweet (store as nested within same Tweet)\n",
    "            reduced_tweet.update({'retweeted_user':{'status_id' : tweet['retweeted_status']['id'], # Store original tweet id\n",
    "                                                    'user_id' : tweet['retweeted_status']['user']['id_str'], # Store user ID of retweeted user\n",
    "                                                    'username' : tweet['retweeted_status']['user']['screen_name']}, # Store username\n",
    "                                  'retweeted_status_id': tweet['retweeted_status']['id_str'], # Store retweeted status_id\n",
    "                                  'retweet_count':tweet['retweeted_status']['retweet_count']}) # Store retweet count\n",
    "        else:\n",
    "            reduced_tweet.update({'retweeted_user': np.nan, # No retweet info\n",
    "                                  'retweeted_status_id': tweet['id_str'], # Store retweeted status_id\n",
    "                                  'retweet_count':tweet['retweet_count'] }) # Store retweet count\n",
    "        \n",
    "        # Create additional dictionary values (eventially df columns) specifically for quoted tweets\n",
    "        if 'quoted_status' in tweet: # If current tweet is quoting a separate tweet (store as nested within same Tweet)\n",
    "            reduced_tweet.update({ # Store key details\n",
    "                'quoted_created_at': tweet['quoted_status']['created_at'], # Time and date of original\n",
    "                'quoted_status_id': tweet['quoted_status']['id_str'], # Unique ID of original\n",
    "                'quoted_username': tweet['quoted_status']['user']['screen_name'], # Username of original Twitter profile\n",
    "                'quoted_user_id': tweet['quoted_status']['user']['id_str'], # Unique ID for original Twtter profile\n",
    "                'quoted_text': tweet['quoted_status']['text'], # Store text of original (140 characters max)\n",
    "                'quoted_country': np.nan, # Location details for quoted tweet not available, so assign NaN to ensure compatibility with non-quoted tweets\n",
    "                'quoted_region': np.nan, # Location details for quoted tweet not available, so assign NaN to ensure compatibility with non-quoted tweets\n",
    "                'quoted_retweeted_user': np.nan, # Details of retweets not available, so assign NaN to ensure compatibility with non-quoted tweets\n",
    "                'quoted_retweeted_status_id': tweet['quoted_status']['id_str'], # Details of retweets not available, so assign NaN to ensure compatibility with non-quoted tweets\n",
    "                'quoted_retweet_count': tweet['quoted_status']['retweet_count'] }) # Retweet_count of quoted tweet\n",
    "            \n",
    "            if 'extended_tweet' in tweet['quoted_status']: # If original text is more than 140 characters (Twitter seperates out old and current tweet lengths)\n",
    "                reduced_tweet.update({'quoted_text':tweet['quoted_status']['extended_tweet']['full_text']}) # Store full text (else cut off)\n",
    "            else: # Else keep 140 characters text\n",
    "                reduced_tweet.update({'quoted_text':tweet['quoted_status']['text']})\n",
    "        \n",
    "        tweets.append(reduced_tweet)\n",
    "        \n",
    "    return (tweets)\n",
    "\n",
    "# Function which converts reduced tweets to a dataframe and preps data for vader lexicon\n",
    "def to_df_vader(x,y,a,b):\n",
    "    \n",
    "    # Create copy of original twitter list\n",
    "    x = x.copy()\n",
    "    \n",
    "    # Create list of names for each type of search\n",
    "    search_type = [\"key_terms\", \"accounts\", \"hashtags\"]\n",
    "    \n",
    "    # for loop which extracts data from each 500 tweet search, then converts to df\n",
    "    for day in range(len(x)): # main loop iterates through each day\n",
    "        for st in range(len(x[day])): # nested loop iterates through each of search type\n",
    "            x[day][st] = reduce_tweets(x[day][st]) # Uses reduced_tweets() to extract essential info from tweets\n",
    "            x[day][st] = pd.DataFrame(x[day][st]) # Converts resulting dictionary to df\n",
    "            x[day][st]['search_type'] = search_type[st] # Creates 'search_type' column and assigned a tupe from previously creates list\n",
    "        x[day] = pd.concat(x[day]) # Concatenates tweets from each search type for a given day\n",
    "    \n",
    "    # Concatenates tweets from across all days called and resets index\n",
    "    x = pd.concat(x).reset_index().drop(['index'], axis=1)\n",
    "    \n",
    "    # Create list of column names specifically assigned to quoted tweets\n",
    "    quoted = ['quoted_created_at', 'quoted_status_id', 'quoted_username', 'quoted_user_id', 'quoted_text', \n",
    "              'quoted_country', 'quoted_region', 'quoted_retweeted_user', 'quoted_retweeted_status_id', 'quoted_retweet_count']\n",
    "    \n",
    "    # Create df for main tweets where these quoted tweet columns have been dropped\n",
    "    df = x.drop(quoted, axis = 1)\n",
    "    \n",
    "    # Assign all rows in this df as not quoted tweets\n",
    "    df['quoted_tweet'] = False\n",
    "    \n",
    "    # Create second df which only includes quoted variables\n",
    "    quoted_df = x[quoted + ['search_type'] ]\n",
    "    \n",
    "    # Drop all NaN rows from quoted_df (rows where no tweet was quoted) as well as duplicates (where same tweet quoted multiple times)\n",
    "    quoted_df = quoted_df[quoted_df['quoted_created_at'].notna()].drop_duplicates()\n",
    "     \n",
    "    # Assign all rows in this df as quoted tweets\n",
    "    quoted_df['quoted_tweet'] = True\n",
    "    \n",
    "    # Change column names to those of the non-quoted df\n",
    "    quoted_df.columns = list(df.columns)\n",
    "    \n",
    "    # Subset to only include tweets which contain at least 1 of the country specific search terms (y)\n",
    "    quoted_df = quoted_df[quoted_df['text'].str.contains('|'.join(y), case = False)]\n",
    "    \n",
    "    # Append dataframes, drop any quoted tweets which have already been captured by the original search, and reset index\n",
    "    df = df.append(quoted_df).drop_duplicates('status_id', keep='first').reset_index(drop = True)\n",
    "    \n",
    "    # Convert 'created_at' to datetime variable\n",
    "    df['created_at'] = df['created_at'].apply(lambda z: z[4:10] + z[25:30] + z[10:19] )\\\n",
    "                                       .apply(lambda z: datetime.strptime(z, '%b %d %Y %H:%M:%S') )\n",
    "    \n",
    "    # Use datetime variable to subset tweets to include only those posted within the study period (some quoted tweets were posted as early at 2013)\n",
    "    # Order by time posted then reset index\n",
    "    df = df[(df['created_at'] >= a) & (df['created_at'] < b)].sort_values(by='created_at').reset_index(drop = True)\n",
    "    \n",
    "    # Drop duplicated retweets (i.e. only keep of retweet of each original tweet not captured in the dataset, \n",
    "    # unless the original tweet is captured, in which case remove all retweets of it)\n",
    "    df = df.drop_duplicates('retweeted_status_id', keep='first').reset_index(drop = True)\n",
    "    # Delete retweeted_status_id\n",
    "    del df['retweeted_status_id']\n",
    "    \n",
    "    # Create column for vader input\n",
    "    df['VADER_text'] = df['text'].apply(lambda x: re.sub('@(.*?) ', '@anonymous ', x)) # Change accounts for '@anonymous'\n",
    "    # Remove urls and line breaks\n",
    "    df['VADER_text'] = df['VADER_text'].apply(lambda x: re.sub(r\"http\\S+\", \"https://url_removed\", str(x)) )\\\n",
    "                                       .apply(lambda x: re.sub('http(.*?) ', \"https://url_removed\", str(x)) )\\\n",
    "                                       .apply(lambda x: re.sub('\\n', ' ', str(x)) )\n",
    "    \n",
    "    # Subset to remove tweets dicussing bird or data migration\n",
    "    nature_and_data = ['bird', 'ornithology', '#wildlife', '#deer', '#buck', '#antlers', '#nature', \n",
    "                       'github', 'Azure', 'Microsoft', 'Ubuntu', 'Python', 'SQL', 'cloud', 'bigdata', 'big data', 'virtual machine', 'DevOps']\n",
    "    # Define phrases prevent tweets from being falsely labeled as noise\n",
    "    exclude_words  = ['cloud cuckoo land', 'two birds one stone', 'two birds with one stone', '@Nigel_Farage', 'NigelFarage', 'Nigel Farage', 'refugee']\n",
    "    # Remove tweets which contain 'nature_and_data', but not 'exclude_words'\n",
    "    df = df[~df['text'].replace(exclude_words,'', regex=True)\n",
    "                       .str.contains('|'.join(nature_and_data), case = False)].reset_index(drop = True)\n",
    "    \n",
    "    # Return final dataframe\n",
    "    return(df)\n",
    "\n",
    "# Convert terms such as OMG to Oh My God - I have also included RT as retweet or HT as hattip\n",
    "# Code From: https://medium.com/nerd-stuff/python-script-to-turn-text-message-abbreviations-into-actual-phrases-d5db6f489222\n",
    "def translator(user_string):\n",
    "    user_string = user_string.split(\" \")\n",
    "    j = 0\n",
    "    for _str in user_string:\n",
    "        # File path which consists of Abbreviations.\n",
    "        fileName = \"./slang.txt\"\n",
    "\n",
    "        # File Access mode [Read Mode]\n",
    "        with open(fileName, \"r\") as myCSVfile:\n",
    "            # Reading file as CSV with delimiter as \"=\", so that abbreviation are stored in row[0] and phrases in row[1]\n",
    "            dataFromFile = csv.reader(myCSVfile, delimiter=\"=\")\n",
    "            # Removing Special Characters.\n",
    "            _str = re.sub('[^a-zA-Z0-9]+', '', _str)\n",
    "            for row in dataFromFile:\n",
    "                # Check if selected word matches short forms[LHS] in text file.\n",
    "                if _str.upper() == row[0]:\n",
    "                    # If match found replace it with its appropriate phrase in text file.\n",
    "                    user_string[j] = row[1]\n",
    "            myCSVfile.close()\n",
    "        j = j + 1\n",
    "    return ' '.join(user_string)\n",
    "\n",
    "# Function which replaces all hashtags in a text with their constituent words\n",
    "def expand_hashtags(x):\n",
    "    # Finds all hashtags and inserts into list\n",
    "    hts = re.findall('#(.*?) ',x) \n",
    "    # Breaks hashtags into their constituent words and convert all characters to lower case\n",
    "    expanded_hts = list(map(lambda y: \" \".join([a for a in re.split('([A-Z][a-z]+)', y) if a]).lower(), hts))\n",
    "    # for loop which replaces hashtags with their constituent words\n",
    "    for i in range(len(expanded_hts)):\n",
    "        x = re.sub('#(.*?) ', expanded_hts[i] + ' ', x, 1)\n",
    "    return(x)\n",
    "\n",
    "# Function which converts emojis to single sentence description\n",
    "def convert_emojis(x):\n",
    "    # If string contains emojis\n",
    "    if len(demoji.findall(x)) > 0:\n",
    "        # Extract all emojis and convert to dictionary (with emojis as keys and descriptions as values)\n",
    "        emojis = demoji.findall(x)\n",
    "        # for loop which replaces each emoji with description, with 1st letter capitalised and a full stop\n",
    "        for i in range(len(emojis)):\n",
    "            x = re.sub(list(emojis)[i], list(emojis.values())[i].capitalize() + '.', x)\n",
    "    return(x)\n",
    "\n",
    "# Function which prepares text for topic analysis\n",
    "def clean_tweet_text(df):\n",
    "    \n",
    "    ## Convert full text to string ##\n",
    "    \n",
    "    # Some tweets are classified by Python as 'floats' so need to convert to string for cleaning process\n",
    "    print('Converting full text to string...')\n",
    "    try: # Check if tweets have been translated\n",
    "        df['translated_text']\n",
    "    except: # If not, clean VADER_text\n",
    "        df['lexicon_text'] = df['VADER_text'].astype(str)\n",
    "    else: # If yes, clean translated_text\n",
    "        df['lexicon_text'] = df['translated_text'].astype(str)\n",
    "    \n",
    "    ## Remove retweet prefix ##\n",
    "    \n",
    "    # Create a new column which removes retweets (except in cases where we do not have the retweet, in which case the first retweet chronologically is kept)\n",
    "    print('Removing retweet prefix...')\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub('RT (.*?): ', '', x)) # Remove text prefixed to some retweets so tweets and retweets can be directly compared by text\n",
    "    \n",
    "    ## Replace abbreviations ##\n",
    "    \n",
    "    # Apply translator function to text \n",
    "    #print('Replacing abbreviations...')\n",
    "    #import time\n",
    "    #start_time = time.time()\n",
    "    #df['lexicon_text'] = df['lexicon_text'].apply(lambda x:  translator(x)  )\n",
    "    #print(\"--- Run Time: %s seconds ---\" % (time.time() - start_time))\n",
    "    \n",
    "    ## Expand hashtags ##\n",
    "    \n",
    "    # Replace hashtags with constituent words\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: expand_hashtags(x) )\n",
    "    \n",
    "    ## Remove accounts and urls  ##\n",
    "    \n",
    "    # Remove @'s but keep the account name. This is so the account name is (hopefully) interpreted by coreNLP as an person, \n",
    "    # thereby maintaining the integrity of the sentence structure to improve sentiment score.\n",
    "    print('Removing accounts and urls...')\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(\"RT @anonymous\", \"\", str(x))  )\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(\"http: // url_removed\", \"\", str(x))  )\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(\"https://url_removed\", \"\", str(x))  )\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(\"@anonymous\", \"\", str(x))  )\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(\"@\", \"\", str(x))  )\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(\"&quot;\", \"\", str(x))  )\n",
    "    \n",
    "    ## Replace &amp; with and ##\n",
    "    \n",
    "    #print('Replacing \\'&amp;\\' with \\'and\\'...')\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(\"&amp;\", \"and\", str(x))  )\n",
    "    \n",
    "    ## Remove hyperlinks ##\n",
    "    \n",
    "    # Remove hyperlinks at the end and embeddeded in the middle of the string\n",
    "    #print('Removing hyperlinks...')\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(r\"http\\S+\", \"\", str(x)) )\\\n",
    "                                           .apply(lambda x: re.sub('http(.*?) ', \"\", str(x)) )\n",
    "    \n",
    "    ## Replace emojis ##\n",
    "    \n",
    "    # Replaces emojis with a single sentence description\n",
    "    #print('Replacing emojis...')\n",
    "    #df['lexicon_text'] = df['lexicon_text'].apply(lambda x: convert_emojis(str(x)) )\n",
    "    \n",
    "    ## Remove tweets related to bird, animal or data migration ##\n",
    "    \n",
    "    #print('Removing tweets related to bird, animal or data migration...')\n",
    "    # Define words associated with noise topics\n",
    "    \n",
    "    \n",
    "    ## Remove \\n's and spaces before punctuation ##\n",
    "    \n",
    "    print('Removing \\n\\'s and spaces before punctuation...')\n",
    "    # Replace line breaks with spaces and remove spaces before punctuation\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub('\\n',' ',str(x)) )\\\n",
    "                                           .apply(lambda x: re.sub(r'\\s([?.!\"](?:\\s|$))', r'\\1', str(x)) )\n",
    "    \n",
    "    ## Remove emoji descriptions with misleading discriptions ## \n",
    "    \n",
    "    # Remove description of 'see/hear/speak no evil monkey' (one-off edit due to observations of emoji descriptions)\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: re.sub(r'-no-evil monkey',' no monkey emoji',str(x)) )\n",
    "    \n",
    "    ## Remove punctuation ##\n",
    "    \n",
    "    # We are not interested in punctuation for analyses so replace them with a space\n",
    "    print('Removing punctuation...')\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x : re.sub(r'[^a-zA-Z ]',' ',str(x)))\n",
    "    # If want to include numbers use re.sub(r'[^a-zA-Z0-9 ]'\n",
    "    \n",
    "    ## Convert all words to lower case ##\n",
    "\n",
    "    # To normalise comparisons else Love and love are treated seperately (for upper case swicth to 'word.upper())\n",
    "    print('Converting all words to lower case...')\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: ' '.join( [ word.lower() for word in x.split() ] ) )\n",
    "    \n",
    "    ## Remove stop words ##\n",
    "    \n",
    "    # Remove common words such as 'a', 'the', 'on' that do not contribute to the meaning of texts through providing unncessary information\n",
    "    print('Removing stop words...')\n",
    "    from nltk.corpus import stopwords\n",
    "    stop = stopwords.words(\"english\") # Define stopwords\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)])) # Remove from tweet \n",
    "    \n",
    "    ## Normalising language ##\n",
    "    \n",
    "    # Lemmatization\n",
    "    # Convert terms to their root dictionary form (or lemma) e.g. runs, running and ran are each forms of run\n",
    "    # Pros: greater context to root terms as uses valid words\n",
    "    # Cons: requires greater memory to run, does not always get to root word\n",
    "    \n",
    "    # We will go with Lemmatization as more useful in interpretation of words\n",
    "    \n",
    "    # nltk.download() # To install WordNet corpora\n",
    "    print('Normalising language...')\n",
    "    from nltk.stem.wordnet import WordNetLemmatizer\n",
    "    lmtzr = WordNetLemmatizer()\n",
    "    df['lexicon_text'] = df['lexicon_text'].apply(lambda x: ' '.join([lmtzr.lemmatize(word,'v') for word in x.split() ]))\n",
    "    \n",
    "    print('Done.')\n",
    "    \n",
    "    # Return reformated dataframe\n",
    "    return(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reformat UK Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Extract key info and format VADER text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "#with open(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020.json') as f:\n",
    "#    uk_tweets_01122019_01052020 = json.load(f)\n",
    "with open(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_max.json') as f:\n",
    "    uk_tweets_01122019_01052020_max = json.load(f)\n",
    "#with open(rp + dp + 'tweets\\\\uk_tweets_01052020_01112020.json') as f:\n",
    "#    uk_tweets_01052020_01112020 = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#uk_tweets_01122019_01052020     = to_df_vader(uk_tweets_01122019_01052020, uk_search_terms, \"2019-12-01\", \"2020-05-01\")\n",
    "uk_tweets_01122019_01052020_max = to_df_vader(uk_tweets_01122019_01052020_max, uk_search_terms, \"2019-12-01\", \"2020-05-01\")\n",
    "#uk_tweets_01052020_01112020     = to_df_vader(uk_tweets_01052020_01112020, uk_search_terms, \"2020-05-01\", \"2020-11-01\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Combine searches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in formatted tweets as dataframe\n",
    "uk_tweets_01122019_01052020 = pd.read_csv(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_VADER_removed_dpl_RT_and_status_id_only.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "uk_tweets_01122019_01052020['created_at'] = pd.to_datetime(uk_tweets_01122019_01052020['created_at'])\n",
    "\n",
    "# Read in formatted tweets as dataframe\n",
    "uk_tweets_01122019_01052020_max = pd.read_csv(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_VADER_max.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "uk_tweets_01122019_01052020_max['created_at'] = pd.to_datetime(uk_tweets_01122019_01052020_max['created_at'])\n",
    "\n",
    "# Read in formatted tweets as dataframe\n",
    "#uk_tweets_01052020_01112020 = pd.read_csv(rp + dp + 'tweets\\\\uk_tweets_01052020_01112020_VADER.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "#uk_tweets_01052020_01112020['created_at'] = pd.to_datetime(uk_tweets_01052020_01112020['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Append dataframes, sort tweets chronologically, drop any tweets which have already been captured by the non-retweet search, and reset index\n",
    "uk_tweets_01122019_01052020 = pd.concat([uk_tweets_01122019_01052020, uk_tweets_01122019_01052020_max]).sort_values(by='created_at').drop_duplicates('status_id', keep='first').reset_index(drop = True)\n",
    "#uk_tweets_01122019_01112020 = pd.concat([uk_tweets_01122019_01052020, uk_tweets_01052020_01112020]).sort_values(by='created_at').drop_duplicates('status_id', keep='first').reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Clean data for topic modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uk_tweets_01122019_01052020 = clean_tweet_text(uk_tweets_01122019_01052020)\n",
    "#uk_tweets_01122019_01112020 = clean_tweet_text(uk_tweets_01122019_01112020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as csv\n",
    "#uk_tweets_01122019_01052020.to_csv(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_VADER_removed_dpl_RT_and_status_id_only.csv')\n",
    "#uk_tweets_01122019_01052020_max.to_csv(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_VADER_max.csv')\n",
    "#uk_tweets_01052020_01112020.to_csv(rp + dp + 'tweets\\\\uk_tweets_01052020_01112020_VADER.csv')\n",
    "uk_tweets_01122019_01052020.to_csv(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_VL.csv')\n",
    "#uk_tweets_01122019_01112020.to_csv(rp + dp + 'tweets\\\\uk_tweets_01122020_01112020_VL.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reformat USA tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Extract key info and format VADER text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "#with open(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020.json') as f:\n",
    "#    usa_tweets_01122019_0105020 = json.load(f)\n",
    "with open(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_with_retweets.json') as f:\n",
    "    usa_tweets_01122019_0105020_wrtwts = json.load(f)\n",
    "#with open(rp + dp + 'tweets\\\\usa_tweets_01052020_01112020.json') as f:\n",
    "#    usa_tweets_01052020_01112020 = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reformat tweets\n",
    "#usa_tweets_01122019_0105020 = to_df_vader(usa_tweets_01122019_0105020, usa_search_terms, \"2019-12-01\", \"2020-05-01\")\n",
    "usa_tweets_01122019_0105020_wrtwts = to_df_vader(usa_tweets_01122019_0105020_wrtwts, usa_search_terms, \"2019-12-01\", \"2020-05-01\")\n",
    "#usa_tweets_01052020_01112020 = to_df_vader(usa_tweets_01052020_01112020, usa_search_terms, \"2020-05-01\", \"2020-11-01\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as csv\n",
    "#usa_tweets_01122019_0105020.to_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_VADER.csv')\n",
    "usa_tweets_01122019_0105020_wrtwts.to_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_VADER_with_retweets.csv')\n",
    "#usa_tweets_01052020_01112020.to_csv(rp + dp + 'tweets\\\\usa_tweets_01052020_01112020_VADER.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Combine searches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in formatted tweets as dataframe\n",
    "usa_tweets_01122019_01052020 = pd.read_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_VADER.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "usa_tweets_01122019_01052020['created_at'] = pd.to_datetime(usa_tweets_01122019_01052020['created_at'])\n",
    "\n",
    "# Read in formatted tweets as dataframe\n",
    "usa_tweets_01122019_01052020_wrtwts = pd.read_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_VADER_with_retweets.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "usa_tweets_01122019_01052020_wrtwts['created_at'] = pd.to_datetime(usa_tweets_01122019_01052020_wrtwts['created_at'])\n",
    "\n",
    "# Read in formatted tweets as dataframe\n",
    "#usa_tweets_01052020_01112020 = pd.read_csv(rp + dp + 'tweets\\\\usa_tweets_01052020_01112020_VADER.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "#usa_tweets_01052020_01112020['created_at'] = pd.to_datetime(usa_tweets_01052020_01112020['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create column which records which call they have come from\n",
    "usa_tweets_01122019_01052020['wrtwts_call'] = False\n",
    "usa_tweets_01122019_01052020_wrtwts['wrtwts_call'] = True\n",
    "#usa_tweets_01052020_01112020['wrtwts_call'] = True\n",
    "\n",
    "# Append dataframes, sort tweets chronologically, drop any tweets which have already been captured by the non-retweet search, and reset index\n",
    "usa_tweets_01122019_01052020 = pd.concat([usa_tweets_01122019_01052020, usa_tweets_01122019_01052020_wrtwts]).sort_values(by='created_at').drop_duplicates('status_id', keep='first').reset_index(drop = True)\n",
    "#usa_tweets_01122019_01112020 = pd.concat([usa_tweets_01122019_01052020, usa_tweets_01122019_01052020_wrtwts, usa_tweets_01052020_01112020]).sort_values(by='created_at').drop_duplicates('status_id', keep='first').reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Clean data for topic modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "usa_tweets_01122019_01052020 = clean_tweet_text(usa_tweets_01122019_01052020)\n",
    "#usa_tweets_01122019_01112020 = clean_tweet_text(usa_tweets_01122019_01112020)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as csv\n",
    "usa_tweets_01122019_01052020.to_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_VL.csv')\n",
    "#usa_tweets_01122019_01112020.to_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01112020_VL.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reformat Spain tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Extract key info and format VADER text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "with open(rp + dp + 'tweets\\\\spain_tweets_01122019_01052020.json') as f:\n",
    "    spain_tweets = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converts tweets to manageable dataframe\n",
    "spain_tweets = to_df_vader(spain_tweets, spain_search_terms, \"2019-12-01\", \"2020-05-01\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Clean data for topic modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in formatted tweets as dataframe\n",
    "spain_tweets = pd.read_csv(rp + dp + 'tweets\\\\spain_tweets_01122019_01052020_VADER_translated.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "spain_tweets['created_at'] = pd.to_datetime(spain_tweets['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spain_tweets = clean_tweet_text(spain_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as csv\n",
    "spain_tweets.to_csv(rp + dp + 'tweets\\\\spain_tweets_01122019_01052020_VTL.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reformat Italy tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Extract key info and format VADER text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "with open(rp + dp + 'tweets\\\\italy_tweets_01122019_01052020.json') as f:\n",
    "    italy_tweets = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converts tweets to manageable dataframe\n",
    "italy_tweets = to_df_vader(italy_tweets, italy_search_terms, \"2019-12-01\", \"2020-05-01\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Clean data for topic modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in formatted tweets as dataframe\n",
    "italy_tweets = pd.read_csv(rp + dp + 'tweets\\\\italy_tweets_01122019_01052020_VADER_translated.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "italy_tweets['created_at'] = pd.to_datetime(italy_tweets['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "italy_tweets = clean_tweet_text(italy_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as csv\n",
    "italy_tweets.to_csv(rp + dp + 'tweets\\\\italy_tweets_01122019_01052020_VTL.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reformat German tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Extract key info and format VADER text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "with open(rp + dp + 'tweets\\\\grman_tweets_01122019_01052020.json') as f:\n",
    "    grman_tweets = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converts tweets to manageable dataframe\n",
    "grman_tweets = to_df_vader(grman_tweets, grman_search_terms, \"2019-12-01\", \"2020-05-01\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Clean data for topic modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in formatted tweets as dataframe\n",
    "grman_tweets = pd.read_csv(rp + dp + 'tweets\\\\grman_tweets_01122019_01052020_VADER_translated.csv').drop(['Unnamed: 0'], axis=1)\n",
    "# Convert date variable from str to datetime\n",
    "grman_tweets['created_at'] = pd.to_datetime(grman_tweets['created_at'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grman_tweets = clean_tweet_text(grman_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Save output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as csv\n",
    "grman_tweets.to_csv(rp + dp + 'tweets\\\\grman_tweets_01122019_01052020_VTL.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Proportion Covid Related"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read in data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def import_csv(x):\n",
    "    # Read in tweets as dataframe\n",
    "    x = pd.read_csv(rp + dp + 'tweets\\\\' + x + '.csv').drop(['Unnamed: 0'], axis=1)\n",
    "    # Convert date variable from str to datetime\n",
    "    x['created_at'] = pd.to_datetime(x['created_at'])\n",
    "    # Return df\n",
    "    return x\n",
    "\n",
    "# Read in data\n",
    "uk_tweets    = import_csv('uk_tweets_01122019_01052020_VL')\n",
    "usa_tweets   = import_csv('usa_tweets_01122019_01052020_VL')\n",
    "#spain_tweets = import_csv('spain_tweets_01122019_01052020_VTL')\n",
    "#italy_tweets = import_csv('italy_tweets_01122019_01052020_VTL')\n",
    "#grman_tweets = import_csv('grman_tweets_01122019_01052020_VTL')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scrape COVID Terms from Twitter's compiled list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import random\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from itertools import cycle\n",
    "import traceback\n",
    "import re\n",
    "\n",
    "# Function to remove html tags\n",
    "def cleanhtml(raw_html):\n",
    "  cleanr = re.compile('<.*?>')\n",
    "  cleantext = re.sub(cleanr, '', raw_html)\n",
    "  return cleantext\n",
    "\n",
    "# Define url where covid terms are listed\n",
    "url = \"https://developer.twitter.com/en/docs/labs/covid19-stream/filtering-rules\"\n",
    "# Get the HTTP response and construct soup object\n",
    "soup = bs(requests.get(url).content, \"html.parser\")\n",
    "# Extract all terms\n",
    "html_terms = soup.find(\"table\").find_all(\"p\")\n",
    "# Create empty list\n",
    "covid_terms = []\n",
    "# For loop which appends terms to empty list\n",
    "for term in html_terms:\n",
    "    term = cleanhtml(str(term))\n",
    "    covid_terms.append(term)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Determine which tweets mention COVID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function which finds tweets which contain covid terms\n",
    "def assign_covid_tweets(df):\n",
    "    # Create variable to record whether tweet contains covid term. Default to False\n",
    "    df['cvd_twt'] = False\n",
    "    # Create subset dataframe of all tweets which contain covid terms\n",
    "    covid_df = df[df['text'].str.contains('|'.join(covid_terms), case = False)]\n",
    "    # Assign 'cvd_twt' in the main dataframe as True for these tweets \n",
    "    df.loc[covid_df.index.values.tolist(),'cvd_twt'] = True\n",
    "    # Return results\n",
    "    return df\n",
    "\n",
    "# Assign covid tweets\n",
    "uk_tweets    = assign_covid_tweets(uk_tweets)\n",
    "usa_tweets   = assign_covid_tweets(usa_tweets)\n",
    "#spain_tweets = assign_covid_tweets(spain_tweets)\n",
    "#italy_tweets = assign_covid_tweets(italy_tweets)\n",
    "#grman_tweets = assign_covid_tweets(grman_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save tweets as csv\n",
    "uk_tweets.to_csv(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_VLC.csv')\n",
    "usa_tweets.to_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_VLC.csv')\n",
    "#spain_tweets.to_csv(rp + dp + 'tweets\\\\spain_tweets_01122019_01052020_VTLC.csv')\n",
    "#italy_tweets.to_csv(rp + dp + 'tweets\\\\italy_tweets_01122019_01052020_VTLC.csv')\n",
    "#grman_tweets.to_csv(rp + dp + 'tweets\\\\grman_tweets_01122019_01052020_VTLC.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uk_tweets    = pd.read_csv(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_VLC.csv').drop(['Unnamed: 0'], axis=1)\n",
    "usa_tweets   = pd.read_csv(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_VLC.csv').drop(['Unnamed: 0'], axis=1)\n",
    "spain_tweets = pd.read_csv(rp + dp + 'tweets\\\\spain_tweets_01122019_01052020_VTLC.csv').drop(['Unnamed: 0'], axis=1)\n",
    "italy_tweets = pd.read_csv(rp + dp + 'tweets\\\\italy_tweets_01122019_01052020_VTLC.csv').drop(['Unnamed: 0'], axis=1)\n",
    "grman_tweets = pd.read_csv(rp + dp + 'tweets\\\\grman_tweets_01122019_01052020_VTLC.csv').drop(['Unnamed: 0'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Tweet IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function which converts reduced tweets to a dataframe and preps data for vader lexicon\n",
    "def to_df(x):\n",
    "    \n",
    "    # Create copy of original twitter list\n",
    "    x = x.copy()\n",
    "    \n",
    "    # Create list of names for each type of search\n",
    "    search_type = [\"key_terms\", \"accounts\", \"hashtags\"]\n",
    "    \n",
    "    # for loop which extracts data from each 500 tweet search, then converts to df\n",
    "    for day in range(len(x)): # main loop iterates through each day\n",
    "        for st in range(len(x[day])): # nested loop iterates through each of search type\n",
    "            x[day][st] = reduce_tweets(x[day][st]) # Uses reduced_tweets() to extract essential info from tweets\n",
    "            x[day][st] = pd.DataFrame(x[day][st]) # Converts resulting dictionary to df\n",
    "            x[day][st]['search_type'] = search_type[st] # Creates 'search_type' column and assigned a tupe from previously creates list\n",
    "        x[day] = pd.concat(x[day]) # Concatenates tweets from each search type for a given day\n",
    "    \n",
    "    # Concatenates tweets from across all days called and resets index\n",
    "    x = pd.concat(x).reset_index().drop(['index'], axis=1)\n",
    "    \n",
    "    x = x['status_id'].tolist()\n",
    "    \n",
    "    return(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save UK Tweet IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "#with open(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020.json') as f:\n",
    "#    uk_tweets_01122019_01052020 = json.load(f)\n",
    "with open(rp + dp + 'tweets\\\\uk_tweets_01122019_01052020_max.json') as f:\n",
    "    uk_tweets_01122019_01052020_max = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#uk_tweet_ids_01122019_01052020     = to_df(uk_tweets_01122019_01052020)\n",
    "uk_tweet_ids_01122019_01052020_max = to_df(uk_tweets_01122019_01052020_max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open(rp + dp + \"tweets\\\\uk_tweet_ids_01122019_01052020_1.txt\", \"w\") as fp:\n",
    "#    json.dump(uk_tweet_ids_01122019_01052020, fp)\n",
    "with open(rp + dp + \"tweets\\\\uk_tweet_ids_01122019_01052020_2.txt\", \"w\") as fp:\n",
    "    json.dump(uk_tweet_ids_01122019_01052020_max, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save USA Tweet IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "#with open(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020.json') as f:\n",
    "#    usa_tweets_01122019_01052020 = json.load(f)\n",
    "with open(rp + dp + 'tweets\\\\usa_tweets_01122019_01052020_with_retweets.json') as f:\n",
    "    usa_tweets_01122019_01052020_wrtwts = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#usa_tweet_ids_01122019_01052020        = to_df(usa_tweets_01122019_01052020)\n",
    "usa_tweet_ids_01122019_01052020_wrtwts = to_df(usa_tweets_01122019_01052020_wrtwts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with open(rp + dp + \"tweets\\\\usa_tweet_ids_01122019_01052020_1.txt\", \"w\") as fp:\n",
    "#    json.dump(usa_tweet_ids_01122019_01052020, fp)\n",
    "with open(rp + dp + \"tweets\\\\usa_tweet_ids_01122019_01052020_2.txt\", \"w\") as fp:\n",
    "    json.dump(usa_tweet_ids_01122019_01052020_wrtwts, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Spain Tweet IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "with open(rp + dp + 'tweets\\\\spain_tweets_01122019_01052020.json') as f:\n",
    "    spain_tweets_01122019_01052020 = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spain_tweet_ids_01122019_01052020 = to_df(spain_tweets_01122019_01052020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(rp + dp + \"tweets\\\\spain_tweet_ids_01122019_01052020.txt\", \"w\") as fp:\n",
    "    json.dump(spain_tweet_ids_01122019_01052020, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Italy Tweet IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "with open(rp + dp + 'tweets\\\\italy_tweets_01122019_01052020.json') as f:\n",
    "    italy_tweets_01122019_01052020 = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "italy_tweet_ids_01122019_01052020 = to_df(italy_tweets_01122019_01052020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(rp + dp + \"tweets\\\\italy_tweet_ids_01122019_01052020.txt\", \"w\") as fp:\n",
    "    json.dump(italy_tweet_ids_01122019_01052020, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Germany Tweet IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in tweets from json file\n",
    "with open(rp + dp + 'tweets\\\\grman_tweets_01122019_01052020.json') as f:\n",
    "    grman_tweets_01122019_01052020 = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grman_tweet_ids_01122019_01052020 = to_df(grman_tweets_01122019_01052020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(rp + dp + \"tweets\\\\grman_tweet_ids_01122019_01052020.txt\", \"w\") as fp:\n",
    "    json.dump(grman_tweet_ids_01122019_01052020, fp)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}