nerd_create_top500_tweet_frequency_count_matrix.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import re\n",
    "import datetime\n",
    "import os\n",
    "import sys\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from matplotlib.pyplot import figure\n",
    "import json\n",
    "# from textblob import TextBlob\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n0 tweet_id\\n1 tweet_created_at\\n2 language\\n3 user_screen_name\\n4 user_created_at\\n5 user_id\\n6 followers_count\\n7 friends_count\\n8 time_zone\\n9 utc_offset\\n10 retweeted_status\\n11 retweet_id\\n12 retweet_user_screen_name\\n13 retweet_user_id\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "0 tweet_id\n",
    "1 tweet_created_at\n",
    "2 language\n",
    "3 user_screen_name\n",
    "4 user_created_at\n",
    "5 user_id\n",
    "6 followers_count\n",
    "7 friends_count\n",
    "8 time_zone\n",
    "9 utc_offset\n",
    "10 retweeted_status\n",
    "11 retweet_id\n",
    "12 retweet_user_screen_name\n",
    "13 retweet_user_id\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start:2018-12-13 09:41:54.262793\n",
      "10000\n",
      "End:2018-12-13 09:41:56.391106\n"
     ]
    }
   ],
   "source": [
    "inc = 0\n",
    "val = 0\n",
    "val_inc = 0\n",
    "all_tweets = []\n",
    "tweet_id = []\n",
    "tweet_created_at = []\n",
    "followers_count = []\n",
    "user_screen_name = []\n",
    "retweet_id = []\n",
    "retweeted_status = []\n",
    "language = []\n",
    "utc_offset = []\n",
    "retweet_created_at = {}\n",
    "tweet_text = {}\n",
    "filename = 'tweet_data/nerd.json'\n",
    "print(\"Start:\" + str(datetime.datetime.now()))\n",
    "\n",
    "with open(filename, 'r') as f:\n",
    "    data = json.load(f)\n",
    "    for tweet in data:\n",
    "#         print(str(inc))\n",
    "        all_tweets.append(tweet)\n",
    "        user_screen_name.append(tweet['user']['screen_name'])\n",
    "        tweet_id.append(tweet['id'])\n",
    "        tweet_created_at.append(tweet['created_at'])\n",
    "        followers_count.append(tweet['user']['followers_count'])\n",
    "        if 'retweeted_status' in tweet:\n",
    "            retweet_id.append(tweet['retweeted_status']['id'])\n",
    "            retweet_created_at[tweet['id']] = tweet['created_at']\n",
    "            tweet_text[tweet['id']] = tweet['retweeted_status']['text']\n",
    "        else:\n",
    "            retweet_id.append(0)\n",
    "            tweet_text[tweet['id']] = tweet['text']\n",
    "        inc += 1\n",
    "        val_inc += 1\n",
    "        if val_inc > 10000:\n",
    "            val = val + 10000\n",
    "            print(str(val))\n",
    "            val_inc = 0\n",
    "f.close()\n",
    "\n",
    "print(\"End:\" + str(datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15369"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15369"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tweet_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "retweet_id_counter=collections.Counter(retweet_id)\n",
    "\n",
    "top1000_retweets = retweet_id_counter.most_common(1000)\n",
    "top500_retweets = retweet_id_counter.most_common(500)\n",
    "top36_retweets = retweet_id_counter.most_common(36)\n",
    "top20_retweets = retweet_id_counter.most_common(20)\n",
    "\n",
    "top20_tweet_color_indices = {}\n",
    "top20_most_common_tweet_ids = []\n",
    "top36_most_common_tweet_ids = []\n",
    "top500_most_common_tweet_ids = []\n",
    "top1000_most_common_tweet_ids = []\n",
    "for i in range(len(top500_retweets)):\n",
    "    top500_most_common_tweet_ids.append(int(top500_retweets[i][0]))\n",
    "for i in range(len(top36_retweets)):\n",
    "    top36_most_common_tweet_ids.append(int(top36_retweets[i][0]))\n",
    "for i in range(len(top20_retweets)):\n",
    "    top20_tweet_color_indices[top20_retweets[i][0]] = i\n",
    "    top20_most_common_tweet_ids.append(top20_retweets[i][0])\n",
    "for i in range(len(top1000_retweets)):\n",
    "    top1000_most_common_tweet_ids.append(top1000_retweets[i][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 1019765122468892672,\n",
       " 1019910849773948928,\n",
       " 1021435234691117057,\n",
       " 1022909350174584832,\n",
       " 1022281439310573568,\n",
       " 1020366819884904448,\n",
       " 1022852817902391296,\n",
       " 1022870158052872193,\n",
       " 1019990534830673920,\n",
       " 1022324481870704640,\n",
       " 1019926613784481792,\n",
       " 1019669712823881728,\n",
       " 1022565302662455296,\n",
       " 1020417349252415488,\n",
       " 1021589547543613440,\n",
       " 1018967936638291968,\n",
       " 1021526708632018944,\n",
       " 1020181165175713793,\n",
       " 1019263737583357953,\n",
       " 1020761965587718144,\n",
       " 1022106799888130049,\n",
       " 974503938174210048,\n",
       " 1018642329983438848,\n",
       " 1022259682218532865,\n",
       " 1022809739707789312,\n",
       " 1022743665758941184,\n",
       " 1020086667179421696,\n",
       " 1022322700684087297,\n",
       " 1019150260864765952,\n",
       " 1021622727478718464,\n",
       " 1019961585341796354,\n",
       " 1019118307587305473,\n",
       " 1019933142898106368,\n",
       " 1019096894461632513,\n",
       " 1020820678319181824]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top36_most_common_tweet_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "¡Es oficial! 😱 el 16 de Agosto será el estreno en cines de #PlanV ✌🏼, necesitamos hacer muchísimo ruido para que la… https://t.co/8EkRxkTr88\n",
      "NEW #PODCAST! This week we’re joined by Steven from @SuperRetroTBR to talk #nerd news and review #TheRock is… https://t.co/l5AJV1M9qV\n",
      "This picture won the Internet for me today! 😆 Repost from transformeraddict on Instagram #transformers… https://t.co/z337ZZv4Lh\n",
      "Did anyone else have to diagram sentences in grade school?\n",
      "\n",
      "That was so fun. 🤓😂 #nerd\n",
      "How do I know I’m losing my eyesight? I CAN NO LONGER READ MY ‘MAGIC, THE GATHERING’ DECK!!! #MTG #Deck #OldAge #Nerd\n",
      "Follow @GeekSquee, Retweet, and Like for a chance to win a signed copy of #warlockholmes. Drawing: July 30th. Good… https://t.co/VWEQSYOESA\n",
      "2018年は苗場での開催が20回目の節目年！フジテレビNEXTではFUJI ROCK FESTIVAL '18の3日間を各5時間ずつ計15時間にわたってお送りします。&lt;DAY1&gt;#nerd \n",
      "#postmalone… https://t.co/LQq64SM926\n",
      "Fairly guilty of this... #DoctorWho #DrWho #Tardis #Geek #Nerd #Whovian #Memes https://t.co/woJ9phVhEh\n",
      "Ya falta poco para el estreno de #PlanV @planvoficial\n",
      "con @natdupeyron @JPMinor @mgabrieladfc @GerardStephanie… https://t.co/5vwzrQ3qkB\n",
      "My newest post is about my reading quirks and habits! https://t.co/BbJA3RB7Nd\n",
      "\n",
      "#bookblog #bookblogger #blog… https://t.co/ynXpU6jUAj\n",
      "Kanojo wa dare to demo sex suru\n",
      "full hentai - https://t.co/LqNtoae4ve\n",
      "\n",
      "#hentai #hentaivideo #ahegao #nerd https://t.co/5d8BqOagY0\n",
      "Me years from now:\n",
      "\n",
      "Spurs fan: Why did Kawhi leave Spurs?\n",
      "\n",
      "Me: (Hesitant) Reportedly a family member, his uncle, he… https://t.co/JRLh2SqvV7\n",
      "Coming for that 4.0 this semester again 💯📚#GramFam #schoolgirl #nerd🤓\n",
      "The “Holy Grail” of my #TMNT collection! I would rate it in solid C9 condition! Designed &amp; built by Krang &amp; enginee… https://t.co/Eovv47nyHt\n",
      "Zatanna makes my believe in magic...and myself 😍🙏👻\n",
      ".\n",
      ".\n",
      "Cosplayers Instagram: @danievedo\n",
      ".\n",
      ".\n",
      "*Tag us in your pics💙… https://t.co/eTbKPMQINg\n",
      "Mmm dónde lo he visto? 🤔 esos diseñadores lo vuelven a hacer  jajajaja 💪🤣\n",
      "#Aquaman #DCUniverse #AquamanMovie… https://t.co/zQzcAnnJ6q\n",
      "Forever Our Princess @Etsy #PrincessLeia #Leia #StarWars #Fandom #zentangle #Zentangle_Arts #art #print #Geek #Nerd… https://t.co/Sh0SsDBIps\n",
      "RT und Like für eine Teilnahme! *Werbung* Ich verlose 1x Black Panther Blu-Ray oder 1x Black Panther DVD. Wünsche a… https://t.co/kHhluj9eG6\n",
      "🤓CUTE Nerd Girl or SEXY Nerd Girl ?🤓\n",
      "❤️Choose your favourite and RT ❤️\n",
      "\n",
      "more on #Patreon ✨ https://t.co/aATr4keSum… https://t.co/s9KzKlMcm4\n",
      "Mamá Imelda Rivera makes an appearance at Comic Con, no conditions\n",
      "#sdcc #sdcc2018 #geek #nerd #nerdlife #wedocons… https://t.co/pflmxI7GUH\n",
      "For #WednesVitaday here is my Vita collection..I know it’s small but it’s not the size that counts, right? 😱😆#Nerd… https://t.co/D7udwZAVR9\n",
      "Who else loves the original Teenage Mutant Ninja Turtles? I’ll do more TMNT collection pics this week! #tmnt… https://t.co/6bo31X1y3g\n",
      "Porqué dios mío? Porquéeeee? 🙈\n",
      "#TeenTitans #dccomics #geek #friki #nerd #Robin  #Starfire #Beastboy #Raven… https://t.co/YJVEADA7dN\n",
      "Pegasus seiya\n",
      ".\n",
      ".\n",
      "Art piece by @aanz_nys (IG)\n",
      ".\n",
      ".\n",
      "#pegasus #saintseiya\n",
      ".\n",
      "*Hashtag Themed*\n",
      "#Draw #Drawing #Art… https://t.co/VTPS2aY647\n",
      "🔥 Hot Offer Ending Soon!\n",
      "\n",
      "Xbox Live Gold 12 Months Membership for $39.99!\n",
      "\n",
      "https://t.co/mreeRx6ZhG \n",
      "\n",
      "#geek #deals… https://t.co/Kfas88wdUu\n",
      "Who else rocked these back in the day? I sure did! Wish I still had them 🤔 #retrosTMNTweek #retrogaming #nerd #geek… https://t.co/MIG4FW3dzf\n",
      "I love being a turtle! #tmnt #teenagemutantninjaturtles #arcade #nerd want to catch me live-streaming the whole gam… https://t.co/oSJ9YXP7yE\n",
      "🔥 Prime Day Special!\n",
      "\n",
      "Echo Spot - Black for $99.99!\n",
      "\n",
      "23% off - https://t.co/qShBiYGFpB\n",
      "\n",
      "#PrimeDay2018 #PrimeDay… https://t.co/ppRQ4mqPti\n",
      "🔥 eBook Deal of the Day!\n",
      "Visual Data Storytelling with Tableau for $19.99 (44% off)!\n",
      "\n",
      "Buy here… https://t.co/Gwksl4BuWp\n",
      "One of the greatest romances in all of #StarTrek is Riker/Troi. @ohtheprofanity and @TheInsaneRobin would be remiss… https://t.co/2fMOssnHZh\n",
      "🔥 Prime Day Special!\n",
      "\n",
      "Echo (2nd Generation) Bundle for $86.99!\n",
      "\n",
      "Save $52.99 off - https://t.co/K3wnZLeF7R… https://t.co/IjAnOfXABs\n",
      "#Rainbows! Rainbows are one of my favorite things :D #EtsySeller #Etsyshop #Etsy #Colors #Colorful  #Handmade… https://t.co/MAWfEpsUiZ\n",
      "🔥 Must Have!\n",
      "Echo Dot Case (fits Echo Dot 2nd Gen. only) for $4.99!\n",
      "\n",
      "50% off! - https://t.co/aJvycWcr1L… https://t.co/S9gmxdskQC\n",
      "𝐓𝐡𝐨𝐮𝐠𝐡𝐭 𝐨𝐟 𝐭𝐡𝐞 𝐝𝐚𝐲...  #DnD #dnd5e #games #gaming #gamergirl #nerd #geek #geekgirl #writing #fantasy #scifi #horror… https://t.co/rwhHSeM5qJ\n"
     ]
    }
   ],
   "source": [
    "top36_most_common_tweet_ids = [1019765122468892672,\n",
    " 1019910849773948928,\n",
    " 1021435234691117057,\n",
    " 1022909350174584832,\n",
    " 1022281439310573568,\n",
    " 1020366819884904448,\n",
    " 1022852817902391296,\n",
    " 1022870158052872193,\n",
    " 1019990534830673920,\n",
    " 1022324481870704640,\n",
    " 1019926613784481792,\n",
    " 1019669712823881728,\n",
    " 1022565302662455296,\n",
    " 1020417349252415488,\n",
    " 1021589547543613440,\n",
    " 1018967936638291968,\n",
    " 1021526708632018944,\n",
    " 1020181165175713793,\n",
    " 1019263737583357953,\n",
    " 1020761965587718144,\n",
    " 1022106799888130049,\n",
    " 1018642329983438848,\n",
    " 1022259682218532865,\n",
    " 1022809739707789312,\n",
    " 1022743665758941184,\n",
    " 1020086667179421696,\n",
    " 1022322700684087297,\n",
    " 1019150260864765952,\n",
    " 1021622727478718464,\n",
    " 1019961585341796354,\n",
    " 1019118307587305473,\n",
    " 1019933142898106368,\n",
    " 1019096894461632513,\n",
    " 1020820678319181824]\n",
    "\n",
    "for i in range(len(top36_most_common_tweet_ids)):\n",
    "    print(tweet_text[top36_most_common_tweet_ids[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sat Jul 14 17:02:02 +0000 2018\n",
      "Sat Jul 28 17:28:55 +0000 2018\n"
     ]
    }
   ],
   "source": [
    "start_date = tweet_created_at[0]\n",
    "end_date = tweet_created_at[len(tweet_created_at)-1]\n",
    "print(start_date)\n",
    "print(end_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31\n"
     ]
    }
   ],
   "source": [
    "days_per_month = {'Jan':31,'Feb':28,'Mar':31,'Apr':30,'May':31,'Jun':30,'Jul':31,'Aug':31,'Sep':30,'Oct':31,'Nov':30,'Dec':31}\n",
    "days_per_month_cumulative = {'Jan':0,'Feb':31,'Mar':59,'Apr':90,'May':120,'Jun':151,'Jul':181,'Aug':212,'Sep':243,'Oct':273,'Nov':304,'Dec':334}\n",
    "print(str(days_per_month['Jan']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "289\n",
      "290\n",
      "291\n",
      "292\n",
      "293\n",
      "294\n",
      "295\n",
      "296\n",
      "297\n",
      "298\n",
      "299\n",
      "300\n",
      "301\n",
      "302\n",
      "303\n",
      "304\n",
      "305\n",
      "306\n",
      "307\n",
      "308\n",
      "309\n",
      "310\n",
      "311\n",
      "312\n",
      "313\n",
      "314\n",
      "315\n",
      "316\n",
      "317\n",
      "318\n",
      "319\n",
      "320\n",
      "321\n",
      "322\n",
      "323\n",
      "324\n",
      "325\n",
      "326\n",
      "327\n",
      "328\n",
      "329\n",
      "330\n",
      "331\n",
      "332\n",
      "333\n",
      "334\n",
      "335\n",
      "336\n",
      "337\n",
      "338\n",
      "339\n",
      "340\n",
      "341\n",
      "342\n",
      "343\n",
      "344\n",
      "345\n",
      "346\n",
      "347\n",
      "348\n",
      "349\n",
      "350\n",
      "351\n",
      "352\n",
      "353\n",
      "354\n",
      "355\n",
      "356\n",
      "357\n",
      "358\n",
      "359\n",
      "360\n",
      "361\n",
      "362\n",
      "363\n",
      "364\n",
      "365\n",
      "366\n",
      "367\n",
      "368\n",
      "369\n",
      "370\n",
      "371\n",
      "372\n",
      "373\n",
      "374\n",
      "375\n",
      "376\n",
      "377\n",
      "378\n",
      "379\n",
      "380\n",
      "381\n",
      "382\n",
      "383\n",
      "384\n",
      "385\n",
      "386\n",
      "387\n",
      "388\n",
      "389\n",
      "390\n",
      "391\n",
      "392\n",
      "393\n",
      "394\n",
      "395\n",
      "396\n",
      "397\n",
      "398\n",
      "399\n",
      "400\n",
      "401\n",
      "402\n",
      "403\n",
      "404\n",
      "405\n",
      "406\n",
      "407\n",
      "408\n",
      "409\n",
      "410\n",
      "411\n",
      "412\n",
      "413\n",
      "414\n",
      "415\n",
      "416\n",
      "417\n",
      "418\n",
      "419\n",
      "420\n",
      "421\n",
      "422\n",
      "423\n",
      "424\n",
      "425\n",
      "426\n",
      "427\n",
      "428\n",
      "429\n",
      "430\n",
      "431\n",
      "432\n",
      "433\n",
      "434\n",
      "435\n",
      "436\n",
      "437\n",
      "438\n",
      "439\n",
      "440\n",
      "441\n",
      "442\n",
      "443\n",
      "444\n",
      "445\n",
      "446\n",
      "447\n",
      "448\n",
      "449\n",
      "450\n",
      "451\n",
      "452\n",
      "453\n",
      "454\n",
      "455\n",
      "456\n",
      "457\n",
      "458\n",
      "459\n",
      "460\n",
      "461\n",
      "462\n",
      "463\n",
      "464\n",
      "465\n",
      "466\n",
      "467\n",
      "468\n",
      "469\n",
      "470\n",
      "471\n",
      "472\n",
      "473\n",
      "474\n",
      "475\n",
      "476\n",
      "477\n",
      "478\n",
      "479\n",
      "480\n",
      "481\n",
      "482\n",
      "483\n",
      "484\n",
      "485\n",
      "486\n",
      "487\n",
      "488\n",
      "489\n",
      "490\n",
      "491\n",
      "492\n",
      "493\n",
      "494\n",
      "495\n",
      "496\n",
      "497\n",
      "498\n",
      "499\n"
     ]
    }
   ],
   "source": [
    "outfile = \"nerd_top500_retweet_frequency_counts_matrix_120sec.csv\"\n",
    "fout = open(outfile, 'w')\n",
    "\n",
    "fc_outfile = \"nerd_top500_retweet_frequency_counts_color_index_matrix_120sec.csv\"\n",
    "fcout = open(fc_outfile, 'w')\n",
    "\n",
    "for tweet_inc in range(1,len(top500_most_common_tweet_ids)):\n",
    "\n",
    "    tweet = top500_most_common_tweet_ids[tweet_inc]\n",
    "    print(str(tweet_inc))\n",
    "    total_time = {}\n",
    "    tweet_frequency_count = []\n",
    "    followers_count_frequency_count = []\n",
    "\n",
    "    threshold_epoch = 1500\n",
    "    threshold = 0\n",
    "    count_temp = 0\n",
    "    followers_count_temp = 0\n",
    "\n",
    "    for i in range(0,len(tweet_id)):\n",
    "\n",
    "        # Parse the date and time into seconds for total_time and use for translate_y\n",
    "        date_array = tweet_created_at[i].split(\" \")\n",
    "        time_var = date_array[3]\n",
    "        time_array = date_array[3].split(\":\")\n",
    "        month_seconds = days_per_month_cumulative[date_array[1]] * 86400 # 31 * 24 * 60 * 60\n",
    "        day_seconds = float(date_array[2]) * 86400 # 24 * 60 * 60\n",
    "        hour_seconds = float(time_array[0]) * 3600\n",
    "        minute_seconds = float(time_array[1]) * 60\n",
    "        seconds = float(time_array[2]) * 1\n",
    "        if i == 0:\n",
    "            start_time = float(seconds) + float(minute_seconds) + float(hour_seconds) + day_seconds + month_seconds\n",
    "            total_time[i] = 0\n",
    "        else:\n",
    "            total_time[i] = float(seconds) + float(minute_seconds) + float(hour_seconds) + day_seconds + month_seconds - start_time\n",
    "        if total_time[i] < threshold:\n",
    "#             print(str(tweet_inc) + \" \" + str(i) + \" \" + str(tweet_id[i] + \" \" + str(tweet)))\n",
    "            if int(tweet_id[i]) == int(tweet):\n",
    "#                 print(\"Hit\")\n",
    "                count_temp += 1\n",
    "                followers_count_temp += int(followers_count[i])\n",
    "            if int(retweet_id[i]) == int(tweet):\n",
    "#                     print(\"Hit\")\n",
    "                    count_temp += 1\n",
    "                    followers_count_temp += int(followers_count[i])\n",
    "        else:\n",
    "            threshold += threshold_epoch\n",
    "            tweet_frequency_count.append(count_temp)\n",
    "            count_temp = 0\n",
    "            followers_count_frequency_count.append(followers_count_temp)\n",
    "            followers_count_temp = 0\n",
    "    for i in range(len(tweet_frequency_count)):\n",
    "        if i == 0:\n",
    "            outputstring = str(tweet_frequency_count[i])\n",
    "            followers_count_outputstring = str(followers_count_frequency_count[i])\n",
    "        else:\n",
    "            outputstring = outputstring + \",\" + str(tweet_frequency_count[i])\n",
    "            followers_count_outputstring = followers_count_outputstring + \",\" + str(followers_count_frequency_count[i])\n",
    "    outputstring = outputstring + \"\\n\"\n",
    "    followers_count_outputstring = followers_count_outputstring + \"\\n\"\n",
    "    fout.write(outputstring)\n",
    "    fcout.write(followers_count_outputstring)\n",
    "\n",
    "fout.close()\n",
    "fcout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create adjacency matrix\n",
    "f_out = open(\"nerd_adjacency_matrix.csv\", \"w\")\n",
    "for i in range(len(all_tweets)):\n",
    "    if 'retweeted_status' in all_tweets[i]:\n",
    "        outputstring = all_tweets[i]['retweeted_status']['user']['screen_name'] + \",\" + all_tweets[i]['user']['screen_name'] + \"\\n\"\n",
    "        f_out.write(outputstring)\n",
    "f_out.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fout.close()\n",
    "fcout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top500_most_common_tweet_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}