data/clean_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from os import listdir, path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "outputs": [],
   "source": [
    "house_files_path = '/Users/sudhamshu.hosamane/Q6/macs-30123/project/house_followers_2019/'"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "outputs": [],
   "source": [
    "sen_files_path = '/Users/sudhamshu.hosamane/Q6/macs-30123/project/senate_followers_2019/'"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "outputs": [],
   "source": [
    "house_handles = set([f[:-4].lower() for f in listdir(house_files_path) if f.endswith('txt')])\n",
    "sen_handles = set([f[:-4].lower() for f in listdir(sen_files_path) if f.endswith('txt')])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "outputs": [
    {
     "data": {
      "text/plain": "{'abby4iowa',\n 'abrindisiny',\n 'aguilarpete',\n 'andy_levin',\n 'angiecraigmn',\n 'ann_kirkpatrick',\n 'anthonygonzalez',\n 'appropsdems',\n 'armstrong_nd',\n 'austinscottga08',\n 'axne4congress',\n 'ayannapressley',\n 'benniegthompson',\n 'bettymccollum04',\n 'billpascrell',\n 'bobbyscott',\n 'boblatta',\n 'bradsherman',\n 'braun4indiana',\n 'bryansteilforwi',\n 'call_me_dutch',\n 'carolmillerwv',\n 'cathymcmorris',\n 'chelliepingree',\n 'chiproytx',\n 'chrispappasnh',\n 'chuyforcongress',\n 'cline4virginia',\n 'colinallredtx',\n 'conawaytx11',\n 'congbillposey',\n 'congboyle',\n 'congmikesimpson',\n 'congpalazzo',\n 'congressmangt',\n 'congressmanhice',\n 'congressmanjvd',\n 'congressmanraja',\n 'congressmanruiz',\n 'curtisut',\n 'dancrenshawtx',\n 'daveloebsack',\n 'davidcicilline',\n 'davidjtrone',\n 'deanbphillips',\n 'deb4congressnm',\n 'debbieforfl',\n 'debbielesko',\n 'delgadoforny19',\n 'denver4va',\n 'desjarlaistn04',\n 'devinnunes',\n 'dinatitus',\n 'donaldnorcross',\n 'donnashalala',\n 'dorismatsui',\n 'drkimschrier',\n 'drmarkgreen4tn',\n 'drnealdunnfl2',\n 'drphilroe',\n 'dwstweets',\n 'edcasehawaii',\n 'edlaborcmte',\n 'edlaborgop',\n 'elaineluriava',\n 'eleanornorton',\n 'electjimbaird',\n 'elissaslotkin',\n 'energycommerce',\n 'financialcmte',\n 'frankdlucas',\n 'frankpallone',\n 'gerryconnolly',\n 'gilcisnerosca',\n 'gkbutterfield',\n 'golden4congress',\n 'gopleader',\n 'gracenapolitano',\n 'gregformontana',\n 'gregpencein',\n 'gregstantonaz',\n 'gregsteube',\n 'greschenthaler',\n 'guestcongress',\n 'haleylive',\n 'harleyrouda',\n 'hascdemocrats',\n 'hascrepublicans',\n 'hawleymo',\n 'herrerabeutler',\n 'hispaniccaucus',\n 'homelanddems',\n 'homelandgop',\n 'hornforcongress',\n 'houlahanforpa',\n 'houseadmngop',\n 'houseaggop',\n 'housebudgetdems',\n 'housecommerce',\n 'housedemocrats',\n 'houseforeign',\n 'houseforeigngop',\n 'housegop',\n 'housejudiciary',\n 'housescience',\n 'housesciencegop',\n 'housevetaffairs',\n 'hurdonthehill',\n 'ilhan',\n 'ilhanmn',\n 'jacksonleetx18',\n 'jahanahayesct',\n 'jahimes',\n 'janschakowsky',\n 'jaredhuffman',\n 'jaredpolis',\n 'jasoncrowco6',\n 'jefffortenberry',\n 'jeffvandrew',\n 'jenniferwexton',\n 'jim_jordan',\n 'jimhagedornmn',\n 'jimlangevin',\n 'jimpressoffice',\n 'joaquincastrotx',\n 'joecrowleyny',\n 'joecunninghamsc',\n 'johnjoyceforpa',\n 'johnrosefortn',\n 'joshua_harder',\n 'judgecarter',\n 'justinamash',\n 'katiehill4ca',\n 'kencalvert',\n 'kevincramer',\n 'krhern',\n 'kycomer',\n 'kyrstensinema',\n 'lacongresista',\n 'lacyclaymo1',\n 'lancegooden',\n 'laurenunderwood',\n 'leaderhoyer',\n 'lizzie4congress',\n 'loritrahanma',\n 'lucymcbath',\n 'mactxpress',\n 'mariodb',\n 'markamodeinv2',\n 'markharrisnc9',\n 'markpocan',\n 'marygayscanlon',\n 'maxrose4ny',\n 'meuser4congress',\n 'michaelcburgess',\n 'michaelgwaltz',\n 'michaeljcloud',\n 'mikekellypa',\n 'mikelevinca',\n 'mikiesherrill',\n 'nitalowey',\n 'normajtorres',\n 'nydiavelazquez',\n 'oversightdems',\n 'patrickmchenry',\n 'pattiberi',\n 'peterwelch',\n 'prayercaucus',\n 'rashidatlaib',\n 'rep_hunter',\n 'rep_stevewomack',\n 'rep_watkins',\n 'repabraham',\n 'repadams',\n 'repadamschiff',\n 'repadamsmith',\n 'repadriansmith',\n 'repagonzalez',\n 'repalexmooney',\n 'repalgreen',\n 'repallawsonjr',\n 'repamata',\n 'repandrecarson',\n 'repandybarr',\n 'repandybiggsaz',\n 'repandyharrismd',\n 'repandykimnj',\n 'repandylevin',\n 'repangiecraig',\n 'repannaeshoo',\n 'repanniekuster',\n 'repannwagner',\n 'repanthonybrown',\n 'repaoc',\n 'reparmstrongnd',\n 'reparrington',\n 'repbalderson',\n 'repbarbaralee',\n 'repbarragan',\n 'repbeatty',\n 'repbencline',\n 'repbenmcadams',\n 'repbenraylujan',\n 'repbera',\n 'repbillflores',\n 'repbillfoster',\n 'repbilljohnson',\n 'repblaine',\n 'repblumenauer',\n 'repbobbyrush',\n 'repbobgibbs',\n 'repbonamici',\n 'repbonnie',\n 'repbost',\n 'repbradwenstrup',\n 'repbrendanboyle',\n 'repbrianbabin',\n 'repbrianfitz',\n 'repbrianhiggins',\n 'repbrianmast',\n 'repbrindisi',\n 'repbrownley',\n 'repbryansteil',\n 'repbuddycarter',\n 'repbyrne',\n 'repcarbajal',\n 'repcardenas',\n 'repcarolmiller',\n 'repcartwright',\n 'repcasten',\n 'repcharliecrist',\n 'repcheri',\n 'repchiproy',\n 'repchriscollins',\n 'repchrispappas',\n 'repchrissmith',\n 'repchrisstewart',\n 'repchuck',\n 'repchuygarcia',\n 'repcicilline',\n 'repcindyaxne',\n 'repclayhiggins',\n 'repcleaver',\n 'repcloudtx',\n 'repcohen',\n 'repcolinallred',\n 'repconorlamb',\n 'repcuellar',\n 'repcummings',\n 'repcunningham',\n 'repdancrenshaw',\n 'repdankildee',\n 'repdannydavis',\n 'repdarrensoto',\n 'repdavejoyce',\n 'repdavid',\n 'repdavideprice',\n 'repdavidkustoff',\n 'repdavidrouzer',\n 'repdavids',\n 'repdavidscott',\n 'repdavidtrone',\n 'repdean',\n 'repdeanphillips',\n 'repdebdingell',\n 'repdebhaaland',\n 'repdelbene',\n 'repdelgado',\n 'repdennyheck',\n 'repderekkilmer',\n 'repdesaulnier',\n 'repdevinnunes',\n 'repdianadegette',\n 'repdinatitus',\n 'repdlamborn',\n 'repdlesko',\n 'repdmp',\n 'repdonaldpayne',\n 'repdonbacon',\n 'repdonbeyer',\n 'repdonyoung',\n 'repdougcollins',\n 'repdrewferguson',\n 'repdustyjohnson',\n 'repdwightevans',\n 'repdwstweets',\n 'repebj',\n 'repedcase',\n 'repelaineluria',\n 'repeliotengel',\n 'repescobar',\n 'repespaillat',\n 'repfilemonvela',\n 'repfinkenauer',\n 'repfletcher',\n 'repfranklucas',\n 'repfredupton',\n 'repfrenchhill',\n 'repgallagher',\n 'repgaramendi',\n 'repgarretgraves',\n 'repgilcisneros',\n 'repgolden',\n 'repgonzalez',\n 'repgosar',\n 'repgracemeng',\n 'repgregorymeeks',\n 'repgregpence',\n 'repgregstanton',\n 'repgregsteube',\n 'repgregwalden',\n 'repgrothman',\n 'repgusbilirakis',\n 'repguthrie',\n 'repgwenmoore',\n 'rephagedorn',\n 'rephaleystevens',\n 'rephalrogers',\n 'rephankjohnson',\n 'repharley',\n 'rephartzler',\n 'rephastingsfl',\n 'repholding',\n 'rephorsford',\n 'rephoulahan',\n 'rephuffman',\n 'rephuizenga',\n 'repjackbergman',\n 'repjahanahayes',\n 'repjasoncrow',\n 'repjasonsmith',\n 'repjayapal',\n 'repjeffduncan',\n 'repjeffries',\n 'repjenniffer',\n 'repjerrynadler',\n 'repjimbaird',\n 'repjimbanks',\n 'repjimcooper',\n 'repjimcosta',\n 'repjimmygomez',\n 'repjimmypanetta',\n 'repjoecourtney',\n 'repjoekennedy',\n 'repjoemorelle',\n 'repjoeneguse',\n 'repjoewilson',\n 'repjohncurtis',\n 'repjohnjoyce',\n 'repjohnkatko',\n 'repjohnlarson',\n 'repjohnlewis',\n 'repjohnrose',\n 'repjohnyarmuth',\n 'repjoseserrano',\n 'repjoshg',\n 'repjoshharder',\n 'repjuanvargas',\n 'repjudychu',\n 'repkarenbass',\n 'repkathleenrice',\n 'repkatiehill',\n 'repkatieporter',\n 'repkaygranger',\n 'repkclark',\n 'repkenbuck',\n 'repkendrahorn',\n 'repkenmarchant',\n 'repkevinbrady',\n 'repkevinhern',\n 'repkimschrier',\n 'repkinzinger',\n 'repkirkpatrick',\n 'replahood',\n 'replamalfa',\n 'replancegooden',\n 'replarrybucshon',\n 'replawrence',\n 'replbr',\n 'repleezeldin',\n 'replindasanchez',\n 'replipinski',\n 'replizcheney',\n 'replloyddoggett',\n 'reploisfrankel',\n 'reploritrahan',\n 'reploucorrea',\n 'reploudermilk',\n 'replouiegohmert',\n 'replowenthal',\n 'replucymcbath',\n 'repmalinowski',\n 'repmaloney',\n 'repmarciafudge',\n 'repmarcykaptur',\n 'repmarkgreen',\n 'repmarkmeadows',\n 'repmarkpocan',\n 'repmarktakano',\n 'repmarkwalker',\n 'repmarshall',\n 'repmartharoby',\n 'repmattgaetz',\n 'repmaxinewaters',\n 'repmaxrose',\n 'repmccaul',\n 'repmcclintock',\n 'repmceachin',\n 'repmcgovern',\n 'repmckinley',\n 'repmcnerney',\n 'repmeuser',\n 'repmgriffith',\n 'repmgs',\n 'repmichaelguest',\n 'repmichaelwaltz',\n 'repmikejohnson',\n 'repmikelevin',\n 'repmikequigley',\n 'repmikerogersal',\n 'repmiketurner',\n 'repmobrooks',\n 'repmoolenaar',\n 'repmullin',\n 'repnewhouse',\n 'repohalleran',\n 'reppaulcook',\n 'reppaulmitchell',\n 'reppaultonko',\n 'repperlmutter',\n 'reppeteaguilar',\n 'reppeteking',\n 'reppeteolson',\n 'reppeterdefazio',\n 'reppetestauber',\n 'reppressley',\n 'repralphnorman',\n 'reprashida',\n 'repraskin',\n 'repratcliffe',\n 'repraulgrijalva',\n 'reprichardneal',\n 'reprichhudson',\n 'reprichmond',\n 'reprickallen',\n 'reprickcrawford',\n 'repricklarsen',\n 'repriggleman',\n 'reprobbishop',\n 'reprobinkelly',\n 'reprobwoodall',\n 'reprokhanna',\n 'repronestes',\n 'repronkind',\n 'repronwright',\n 'reprooney',\n 'reprossspano',\n 'reproybalallard',\n 'reprubengallego',\n 'reprussfulcher',\n 'reprutherfordfl',\n 'reprwilliams',\n 'repsamgraves',\n 'repsarbanes',\n 'repschneider',\n 'repschrader',\n 'repscottperry',\n 'repscottpeters',\n 'repseanduffy',\n 'repseanmaloney',\n 'repshalala',\n 'repsherrill',\n 'repshimkus',\n 'repsires',\n 'repslotkin',\n 'repsmucker',\n 'repspanberger',\n 'repspeier',\n 'repstefanik',\n 'repstephenlynch',\n 'repstephmurphy',\n 'repstevechabot',\n 'repstevestivers',\n 'repsusandavis',\n 'repsusanwild',\n 'repsusielee',\n 'repswalwell',\n 'repsylviagarcia',\n 'reptedbudd',\n 'repteddeutch',\n 'reptedlieu',\n 'reptedyoho',\n 'repterrisewell',\n 'repthomasmassie',\n 'repthompson',\n 'reptimburchett',\n 'reptimmons',\n 'reptimryan',\n 'reptipton',\n 'reptjcox',\n 'reptomemmer',\n 'reptomgraves',\n 'reptomreed',\n 'reptomrice',\n 'reptomsuozzi',\n 'reptorressmall',\n 'reptrentkelly',\n 'reptrey',\n 'republicanstudy',\n 'repunderwood',\n 'repvaldemings',\n 'repvantaylor',\n 'repveasey',\n 'repvisclosky',\n 'repwalberg',\n 'repwalorski',\n 'repwalterjones',\n 'repwebster',\n 'repwesterman',\n 'repwexton',\n 'repwilson',\n 'repyvetteclarke',\n 'repzoelofgren',\n 'reschenthaler',\n 'robert_aderholt',\n 'robinlynnekelly',\n 'robwittman',\n 'rodneydavis',\n 'rokhanna',\n 'ron4congress',\n 'rosadelauro',\n 'rosenfornevada',\n 'rothfusforpa',\n 'rulesreps',\n 'sanfordbishop',\n 'scottforflorida',\n 'seancasten',\n 'sethmoulton',\n 'sharicedavids',\n 'spanbergerva07',\n 'speakerpelosi',\n 'staceyplaskett',\n 'steve4kansas',\n 'stevekingia',\n 'stevenhorsford',\n 'stevescalise',\n 'susanwbrooks',\n 'susieleenv',\n 'tim_walz',\n 'timburchett',\n 'tjcoxcongress',\n 'tomcoleok04',\n 'transportdems',\n 'transportgop',\n 'troy_balderson',\n 'tulsipress',\n 'txrandy14',\n 'usrepgarypalmer',\n 'usrepkcastor',\n 'usrepkeating',\n 'usreplong',\n 'usrepmikedoyle',\n 'vantaylortx',\n 'vernbuchanan',\n 'vgescobar',\n 'virginiafoxx',\n 'votemarsha',\n 'votemorelle',\n 'votetimmons',\n 'warrendavidson',\n 'waysandmeansgop',\n 'waysmeanscmte',\n 'whipclyburn',\n 'wildforcongress',\n 'xochforcongress'}"
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(house_handles)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "outputs": [],
   "source": [
    "congress_df = pd.read_csv(house_files_path+'../116Congress.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "outputs": [
    {
     "data": {
      "text/plain": "True"
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "congress_df['ODU-WSDL'].isnull().values.any()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "outputs": [
    {
     "data": {
      "text/plain": "[404]"
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(np.where(congress_df['ODU-WSDL'].isnull())[0])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "outputs": [
    {
     "data": {
      "text/plain": "         Wikipedia  Names Position Joined Congress         ODU-WSDL  \\\n0             Doug Jones       Sen        3-Jan-18     sendougjones   \n1         Richard Shelby       Sen        3-Jan-87        SenShelby   \n2           Dan Sullivan       Sen        3-Jan-15   SenDanSullivan   \n3         Lisa Murkowski       Sen       20-Dec-02    LisaMurkowski   \n4         Kyrsten Sinema       Sen        3-Jan-19    SenatorSinema   \n..                    ...      ...             ...              ...   \n533            Will Hurd       Rep        3-Jan-15    hurdonthehill   \n534      William Timmons       Rep        3-Jan-19       RepTimmons   \n535  Xochitl Torres Small      Rep        3-Jan-19   RepTorresSmall   \n536        Yvette Clarke       Rep        3-Jan-13  RepYvetteClarke   \n537          Zoe Lofgren       Rep        3-Jan-95    RepZoeLofgren   \n\n               CSPAN    TweetCongress           Github REMARK  \n0       sendougjones     sendougjones     sendougjones    NaN  \n1          SenShelby        SenShelby        SenShelby    NaN  \n2     SenDanSullivan   SenDanSullivan   SenDanSullivan    NaN  \n3      LisaMurkowski    LisaMurkowski    LisaMurkowski    NaN  \n4      SenatorSinema    SenatorSinema    SenatorSinema    NaN  \n..               ...              ...              ...    ...  \n533    hurdonthehill    hurdonthehill    hurdonthehill    NaN  \n534       RepTimmons       RepTimmons       RepTimmons    NaN  \n535   RepTorresSmall   RepTorresSmall   RepTorresSmall    NaN  \n536  RepYvetteClarke  RepYvetteClarke  RepYvetteClarke    NaN  \n537    RepZoeLofgren    RepZoeLofgren    RepZoeLofgren    NaN  \n\n[538 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Wikipedia  Names</th>\n      <th>Position</th>\n      <th>Joined Congress</th>\n      <th>ODU-WSDL</th>\n      <th>CSPAN</th>\n      <th>TweetCongress</th>\n      <th>Github</th>\n      <th>REMARK</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Doug Jones</td>\n      <td>Sen</td>\n      <td>3-Jan-18</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Richard Shelby</td>\n      <td>Sen</td>\n      <td>3-Jan-87</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Dan Sullivan</td>\n      <td>Sen</td>\n      <td>3-Jan-15</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Lisa Murkowski</td>\n      <td>Sen</td>\n      <td>20-Dec-02</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Kyrsten Sinema</td>\n      <td>Sen</td>\n      <td>3-Jan-19</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>533</th>\n      <td>Will Hurd</td>\n      <td>Rep</td>\n      <td>3-Jan-15</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>534</th>\n      <td>William Timmons</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>535</th>\n      <td>Xochitl Torres Small</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>536</th>\n      <td>Yvette Clarke</td>\n      <td>Rep</td>\n      <td>3-Jan-13</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>537</th>\n      <td>Zoe Lofgren</td>\n      <td>Rep</td>\n      <td>3-Jan-95</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n<p>538 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "congress_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "outputs": [
    {
     "data": {
      "text/plain": "Wikipedia  Names    Michael San Nicolas\nPosition                            Rep\nJoined Congress                3-Jan-19\nODU-WSDL                            NaN\nCSPAN                               NaN\nTweetCongress                       NaN\nGithub                              NaN\nREMARK                       NO TWITTER\nName: 404, dtype: object"
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "congress_df.loc[404]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "outputs": [],
   "source": [
    "congress_set = set(congress_df['ODU-WSDL'].apply(lambda x: str(x).lower()).tolist())"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "outputs": [
    {
     "data": {
      "text/plain": "{'collinpeterson', 'kilili_sablan', 'nan'}"
     },
     "execution_count": 366,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(congress_set - (house_handles | sen_handles))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "outputs": [
    {
     "data": {
      "text/plain": "{'abby4iowa',\n 'abrindisiny',\n 'aguilarpete',\n 'andy_levin',\n 'angiecraigmn',\n 'ann_kirkpatrick',\n 'anthonygonzalez',\n 'appropsdems',\n 'armstrong_nd',\n 'axne4congress',\n 'ayannapressley',\n 'braun4indiana',\n 'brianschatz',\n 'bryansteilforwi',\n 'carolmillerwv',\n 'chiproytx',\n 'chrispappasnh',\n 'chuyforcongress',\n 'cline4virginia',\n 'colinallredtx',\n 'curtisut',\n 'dancrenshawtx',\n 'davidcicilline',\n 'davidjtrone',\n 'deanbphillips',\n 'deb4congressnm',\n 'debbieforfl',\n 'debbielesko',\n 'delgadoforny19',\n 'denver4va',\n 'devinnunes',\n 'dinatitus',\n 'donnashalala',\n 'drkimschrier',\n 'drmarkgreen4tn',\n 'dwstweets',\n 'edcasehawaii',\n 'edlaborcmte',\n 'edlaborgop',\n 'elaineluriava',\n 'electjimbaird',\n 'elissaslotkin',\n 'energycommerce',\n 'financialcmte',\n 'frankdlucas',\n 'gilcisnerosca',\n 'golden4congress',\n 'gregpencein',\n 'gregstantonaz',\n 'gregsteube',\n 'guestcongress',\n 'haleylive',\n 'harleyrouda',\n 'hascdemocrats',\n 'hascrepublicans',\n 'hawleymo',\n 'hispaniccaucus',\n 'homelanddems',\n 'homelandgop',\n 'hornforcongress',\n 'houlahanforpa',\n 'houseadmngop',\n 'houseaggop',\n 'housebudgetdems',\n 'housecommerce',\n 'housedemocrats',\n 'houseforeign',\n 'houseforeigngop',\n 'housegop',\n 'housejudiciary',\n 'housescience',\n 'housesciencegop',\n 'housevetaffairs',\n 'ilhanmn',\n 'jahanahayesct',\n 'jaredhuffman',\n 'jaredpolis',\n 'jasoncrowco6',\n 'jeffvandrew',\n 'jenniferwexton',\n 'jimhagedornmn',\n 'joecrowleyny',\n 'joecunninghamsc',\n 'johnjoyceforpa',\n 'johnrosefortn',\n 'joshua_harder',\n 'katiehill4ca',\n 'kevincramer',\n 'krhern',\n 'kyrstensinema',\n 'lacongresista',\n 'lancegooden',\n 'laurenunderwood',\n 'lindseygrahamsc',\n 'lizzie4congress',\n 'loritrahanma',\n 'lucymcbath',\n 'markharrisnc9',\n 'markpocan',\n 'marygayscanlon',\n 'maxrose4ny',\n 'meuser4congress',\n 'michaelgwaltz',\n 'michaeljcloud',\n 'mikelevinca',\n 'mikiesherrill',\n 'mittromney',\n 'oversightdems',\n 'pattiberi',\n 'prayercaucus',\n 'rashidatlaib',\n 'repbrendanboyle',\n 'republicanstudy',\n 'repwalterjones',\n 'reschenthaler',\n 'robinlynnekelly',\n 'rokhanna',\n 'ron4congress',\n 'rosenfornevada',\n 'rothfusforpa',\n 'rulesreps',\n 'scottforflorida',\n 'seancasten',\n 'senatemajldr',\n 'sharicedavids',\n 'spanbergerva07',\n 'steve4kansas',\n 'stevenhorsford',\n 'susieleenv',\n 'tim_walz',\n 'timburchett',\n 'tjcoxcongress',\n 'transportdems',\n 'transportgop',\n 'troy_balderson',\n 'vantaylortx',\n 'vgescobar',\n 'votemarsha',\n 'votemorelle',\n 'votetimmons',\n 'waysandmeansgop',\n 'waysmeanscmte',\n 'wildforcongress',\n 'xochforcongress'}"
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "((house_handles | sen_handles) - congress_set)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "outputs": [
    {
     "data": {
      "text/plain": "535"
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len((house_handles | sen_handles).intersection(congress_set))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "outputs": [
    {
     "data": {
      "text/plain": "False"
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'BobbyScott' in congress_set"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "outputs": [
    {
     "data": {
      "text/plain": "False"
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'BobbyScott' in house_handles"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "outputs": [],
   "source": [
    "df_dw = pd.read_csv(house_files_path+'../Congress116DWscores.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "outputs": [
    {
     "data": {
      "text/plain": "     congress    chamber  icpsr  state_icpsr  district_code state_abbrev  \\\n0         116  President  99912           99              0          USA   \n1         116      House  20301           41              3           AL   \n2         116      House  21102           41              7           AL   \n3         116      House  21192           41              2           AL   \n4         116      House  21193           41              5           AL   \n..        ...        ...    ...          ...            ...          ...   \n548       116     Senate  40915           56              0           WV   \n549       116     Senate  29940           25              0           WI   \n550       116     Senate  41111           25              0           WI   \n551       116     Senate  40707           68              0           WY   \n552       116     Senate  49706           68              0           WY   \n\n     party_code  occupancy  last_means              bioname  ...    died  \\\n0           200        0.0         0.0   TRUMP, Donald John  ...     NaN   \n1           200        NaN         NaN  ROGERS, Mike Dennis  ...     NaN   \n2           100        NaN         NaN        SEWELL, Terri  ...     NaN   \n3           200        NaN         NaN         ROBY, Martha  ...     NaN   \n4           200        NaN         NaN           BROOKS, Mo  ...     NaN   \n..          ...        ...         ...                  ...  ...     ...   \n548         100        NaN         NaN         MANCHIN, Joe  ...     NaN   \n549         100        NaN         NaN       BALDWIN, Tammy  ...     NaN   \n550         200        NaN         NaN         JOHNSON, Ron  ...     NaN   \n551         200        NaN         NaN       BARRASSO, John  ...     NaN   \n552         200        NaN         NaN           ENZI, Mike  ...  2021.0   \n\n     nominate_dim1  nominate_dim2  nominate_log_likelihood  \\\n0            0.403          0.162                      NaN   \n1            0.361          0.462               -166.79338   \n2           -0.393          0.398                -28.41563   \n3            0.362          0.658                -90.42097   \n4            0.652         -0.417               -140.71682   \n..             ...            ...                      ...   \n548         -0.060          0.417                -70.67489   \n549         -0.493         -0.136               -132.28905   \n550          0.626         -0.160                -45.72966   \n551          0.536          0.216                -30.88484   \n552          0.545          0.199                -42.60100   \n\n     nominate_geo_mean_probability  nominate_number_of_votes  \\\n0                              NaN                       NaN   \n1                          0.80364                     763.0   \n2                          0.96463                     789.0   \n3                          0.88244                     723.0   \n4                          0.83962                     805.0   \n..                             ...                       ...   \n548                        0.89772                     655.0   \n549                        0.81985                     666.0   \n550                        0.93135                     643.0   \n551                        0.95482                     668.0   \n552                        0.93551                     639.0   \n\n     nominate_number_of_errors  conditional  nokken_poole_dim1  \\\n0                          NaN          NaN                NaN   \n1                         69.0          NaN              0.520   \n2                         11.0          NaN             -0.430   \n3                         31.0          NaN              0.346   \n4                         57.0          NaN              0.772   \n..                         ...          ...                ...   \n548                       19.0          NaN             -0.052   \n549                       65.0          NaN             -0.396   \n550                       25.0          NaN              0.610   \n551                       16.0          NaN              0.602   \n552                       16.0          NaN              0.587   \n\n     nokken_poole_dim2  \n0                  NaN  \n1                0.388  \n2                0.384  \n3                0.672  \n4               -0.337  \n..                 ...  \n548              0.388  \n549             -0.001  \n550              0.017  \n551              0.323  \n552              0.414  \n\n[553 rows x 22 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>congress</th>\n      <th>chamber</th>\n      <th>icpsr</th>\n      <th>state_icpsr</th>\n      <th>district_code</th>\n      <th>state_abbrev</th>\n      <th>party_code</th>\n      <th>occupancy</th>\n      <th>last_means</th>\n      <th>bioname</th>\n      <th>...</th>\n      <th>died</th>\n      <th>nominate_dim1</th>\n      <th>nominate_dim2</th>\n      <th>nominate_log_likelihood</th>\n      <th>nominate_geo_mean_probability</th>\n      <th>nominate_number_of_votes</th>\n      <th>nominate_number_of_errors</th>\n      <th>conditional</th>\n      <th>nokken_poole_dim1</th>\n      <th>nokken_poole_dim2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>116</td>\n      <td>President</td>\n      <td>99912</td>\n      <td>99</td>\n      <td>0</td>\n      <td>USA</td>\n      <td>200</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>TRUMP, Donald John</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.403</td>\n      <td>0.162</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>116</td>\n      <td>House</td>\n      <td>20301</td>\n      <td>41</td>\n      <td>3</td>\n      <td>AL</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ROGERS, Mike Dennis</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.361</td>\n      <td>0.462</td>\n      <td>-166.79338</td>\n      <td>0.80364</td>\n      <td>763.0</td>\n      <td>69.0</td>\n      <td>NaN</td>\n      <td>0.520</td>\n      <td>0.388</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>116</td>\n      <td>House</td>\n      <td>21102</td>\n      <td>41</td>\n      <td>7</td>\n      <td>AL</td>\n      <td>100</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>SEWELL, Terri</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.393</td>\n      <td>0.398</td>\n      <td>-28.41563</td>\n      <td>0.96463</td>\n      <td>789.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>-0.430</td>\n      <td>0.384</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>116</td>\n      <td>House</td>\n      <td>21192</td>\n      <td>41</td>\n      <td>2</td>\n      <td>AL</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ROBY, Martha</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.362</td>\n      <td>0.658</td>\n      <td>-90.42097</td>\n      <td>0.88244</td>\n      <td>723.0</td>\n      <td>31.0</td>\n      <td>NaN</td>\n      <td>0.346</td>\n      <td>0.672</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>116</td>\n      <td>House</td>\n      <td>21193</td>\n      <td>41</td>\n      <td>5</td>\n      <td>AL</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>BROOKS, Mo</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.652</td>\n      <td>-0.417</td>\n      <td>-140.71682</td>\n      <td>0.83962</td>\n      <td>805.0</td>\n      <td>57.0</td>\n      <td>NaN</td>\n      <td>0.772</td>\n      <td>-0.337</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>548</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>40915</td>\n      <td>56</td>\n      <td>0</td>\n      <td>WV</td>\n      <td>100</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>MANCHIN, Joe</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.060</td>\n      <td>0.417</td>\n      <td>-70.67489</td>\n      <td>0.89772</td>\n      <td>655.0</td>\n      <td>19.0</td>\n      <td>NaN</td>\n      <td>-0.052</td>\n      <td>0.388</td>\n    </tr>\n    <tr>\n      <th>549</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>29940</td>\n      <td>25</td>\n      <td>0</td>\n      <td>WI</td>\n      <td>100</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>BALDWIN, Tammy</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.493</td>\n      <td>-0.136</td>\n      <td>-132.28905</td>\n      <td>0.81985</td>\n      <td>666.0</td>\n      <td>65.0</td>\n      <td>NaN</td>\n      <td>-0.396</td>\n      <td>-0.001</td>\n    </tr>\n    <tr>\n      <th>550</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>41111</td>\n      <td>25</td>\n      <td>0</td>\n      <td>WI</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>JOHNSON, Ron</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.626</td>\n      <td>-0.160</td>\n      <td>-45.72966</td>\n      <td>0.93135</td>\n      <td>643.0</td>\n      <td>25.0</td>\n      <td>NaN</td>\n      <td>0.610</td>\n      <td>0.017</td>\n    </tr>\n    <tr>\n      <th>551</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>40707</td>\n      <td>68</td>\n      <td>0</td>\n      <td>WY</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>BARRASSO, John</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.536</td>\n      <td>0.216</td>\n      <td>-30.88484</td>\n      <td>0.95482</td>\n      <td>668.0</td>\n      <td>16.0</td>\n      <td>NaN</td>\n      <td>0.602</td>\n      <td>0.323</td>\n    </tr>\n    <tr>\n      <th>552</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>49706</td>\n      <td>68</td>\n      <td>0</td>\n      <td>WY</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ENZI, Mike</td>\n      <td>...</td>\n      <td>2021.0</td>\n      <td>0.545</td>\n      <td>0.199</td>\n      <td>-42.60100</td>\n      <td>0.93551</td>\n      <td>639.0</td>\n      <td>16.0</td>\n      <td>NaN</td>\n      <td>0.587</td>\n      <td>0.414</td>\n    </tr>\n  </tbody>\n</table>\n<p>553 rows × 22 columns</p>\n</div>"
     },
     "execution_count": 372,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dw"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "outputs": [],
   "source": [
    "dw_dataset_names = set(df_dw['bioname'].apply(lambda x: str(x).split(',')).apply(lambda x: x[1].split()[0].lower() + ' ' + x[0].lower()))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "outputs": [],
   "source": [
    "df_dw['names'] = df_dw['bioname'].apply(lambda x: str(x).split(',')).apply(lambda x: x[1].split()[0].lower() + ' ' + x[0].lower())"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "outputs": [],
   "source": [
    "df_dw.set_index('names', inplace=True)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "outputs": [
    {
     "data": {
      "text/plain": "               congress    chamber  icpsr  state_icpsr  district_code  \\\nnames                                                                   \ndonald trump        116  President  99912           99              0   \nmike rogers         116      House  20301           41              3   \nterri sewell        116      House  21102           41              7   \nmartha roby         116      House  21192           41              2   \nmo brooks           116      House  21193           41              5   \n...                 ...        ...    ...          ...            ...   \njoe manchin         116     Senate  40915           56              0   \ntammy baldwin       116     Senate  29940           25              0   \nron johnson         116     Senate  41111           25              0   \njohn barrasso       116     Senate  40707           68              0   \nmike enzi           116     Senate  49706           68              0   \n\n              state_abbrev  party_code  occupancy  last_means  \\\nnames                                                           \ndonald trump           USA         200        0.0         0.0   \nmike rogers             AL         200        NaN         NaN   \nterri sewell            AL         100        NaN         NaN   \nmartha roby             AL         200        NaN         NaN   \nmo brooks               AL         200        NaN         NaN   \n...                    ...         ...        ...         ...   \njoe manchin             WV         100        NaN         NaN   \ntammy baldwin           WI         100        NaN         NaN   \nron johnson             WI         200        NaN         NaN   \njohn barrasso           WY         200        NaN         NaN   \nmike enzi               WY         200        NaN         NaN   \n\n                           bioname  ...    died  nominate_dim1  nominate_dim2  \\\nnames                               ...                                         \ndonald trump    TRUMP, Donald John  ...     NaN          0.403          0.162   \nmike rogers    ROGERS, Mike Dennis  ...     NaN          0.361          0.462   \nterri sewell         SEWELL, Terri  ...     NaN         -0.393          0.398   \nmartha roby           ROBY, Martha  ...     NaN          0.362          0.658   \nmo brooks               BROOKS, Mo  ...     NaN          0.652         -0.417   \n...                            ...  ...     ...            ...            ...   \njoe manchin           MANCHIN, Joe  ...     NaN         -0.060          0.417   \ntammy baldwin       BALDWIN, Tammy  ...     NaN         -0.493         -0.136   \nron johnson           JOHNSON, Ron  ...     NaN          0.626         -0.160   \njohn barrasso       BARRASSO, John  ...     NaN          0.536          0.216   \nmike enzi               ENZI, Mike  ...  2021.0          0.545          0.199   \n\n               nominate_log_likelihood  nominate_geo_mean_probability  \\\nnames                                                                   \ndonald trump                       NaN                            NaN   \nmike rogers                 -166.79338                        0.80364   \nterri sewell                 -28.41563                        0.96463   \nmartha roby                  -90.42097                        0.88244   \nmo brooks                   -140.71682                        0.83962   \n...                                ...                            ...   \njoe manchin                  -70.67489                        0.89772   \ntammy baldwin               -132.28905                        0.81985   \nron johnson                  -45.72966                        0.93135   \njohn barrasso                -30.88484                        0.95482   \nmike enzi                    -42.60100                        0.93551   \n\n               nominate_number_of_votes  nominate_number_of_errors  \\\nnames                                                                \ndonald trump                        NaN                        NaN   \nmike rogers                       763.0                       69.0   \nterri sewell                      789.0                       11.0   \nmartha roby                       723.0                       31.0   \nmo brooks                         805.0                       57.0   \n...                                 ...                        ...   \njoe manchin                       655.0                       19.0   \ntammy baldwin                     666.0                       65.0   \nron johnson                       643.0                       25.0   \njohn barrasso                     668.0                       16.0   \nmike enzi                         639.0                       16.0   \n\n               conditional  nokken_poole_dim1  nokken_poole_dim2  \nnames                                                             \ndonald trump           NaN                NaN                NaN  \nmike rogers            NaN              0.520              0.388  \nterri sewell           NaN             -0.430              0.384  \nmartha roby            NaN              0.346              0.672  \nmo brooks              NaN              0.772             -0.337  \n...                    ...                ...                ...  \njoe manchin            NaN             -0.052              0.388  \ntammy baldwin          NaN             -0.396             -0.001  \nron johnson            NaN              0.610              0.017  \njohn barrasso          NaN              0.602              0.323  \nmike enzi              NaN              0.587              0.414  \n\n[553 rows x 22 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>congress</th>\n      <th>chamber</th>\n      <th>icpsr</th>\n      <th>state_icpsr</th>\n      <th>district_code</th>\n      <th>state_abbrev</th>\n      <th>party_code</th>\n      <th>occupancy</th>\n      <th>last_means</th>\n      <th>bioname</th>\n      <th>...</th>\n      <th>died</th>\n      <th>nominate_dim1</th>\n      <th>nominate_dim2</th>\n      <th>nominate_log_likelihood</th>\n      <th>nominate_geo_mean_probability</th>\n      <th>nominate_number_of_votes</th>\n      <th>nominate_number_of_errors</th>\n      <th>conditional</th>\n      <th>nokken_poole_dim1</th>\n      <th>nokken_poole_dim2</th>\n    </tr>\n    <tr>\n      <th>names</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>donald trump</th>\n      <td>116</td>\n      <td>President</td>\n      <td>99912</td>\n      <td>99</td>\n      <td>0</td>\n      <td>USA</td>\n      <td>200</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>TRUMP, Donald John</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.403</td>\n      <td>0.162</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>mike rogers</th>\n      <td>116</td>\n      <td>House</td>\n      <td>20301</td>\n      <td>41</td>\n      <td>3</td>\n      <td>AL</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ROGERS, Mike Dennis</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.361</td>\n      <td>0.462</td>\n      <td>-166.79338</td>\n      <td>0.80364</td>\n      <td>763.0</td>\n      <td>69.0</td>\n      <td>NaN</td>\n      <td>0.520</td>\n      <td>0.388</td>\n    </tr>\n    <tr>\n      <th>terri sewell</th>\n      <td>116</td>\n      <td>House</td>\n      <td>21102</td>\n      <td>41</td>\n      <td>7</td>\n      <td>AL</td>\n      <td>100</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>SEWELL, Terri</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.393</td>\n      <td>0.398</td>\n      <td>-28.41563</td>\n      <td>0.96463</td>\n      <td>789.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>-0.430</td>\n      <td>0.384</td>\n    </tr>\n    <tr>\n      <th>martha roby</th>\n      <td>116</td>\n      <td>House</td>\n      <td>21192</td>\n      <td>41</td>\n      <td>2</td>\n      <td>AL</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ROBY, Martha</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.362</td>\n      <td>0.658</td>\n      <td>-90.42097</td>\n      <td>0.88244</td>\n      <td>723.0</td>\n      <td>31.0</td>\n      <td>NaN</td>\n      <td>0.346</td>\n      <td>0.672</td>\n    </tr>\n    <tr>\n      <th>mo brooks</th>\n      <td>116</td>\n      <td>House</td>\n      <td>21193</td>\n      <td>41</td>\n      <td>5</td>\n      <td>AL</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>BROOKS, Mo</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.652</td>\n      <td>-0.417</td>\n      <td>-140.71682</td>\n      <td>0.83962</td>\n      <td>805.0</td>\n      <td>57.0</td>\n      <td>NaN</td>\n      <td>0.772</td>\n      <td>-0.337</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>joe manchin</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>40915</td>\n      <td>56</td>\n      <td>0</td>\n      <td>WV</td>\n      <td>100</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>MANCHIN, Joe</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.060</td>\n      <td>0.417</td>\n      <td>-70.67489</td>\n      <td>0.89772</td>\n      <td>655.0</td>\n      <td>19.0</td>\n      <td>NaN</td>\n      <td>-0.052</td>\n      <td>0.388</td>\n    </tr>\n    <tr>\n      <th>tammy baldwin</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>29940</td>\n      <td>25</td>\n      <td>0</td>\n      <td>WI</td>\n      <td>100</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>BALDWIN, Tammy</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.493</td>\n      <td>-0.136</td>\n      <td>-132.28905</td>\n      <td>0.81985</td>\n      <td>666.0</td>\n      <td>65.0</td>\n      <td>NaN</td>\n      <td>-0.396</td>\n      <td>-0.001</td>\n    </tr>\n    <tr>\n      <th>ron johnson</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>41111</td>\n      <td>25</td>\n      <td>0</td>\n      <td>WI</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>JOHNSON, Ron</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.626</td>\n      <td>-0.160</td>\n      <td>-45.72966</td>\n      <td>0.93135</td>\n      <td>643.0</td>\n      <td>25.0</td>\n      <td>NaN</td>\n      <td>0.610</td>\n      <td>0.017</td>\n    </tr>\n    <tr>\n      <th>john barrasso</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>40707</td>\n      <td>68</td>\n      <td>0</td>\n      <td>WY</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>BARRASSO, John</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.536</td>\n      <td>0.216</td>\n      <td>-30.88484</td>\n      <td>0.95482</td>\n      <td>668.0</td>\n      <td>16.0</td>\n      <td>NaN</td>\n      <td>0.602</td>\n      <td>0.323</td>\n    </tr>\n    <tr>\n      <th>mike enzi</th>\n      <td>116</td>\n      <td>Senate</td>\n      <td>49706</td>\n      <td>68</td>\n      <td>0</td>\n      <td>WY</td>\n      <td>200</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>ENZI, Mike</td>\n      <td>...</td>\n      <td>2021.0</td>\n      <td>0.545</td>\n      <td>0.199</td>\n      <td>-42.60100</td>\n      <td>0.93551</td>\n      <td>639.0</td>\n      <td>16.0</td>\n      <td>NaN</td>\n      <td>0.587</td>\n      <td>0.414</td>\n    </tr>\n  </tbody>\n</table>\n<p>553 rows × 22 columns</p>\n</div>"
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dw"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "outputs": [
    {
     "data": {
      "text/plain": "{'abby finkenauer',\n 'abigail spanberger',\n 'adam kinzinger',\n 'adam schiff',\n 'adam smith',\n 'adrian smith',\n 'adriano espaillat',\n 'al green',\n 'al lawson',\n 'alan lowenthal',\n 'albio sires',\n 'alcee hastings',\n 'alex mooney',\n 'alexandria ocasio-cortez',\n 'alma adams',\n 'amata radewagen',\n 'ami bera',\n 'amy klobuchar',\n 'andr� carson',\n 'andy barr',\n 'andy biggs',\n 'andy harris',\n 'andy kim',\n 'andy levin',\n 'angie craig',\n 'angus king',\n 'ann kirkpatrick',\n 'ann kuster',\n 'ann wagner',\n 'anna eshoo',\n 'anthony brindisi',\n 'anthony brown',\n 'anthony gonzalez',\n 'antonio delgado',\n 'austin scott',\n 'ayanna pressley',\n 'barbara lee',\n 'barry loudermilk',\n 'ben cardin',\n 'ben cline',\n 'ben luj�n',\n 'ben mcadams',\n 'ben sasse',\n 'bennie thompson',\n 'bernie sanders',\n 'betty mccollum',\n 'bill cassidy',\n 'bill flores',\n 'bill foster',\n 'bill huizenga',\n 'bill johnson',\n 'bill keating',\n 'bill pascrell',\n 'bill posey',\n 'billy long',\n 'blaine luetkemeyer',\n 'bob gibbs',\n 'bob jr.',\n 'bob latta',\n 'bobby rush',\n 'bobby scott',\n 'bonnie coleman',\n 'brad schneider',\n 'brad sherman',\n 'brad wenstrup',\n 'bradley byrne',\n 'brenda lawrence',\n 'brendan boyle',\n 'brett guthrie',\n 'brian babin',\n 'brian fitzpatrick',\n 'brian higgins',\n 'brian mast',\n 'brian schatz',\n 'bruce westerman',\n 'bryan steil',\n 'buddy carter',\n 'carol miller',\n 'carolyn maloney',\n 'catherine masto',\n 'cathy rodgers',\n 'cedric richmond',\n 'charlie crist',\n 'chellie pingree',\n 'cheri bustos',\n 'chip roy',\n 'chris collins',\n 'chris coons',\n 'chris hollen',\n 'chris jacobs',\n 'chris murphy',\n 'chris pappas',\n 'chris stewart',\n 'chrissy houlahan',\n 'christopher smith',\n 'chuck fleischmann',\n 'chuck grassley',\n 'chuck schumer',\n 'cindy axne',\n 'cindy hyde-smith',\n 'clay higgins',\n 'colin allred',\n 'collin peterson',\n 'conor lamb',\n 'cory booker',\n 'cory gardner',\n 'dan bishop',\n 'dan crenshaw',\n 'dan kildee',\n 'dan lipinski',\n 'dan meuser',\n 'dan newhouse',\n 'dan sullivan',\n 'daniel webster',\n 'danny davis',\n 'darin lahood',\n 'darren soto',\n 'dave loebsack',\n 'david cicilline',\n 'david joyce',\n 'david kustoff',\n 'david mckinley',\n 'david perdue',\n 'david price',\n 'david rouzer',\n 'david schweikert',\n 'david scott',\n 'david trone',\n 'dean phillips',\n 'deb fischer',\n 'deb haaland',\n 'debbie dingell',\n 'debbie lesko',\n 'debbie mucarsel-powell',\n 'debbie schultz',\n 'debbie stabenow',\n 'dennis heck',\n 'denver riggleman',\n 'derek kilmer',\n 'devin nunes',\n 'diana degette',\n 'dianne feinstein',\n 'dick durbin',\n 'dina titus',\n 'don bacon',\n 'don beyer',\n 'don young',\n 'donald jr.',\n 'donald mceachin',\n 'donald norcross',\n 'donald trump',\n 'donna shalala',\n 'doris matsui',\n 'doug collins',\n 'doug jones',\n 'doug lamalfa',\n 'doug lamborn',\n 'drew ferguson',\n 'duncan hunter',\n 'dusty johnson',\n 'dutch ruppersberger',\n 'dwight evans',\n 'earl blumenauer',\n 'ed case',\n 'ed markey',\n 'ed perlmutter',\n 'eddie johnson',\n 'elaine luria',\n 'eleanor norton',\n 'elijah cummings',\n 'eliot engel',\n 'elise stefanik',\n 'elissa slotkin',\n 'elizabeth warren',\n 'emanuel cleaver',\n 'eric swalwell',\n 'filemon jr.',\n 'francis rooney',\n 'frank lucas',\n 'frank pallone',\n 'fred keller',\n 'fred upton',\n 'frederica wilson',\n 'french hill',\n 'g. butterfield',\n 'garret graves',\n 'gary palmer',\n 'gary peters',\n 'george holding',\n 'gerry connolly',\n 'gil cisneros',\n 'glenn grothman',\n 'glenn thompson',\n 'grace meng',\n 'grace napolitano',\n 'greg gianforte',\n 'greg pence',\n 'greg stanton',\n 'greg steube',\n 'greg walden',\n 'gregorio sablan',\n 'gregory meeks',\n 'gregory murphy',\n 'gus bilirakis',\n 'guy reschenthaler',\n 'gwen moore',\n 'hakeem jeffries',\n 'hal rogers',\n 'haley stevens',\n 'hank johnson',\n 'harley rouda',\n 'henry cuellar',\n 'ilhan omar',\n 'jack bergman',\n 'jack reed',\n 'jackie speier',\n 'jackie walorski',\n 'jacky rosen',\n 'jahana hayes',\n 'jaime beutler',\n 'james comer',\n 'james inhofe',\n 'james langevin',\n 'james lankford',\n 'jamie raskin',\n 'jan schakowsky',\n 'jared golden',\n 'jared huffman',\n 'jason crow',\n 'jason smith',\n 'jeanne shaheen',\n 'jeff drew',\n 'jeff duncan',\n 'jeff fortenberry',\n 'jeff merkley',\n 'jeff van drew',\n 'jennifer wexton',\n 'jenniffer gonz�lez',\n 'jerrold nadler',\n 'jerry mcnerney',\n 'jerry moran',\n 'jes�s garc�a',\n 'jim baird',\n 'jim banks',\n 'jim clyburn',\n 'jim cooper',\n 'jim costa',\n 'jim hagedorn',\n 'jim himes',\n 'jim jordan',\n 'jim mcgovern',\n 'jim risch',\n 'jim sensenbrenner',\n 'jimmy gomez',\n 'jimmy panetta',\n 'joaqu�n castro',\n 'jodey arrington',\n 'jody hice',\n 'joe courtney',\n 'joe cunningham',\n 'joe manchin',\n 'joe neguse',\n 'joe wilson',\n 'john barrasso',\n 'john boozman',\n 'john carter',\n 'john cornyn',\n 'john curtis',\n 'john garamendi',\n 'john hoeven',\n 'john joyce',\n 'john katko',\n 'john kennedy',\n 'john larson',\n 'john lewis',\n 'john moolenaar',\n 'john ratcliffe',\n 'john rose',\n 'john rutherford',\n 'john sarbanes',\n 'john shimkus',\n 'john thune',\n 'john yarmuth',\n 'johnny isakson',\n 'jon tester',\n 'joni ernst',\n 'joseph kennedy',\n 'joseph morelle',\n 'josh gottheimer',\n 'josh harder',\n 'josh hawley',\n 'jos� serrano',\n 'joyce beatty',\n 'juan vargas',\n 'judy chu',\n 'julia brownley',\n 'justin amash',\n 'kamala harris',\n 'karen bass',\n 'katherine clark',\n 'kathleen rice',\n 'kathy castor',\n 'katie hill',\n 'katie porter',\n 'kay granger',\n 'kelly armstrong',\n 'kelly loeffler',\n 'ken buck',\n 'ken calvert',\n 'kendra horn',\n 'kenny marchant',\n 'kevin brady',\n 'kevin cramer',\n 'kevin hern',\n 'kevin mccarthy',\n 'kim schrier',\n 'kirsten gillibrand',\n 'kurt schrader',\n 'kwanza hall',\n 'kweisi mfume',\n 'kyrsten sinema',\n 'lacy clay',\n 'lamar alexander',\n 'lance gooden',\n 'larry bucshon',\n 'lauren underwood',\n 'lee zeldin',\n 'linda s�nchez',\n 'lindsey graham',\n 'lisa murkowski',\n 'lisa rochester',\n 'liz cheney',\n 'lizzie fletcher',\n 'lloyd doggett',\n 'lloyd smucker',\n 'lois frankel',\n 'lori trahan',\n 'lou correa',\n 'louie gohmert',\n 'lucille roybal-allard',\n 'lucy mcbath',\n 'mac thornberry',\n 'madeleine dean',\n 'maggie hassan',\n 'marc veasey',\n 'marcia fudge',\n 'marco rubio',\n 'marcy kaptur',\n 'maria cantwell',\n 'mario diaz-balart',\n 'mark amodei',\n 'mark desaulnier',\n 'mark green',\n 'mark kelly',\n 'mark meadows',\n 'mark pocan',\n 'mark takano',\n 'mark walker',\n 'mark warner',\n 'markwayne mullin',\n 'marsha blackburn',\n 'martha mcsally',\n 'martha roby',\n 'martin heinrich',\n 'mary scanlon',\n 'matt cartwright',\n 'matt gaetz',\n 'max rose',\n 'maxine waters',\n 'mazie hirono',\n 'michael bennet',\n 'michael burgess',\n 'michael cloud',\n 'michael doyle',\n 'michael guest',\n 'michael mccaul',\n 'michael nicolas',\n 'michael waltz',\n 'mike bost',\n 'mike braun',\n 'mike conaway',\n 'mike crapo',\n 'mike enzi',\n 'mike gallagher',\n 'mike garcia',\n 'mike johnson',\n 'mike kelly',\n 'mike lee',\n 'mike levin',\n 'mike quigley',\n 'mike rogers',\n 'mike rounds',\n 'mike simpson',\n 'mike thompson',\n 'mike turner',\n 'mikie sherrill',\n 'mitch mcconnell',\n 'mitt romney',\n 'mo brooks',\n 'morgan griffith',\n 'nancy pelosi',\n 'nanette barrag�n',\n 'neal dunn',\n 'nita lowey',\n 'norma torres',\n 'nydia vel�zquez',\n 'pat roberts',\n 'pat toomey',\n 'patrick leahy',\n 'patrick mchenry',\n 'patty murray',\n 'paul cook',\n 'paul gosar',\n 'paul mitchell',\n 'paul tonko',\n 'pete aguilar',\n 'pete olson',\n 'pete stauber',\n 'pete visclosky',\n 'peter defazio',\n 'peter king',\n 'peter welch',\n 'phil roe',\n 'pramila jayapal',\n 'raja krishnamoorthi',\n 'ralph abraham',\n 'ralph norman',\n 'rand paul',\n 'randy weber',\n 'rashida tlaib',\n 'raul ruiz',\n 'ra�l grijalva',\n 'richard blumenthal',\n 'richard burr',\n 'richard hudson',\n 'richard neal',\n 'richard shelby',\n 'rick allen',\n 'rick crawford',\n 'rick larsen',\n 'rick scott',\n 'ro khanna',\n 'rob bishop',\n 'rob portman',\n 'rob wittman',\n 'rob woodall',\n 'robert aderholt',\n 'robert menendez',\n 'robin kelly',\n 'rodney davis',\n 'roger marshall',\n 'roger wicker',\n 'roger williams',\n 'ron estes',\n 'ron johnson',\n 'ron kind',\n 'ron wright',\n 'ron wyden',\n 'rosa delauro',\n 'ross spano',\n 'roy blunt',\n 'ruben gallego',\n 'russ fulcher',\n 'salud carbajal',\n 'sam graves',\n 'sanford bishop',\n 'scott desjarlais',\n 'scott perry',\n 'scott peters',\n 'scott tipton',\n 'sean casten',\n 'sean duffy',\n 'sean maloney',\n 'seth moulton',\n 'sharice davids',\n 'sheila lee',\n 'sheldon whitehouse',\n 'shelley capito',\n 'sherrod brown',\n 'stacey plaskett',\n 'steny hoyer',\n 'stephanie murphy',\n 'stephen lynch',\n 'steve chabot',\n 'steve cohen',\n 'steve daines',\n 'steve king',\n 'steve scalise',\n 'steve stivers',\n 'steve watkins',\n 'steve womack',\n 'steven horsford',\n 'steven palazzo',\n 'susan brooks',\n 'susan collins',\n 'susan davis',\n 'susan wild',\n 'susie lee',\n 'suzan delbene',\n 'suzanne bonamici',\n 'sylvia garcia',\n 'tammy baldwin',\n 'tammy duckworth',\n 'ted budd',\n 'ted cruz',\n 'ted deutch',\n 'ted lieu',\n 'ted yoho',\n 'terri sewell',\n 'thom tillis',\n 'thomas marino',\n 'thomas massie',\n 'thomas suozzi',\n 'thomas tiffany',\n 'tim burchett',\n 'tim kaine',\n 'tim ryan',\n 'tim scott',\n 'tim walberg',\n 'tina smith',\n 'tj cox',\n 'todd young',\n 'tom carper',\n 'tom cole',\n 'tom cotton',\n 'tom emmer',\n 'tom graves',\n 'tom malinowski',\n 'tom mcclintock',\n \"tom o'halleran\",\n 'tom reed',\n 'tom rice',\n 'tom udall',\n 'tony c�rdenas',\n 'trent kelly',\n 'trey hollingsworth',\n 'troy balderson',\n 'tulsi gabbard',\n 'val demings',\n 'van taylor',\n 'vern buchanan',\n 'veronica escobar',\n 'vicente gonz�lez',\n 'vicky hartzler',\n 'virginia foxx',\n 'walter jones',\n 'warren davidson',\n 'will hurd',\n 'william timmons',\n 'xochitl small',\n 'yvette clarke',\n 'zoe lofgren'}"
     },
     "execution_count": 377,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dw_dataset_names"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "outputs": [],
   "source": [
    "congress_df['names'] = congress_df['Wikipedia  Names'].apply(lambda x: x.lower().split()).apply(lambda x: x[0]+' '+x[-1])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "outputs": [],
   "source": [
    "twitter_dataset_names = set(congress_df['Wikipedia  Names'].apply(lambda x: x.lower().split()).apply(lambda x: x[0]+' '+x[-1]))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "outputs": [],
   "source": [
    "congress_df.set_index('names', inplace=True)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "outputs": [
    {
     "data": {
      "text/plain": "                     Wikipedia  Names Position Joined Congress  \\\nnames                                                            \ndoug jones                Doug Jones       Sen        3-Jan-18   \nrichard shelby        Richard Shelby       Sen        3-Jan-87   \ndan sullivan            Dan Sullivan       Sen        3-Jan-15   \nlisa murkowski        Lisa Murkowski       Sen       20-Dec-02   \nkyrsten sinema        Kyrsten Sinema       Sen        3-Jan-19   \n...                               ...      ...             ...   \nwill hurd                  Will Hurd       Rep        3-Jan-15   \nwilliam timmons      William Timmons       Rep        3-Jan-19   \nxochitl small    Xochitl Torres Small      Rep        3-Jan-19   \nyvette clarke          Yvette Clarke       Rep        3-Jan-13   \nzoe lofgren              Zoe Lofgren       Rep        3-Jan-95   \n\n                        ODU-WSDL            CSPAN    TweetCongress  \\\nnames                                                                \ndoug jones          sendougjones     sendougjones     sendougjones   \nrichard shelby         SenShelby        SenShelby        SenShelby   \ndan sullivan      SenDanSullivan   SenDanSullivan   SenDanSullivan   \nlisa murkowski     LisaMurkowski    LisaMurkowski    LisaMurkowski   \nkyrsten sinema     SenatorSinema    SenatorSinema    SenatorSinema   \n...                          ...              ...              ...   \nwill hurd          hurdonthehill    hurdonthehill    hurdonthehill   \nwilliam timmons       RepTimmons       RepTimmons       RepTimmons   \nxochitl small     RepTorresSmall   RepTorresSmall   RepTorresSmall   \nyvette clarke    RepYvetteClarke  RepYvetteClarke  RepYvetteClarke   \nzoe lofgren        RepZoeLofgren    RepZoeLofgren    RepZoeLofgren   \n\n                          Github REMARK  \nnames                                    \ndoug jones          sendougjones    NaN  \nrichard shelby         SenShelby    NaN  \ndan sullivan      SenDanSullivan    NaN  \nlisa murkowski     LisaMurkowski    NaN  \nkyrsten sinema     SenatorSinema    NaN  \n...                          ...    ...  \nwill hurd          hurdonthehill    NaN  \nwilliam timmons       RepTimmons    NaN  \nxochitl small     RepTorresSmall    NaN  \nyvette clarke    RepYvetteClarke    NaN  \nzoe lofgren        RepZoeLofgren    NaN  \n\n[538 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Wikipedia  Names</th>\n      <th>Position</th>\n      <th>Joined Congress</th>\n      <th>ODU-WSDL</th>\n      <th>CSPAN</th>\n      <th>TweetCongress</th>\n      <th>Github</th>\n      <th>REMARK</th>\n    </tr>\n    <tr>\n      <th>names</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>doug jones</th>\n      <td>Doug Jones</td>\n      <td>Sen</td>\n      <td>3-Jan-18</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>richard shelby</th>\n      <td>Richard Shelby</td>\n      <td>Sen</td>\n      <td>3-Jan-87</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>dan sullivan</th>\n      <td>Dan Sullivan</td>\n      <td>Sen</td>\n      <td>3-Jan-15</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>lisa murkowski</th>\n      <td>Lisa Murkowski</td>\n      <td>Sen</td>\n      <td>20-Dec-02</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>kyrsten sinema</th>\n      <td>Kyrsten Sinema</td>\n      <td>Sen</td>\n      <td>3-Jan-19</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>will hurd</th>\n      <td>Will Hurd</td>\n      <td>Rep</td>\n      <td>3-Jan-15</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>william timmons</th>\n      <td>William Timmons</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>xochitl small</th>\n      <td>Xochitl Torres Small</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>yvette clarke</th>\n      <td>Yvette Clarke</td>\n      <td>Rep</td>\n      <td>3-Jan-13</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>zoe lofgren</th>\n      <td>Zoe Lofgren</td>\n      <td>Rep</td>\n      <td>3-Jan-95</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n<p>538 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "congress_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "outputs": [
    {
     "data": {
      "text/plain": "{'abby finkenauer',\n 'abigail spanberger',\n 'adam kinzinger',\n 'adam schiff',\n 'adam smith',\n 'adrian smith',\n 'adriano espaillat',\n 'al green',\n 'al lawson',\n 'alan lowenthal',\n 'albio sires',\n 'alcee hastings',\n 'alex mooney',\n 'alexandria ocasio-cortez',\n 'alma adams',\n 'amata radewagen',\n 'ami bera',\n 'amy klobuchar',\n 'andr� carson',\n 'andy barr',\n 'andy biggs',\n 'andy harris',\n 'andy kim',\n 'andy levin',\n 'angie craig',\n 'angus king',\n 'ann kirkpatrick',\n 'ann kuster',\n 'ann wagner',\n 'anna eshoo',\n 'anthony brindisi',\n 'anthony brown',\n 'anthony gonzalez',\n 'antonio delgado',\n 'austin scott',\n 'ayanna pressley',\n 'barbara lee',\n 'barry loudermilk',\n 'ben cardin',\n 'ben cline',\n 'ben luj�n',\n 'ben mcadams',\n 'ben sasse',\n 'bennie thompson',\n 'bernie sanders',\n 'betty mccollum',\n 'bill cassidy',\n 'bill flores',\n 'bill foster',\n 'bill huizenga',\n 'bill johnson',\n 'bill keating',\n 'bill pascrell',\n 'bill posey',\n 'billy long',\n 'blaine luetkemeyer',\n 'bob gibbs',\n 'bob jr.',\n 'bob latta',\n 'bobby rush',\n 'bobby scott',\n 'bonnie coleman',\n 'brad schneider',\n 'brad sherman',\n 'brad wenstrup',\n 'bradley byrne',\n 'brenda lawrence',\n 'brendan boyle',\n 'brett guthrie',\n 'brian babin',\n 'brian fitzpatrick',\n 'brian higgins',\n 'brian mast',\n 'brian schatz',\n 'bruce westerman',\n 'bryan steil',\n 'buddy carter',\n 'carol miller',\n 'carolyn maloney',\n 'catherine masto',\n 'cathy rodgers',\n 'cedric richmond',\n 'charlie crist',\n 'chellie pingree',\n 'cheri bustos',\n 'chip roy',\n 'chris collins',\n 'chris coons',\n 'chris hollen',\n 'chris murphy',\n 'chris pappas',\n 'chris stewart',\n 'chrissy houlahan',\n 'christopher smith',\n 'chuck fleischmann',\n 'chuck grassley',\n 'chuck schumer',\n 'cindy axne',\n 'cindy hyde-smith',\n 'clay higgins',\n 'colin allred',\n 'collin peterson',\n 'conor lamb',\n 'cory booker',\n 'cory gardner',\n 'dan crenshaw',\n 'dan kildee',\n 'dan lipinski',\n 'dan meuser',\n 'dan newhouse',\n 'dan sullivan',\n 'daniel webster',\n 'danny davis',\n 'darin lahood',\n 'darren soto',\n 'dave loebsack',\n 'david cicilline',\n 'david joyce',\n 'david kustoff',\n 'david mckinley',\n 'david perdue',\n 'david price',\n 'david rouzer',\n 'david schweikert',\n 'david scott',\n 'david trone',\n 'dean phillips',\n 'deb fischer',\n 'deb haaland',\n 'debbie dingell',\n 'debbie lesko',\n 'debbie mucarsel-powell',\n 'debbie schultz',\n 'debbie stabenow',\n 'dennis heck',\n 'denver riggleman',\n 'derek kilmer',\n 'devin nunes',\n 'diana degette',\n 'dianne feinstein',\n 'dick durbin',\n 'dina titus',\n 'don bacon',\n 'don beyer',\n 'don young',\n 'donald jr.',\n 'donald mceachin',\n 'donald norcross',\n 'donna shalala',\n 'doris matsui',\n 'doug collins',\n 'doug jones',\n 'doug lamalfa',\n 'doug lamborn',\n 'drew ferguson',\n 'duncan hunter',\n 'dusty johnson',\n 'dutch ruppersberger',\n 'dwight evans',\n 'earl blumenauer',\n 'ed case',\n 'ed markey',\n 'ed perlmutter',\n 'eddie johnson',\n 'elaine luria',\n 'eleanor norton',\n 'elijah cummings',\n 'eliot engel',\n 'elise stefanik',\n 'elissa slotkin',\n 'elizabeth warren',\n 'emanuel cleaver',\n 'eric swalwell',\n 'filemon jr.',\n 'francis rooney',\n 'frank lucas',\n 'frank pallone',\n 'fred upton',\n 'frederica wilson',\n 'french hill',\n 'g. butterfield',\n 'garret graves',\n 'gary palmer',\n 'gary peters',\n 'george holding',\n 'gerry connolly',\n 'gil cisneros',\n 'glenn grothman',\n 'glenn thompson',\n 'grace meng',\n 'grace napolitano',\n 'greg gianforte',\n 'greg pence',\n 'greg stanton',\n 'greg steube',\n 'greg walden',\n 'gregorio sablan',\n 'gregory meeks',\n 'gus bilirakis',\n 'guy reschenthaler',\n 'gwen moore',\n 'hakeem jeffries',\n 'hal rogers',\n 'haley stevens',\n 'hank johnson',\n 'harley rouda',\n 'henry cuellar',\n 'ilhan omar',\n 'jack bergman',\n 'jack reed',\n 'jackie speier',\n 'jackie walorski',\n 'jacky rosen',\n 'jahana hayes',\n 'jaime beutler',\n 'james comer',\n 'james inhofe',\n 'james langevin',\n 'james lankford',\n 'jamie raskin',\n 'jan schakowsky',\n 'jared golden',\n 'jared huffman',\n 'jason crow',\n 'jason smith',\n 'jeanne shaheen',\n 'jeff drew',\n 'jeff duncan',\n 'jeff fortenberry',\n 'jeff merkley',\n 'jennifer wexton',\n 'jenniffer gonz�lez',\n 'jerrold nadler',\n 'jerry mcnerney',\n 'jerry moran',\n 'jes�s garc�a',\n 'jim baird',\n 'jim banks',\n 'jim clyburn',\n 'jim cooper',\n 'jim costa',\n 'jim hagedorn',\n 'jim himes',\n 'jim jordan',\n 'jim mcgovern',\n 'jim risch',\n 'jim sensenbrenner',\n 'jimmy gomez',\n 'jimmy panetta',\n 'joaqu�n castro',\n 'jodey arrington',\n 'jody hice',\n 'joe courtney',\n 'joe cunningham',\n 'joe manchin',\n 'joe neguse',\n 'joe wilson',\n 'john barrasso',\n 'john boozman',\n 'john carter',\n 'john cornyn',\n 'john curtis',\n 'john garamendi',\n 'john hoeven',\n 'john joyce',\n 'john katko',\n 'john kennedy',\n 'john larson',\n 'john lewis',\n 'john moolenaar',\n 'john ratcliffe',\n 'john rose',\n 'john rutherford',\n 'john sarbanes',\n 'john shimkus',\n 'john thune',\n 'john yarmuth',\n 'johnny isakson',\n 'jon tester',\n 'joni ernst',\n 'joseph kennedy',\n 'joseph morelle',\n 'josh gottheimer',\n 'josh harder',\n 'josh hawley',\n 'jos� serrano',\n 'joyce beatty',\n 'juan vargas',\n 'judy chu',\n 'julia brownley',\n 'justin amash',\n 'kamala harris',\n 'karen bass',\n 'katherine clark',\n 'kathleen rice',\n 'kathy castor',\n 'katie hill',\n 'katie porter',\n 'kay granger',\n 'kelly armstrong',\n 'ken buck',\n 'ken calvert',\n 'kendra horn',\n 'kenny marchant',\n 'kevin brady',\n 'kevin cramer',\n 'kevin hern',\n 'kevin mccarthy',\n 'kim schrier',\n 'kirsten gillibrand',\n 'kurt schrader',\n 'kyrsten sinema',\n 'lacy clay',\n 'lamar alexander',\n 'lance gooden',\n 'larry bucshon',\n 'lauren underwood',\n 'lee zeldin',\n 'linda s�nchez',\n 'lindsey graham',\n 'lisa murkowski',\n 'lisa rochester',\n 'liz cheney',\n 'lizzie fletcher',\n 'lloyd doggett',\n 'lloyd smucker',\n 'lois frankel',\n 'lori trahan',\n 'lou correa',\n 'louie gohmert',\n 'lucille roybal-allard',\n 'lucy mcbath',\n 'mac thornberry',\n 'madeleine dean',\n 'maggie hassan',\n 'marc veasey',\n 'marcia fudge',\n 'marco rubio',\n 'marcy kaptur',\n 'maria cantwell',\n 'mario diaz-balart',\n 'mark amodei',\n 'mark desaulnier',\n 'mark green',\n 'mark meadows',\n 'mark pocan',\n 'mark takano',\n 'mark walker',\n 'mark warner',\n 'markwayne mullin',\n 'marsha blackburn',\n 'martha mcsally',\n 'martha roby',\n 'martin heinrich',\n 'mary scanlon',\n 'matt cartwright',\n 'matt gaetz',\n 'max rose',\n 'maxine waters',\n 'mazie hirono',\n 'michael bennet',\n 'michael burgess',\n 'michael cloud',\n 'michael doyle',\n 'michael guest',\n 'michael mccaul',\n 'michael nicolas',\n 'michael waltz',\n 'mike bost',\n 'mike braun',\n 'mike conaway',\n 'mike crapo',\n 'mike enzi',\n 'mike gallagher',\n 'mike johnson',\n 'mike kelly',\n 'mike lee',\n 'mike levin',\n 'mike quigley',\n 'mike rogers',\n 'mike rounds',\n 'mike simpson',\n 'mike thompson',\n 'mike turner',\n 'mikie sherrill',\n 'mitch mcconnell',\n 'mitt romney',\n 'mo brooks',\n 'morgan griffith',\n 'nancy pelosi',\n 'nanette barrag�n',\n 'neal dunn',\n 'nita lowey',\n 'norma torres',\n 'nydia vel�zquez',\n 'pat roberts',\n 'pat toomey',\n 'patrick leahy',\n 'patrick mchenry',\n 'patty murray',\n 'paul cook',\n 'paul gosar',\n 'paul mitchell',\n 'paul tonko',\n 'pete aguilar',\n 'pete olson',\n 'pete stauber',\n 'pete visclosky',\n 'peter defazio',\n 'peter king',\n 'peter welch',\n 'phil roe',\n 'pramila jayapal',\n 'raja krishnamoorthi',\n 'ralph abraham',\n 'ralph norman',\n 'rand paul',\n 'randy weber',\n 'rashida tlaib',\n 'raul ruiz',\n 'ra�l grijalva',\n 'richard blumenthal',\n 'richard burr',\n 'richard hudson',\n 'richard neal',\n 'richard shelby',\n 'rick allen',\n 'rick crawford',\n 'rick larsen',\n 'rick scott',\n 'ro khanna',\n 'rob bishop',\n 'rob portman',\n 'rob wittman',\n 'rob woodall',\n 'robert aderholt',\n 'robert menendez',\n 'robin kelly',\n 'rodney davis',\n 'roger marshall',\n 'roger wicker',\n 'roger williams',\n 'ron estes',\n 'ron johnson',\n 'ron kind',\n 'ron wright',\n 'ron wyden',\n 'rosa delauro',\n 'ross spano',\n 'roy blunt',\n 'ruben gallego',\n 'russ fulcher',\n 'salud carbajal',\n 'sam graves',\n 'sanford bishop',\n 'scott desjarlais',\n 'scott perry',\n 'scott peters',\n 'scott tipton',\n 'sean casten',\n 'sean duffy',\n 'sean maloney',\n 'seth moulton',\n 'sharice davids',\n 'sheila lee',\n 'sheldon whitehouse',\n 'shelley capito',\n 'sherrod brown',\n 'stacey plaskett',\n 'steny hoyer',\n 'stephanie murphy',\n 'stephen lynch',\n 'steve chabot',\n 'steve cohen',\n 'steve daines',\n 'steve king',\n 'steve scalise',\n 'steve stivers',\n 'steve watkins',\n 'steve womack',\n 'steven horsford',\n 'steven palazzo',\n 'susan brooks',\n 'susan collins',\n 'susan davis',\n 'susan wild',\n 'susie lee',\n 'suzan delbene',\n 'suzanne bonamici',\n 'sylvia garcia',\n 'tammy baldwin',\n 'tammy duckworth',\n 'ted budd',\n 'ted cruz',\n 'ted deutch',\n 'ted lieu',\n 'ted yoho',\n 'terri sewell',\n 'thom tillis',\n 'thomas massie',\n 'thomas suozzi',\n 'tim burchett',\n 'tim kaine',\n 'tim ryan',\n 'tim scott',\n 'tim walberg',\n 'tina smith',\n 'tj cox',\n 'todd young',\n 'tom carper',\n 'tom cole',\n 'tom cotton',\n 'tom emmer',\n 'tom graves',\n 'tom malinowski',\n 'tom mcclintock',\n \"tom o'halleran\",\n 'tom reed',\n 'tom rice',\n 'tom udall',\n 'tony c�rdenas',\n 'trent kelly',\n 'trey hollingsworth',\n 'troy balderson',\n 'tulsi gabbard',\n 'val demings',\n 'van taylor',\n 'vern buchanan',\n 'veronica escobar',\n 'vicente gonz�lez',\n 'vicky hartzler',\n 'virginia foxx',\n 'warren davidson',\n 'will hurd',\n 'william timmons',\n 'xochitl small',\n 'yvette clarke',\n 'zoe lofgren'}"
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_dataset_names"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "outputs": [
    {
     "data": {
      "text/plain": "set()"
     },
     "execution_count": 383,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_dataset_names - dw_dataset_names"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "outputs": [],
   "source": [
    "data = congress_df.merge(df_dw, on='names', how='left')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "outputs": [
    {
     "data": {
      "text/plain": "                     Wikipedia  Names Position Joined Congress  \\\nnames                                                            \ndoug jones                Doug Jones       Sen        3-Jan-18   \nrichard shelby        Richard Shelby       Sen        3-Jan-87   \ndan sullivan            Dan Sullivan       Sen        3-Jan-15   \nlisa murkowski        Lisa Murkowski       Sen       20-Dec-02   \nkyrsten sinema        Kyrsten Sinema       Sen        3-Jan-19   \n...                               ...      ...             ...   \nwill hurd                  Will Hurd       Rep        3-Jan-15   \nwilliam timmons      William Timmons       Rep        3-Jan-19   \nxochitl small    Xochitl Torres Small      Rep        3-Jan-19   \nyvette clarke          Yvette Clarke       Rep        3-Jan-13   \nzoe lofgren              Zoe Lofgren       Rep        3-Jan-95   \n\n                        ODU-WSDL            CSPAN    TweetCongress  \\\nnames                                                                \ndoug jones          sendougjones     sendougjones     sendougjones   \nrichard shelby         SenShelby        SenShelby        SenShelby   \ndan sullivan      SenDanSullivan   SenDanSullivan   SenDanSullivan   \nlisa murkowski     LisaMurkowski    LisaMurkowski    LisaMurkowski   \nkyrsten sinema     SenatorSinema    SenatorSinema    SenatorSinema   \n...                          ...              ...              ...   \nwill hurd          hurdonthehill    hurdonthehill    hurdonthehill   \nwilliam timmons       RepTimmons       RepTimmons       RepTimmons   \nxochitl small     RepTorresSmall   RepTorresSmall   RepTorresSmall   \nyvette clarke    RepYvetteClarke  RepYvetteClarke  RepYvetteClarke   \nzoe lofgren        RepZoeLofgren    RepZoeLofgren    RepZoeLofgren   \n\n                          Github REMARK  congress chamber  ...  died  \\\nnames                                                      ...         \ndoug jones          sendougjones    NaN       116  Senate  ...   NaN   \nrichard shelby         SenShelby    NaN       116  Senate  ...   NaN   \ndan sullivan      SenDanSullivan    NaN       116  Senate  ...   NaN   \nlisa murkowski     LisaMurkowski    NaN       116  Senate  ...   NaN   \nkyrsten sinema     SenatorSinema    NaN       116  Senate  ...   NaN   \n...                          ...    ...       ...     ...  ...   ...   \nwill hurd          hurdonthehill    NaN       116   House  ...   NaN   \nwilliam timmons       RepTimmons    NaN       116   House  ...   NaN   \nxochitl small     RepTorresSmall    NaN       116   House  ...   NaN   \nyvette clarke    RepYvetteClarke    NaN       116   House  ...   NaN   \nzoe lofgren        RepZoeLofgren    NaN       116   House  ...   NaN   \n\n                 nominate_dim1  nominate_dim2 nominate_log_likelihood  \\\nnames                                                                   \ndoug jones              -0.088          0.187               -80.32594   \nrichard shelby           0.446          0.570               -45.74049   \ndan sullivan             0.485          0.061               -53.27224   \nlisa murkowski           0.211         -0.304               -47.53006   \nkyrsten sinema          -0.105          0.076               -86.25793   \n...                        ...            ...                     ...   \nwill hurd                0.287          0.228              -175.54897   \nwilliam timmons          0.591          0.225               -84.32677   \nxochitl small           -0.163          0.314               -99.88538   \nyvette clarke           -0.612         -0.225               -32.45444   \nzoe lofgren             -0.400         -0.435               -32.74495   \n\n                 nominate_geo_mean_probability  nominate_number_of_votes  \\\nnames                                                                      \ndoug jones                             0.88065                     632.0   \nrichard shelby                         0.93334                     663.0   \ndan sullivan                           0.92119                     649.0   \nlisa murkowski                         0.92655                     623.0   \nkyrsten sinema                         0.86773                     608.0   \n...                                        ...                       ...   \nwill hurd                              0.80231                     797.0   \nwilliam timmons                        0.88465                     688.0   \nxochitl small                          0.88358                     807.0   \nyvette clarke                          0.96049                     805.0   \nzoe lofgren                            0.95935                     789.0   \n\n                 nominate_number_of_errors conditional nokken_poole_dim1  \\\nnames                                                                      \ndoug jones                            33.0         NaN            -0.081   \nrichard shelby                        27.0         NaN             0.437   \ndan sullivan                          23.0         NaN             0.510   \nlisa murkowski                        11.0         NaN             0.283   \nkyrsten sinema                        27.0         NaN            -0.065   \n...                                    ...         ...               ...   \nwill hurd                             77.0         NaN             0.219   \nwilliam timmons                       34.0         NaN             0.599   \nxochitl small                         42.0         NaN            -0.163   \nyvette clarke                         14.0         NaN            -0.520   \nzoe lofgren                           11.0         NaN            -0.461   \n\n                 nokken_poole_dim2  \nnames                               \ndoug jones                   0.234  \nrichard shelby               0.219  \ndan sullivan                -0.040  \nlisa murkowski              -0.424  \nkyrsten sinema               0.217  \n...                            ...  \nwill hurd                    0.277  \nwilliam timmons              0.145  \nxochitl small                0.314  \nyvette clarke               -0.623  \nzoe lofgren                 -0.466  \n\n[539 rows x 30 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Wikipedia  Names</th>\n      <th>Position</th>\n      <th>Joined Congress</th>\n      <th>ODU-WSDL</th>\n      <th>CSPAN</th>\n      <th>TweetCongress</th>\n      <th>Github</th>\n      <th>REMARK</th>\n      <th>congress</th>\n      <th>chamber</th>\n      <th>...</th>\n      <th>died</th>\n      <th>nominate_dim1</th>\n      <th>nominate_dim2</th>\n      <th>nominate_log_likelihood</th>\n      <th>nominate_geo_mean_probability</th>\n      <th>nominate_number_of_votes</th>\n      <th>nominate_number_of_errors</th>\n      <th>conditional</th>\n      <th>nokken_poole_dim1</th>\n      <th>nokken_poole_dim2</th>\n    </tr>\n    <tr>\n      <th>names</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>doug jones</th>\n      <td>Doug Jones</td>\n      <td>Sen</td>\n      <td>3-Jan-18</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.088</td>\n      <td>0.187</td>\n      <td>-80.32594</td>\n      <td>0.88065</td>\n      <td>632.0</td>\n      <td>33.0</td>\n      <td>NaN</td>\n      <td>-0.081</td>\n      <td>0.234</td>\n    </tr>\n    <tr>\n      <th>richard shelby</th>\n      <td>Richard Shelby</td>\n      <td>Sen</td>\n      <td>3-Jan-87</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.446</td>\n      <td>0.570</td>\n      <td>-45.74049</td>\n      <td>0.93334</td>\n      <td>663.0</td>\n      <td>27.0</td>\n      <td>NaN</td>\n      <td>0.437</td>\n      <td>0.219</td>\n    </tr>\n    <tr>\n      <th>dan sullivan</th>\n      <td>Dan Sullivan</td>\n      <td>Sen</td>\n      <td>3-Jan-15</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.485</td>\n      <td>0.061</td>\n      <td>-53.27224</td>\n      <td>0.92119</td>\n      <td>649.0</td>\n      <td>23.0</td>\n      <td>NaN</td>\n      <td>0.510</td>\n      <td>-0.040</td>\n    </tr>\n    <tr>\n      <th>lisa murkowski</th>\n      <td>Lisa Murkowski</td>\n      <td>Sen</td>\n      <td>20-Dec-02</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.211</td>\n      <td>-0.304</td>\n      <td>-47.53006</td>\n      <td>0.92655</td>\n      <td>623.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>0.283</td>\n      <td>-0.424</td>\n    </tr>\n    <tr>\n      <th>kyrsten sinema</th>\n      <td>Kyrsten Sinema</td>\n      <td>Sen</td>\n      <td>3-Jan-19</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.105</td>\n      <td>0.076</td>\n      <td>-86.25793</td>\n      <td>0.86773</td>\n      <td>608.0</td>\n      <td>27.0</td>\n      <td>NaN</td>\n      <td>-0.065</td>\n      <td>0.217</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>will hurd</th>\n      <td>Will Hurd</td>\n      <td>Rep</td>\n      <td>3-Jan-15</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.287</td>\n      <td>0.228</td>\n      <td>-175.54897</td>\n      <td>0.80231</td>\n      <td>797.0</td>\n      <td>77.0</td>\n      <td>NaN</td>\n      <td>0.219</td>\n      <td>0.277</td>\n    </tr>\n    <tr>\n      <th>william timmons</th>\n      <td>William Timmons</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.591</td>\n      <td>0.225</td>\n      <td>-84.32677</td>\n      <td>0.88465</td>\n      <td>688.0</td>\n      <td>34.0</td>\n      <td>NaN</td>\n      <td>0.599</td>\n      <td>0.145</td>\n    </tr>\n    <tr>\n      <th>xochitl small</th>\n      <td>Xochitl Torres Small</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.163</td>\n      <td>0.314</td>\n      <td>-99.88538</td>\n      <td>0.88358</td>\n      <td>807.0</td>\n      <td>42.0</td>\n      <td>NaN</td>\n      <td>-0.163</td>\n      <td>0.314</td>\n    </tr>\n    <tr>\n      <th>yvette clarke</th>\n      <td>Yvette Clarke</td>\n      <td>Rep</td>\n      <td>3-Jan-13</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.612</td>\n      <td>-0.225</td>\n      <td>-32.45444</td>\n      <td>0.96049</td>\n      <td>805.0</td>\n      <td>14.0</td>\n      <td>NaN</td>\n      <td>-0.520</td>\n      <td>-0.623</td>\n    </tr>\n    <tr>\n      <th>zoe lofgren</th>\n      <td>Zoe Lofgren</td>\n      <td>Rep</td>\n      <td>3-Jan-95</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.400</td>\n      <td>-0.435</td>\n      <td>-32.74495</td>\n      <td>0.95935</td>\n      <td>789.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>-0.461</td>\n      <td>-0.466</td>\n    </tr>\n  </tbody>\n</table>\n<p>539 rows × 30 columns</p>\n</div>"
     },
     "execution_count": 385,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "outputs": [],
   "source": [
    "# data.to_csv(house_files_path+'../data.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "outputs": [],
   "source": [
    "data_df = pd.read_csv(house_files_path+'../data.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "outputs": [
    {
     "data": {
      "text/plain": "                     Wikipedia  Names Position Joined Congress  \\\nnames                                                            \ndoug jones                Doug Jones       Sen        3-Jan-18   \nrichard shelby        Richard Shelby       Sen        3-Jan-87   \ndan sullivan            Dan Sullivan       Sen        3-Jan-15   \nlisa murkowski        Lisa Murkowski       Sen       20-Dec-02   \nkyrsten sinema        Kyrsten Sinema       Sen        3-Jan-19   \n...                               ...      ...             ...   \nwill hurd                  Will Hurd       Rep        3-Jan-15   \nwilliam timmons      William Timmons       Rep        3-Jan-19   \nxochitl small    Xochitl Torres Small      Rep        3-Jan-19   \nyvette clarke          Yvette Clarke       Rep        3-Jan-13   \nzoe lofgren              Zoe Lofgren       Rep        3-Jan-95   \n\n                        ODU-WSDL            CSPAN    TweetCongress  \\\nnames                                                                \ndoug jones          sendougjones     sendougjones     sendougjones   \nrichard shelby         SenShelby        SenShelby        SenShelby   \ndan sullivan      SenDanSullivan   SenDanSullivan   SenDanSullivan   \nlisa murkowski     LisaMurkowski    LisaMurkowski    LisaMurkowski   \nkyrsten sinema     SenatorSinema    SenatorSinema    SenatorSinema   \n...                          ...              ...              ...   \nwill hurd          hurdonthehill    hurdonthehill    hurdonthehill   \nwilliam timmons       RepTimmons       RepTimmons       RepTimmons   \nxochitl small     RepTorresSmall   RepTorresSmall   RepTorresSmall   \nyvette clarke    RepYvetteClarke  RepYvetteClarke  RepYvetteClarke   \nzoe lofgren        RepZoeLofgren    RepZoeLofgren    RepZoeLofgren   \n\n                          Github REMARK  congress chamber  ...  died  \\\nnames                                                      ...         \ndoug jones          sendougjones    NaN       116  Senate  ...   NaN   \nrichard shelby         SenShelby    NaN       116  Senate  ...   NaN   \ndan sullivan      SenDanSullivan    NaN       116  Senate  ...   NaN   \nlisa murkowski     LisaMurkowski    NaN       116  Senate  ...   NaN   \nkyrsten sinema     SenatorSinema    NaN       116  Senate  ...   NaN   \n...                          ...    ...       ...     ...  ...   ...   \nwill hurd          hurdonthehill    NaN       116   House  ...   NaN   \nwilliam timmons       RepTimmons    NaN       116   House  ...   NaN   \nxochitl small     RepTorresSmall    NaN       116   House  ...   NaN   \nyvette clarke    RepYvetteClarke    NaN       116   House  ...   NaN   \nzoe lofgren        RepZoeLofgren    NaN       116   House  ...   NaN   \n\n                 nominate_dim1  nominate_dim2 nominate_log_likelihood  \\\nnames                                                                   \ndoug jones              -0.088          0.187               -80.32594   \nrichard shelby           0.446          0.570               -45.74049   \ndan sullivan             0.485          0.061               -53.27224   \nlisa murkowski           0.211         -0.304               -47.53006   \nkyrsten sinema          -0.105          0.076               -86.25793   \n...                        ...            ...                     ...   \nwill hurd                0.287          0.228              -175.54897   \nwilliam timmons          0.591          0.225               -84.32677   \nxochitl small           -0.163          0.314               -99.88538   \nyvette clarke           -0.612         -0.225               -32.45444   \nzoe lofgren             -0.400         -0.435               -32.74495   \n\n                 nominate_geo_mean_probability  nominate_number_of_votes  \\\nnames                                                                      \ndoug jones                             0.88065                     632.0   \nrichard shelby                         0.93334                     663.0   \ndan sullivan                           0.92119                     649.0   \nlisa murkowski                         0.92655                     623.0   \nkyrsten sinema                         0.86773                     608.0   \n...                                        ...                       ...   \nwill hurd                              0.80231                     797.0   \nwilliam timmons                        0.88465                     688.0   \nxochitl small                          0.88358                     807.0   \nyvette clarke                          0.96049                     805.0   \nzoe lofgren                            0.95935                     789.0   \n\n                 nominate_number_of_errors conditional nokken_poole_dim1  \\\nnames                                                                      \ndoug jones                            33.0         NaN            -0.081   \nrichard shelby                        27.0         NaN             0.437   \ndan sullivan                          23.0         NaN             0.510   \nlisa murkowski                        11.0         NaN             0.283   \nkyrsten sinema                        27.0         NaN            -0.065   \n...                                    ...         ...               ...   \nwill hurd                             77.0         NaN             0.219   \nwilliam timmons                       34.0         NaN             0.599   \nxochitl small                         42.0         NaN            -0.163   \nyvette clarke                         14.0         NaN            -0.520   \nzoe lofgren                           11.0         NaN            -0.461   \n\n                 nokken_poole_dim2  \nnames                               \ndoug jones                   0.234  \nrichard shelby               0.219  \ndan sullivan                -0.040  \nlisa murkowski              -0.424  \nkyrsten sinema               0.217  \n...                            ...  \nwill hurd                    0.277  \nwilliam timmons              0.145  \nxochitl small                0.314  \nyvette clarke               -0.623  \nzoe lofgren                 -0.466  \n\n[539 rows x 30 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Wikipedia  Names</th>\n      <th>Position</th>\n      <th>Joined Congress</th>\n      <th>ODU-WSDL</th>\n      <th>CSPAN</th>\n      <th>TweetCongress</th>\n      <th>Github</th>\n      <th>REMARK</th>\n      <th>congress</th>\n      <th>chamber</th>\n      <th>...</th>\n      <th>died</th>\n      <th>nominate_dim1</th>\n      <th>nominate_dim2</th>\n      <th>nominate_log_likelihood</th>\n      <th>nominate_geo_mean_probability</th>\n      <th>nominate_number_of_votes</th>\n      <th>nominate_number_of_errors</th>\n      <th>conditional</th>\n      <th>nokken_poole_dim1</th>\n      <th>nokken_poole_dim2</th>\n    </tr>\n    <tr>\n      <th>names</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>doug jones</th>\n      <td>Doug Jones</td>\n      <td>Sen</td>\n      <td>3-Jan-18</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.088</td>\n      <td>0.187</td>\n      <td>-80.32594</td>\n      <td>0.88065</td>\n      <td>632.0</td>\n      <td>33.0</td>\n      <td>NaN</td>\n      <td>-0.081</td>\n      <td>0.234</td>\n    </tr>\n    <tr>\n      <th>richard shelby</th>\n      <td>Richard Shelby</td>\n      <td>Sen</td>\n      <td>3-Jan-87</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.446</td>\n      <td>0.570</td>\n      <td>-45.74049</td>\n      <td>0.93334</td>\n      <td>663.0</td>\n      <td>27.0</td>\n      <td>NaN</td>\n      <td>0.437</td>\n      <td>0.219</td>\n    </tr>\n    <tr>\n      <th>dan sullivan</th>\n      <td>Dan Sullivan</td>\n      <td>Sen</td>\n      <td>3-Jan-15</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.485</td>\n      <td>0.061</td>\n      <td>-53.27224</td>\n      <td>0.92119</td>\n      <td>649.0</td>\n      <td>23.0</td>\n      <td>NaN</td>\n      <td>0.510</td>\n      <td>-0.040</td>\n    </tr>\n    <tr>\n      <th>lisa murkowski</th>\n      <td>Lisa Murkowski</td>\n      <td>Sen</td>\n      <td>20-Dec-02</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.211</td>\n      <td>-0.304</td>\n      <td>-47.53006</td>\n      <td>0.92655</td>\n      <td>623.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>0.283</td>\n      <td>-0.424</td>\n    </tr>\n    <tr>\n      <th>kyrsten sinema</th>\n      <td>Kyrsten Sinema</td>\n      <td>Sen</td>\n      <td>3-Jan-19</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>Senate</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.105</td>\n      <td>0.076</td>\n      <td>-86.25793</td>\n      <td>0.86773</td>\n      <td>608.0</td>\n      <td>27.0</td>\n      <td>NaN</td>\n      <td>-0.065</td>\n      <td>0.217</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>will hurd</th>\n      <td>Will Hurd</td>\n      <td>Rep</td>\n      <td>3-Jan-15</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.287</td>\n      <td>0.228</td>\n      <td>-175.54897</td>\n      <td>0.80231</td>\n      <td>797.0</td>\n      <td>77.0</td>\n      <td>NaN</td>\n      <td>0.219</td>\n      <td>0.277</td>\n    </tr>\n    <tr>\n      <th>william timmons</th>\n      <td>William Timmons</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.591</td>\n      <td>0.225</td>\n      <td>-84.32677</td>\n      <td>0.88465</td>\n      <td>688.0</td>\n      <td>34.0</td>\n      <td>NaN</td>\n      <td>0.599</td>\n      <td>0.145</td>\n    </tr>\n    <tr>\n      <th>xochitl small</th>\n      <td>Xochitl Torres Small</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.163</td>\n      <td>0.314</td>\n      <td>-99.88538</td>\n      <td>0.88358</td>\n      <td>807.0</td>\n      <td>42.0</td>\n      <td>NaN</td>\n      <td>-0.163</td>\n      <td>0.314</td>\n    </tr>\n    <tr>\n      <th>yvette clarke</th>\n      <td>Yvette Clarke</td>\n      <td>Rep</td>\n      <td>3-Jan-13</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.612</td>\n      <td>-0.225</td>\n      <td>-32.45444</td>\n      <td>0.96049</td>\n      <td>805.0</td>\n      <td>14.0</td>\n      <td>NaN</td>\n      <td>-0.520</td>\n      <td>-0.623</td>\n    </tr>\n    <tr>\n      <th>zoe lofgren</th>\n      <td>Zoe Lofgren</td>\n      <td>Rep</td>\n      <td>3-Jan-95</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>House</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.400</td>\n      <td>-0.435</td>\n      <td>-32.74495</td>\n      <td>0.95935</td>\n      <td>789.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>-0.461</td>\n      <td>-0.466</td>\n    </tr>\n  </tbody>\n</table>\n<p>539 rows × 30 columns</p>\n</div>"
     },
     "execution_count": 388,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "outputs": [
    {
     "data": {
      "text/plain": "               names      Wikipedia  Names Position Joined Congress  \\\n0         doug jones           Doug Jones       Sen        3-Jan-18   \n1     richard shelby       Richard Shelby       Sen        3-Jan-87   \n2       dan sullivan         Dan Sullivan       Sen        3-Jan-15   \n3     lisa murkowski       Lisa Murkowski       Sen       20-Dec-02   \n4     kyrsten sinema       Kyrsten Sinema       Sen        3-Jan-19   \n..               ...                   ...      ...             ...   \n534        will hurd            Will Hurd       Rep        3-Jan-15   \n535  william timmons      William Timmons       Rep        3-Jan-19   \n536    xochitl small  Xochitl Torres Small      Rep        3-Jan-19   \n537    yvette clarke        Yvette Clarke       Rep        3-Jan-13   \n538      zoe lofgren          Zoe Lofgren       Rep        3-Jan-95   \n\n            ODU-WSDL            CSPAN    TweetCongress           Github  \\\n0       sendougjones     sendougjones     sendougjones     sendougjones   \n1          SenShelby        SenShelby        SenShelby        SenShelby   \n2     SenDanSullivan   SenDanSullivan   SenDanSullivan   SenDanSullivan   \n3      LisaMurkowski    LisaMurkowski    LisaMurkowski    LisaMurkowski   \n4      SenatorSinema    SenatorSinema    SenatorSinema    SenatorSinema   \n..               ...              ...              ...              ...   \n534    hurdonthehill    hurdonthehill    hurdonthehill    hurdonthehill   \n535       RepTimmons       RepTimmons       RepTimmons       RepTimmons   \n536   RepTorresSmall   RepTorresSmall   RepTorresSmall   RepTorresSmall   \n537  RepYvetteClarke  RepYvetteClarke  RepYvetteClarke  RepYvetteClarke   \n538    RepZoeLofgren    RepZoeLofgren    RepZoeLofgren    RepZoeLofgren   \n\n    REMARK  congress  ... died  nominate_dim1  nominate_dim2  \\\n0      NaN       116  ...  NaN         -0.088          0.187   \n1      NaN       116  ...  NaN          0.446          0.570   \n2      NaN       116  ...  NaN          0.485          0.061   \n3      NaN       116  ...  NaN          0.211         -0.304   \n4      NaN       116  ...  NaN         -0.105          0.076   \n..     ...       ...  ...  ...            ...            ...   \n534    NaN       116  ...  NaN          0.287          0.228   \n535    NaN       116  ...  NaN          0.591          0.225   \n536    NaN       116  ...  NaN         -0.163          0.314   \n537    NaN       116  ...  NaN         -0.612         -0.225   \n538    NaN       116  ...  NaN         -0.400         -0.435   \n\n     nominate_log_likelihood nominate_geo_mean_probability  \\\n0                  -80.32594                       0.88065   \n1                  -45.74049                       0.93334   \n2                  -53.27224                       0.92119   \n3                  -47.53006                       0.92655   \n4                  -86.25793                       0.86773   \n..                       ...                           ...   \n534               -175.54897                       0.80231   \n535                -84.32677                       0.88465   \n536                -99.88538                       0.88358   \n537                -32.45444                       0.96049   \n538                -32.74495                       0.95935   \n\n     nominate_number_of_votes  nominate_number_of_errors  conditional  \\\n0                       632.0                       33.0          NaN   \n1                       663.0                       27.0          NaN   \n2                       649.0                       23.0          NaN   \n3                       623.0                       11.0          NaN   \n4                       608.0                       27.0          NaN   \n..                        ...                        ...          ...   \n534                     797.0                       77.0          NaN   \n535                     688.0                       34.0          NaN   \n536                     807.0                       42.0          NaN   \n537                     805.0                       14.0          NaN   \n538                     789.0                       11.0          NaN   \n\n    nokken_poole_dim1 nokken_poole_dim2  \n0              -0.081             0.234  \n1               0.437             0.219  \n2               0.510            -0.040  \n3               0.283            -0.424  \n4              -0.065             0.217  \n..                ...               ...  \n534             0.219             0.277  \n535             0.599             0.145  \n536            -0.163             0.314  \n537            -0.520            -0.623  \n538            -0.461            -0.466  \n\n[539 rows x 31 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>names</th>\n      <th>Wikipedia  Names</th>\n      <th>Position</th>\n      <th>Joined Congress</th>\n      <th>ODU-WSDL</th>\n      <th>CSPAN</th>\n      <th>TweetCongress</th>\n      <th>Github</th>\n      <th>REMARK</th>\n      <th>congress</th>\n      <th>...</th>\n      <th>died</th>\n      <th>nominate_dim1</th>\n      <th>nominate_dim2</th>\n      <th>nominate_log_likelihood</th>\n      <th>nominate_geo_mean_probability</th>\n      <th>nominate_number_of_votes</th>\n      <th>nominate_number_of_errors</th>\n      <th>conditional</th>\n      <th>nokken_poole_dim1</th>\n      <th>nokken_poole_dim2</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>doug jones</td>\n      <td>Doug Jones</td>\n      <td>Sen</td>\n      <td>3-Jan-18</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>sendougjones</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.088</td>\n      <td>0.187</td>\n      <td>-80.32594</td>\n      <td>0.88065</td>\n      <td>632.0</td>\n      <td>33.0</td>\n      <td>NaN</td>\n      <td>-0.081</td>\n      <td>0.234</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>richard shelby</td>\n      <td>Richard Shelby</td>\n      <td>Sen</td>\n      <td>3-Jan-87</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>SenShelby</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.446</td>\n      <td>0.570</td>\n      <td>-45.74049</td>\n      <td>0.93334</td>\n      <td>663.0</td>\n      <td>27.0</td>\n      <td>NaN</td>\n      <td>0.437</td>\n      <td>0.219</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>dan sullivan</td>\n      <td>Dan Sullivan</td>\n      <td>Sen</td>\n      <td>3-Jan-15</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>SenDanSullivan</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.485</td>\n      <td>0.061</td>\n      <td>-53.27224</td>\n      <td>0.92119</td>\n      <td>649.0</td>\n      <td>23.0</td>\n      <td>NaN</td>\n      <td>0.510</td>\n      <td>-0.040</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>lisa murkowski</td>\n      <td>Lisa Murkowski</td>\n      <td>Sen</td>\n      <td>20-Dec-02</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>LisaMurkowski</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.211</td>\n      <td>-0.304</td>\n      <td>-47.53006</td>\n      <td>0.92655</td>\n      <td>623.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>0.283</td>\n      <td>-0.424</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>kyrsten sinema</td>\n      <td>Kyrsten Sinema</td>\n      <td>Sen</td>\n      <td>3-Jan-19</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>SenatorSinema</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.105</td>\n      <td>0.076</td>\n      <td>-86.25793</td>\n      <td>0.86773</td>\n      <td>608.0</td>\n      <td>27.0</td>\n      <td>NaN</td>\n      <td>-0.065</td>\n      <td>0.217</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>534</th>\n      <td>will hurd</td>\n      <td>Will Hurd</td>\n      <td>Rep</td>\n      <td>3-Jan-15</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>hurdonthehill</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.287</td>\n      <td>0.228</td>\n      <td>-175.54897</td>\n      <td>0.80231</td>\n      <td>797.0</td>\n      <td>77.0</td>\n      <td>NaN</td>\n      <td>0.219</td>\n      <td>0.277</td>\n    </tr>\n    <tr>\n      <th>535</th>\n      <td>william timmons</td>\n      <td>William Timmons</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>RepTimmons</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>0.591</td>\n      <td>0.225</td>\n      <td>-84.32677</td>\n      <td>0.88465</td>\n      <td>688.0</td>\n      <td>34.0</td>\n      <td>NaN</td>\n      <td>0.599</td>\n      <td>0.145</td>\n    </tr>\n    <tr>\n      <th>536</th>\n      <td>xochitl small</td>\n      <td>Xochitl Torres Small</td>\n      <td>Rep</td>\n      <td>3-Jan-19</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>RepTorresSmall</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.163</td>\n      <td>0.314</td>\n      <td>-99.88538</td>\n      <td>0.88358</td>\n      <td>807.0</td>\n      <td>42.0</td>\n      <td>NaN</td>\n      <td>-0.163</td>\n      <td>0.314</td>\n    </tr>\n    <tr>\n      <th>537</th>\n      <td>yvette clarke</td>\n      <td>Yvette Clarke</td>\n      <td>Rep</td>\n      <td>3-Jan-13</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>RepYvetteClarke</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.612</td>\n      <td>-0.225</td>\n      <td>-32.45444</td>\n      <td>0.96049</td>\n      <td>805.0</td>\n      <td>14.0</td>\n      <td>NaN</td>\n      <td>-0.520</td>\n      <td>-0.623</td>\n    </tr>\n    <tr>\n      <th>538</th>\n      <td>zoe lofgren</td>\n      <td>Zoe Lofgren</td>\n      <td>Rep</td>\n      <td>3-Jan-95</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>RepZoeLofgren</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>-0.400</td>\n      <td>-0.435</td>\n      <td>-32.74495</td>\n      <td>0.95935</td>\n      <td>789.0</td>\n      <td>11.0</td>\n      <td>NaN</td>\n      <td>-0.461</td>\n      <td>-0.466</td>\n    </tr>\n  </tbody>\n</table>\n<p>539 rows × 31 columns</p>\n</div>"
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "outputs": [
    {
     "data": {
      "text/plain": "435"
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(house_handles.intersection(congress_set))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "outputs": [
    {
     "data": {
      "text/plain": "set()"
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(sen_handles.intersection(house_handles))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "outputs": [],
   "source": [
    "df_house = pd.DataFrame(columns=['handle', 'followers'])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "cannot concatenate object of type '<class 'set'>'; only Series and DataFrame objs are valid",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[0;32m<ipython-input-394-d6ad8e368ee0>\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[0;32mfor\u001B[0m \u001B[0mrep\u001B[0m \u001B[0;32min\u001B[0m \u001B[0mhouse_handles\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mintersection\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mcongress_set\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      2\u001B[0m     \u001B[0;32mwith\u001B[0m \u001B[0mopen\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mhouse_files_path\u001B[0m\u001B[0;34m+\u001B[0m\u001B[0mrep\u001B[0m\u001B[0;34m+\u001B[0m\u001B[0;34m'.txt'\u001B[0m\u001B[0;34m)\u001B[0m \u001B[0;32mas\u001B[0m \u001B[0mf\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m----> 3\u001B[0;31m         \u001B[0mdf_house\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0mdf_house\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mappend\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0;34m{\u001B[0m\u001B[0;34m'handle'\u001B[0m\u001B[0;34m}\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mignore_index\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;32mTrue\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      4\u001B[0m         \u001B[0;31m# print(f.readlines())\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      5\u001B[0m     \u001B[0;32mbreak\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/frame.py\u001B[0m in \u001B[0;36mappend\u001B[0;34m(self, other, ignore_index, verify_integrity, sort)\u001B[0m\n\u001B[1;32m   7980\u001B[0m             \u001B[0mto_concat\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0mself\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mother\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   7981\u001B[0m         return (\n\u001B[0;32m-> 7982\u001B[0;31m             concat(\n\u001B[0m\u001B[1;32m   7983\u001B[0m                 \u001B[0mto_concat\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m   7984\u001B[0m                 \u001B[0mignore_index\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0mignore_index\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/reshape/concat.py\u001B[0m in \u001B[0;36mconcat\u001B[0;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001B[0m\n\u001B[1;32m    283\u001B[0m     \u001B[0mValueError\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0mIndexes\u001B[0m \u001B[0mhave\u001B[0m \u001B[0moverlapping\u001B[0m \u001B[0mvalues\u001B[0m\u001B[0;34m:\u001B[0m \u001B[0;34m[\u001B[0m\u001B[0;34m'a'\u001B[0m\u001B[0;34m]\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    284\u001B[0m     \"\"\"\n\u001B[0;32m--> 285\u001B[0;31m     op = _Concatenator(\n\u001B[0m\u001B[1;32m    286\u001B[0m         \u001B[0mobjs\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    287\u001B[0m         \u001B[0maxis\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0maxis\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/opt/anaconda3/lib/python3.8/site-packages/pandas/core/reshape/concat.py\u001B[0m in \u001B[0;36m__init__\u001B[0;34m(self, objs, axis, join, keys, levels, names, ignore_index, verify_integrity, copy, sort)\u001B[0m\n\u001B[1;32m    368\u001B[0m                     \u001B[0;34m\"only Series and DataFrame objs are valid\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    369\u001B[0m                 )\n\u001B[0;32m--> 370\u001B[0;31m                 \u001B[0;32mraise\u001B[0m \u001B[0mTypeError\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mmsg\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m    371\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m    372\u001B[0m             \u001B[0;31m# consolidate\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;31mTypeError\u001B[0m: cannot concatenate object of type '<class 'set'>'; only Series and DataFrame objs are valid"
     ]
    }
   ],
   "source": [
    "for rep in house_handles.intersection(congress_set):\n",
    "    with open(house_files_path+rep+'.txt') as f:\n",
    "        df_house = df_house.append({'handle'}, ignore_index=True)\n",
    "        # print(f.readlines())\n",
    "    break"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "with open(house_files_path+'repbarbaralee'+'.txt') as f:\n",
    "    print(rep)\n",
    "    x = f.readlines()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "len(twitter_dataset_names)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "outputs": [
    {
     "data": {
      "text/plain": "[405]"
     },
     "execution_count": 395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(np.where(data['ODU-WSDL'].isnull())[0])\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "outputs": [
    {
     "data": {
      "text/plain": "Wikipedia  Names                 Michael San Nicolas\nPosition                                         Rep\nJoined Congress                             3-Jan-19\nODU-WSDL                                         NaN\nCSPAN                                            NaN\nTweetCongress                                    NaN\nGithub                                           NaN\nREMARK                                    NO TWITTER\ncongress                                         116\nchamber                                        House\nicpsr                                          21997\nstate_icpsr                                       94\ndistrict_code                                      0\nstate_abbrev                                      GU\nparty_code                                       100\noccupancy                                        NaN\nlast_means                                       NaN\nbioname                             NICOLAS, Michael\nbioguide_id                                  S001204\nborn                                            1981\ndied                                             NaN\nnominate_dim1                                  -0.41\nnominate_dim2                                  -0.52\nnominate_log_likelihood                     -0.92032\nnominate_geo_mean_probability                0.99189\nnominate_number_of_votes                       113.0\nnominate_number_of_errors                        0.0\nconditional                                      NaN\nnokken_poole_dim1                             -0.407\nnokken_poole_dim2                             -0.494\nName: michael nicolas, dtype: object"
     },
     "execution_count": 396,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.iloc[405]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}