12231 rows × 2 columns
\n", + "189685 rows × 10 columns
\n", "" ], "text/plain": [ - " message label\n", - "10779 $WMT how are you bearish fools doing? 1\n", - "7731 $WMT more pain for stubborn shorts. -signed st... 1\n", - "3159 $WMT layoffs and cost cutting measures coming ... -1\n", - "561 fyi: Huge (over 1 million) call vol in $T and ... 1\n", - "12165 The 6 Most Shorted Dow Stocks\\n\\n$MSFT $CSCO ... 0\n", - "... ... ...\n", - "7153 Yesterdays Sell On Close Closing Order Imbal... -1\n", - "7790 Why XPO Logistics Inc Stock Is More Than a Spe... 1\n", - "10965 Newport Wealth Strategies Inc.,has filed Form ... 1\n", - "12201 $WMT Walmart needs to show us some action today 1\n", - "1225 #Tesco 2nd of Top 3 global grocers to quit Jap... 1\n", + " Unnamed: 0 symbol ... Day_of_week polarity\n", + "16590 16590 DIA ... Tuesday 0\n", + "154988 154988 DIA ... Friday 1\n", + "174141 174141 DIA ... Friday 0\n", + "94680 94680 DIA ... Wednesday 0\n", + "64506 64506 DIA ... Wednesday 0\n", + "... ... ... ... ... ...\n", + "166025 166025 DIA ... Monday 0\n", + "182008 182008 DIA ... Wednesday 0\n", + "36692 36692 DIA ... Thursday 0\n", + "144330 144330 DIA ... Thursday 0\n", + "49227 49227 DIA ... Monday 0\n", "\n", - "[12231 rows x 2 columns]" + "[189685 rows x 10 columns]" ] }, "metadata": { @@ -288,11 +427,24 @@ "## Preprocess data" ] }, + { + "cell_type": "code", + "metadata": { + "id": "i8TteZhWQ0Ba" + }, + "source": [ + "# fill nan values in file with '0'\n", + "combine_ds.isna().values.any()\n", + "combine_ds['message'] = combine_ds['message'].fillna('0')" + ], + "execution_count": 8, + "outputs": [] + }, { "cell_type": "code", "metadata": { "id": "z70r1eYrnoB_", - "outputId": "ea37ebdc-ea6a-4fa9-ce8c-610b7bbac64f", + "outputId": "62b8b512-a7fa-411d-de03-cebd62463cb4", "colab": { "base_uri": "https://localhost:8080/", "height": 54 @@ -304,12 +456,12 @@ "\n", "print(message[:10])" ], - "execution_count": 4, + "execution_count": 9, "outputs": [ { "output_type": "stream", "text": [ - "['$wmt how are you bearish fools doing?', '$wmt more pain for stubborn shorts. -signed stubborn long', '$wmt layoffs and cost cutting measures coming -brazil is a disaster.no position. if it gets to 45, maybe..lol', 'fyi: huge (over 1 million) call vol in $t and $vz itm july calls yesterday. chain: http://stk.ly/owk4r5 , http://stk.ly/mqiyye > what's up?', 'the 6 most shorted dow stocks\\n\\n$msft $csco $vz $ko $v \\n\\nhttps://247wallst.com/investing/2019/12/26/6-most-shorted-dow-stocks-microsoft-is-king-of-the-hill-in-december/1/', '02:57:30 pm manual exit. closing my $wmt position of 672 shares that was opened oct 29 for a 0.07% gain.', '@sspencer_smb tmobile/sprint still a foreign owed enterprise, either way.. buy american $t and $vz ..', '@scarborory agree .... $s owns the most spectrum in the us comparing w $t , $vz and $tmus', '$baba just had some positive pr on fox news radio. talked about how $baba is destroying $wmt due to it's success.', '$wmt $ostk $msft employees chant this in the bathroom']\n" + "['$dia bets on market tomorrow?', 'municipal bond funds reported an inflow for the 23rd time in the past 24wks (+$444mn). $dia $qqq $spy $mub $hyd $tlt $tbt', 'ts tradeplan for ym http://stks.co/hbrm - friday june 28 2013 $es_f $ym_f $6e_f $cl_f $gc_f sent to members $dia $indu #futures #trading', '$spx $spy $qqq $ndx $iwm $vix $dia $djia nasdaq market internals summary', 'although the technical rating is bad, $dia does present a nice setup opportunity. https://www.chartmill.com/analyze.php?utm_source=stocktwits&utm_medium=ta&utm_content=setup&utm_campaign=social_tracking#/dia?key=810cb939-dc70-485b-b7ec-a9a1fe911f70', 'new hod on $tick on that move over 15459 as well $dia $indu', '$iwm $spy $qqq $dia $spx markets makers move prices flat into the sun. vol. event. as close to middle as possible.', '$dia they will not rollback tariffs . c'mon look at the news man! the asian markets usa everyone this will be no deal!!!!', '$spy $dia panic selling in overall markets however not all stocks selling off here and continue to focus on stronger positions #livetradepro', 'bearish persistence on the aaii survey http://stks.co/dmvy via @ppearlman $spy $dia']\n" ], "name": "stdout" } @@ -348,19 +500,46 @@ " # print(re_t)\n", " return re_t" ], - "execution_count": 5, + "execution_count": 10, "outputs": [] }, { "cell_type": "code", "metadata": { - "id": "_3YWRwzLBGs1", - "outputId": "25a76e0b-f43d-4419-bfc5-77f4da53ff49", + "id": "gfz_YC4eQ6nk", + "outputId": "3faa5c5c-5dd2-4a21-b2c7-664c044b4887", "colab": { "base_uri": "https://localhost:8080/", "height": 51 } }, + "source": [ + "print(sys.getrecursionlimit())\n", + "sys.setrecursionlimit(1800)\n", + "print(sys.getrecursionlimit())" + ], + "execution_count": 11, + "outputs": [ + { + "output_type": "stream", + "text": [ + "1000\n", + "1800\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "_3YWRwzLBGs1", + "outputId": "3127d551-45f8-4435-f173-92abbd01342d", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 170 + } + }, "source": [ "stop_words = sw.words(\"english\")\n", "tweet_tokenizer = TweetTokenizer()\n", @@ -373,13 +552,20 @@ "# segmenter using the word statistics from Twitter\n", "seg_tw = Segmenter(corpus=\"twitter\")\n" ], - "execution_count": 6, + "execution_count": 12, "outputs": [ { "output_type": "stream", "text": [ + "Word statistics files not found!\n", + "Downloading... done!\n", + "Unpacking... done!\n", "Reading twitter - 1grams ...\n", - "Reading twitter - 2grams ...\n" + "generating cache file for faster loading...\n", + "reading ngrams /root/.ekphrasis/stats/twitter/counts_1grams.txt\n", + "Reading twitter - 2grams ...\n", + "generating cache file for faster loading...\n", + "reading ngrams /root/.ekphrasis/stats/twitter/counts_2grams.txt\n" ], "name": "stdout" } @@ -393,46 +579,51 @@ "source": [ "message_p = []\n", "for msg in message:\n", - " # remove emojis\n", - " msg = emoji.demojize(msg)\n", "\n", - " # fix contractions\n", - " msg = contractions.fix(msg)\n", + " if msg == '0': #nan replaced by '0'\n", + " message_p.append('-1')\n", "\n", - " # remove punctuations\n", - " msg = remove_punctuation_re(msg) \n", + " else:\n", + " # remove emojis\n", + " msg = emoji.demojize(msg)\n", "\n", - " #tokenize\n", - " msg_tokens = tweet_tokenizer.tokenize(msg)\n", + " # fix contractions\n", + " msg = contractions.fix(msg)\n", "\n", - " #For Hashtags elongated words using Word segmenter\n", - " message_seg = []\n", - " for w in msg_tokens:\n", - " message_seg.append(seg_tw.segment(w))\n", + " # remove punctuations\n", + " msg = remove_punctuation_re(msg) \n", "\n", - " # remove stopwords\n", - " msg = remove_stopwords(message_seg)\n", + " #tokenize\n", + " msg_tokens = tweet_tokenizer.tokenize(msg)\n", "\n", - " if 'rt' in msg:\n", - " # remove retweets\n", - " message_p.append('-1')\n", - " else: \n", - " # detokenize\n", - " msg = detokenizer.detokenize(msg)\n", + " #For Hashtags elongated words using Word segmenter\n", + " message_seg = []\n", + " for w in msg_tokens:\n", + " message_seg.append(seg_tw.segment(w))\n", + "\n", + " # remove stopwords\n", + " msg = remove_stopwords(message_seg)\n", "\n", - " # removing repeating characters like hurrrryyyyyy-- worrks on tokenized list\n", - " msg = processRepeatings(msg)\n", + " if 'rt' in msg:\n", + " # remove retweets\n", + " message_p.append('-1')\n", + " else: \n", + " # detokenize\n", + " msg = detokenizer.detokenize(msg)\n", "\n", - " message_p.append(msg)\n" + " # removing repeating characters like hurrrryyyyyy-- worrks on tokenized list\n", + " msg = processRepeatings(msg)\n", + "\n", + " message_p.append(msg)\n" ], - "execution_count": 7, + "execution_count": 13, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "pXWjHAcQoIEh", - "outputId": "346a4824-d1fd-485f-8280-64eb64327499", + "outputId": "da41adaf-1b9a-46ee-80a0-7bcb935a31c6", "colab": { "base_uri": "https://localhost:8080/", "height": 187 @@ -441,28 +632,28 @@ "source": [ "message_p[:10]" ], - "execution_count": 8, + "execution_count": 14, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "['wmt bearish fools',\n", - " 'wmt pain stubborn shorts signed stubborn long',\n", - " 'wmt layoffs cost cutting measures coming brazil disaster position gets 45 maybe lol',\n", - " 'fyi huge 1 million call vol vz itm july calls yesterday chain gt 39',\n", - " '6 shorted dow stocks msft csco vz ko v',\n", - " '02 57 30 pm manual exit closing wmt position 672 shares opened oct 29 0 07 gain',\n", - " 'tmobile sprint still foreign owed enterprise either way buy american vz',\n", - " 'agree owns spectrum us comparing w vz tmus',\n", - " 'baba positive pr fox news radio talked baba destroying wmt due 39 success',\n", - " 'wmt ostk msft employees chant bathroom']" + "['dia bets market tomorrow',\n", + " 'municipal bond funds reported inflow 23rd time past 24wks 44mn dia qq spy mub hyd tlt tbt',\n", + " 'ts tradeplan ym friday june 28 2013 es f ym f 6e f cl f gc f sent members dia indu futures trading',\n", + " 'spx spy qq ndx iwm vix dia djia nasdaq market internals summary',\n", + " 'although technical rating bad dia present nice setup opportunity',\n", + " 'new hod tick move 15459 well dia indu',\n", + " 'iwm spy qq dia spx markets makers move prices flat sun vol event close middle possible',\n", + " 'dia rollback tariffs c 39 mon look news man asian markets usa everyone deal',\n", + " 'spy dia panic selling overall markets however stocks selling continue focus stronger positions livetradepro',\n", + " 'bearish persistence aaii survey via spy dia']" ] }, "metadata": { "tags": [] }, - "execution_count": 8 + "execution_count": 14 } ] }, @@ -470,17 +661,17 @@ "cell_type": "code", "metadata": { "id": "KkSpMpJWAG4u", - "outputId": "434cb601-02ad-4600-d726-93487cc11064", + "outputId": "7ac5fba5-e2a0-4599-af4f-8d41f5dcd91f", "colab": { "base_uri": "https://localhost:8080/", - "height": 419 + "height": 623 } }, "source": [ "combine_ds['message'] = message_p\n", "combine_ds" ], - "execution_count": 10, + "execution_count": 15, "outputs": [ { "output_type": "execute_result", @@ -504,92 +695,219 @@ " \n", "12231 rows × 2 columns
\n", + "189685 rows × 10 columns
\n", "" ], "text/plain": [ - " message label\n", - "10779 wmt bearish fools 1\n", - "7731 wmt pain stubborn shorts signed stubborn long 1\n", - "3159 wmt layoffs cost cutting measures coming brazi... -1\n", - "561 fyi huge 1 million call vol vz itm july calls ... 1\n", - "12165 6 shorted dow stocks msft csco vz ko v 0\n", - "... ... ...\n", - "7153 yesterdays sell close closing order imbalance ... -1\n", - "7790 xpo logistics inc stock speculative buy xpo tg... 1\n", - "10965 newport wealth strategies inc filed form 13f q... 1\n", - "12201 wmt walmart needs show us action today 1\n", - "1225 tesco 2nd top 3 global grocers quit japan carr... 1\n", + " Unnamed: 0 symbol ... Day_of_week polarity\n", + "16590 16590 DIA ... Tuesday 0\n", + "154988 154988 DIA ... Friday 1\n", + "174141 174141 DIA ... Friday 0\n", + "94680 94680 DIA ... Wednesday 0\n", + "64506 64506 DIA ... Wednesday 0\n", + "... ... ... ... ... ...\n", + "166025 166025 DIA ... Monday 0\n", + "182008 182008 DIA ... Wednesday 0\n", + "36692 36692 DIA ... Thursday 0\n", + "144330 144330 DIA ... Thursday 0\n", + "49227 49227 DIA ... Monday 0\n", "\n", - "[12231 rows x 2 columns]" + "[189685 rows x 10 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 15 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nTSMxP_VRRQu", + "outputId": "cf9d2165-28df-45d8-a04e-6b5122e98f83", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 68 + } + }, + "source": [ + "combine_ds['polarity'].value_counts()" + ], + "execution_count": 16, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "1 99652\n", + "0 90033\n", + "Name: polarity, dtype: int64" ] }, "metadata": { "tags": [] }, - "execution_count": 10 + "execution_count": 16 } ] }, @@ -597,10 +915,10 @@ "cell_type": "code", "metadata": { "id": "3DraQK_uDcBU", - "outputId": "1413d0b6-54b1-49e4-8fcb-26c8302a2c30", + "outputId": "e80c6e22-7af4-4050-cbcb-c7c7f5393a90", "colab": { "base_uri": "https://localhost:8080/", - "height": 419 + "height": 623 } }, "source": [ @@ -608,7 +926,7 @@ "combine_ds.drop(combine_ds[combine_ds['message'] == '-1'].index, inplace = True) \n", "combine_ds" ], - "execution_count": 12, + "execution_count": 17, "outputs": [ { "output_type": "execute_result", @@ -632,94 +950,272 @@ " \n", "12082 rows × 2 columns
\n", + "188599 rows × 10 columns
\n", "" ], "text/plain": [ - " message label\n", - "10779 wmt bearish fools 1\n", - "7731 wmt pain stubborn shorts signed stubborn long 1\n", - "3159 wmt layoffs cost cutting measures coming brazi... -1\n", - "561 fyi huge 1 million call vol vz itm july calls ... 1\n", - "12165 6 shorted dow stocks msft csco vz ko v 0\n", - "... ... ...\n", - "7153 yesterdays sell close closing order imbalance ... -1\n", - "7790 xpo logistics inc stock speculative buy xpo tg... 1\n", - "10965 newport wealth strategies inc filed form 13f q... 1\n", - "12201 wmt walmart needs show us action today 1\n", - "1225 tesco 2nd top 3 global grocers quit japan carr... 1\n", + " Unnamed: 0 symbol ... Day_of_week polarity\n", + "16590 16590 DIA ... Tuesday 0\n", + "154988 154988 DIA ... Friday 1\n", + "174141 174141 DIA ... Friday 0\n", + "94680 94680 DIA ... Wednesday 0\n", + "64506 64506 DIA ... Wednesday 0\n", + "... ... ... ... ... ...\n", + "166025 166025 DIA ... Monday 0\n", + "182008 182008 DIA ... Wednesday 0\n", + "36692 36692 DIA ... Thursday 0\n", + "144330 144330 DIA ... Thursday 0\n", + "49227 49227 DIA ... Monday 0\n", "\n", - "[12082 rows x 2 columns]" + "[188599 rows x 10 columns]" ] }, "metadata": { "tags": [] }, - "execution_count": 12 + "execution_count": 17 } ] + }, + { + "cell_type": "code", + "metadata": { + "id": "0L1EiPK6RTOJ", + "outputId": "143de1eb-8beb-470f-8ca9-13b16d3df5f1", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 68 + } + }, + "source": [ + "combine_ds['polarity'].value_counts()" + ], + "execution_count": 18, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "1 99069\n", + "0 89530\n", + "Name: polarity, dtype: int64" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 18 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ExuSBMADRVWx" + }, + "source": [ + "# Uncomment for FB\n", + "# FB_IPO = '2012-05-21'\n", + "# combine_ds.drop(combine_ds[combine_ds['Date'] < '2012-05-21'].index, inplace = True) \n", + "# combine_ds" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "SzbU2bZgRWfg" + }, + "source": [ + "# Uncomment for TSLA\n", + "# TSLA_IPO = '2010-06-30'\n", + "# combine_ds.drop(combine_ds[combine_ds['Date'] < TSLA_IPO].index, inplace = True) \n", + "# combine_ds" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "hMWwq5dKRX2X" + }, + "source": [ + "# combine_ds['polarity'].value_counts()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "uOx_0zmyRZbT" + }, + "source": [ + "combine_ds.to_csv('prep_WMT1.csv')\n", + "!cp prep_WMT1.csv \"/content/drive/My Drive\"" + ], + "execution_count": 19, + "outputs": [] } ] } \ No newline at end of file