Skip to content

Commit

Permalink
minor fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@helios2003
helios2003 committed Aug 19, 2023
1 parent a4a3c4f commit 67f97f8
Show file tree
Hide file tree
Showing 12 changed files with 56 additions and 41 deletions.
2 changes: 1 addition & 1 deletion flask/app.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from flask import Flask, render_template, request
from flask import Flask, render_template
import pandas as pd
import sys

Expand Down
2 changes: 1 addition & 1 deletion flask/data/data_#BankNiftyOption_urls.txt
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion flask/data/data_#KrithiShetty_urls.txt
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion flask/data/data_#TataMotors_urls.txt
View file

Large diffs are not rendered by default.

12 changes: 5 additions & 7 deletions flask/source files/#BankNiftyOption_urls.txt
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
https://www.bankniftyoption.in/
https://www.nseindia.com/get-quotes/derivatives?symbol=BANKNIFTY
https://www.niftytrader.in/nse-option-chain/banknifty
https://www.indiainfoline.com/knowledge-center/share-market/bank-nifty-option-tips-and-strategy
https://in.tradingview.com/ideas/bankniftyoptions/
https://upstox.com/option-chain/banknifty/
https://in.linkedin.com/in/banknifty-option-tips-4041b8213
https://m.facebook.com/127235975750499/
https://www.instagram.com/indexanalyst618/?hl=en
https://www.smarttradingtips.com/bank-nifty-option-tips.php
https://www.instagram.com/indexanalyst618/?hl=en
https://www.facebook.com/134198911720872
https://www.smarttradingtips.com/bank-nifty-option-tips.php
https://www.instagram.com/banknifty_option_calls/?hl=en
https://www.niftyoptiontips.in/bank-nifty-tips.html
https://upstox.com/options-live/banknifty-call-put-option-price/
https://www.shyamadvisory.com/bank-nifty-tips/
18 changes: 9 additions & 9 deletions flask/source files/#KrithiShetty_urls.txt
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
https://www.instagram.com/krithi.shetty_official/?hl=en
https://www.instagram.com/krithi.shetty_official/
https://twitter.com/IamKrithiShetty
https://twitter.com/IamKrithiShetty
https://www.facebook.com/urkrithishetty/
https://www.facebook.com/urkrithishetty/
https://www.pinterest.com/pin/606086062358794281/
https://timesofindia.indiatimes.com/topic/Krithi-Shetty/news
https://www.pinterest.com/pin/606086062358794281/
https://timesofindia.indiatimes.com/topic/Krithi-Shetty/news
https://www.pinterest.com/pin/606086062358794281/
https://www.youtube.com/channel/UCnUlDS09sWSsRAMCgvFBc-g
https://www.facebook.com/IamKrithiShettyOfficial/
https://timesofindia.indiatimes.com/entertainment/malayalam/web-stories/krithi-shetty-alluring-pictures/photostory/101800289.cms
https://www.youtube.com/channel/UCnUlDS09sWSsRAMCgvFBc-g
https://www.facebook.com/IamKrithiShettyOfficial/
https://timesofindia.indiatimes.com/topic/Krithi-Shetty
https://www.facebook.com/IamKrithiShettyOfficial/
https://timesofindia.indiatimes.com/entertainment/malayalam/web-stories/krithi-shetty-alluring-pictures/photostory/101800289.cms
9 changes: 5 additions & 4 deletions flask/source files/#TataMotors_urls.txt
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
https://www.tatamotors.com/
https://www.tatamotors.com/
https://en.wikipedia.org/wiki/Tata_Motors
https://twitter.com/tatamotors?lang=en
https://www.tata.com/business/tata-motors
https://www.facebook.com/TataMotorsGroup/
https://twitter.com/tatamotors?lang=en
https://www.moneycontrol.com/india/stockpricequote/auto-lcvshcvs/tatamotors/TM03
https://www.bseindia.com/stock-share-price/tata-motors-ltd/tatamotors/500570/
https://www.youtube.com/channel/UCGk_EOyEtbl-XdbVHRPwnfQ
https://www.livemint.com/market/market-stats/stocks-tata-motors-share-price-nse-bse-s0003022
https://www.facebook.com/TataMotorsGroup/
https://economictimes.indiatimes.com/tata-motors-ltd/stocks/companyid-12934.cms
https://www.cnbc.com/quotes/TATAMOTORS-IN
https://www.nseindia.com/get-quotes/equity?symbol=TATAMOTORS
4 changes: 2 additions & 2 deletions go/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@ module github.com/helios2003/twitter-scraper.git
go 1.20

require (
github.com/n0madic/twitter-scraper v0.0.0-20230508101308-2b607217fd11 // indirect
golang.org/x/net v0.10.0 // indirect
github.com/n0madic/twitter-scraper v0.0.0-20230711213008-94503a2bc36c // indirect
golang.org/x/net v0.14.0 // indirect
)
25 changes: 25 additions & 0 deletions go/go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/n0madic/twitter-scraper v0.0.0-20230323164130-5ff4cd0d1ba9 h1:fEhPYmREKdkZtEFXqFFdSuEqa7BBHs8eaBgEn5FhyaY=
github.com/n0madic/twitter-scraper v0.0.0-20230323164130-5ff4cd0d1ba9/go.mod h1:77EevZ6tTXilMv+nT1AyXxgZx9frQzayF5UfG24x6+s=
github.com/n0madic/twitter-scraper v0.0.0-20230508101308-2b607217fd11 h1:SNhcA85OrGAweJ2QSSNC7GWMV8tQwjIFRLs7n/jWhBo=
github.com/n0madic/twitter-scraper v0.0.0-20230508101308-2b607217fd11/go.mod h1:ojY32DnRD4duQuvyreLPHBXBeQZuLjzJqGb33JMZUJ8=
github.com/n0madic/twitter-scraper v0.0.0-20230711213008-94503a2bc36c h1:pcEcP5jkXQuze3pkUDc7qtDZca1TP208LgkgXzVn/lc=
github.com/n0madic/twitter-scraper v0.0.0-20230711213008-94503a2bc36c/go.mod h1:VS1i48kYWIoiS0vz6k7GU6iuwcr6deD3CovldOR2rQE=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
Expand All @@ -17,6 +27,10 @@ golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand All @@ -27,17 +41,28 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
6 changes: 3 additions & 3 deletions go/tweets_scraper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

func not_main() {
scraper := twitterscraper.New()
//scraper.LoginOpenAccount()
trends, err := scraper.GetTrends()
if err != nil {
panic(err)
Expand All @@ -37,7 +38,7 @@ func not_main() {
func main() {
not_main()
scraper := twitterscraper.New()
// Read the queries from a text file named "trends.txt"
//scraper.IsLoggedIn()
queryFile, err := os.Open("files/trends.txt")
if err != nil {
panic(err)
Expand All @@ -61,8 +62,7 @@ func main() {
text := strings.Split(tweet.Text, "http")[0]
// Remove "@" symbols from the tweet text
text = strings.ReplaceAll(text, "@", "")

// Add the tweet to the corresponding trend in the map
// add it to the map
trendTweets[query] = append(trendTweets[query], text)
}
}
Expand Down
14 changes: 2 additions & 12 deletions src/python/embeddings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
import pandas as pd
from transformers import BertTokenizer, BertModel

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained("bert-base-uncased")
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', from_tf=True)
model = BertModel.from_pretrained("bert-base-uncased", from_tf=True)

def create_avg_embeddings(tweet):
# Tokenize the tweet and convert tokens to IDs
Expand Down Expand Up @@ -73,13 +73,10 @@ def embeddings():
output_filepath = os.path.join(output_directory, output_filename)
df.to_csv(output_filepath, index=False)

# Define the directory path containing the CSV files
directory = 'embeddings/'

# Initialize the list for mean embedding differences
mean_embedding_diff = []

# Loop through all the CSV files in the directory
for i, filename in enumerate(os.listdir(directory)):
if filename.endswith('.csv'):
# Read the CSV file
Expand All @@ -88,11 +85,8 @@ def embeddings():

# Get the current element from the list
current_element = avg_embeddings[i]

# Initialize the list for embedding differences
embedding_diff = []

# Calculate the absolute difference for each tweet in the current file
for tweet_embedding in df['embedding']:
embedding_diff.append(abs(tweet_embedding - current_element))

Expand All @@ -115,8 +109,6 @@ def embeddings():

# Get the current element from the list
current_element = avg_embeddings[i]

# Initialize the list for embedding differences
embedding_diff = []

# Calculate the absolute difference for each tweet in the current file
Expand All @@ -134,7 +126,6 @@ def embeddings():
# Loop through all the CSV files in the directory
for i, filename in enumerate(os.listdir(directory)):
if filename.endswith('.csv'):
# Read the CSV file
file_path = os.path.join(directory, filename)
df = pd.read_csv(file_path)

Expand All @@ -151,7 +142,6 @@ def embeddings():
# Extract the trend name from the original filename
trend = filename.split('_')[1].split('.')[0]

# Define the output filename
output_filename = f"answer_{trend}.csv"

# Save the "Tweet" column to a new CSV file in the answers directory
Expand Down
1 change: 1 addition & 0 deletions src/python/requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ langdetect==1.0.9
nltk==3.8.1
Requests==2.31.0
transformers
tensorflow
gensim
torch
pandas

0 comments on commit 67f97f8

Please sign in to comment.