-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsd.py
144 lines (129 loc) · 6.06 KB
/
sd.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
import heapq
# import daemon
import time
from bs4 import BeautifulSoup
import urllib.request
import urllib.parse
import mysql.connector
KEEP_LIVE = 12*60*60
DELAY = 1.5
UPVOTE_RATE_PER_HOUR = 5
VIEW_RATE_PER_MINUTE = 30
mydb = mysql.connector.connect(
host="mysql",
user="khangly",
passwd="91FEyhBOOrFIhzba3TVANijH",
database="khangly"
)
mycursor = mydb.cursor()
class Item:
def __init__(self, deal_num, url, name):
self.deal_num = deal_num
self.url = url
self.name = name
self.created = time.time()
self.ratings = list()
self.cancels = 0
self.price_mistakes = 0
sql_insert_query = "REPLACE INTO Deals VALUES (%s, %s, %s, %s, %s, %s)"
insert_tuple = (deal_num, url, name, self.created, 0, 0)
mycursor.execute(sql_insert_query, insert_tuple)
mycursor.execute("CREATE TABLE IF NOT EXISTS `" + str(deal_num) + "` (Time INT UNSIGNED PRIMARY KEY, Ratings SMALLINT, Replies SMALLINT UNSIGNED, Views MEDIUMINT UNSIGNED)")
mydb.commit()
def update_rating(self, upvotes, replies, views):
insert_tuple = (time.time(), upvotes, replies, views)
self.ratings.append(insert_tuple)
sql_insert_query = "REPLACE INTO `" + str(self.deal_num) + "` VALUES (%s, %s, %s, %s)"
mycursor.execute(sql_insert_query, insert_tuple)
life_time = insert_tuple[0] - self.created
if life_time and upvotes * 3600 / life_time > UPVOTE_RATE_PER_HOUR and views * 60 / life_time > VIEW_RATE_PER_MINUTE:
sql_insert_query = "INSERT IGNORE INTO PM (ID) VALUES " + str(self.deal_num)
mycursor.execute(sql_insert_query)
mydb.commit()
def update_cancels(self):
self.cancels += 1
sql_update_query = "UPDATE Deals SET Cancels = %s WHERE ID = %s"
mycursor.execute(sql_update_query, (self.cancels, self.deal_num))
if self.ratings and self.ratings[-1][1] > 1:
sql_insert_query = "INSERT IGNORE INTO PM (ID) VALUES " + str(self.deal_num)
mycursor.execute(sql_insert_query)
mydb.commit()
def update_price_mistakes(self):
self.price_mistakes += 1
sql_update_query = "UPDATE Deals SET Mistakes = %s WHERE ID = %s"
mycursor.execute(sql_update_query, (self.price_mistakes, self.deal_num))
if self.ratings and self.ratings[-1][1] > 0:
sql_insert_query = "INSERT IGNORE INTO PM (ID) VALUES " + str(self.deal_num)
mycursor.execute(sql_insert_query)
mydb.commit()
#def
def fetch():
items = dict()
hq = list()
param = {"thread": 1, "post": 1, "threadrate": 1, "time": int(time.time()), 'maxitems': 20}
spy = 'https://slickdeals.net/live/spy.php'
while True:
try:
html = urllib.request.urlopen(spy).read()
soup = BeautifulSoup(html, 'lxml')
bits = soup.find_all('htmlbit')
# print("Len of bits:", len(bits))
for i in range(len(bits) - 1, -1, -1):
piece = bits[i]
identifier = piece['id']
a = piece.a
if not identifier.startswith('threadrate_'):
a = piece.a.find_next('a')
url = a['href']
name = str(a.string)
if identifier.startswith('thread_'):
deal_num = int(identifier[7:])
if param['thread'] < deal_num:
param['thread'] = deal_num
if deal_num not in items:
items[deal_num] = Item(deal_num, url, name)
heapq.heappush(hq, deal_num)
if name.find("pric") != -1 and name.find("mistake") != -1:
items[deal_num].update_price_mistakes()
try:
deal_content = BeautifulSoup(urllib.request.urlopen(spy).read(), 'lxml').find("div", class_="threadText").getText()
if deal_content.find("pric") != -1 and deal_content.find("mistake") != -1:
items[deal_num].update_price_mistakes()
except urllib.error.URLError:
pass
print(name)
else:
if identifier.startswith('threadrate_'):
num = int(identifier[11:])
if param['threadrate'] < num:
param['threadrate'] = num
else:
num = int(identifier[5:])
if param['post'] < num:
param['post'] = num
deal_num = int(url[3:url.find('-')])
if deal_num in items:
index = 0
if identifier.startswith('threadrate_'):
info = piece.span.getText().split(', ')
index = 1
else:
info = piece.span.find_next('span').getText().split(', ')
p = str(piece.p.string)
if p.find("cancel") != -1:
items[deal_num].update_cancels()
if p.find("pric") != -1 and p.find("mistake") != -1:
items[deal_num].update_price_mistakes()
replies = int(info[index][0:info[index].find(' ')])
views = int(info[index + 1][0:info[index + 1].find(' ')])
upvotes = int(info[index + 2][0:info[index + 2].find(' ')])
items[deal_num].update_rating(upvotes, replies, views)
if time.time() - items[deal_num].created > KEEP_LIVE:
while hq and hq[0] <= deal_num:
items.pop(heapq.heappop(hq))
query_string = urllib.parse.urlencode(param)
spy = 'https://slickdeals.net/live/spy.php?' + query_string
time.sleep(DELAY)
except urllib.error.URLError:
time.sleep(DELAY)
fetch()