-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
190 lines (166 loc) · 8.77 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
from flask import flash, render_template, request, redirect, url_for
from formencode import variabledecode
from app import app
from app import db
import models
import forms
import sclogic
@app.route('/', methods=['GET', 'POST'])
def index():
if "run" in request.args:
flash('run...')
sclogic.runall()
flash('run finished!')
elif "clear" in request.args:
flash('update...')
sclogic.clear()
flash('updated!')
elif "connectvpn" in request.args:
flash('connect to VPN...')
sclogic.connectvpn()
flash('VPN completed')
elif "refresh" in request.args:
#return redirect(url_for('index'))
pass
showhidden = True if "showhidden" in request.args else False
onlysaved = True if 'onlysaved' in request.args else False
return index_engine(showhidden = showhidden, onlysaved = onlysaved)
@app.route('/filtered/<spiderbotids>', methods=['GET', 'POST'])
def index_filtered(spiderbotids):
showhidden = True if 'showhidden' in request.args else False
onlysaved = True if 'onlysaved' in request.args else False
return index_engine(showhidden = showhidden, onlysaved = onlysaved, spiderbotids=spiderbotids)
def index_engine(showhidden, onlysaved, spiderbotids = None):
crawlers = models.Crawler.query.order_by(models.Crawler.name).all()
if spiderbotids:
matches = None
for spiderbotid in spiderbotids:
if onlysaved:
rm = models.Match.query.filter_by(spiderbotid=spiderbotid).filter_by(saved=True).all()
else:
rm = models.Match.query.filter_by(spiderbotid=spiderbotid).filter_by(hide=showhidden).all()
if matches:
matches = matches + rm
else:
matches = rm
flash('Matches are filtered for one Crawler...')
else:
if onlysaved:
matches = models.Match.query.filter_by(saved=True).order_by(models.Match.price).all()
else:
matches = models.Match.query.filter_by(hide=showhidden).order_by(models.Match.price).all()
return render_template('main.html', matches=matches, crawlers=crawlers, showhidden = showhidden, onlysaved = onlysaved)
@app.route('/match-update', methods=['GET', 'POST'])
def match_update():
if request.method == 'POST':
postvars = variabledecode.variable_decode(request.form, dict_char='_')
for k, v in postvars.items():
match = models.Match.query.filter_by(matchid=int(k)).first()
if match:
match.hide = True if ("hide" in v and v["hide"] == "on") else False
match.saved = True if ("saved" in v and v["saved"] == "on") else False
db.session.commit()
return redirect('/')
@app.route('/crawler-update', methods=['GET', 'POST'])
def crawler_update():
if request.method == 'POST':
spiderbotids = []
postvars = variabledecode.variable_decode(request.form, dict_char='_')
for ids, values in postvars.items():
selected = True if "selected" in values and values["selected"] == "on" else False
id = int(ids) if ids.isnumeric() else None
if id:
if selected:
if 'delete_button' in request.form:
flash('Delete crawler...')
sclogic.crawlerDelete(id)
flash('Delete crawler finished!')
elif 'filter_button' in request.form:
for spiderbot in models.Spiderbot.query.filter_by(crawlerid=id).all():
spiderbotids.append(spiderbot.spiderbotid)
elif 'save_button' in request.form:
print("itt: "+values["maxprice"])
crawler = models.Crawler.query.filter_by(crawlerid=id).first()
if crawler:
crawler.active = True if ("active" in values and values["active"] == "on") else False
crawler.name = values["name"]
crawler.notes = values["notes"]
crawler.runcadence = float(values["runcadence"])
crawler.maxprice = float(values["maxprice"]) if values["maxprice"] and values["maxprice"] != '' and float(values["maxprice"]) !=0 else None
crawler.minprice = float(values["minprice"]) if values["minprice"] and values["minprice"] != '' and float(values["minprice"]) !=0 else None
db.session.commit()
if len(spiderbotids)>0:
showhidden = True if 'showhidden' in request.args else False
onlysaved = True if 'onlysaved' in request.args else False
return index_engine(spiderbotids=spiderbotids, showhidden=showhidden, onlysaved=onlysaved)
return redirect('/')
@app.route('/new-crawler', methods=['GET', 'POST'])
def new_crawler():
form = forms.CrawlerForm(request.form)
if request.method == 'POST' and form.validate():
if not ('from_main' in request.form):
sclogic.crawlerAdd(form.name.data)
flash('Crawler created successfully!')
return redirect('/')
return render_template('new_crawler.html', form=form)
@app.route('/spiderbots/<crawlerid>', methods=['GET', 'POST'])
def spiderbots(crawlerid):
spiderbots = models.Spiderbot.query.filter_by(crawlerid=crawlerid).order_by(models.Spiderbot.spiderbotid).all()
crawler = models.Crawler.query.filter_by(crawlerid=crawlerid).first()
return render_template('spiderbots.html', spiderbots=spiderbots, crawler=crawler)
def isfloat(value):
try:
float(value)
return True
except ValueError:
return False
@app.route('/new-spiderbots/<crawlerid>', methods=['GET', 'POST'])
def new_spiderbots(crawlerid):
crawler = models.Crawler.query.filter_by(crawlerid=crawlerid).first()
form = forms.SpiderbotForm(request.form)
if request.method == 'POST' and form.validate():
if not ('from_spiderbots' in request.form):
flash('Create spiders...')
for spider in form.spiders.data:
minprice = float(form.minprice.data) if isfloat(form.minprice.data) and float(form.minprice.data) !=0 else None
maxprice = float(form.maxprice.data) if isfloat(form.maxprice.data) and float(form.maxprice.data) !=0 else None
sclogic.spiderbotAdd(spider, crawlerid, form.searchterm.data, form.fullink.data, minprice, maxprice)
flash('Spiders created successfully!')
return redirect(url_for('spiderbots', crawlerid=crawlerid))
if crawler.maxprice:
form.maxprice.data = str(crawler.maxprice)
if crawler.minprice:
form.minprice.data = str(crawler.minprice)
form.spiders.data = sclogic.getSpiders().keys()
return render_template('new_spiderbot.html', form=form, crawler = crawler)
@app.route('/spiderbot-update/<crawlerid>', methods=['GET', 'POST'])
def spiderbot_update(crawlerid):
if request.method == 'POST':
postvars = variabledecode.variable_decode(request.form, dict_char='_')
for ids, values in postvars.items():
selected = True if "selected" in values and values["selected"] == "on" else False
id = int(ids) if ids.isnumeric() else None
if id:
if 'delete_button' in request.form:
if selected:
flash('Delete spiderbot...')
sclogic.spiderbotDelete(id)
flash('Delete spiderbot finished!')
elif 'save_button' in request.form:
spiderbot = models.Spiderbot.query.filter_by(spiderbotid=id).first()
if spiderbot:
spiderbot.active = True if ("active" in values and values["active"] == "on") else False
spiderbot.searchterm = None if values["searchterm"] == "" or values["searchterm"] == "None" else values["searchterm"]
spiderbot.fullink = None if values["fullink"] == "" or values["fullink"] == "None" else values["fullink"]
spiderbot.minprice = float(values["minprice"]) if isfloat(values["minprice"]) and float(values["minprice"]) !=0 else None
spiderbot.maxprice = float(values["maxprice"]) if isfloat(values["maxprice"]) and float(values["maxprice"]) !=0 else None
db.session.commit()
return redirect(url_for('spiderbots', crawlerid=crawlerid))
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--debug', '-d', action='store_true')
parser.add_argument('--port', '-p', default=5000, type=int)
parser.add_argument('--host', default='0.0.0.0')
args = parser.parse_args()
app.run(args.host, args.port, debug=args.debug)