forked from Charcoal-SE/SmokeDetector
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathglobalvars.py
352 lines (291 loc) · 13.8 KB
/
globalvars.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
# -*- coding: utf-8 -*-
import sys
import os
from collections import namedtuple
from datetime import datetime
from html.parser import HTMLParser
from html import unescape
from hashlib import md5
from configparser import NoOptionError, ConfigParser
import threading
# noinspection PyCompatibility
import regex
import subprocess as sp
import platform
if 'windows' in platform.platform().lower():
# noinspection PyPep8Naming
from _Git_Windows import git, GitError
else:
# noinspection PyUnresolvedReferences
from sh.contrib import git
CommitInfo = namedtuple('CommitInfo', ['id', 'id_full', 'author', 'message'])
git_url = git.config("--get", "remote.origin.url").strip()
git_url_split = git_url.split("/")
git_user_repo = "Charcoal-SE/SmokeDetector"
if git_url[0:19] == "https://github.com/":
git_user_repo = "{}/{}".format(git_url_split[3], git_url_split[4][0:-4])
def git_commit_info():
try:
data = sp.check_output(['git', 'rev-list', '-1', '--pretty=%h%n%H%n%an%n%s', 'HEAD'],
stderr=sp.STDOUT).decode('utf-8')
except sp.CalledProcessError as e:
raise OSError("Git error:\n" + e.output) from e
_, abbrev_id, full_id, author, message = data.strip().split("\n")
return CommitInfo(id=abbrev_id, id_full=full_id, author=author, message=message)
def git_ref():
git_cp = sp.run(['git', 'symbolic-ref', '--short', '-q', 'HEAD'], stdout=sp.PIPE)
return git_cp.stdout.decode("utf-8").strip() # not on branch = empty output
# We don't need strip_escape_chars() anymore, see commit message
# of 1931d30804a675df07887ce0466e558167feae57
# noinspection PyClassHasNoInit,PyDeprecation,PyUnresolvedReferences
class GlobalVars:
on_windows = 'windows' in platform.platform().lower()
false_positives = []
whitelisted_users = set()
blacklisted_users = dict()
blacklisted_usernames = []
blacklisted_websites = []
blacklisted_numbers = []
watched_numbers = []
blacklisted_numbers_normalized = None
watched_numbers_normalized = None
bad_keywords = []
watched_keywords = {}
ignored_posts = []
auto_ignored_posts = []
startup_utc_date = datetime.utcnow()
startup_utc = startup_utc_date.strftime("%H:%M:%S")
latest_questions = []
api_backoff_time = 0
deletion_watcher = None
not_privileged_warning = \
"You are not a privileged user. Please see " \
"[the privileges wiki page](https://charcoal-se.org/smokey/Privileges) for " \
"information on what privileges are and what is expected of privileged users."
experimental_reasons = { # Don't widely report these
"potentially bad keyword in answer",
"potentially bad keyword in body",
"potentially bad keyword in title",
"potentially bad keyword in username",
"potentially bad NS for domain in title",
"potentially bad NS for domain in body",
"potentially bad NS for domain in answer",
"potentially bad ASN for hostname in title",
"potentially bad ASN for hostname in body",
"potentially bad ASN for hostname in answer",
"potentially bad IP for hostname in title",
"potentially bad IP for hostname in body",
"potentially bad IP for hostname in answer",
"potentially problematic NS configuration in title",
"potentially problematic NS configuration in body",
"potentially problematic NS configuration in answer",
"toxic body detected",
"toxic answer detected",
}
parser = HTMLParser()
parser.unescape = unescape
code_privileged_users = None
# these are loaded in GlobalVars.reload()
commit = None
commit_with_author = None
on_branch = None
s = ""
s_reverted = ""
s_norestart_blacklists = ""
s_norestart_findspam = ""
apiquota = -1
bodyfetcher = None
cookies = {}
se_sites = []
why_data = []
notifications = []
listen_to_these_if_edited = []
multiple_reporters = []
api_calls_per_site = {}
reason_weights = {}
metasmoke_ids = {}
standby_message = ""
standby_mode = False
no_se_activity_scan = False
no_deletion_watcher = False
ignore_no_se_websocket_activity_lock = threading.Lock()
ignore_no_se_websocket_activity = False
api_request_lock = threading.Lock() # Get this lock before making API requests
apiquota_rw_lock = threading.Lock() # Get this lock before reading/writing apiquota
class PostScanStat:
""" Tracking post scanning data """
num_posts_scanned = 0
post_scan_time = 0
rw_lock = threading.Lock()
@staticmethod
def add_stat(posts_scanned, scan_time):
""" Adding post scanning data """
with GlobalVars.PostScanStat.rw_lock:
GlobalVars.PostScanStat.num_posts_scanned += posts_scanned
GlobalVars.PostScanStat.post_scan_time += scan_time
@staticmethod
def get_stat():
""" Getting post scanning statistics """
with GlobalVars.PostScanStat.rw_lock:
posts_scanned = GlobalVars.PostScanStat.num_posts_scanned
scan_time = GlobalVars.PostScanStat.post_scan_time
if scan_time == 0:
posts_per_second = None
else:
posts_per_second = posts_scanned / scan_time
return (posts_scanned, scan_time, posts_per_second)
@staticmethod
def reset_stat():
""" Resetting post scanning data """
with GlobalVars.PostScanStat.rw_lock:
GlobalVars.PostScanStat.num_posts_scanned = 0
GlobalVars.PostScanStat.post_scan_time = 0
config_parser = ConfigParser(interpolation=None)
if os.path.isfile('config') and "pytest" not in sys.modules:
config_parser.read('config')
else:
config_parser.read('config.ci')
config = config_parser["Config"] # It's a collections.OrderedDict now
site_id_dict = {}
post_site_id_to_question = {}
location = config.get("location", "Continuous Integration")
# DNS Configuration
# Configure resolver based on config options, or System, configure DNS Cache in
# thread-safe cache as part of dnspython's resolver system as init options,
# control cleanup interval based on **TIME** like a regular DNS server does.
#
# # Explicitly defining fallback= for fallback values in bool and float getters, in order to
# # avoid IDE complaints -- tward
dns_nameservers = config.get("dns_resolver", "system").lower()
dns_cache_enabled = config.getboolean("dns_cache_enabled", fallback=True)
dns_cache_interval = config.getfloat("dns_cache_cleanup_interval", fallback=300.0)
class MSStatus:
""" Tracking metasmoke status """
ms_is_up = True
counter = 0
rw_lock = threading.Lock()
@staticmethod
def set_up():
""" Set metasmoke status to up """
# Private to metasmoke.py
with GlobalVars.MSStatus.rw_lock:
GlobalVars.MSStatus.ms_is_up = True
@staticmethod
def set_down():
""" Set metasmoke status to down """
# Private to metasmoke.py
with GlobalVars.MSStatus.rw_lock:
GlobalVars.MSStatus.ms_is_up = False
@staticmethod
def is_up():
""" Query if metasmoke status is up """
with GlobalVars.MSStatus.rw_lock:
current_ms_status = GlobalVars.MSStatus.ms_is_up
return current_ms_status
@staticmethod
def is_down():
""" Query if metasmoke status is down """
return not GlobalVars.MSStatus.is_up()
# Why implement failed() and succeeded() here, as they will only be called in metasmoke.py?
# Because get_failure_count() need to be exposed to global, so it is more convenient
# to implement failed() and succeeded() here.
@staticmethod
def failed():
""" Indicate a metasmoke connection failure """
with GlobalVars.MSStatus.rw_lock:
GlobalVars.MSStatus.counter += 1
@staticmethod
def succeeded():
""" Indicate a metasmoke connection success """
with GlobalVars.MSStatus.rw_lock:
GlobalVars.MSStatus.counter = 0
@staticmethod
def get_failure_count():
""" Get consecutive metasmoke connection failure count """
with GlobalVars.MSStatus.rw_lock:
failure_count = GlobalVars.MSStatus.counter
return failure_count
@staticmethod
def reset_ms_status():
""" Reset class GlobalVars.MSStatus to default values """
with GlobalVars.MSStatus.rw_lock:
GlobalVars.MSStatus.ms_is_up = True
GlobalVars.MSStatus.counter = 0
chatexchange_u = config.get("ChatExchangeU")
chatexchange_p = config.get("ChatExchangeP")
metasmoke_host = config.get("metasmoke_host")
metasmoke_key = config.get("metasmoke_key")
metasmoke_ws_host = config.get("metasmoke_ws_host")
git_name = config.get("git_username", "SmokeDetector")
git_email = config.get("git_useremail", "smokey@erwaysoftware.com")
github_username = config.get("github_username")
github_password = config.get("github_password")
github_access_token = config.get("github_access_token")
perspective_key = config.get("perspective_key")
flovis_host = config.get("flovis_host")
flovis = None
# Miscellaneous
log_time_format = config.get("log_time_format", "%H:%M:%S")
# Blacklist privileged users from config
se_blacklisters = regex.sub(r"[^\d,]", "", config.get("se_blacklisters", "")).split(",")
mse_blacklisters = regex.sub(r"[^\d,]", "", config.get("mse_blacklisters", "")).split(",")
so_blacklisters = regex.sub(r"[^\d,]", "", config.get("so_blacklisters", "")).split(",")
# Create a set of blacklisters equivalent to what's used in code_privileged_users.
config_blacklisters = set()
for id in se_blacklisters:
if id:
config_blacklisters.add(("stackexchange.com", int(id)))
for id in mse_blacklisters:
if id:
config_blacklisters.add(("meta.stackexchange.com", int(id)))
for id in so_blacklisters:
if id:
config_blacklisters.add(("stackoverflow.com", int(id)))
# environ_or_none replaced by os.environ.get (essentially dict.get)
bot_name = os.environ.get("SMOKEDETECTOR_NAME", git_name)
bot_repo_slug = os.environ.get("SMOKEDETECTOR_REPO", git_user_repo)
bot_repository = "//github.com/{}".format(bot_repo_slug)
chatmessage_prefix = "[{}]({})".format(bot_name, bot_repository)
valid_content = """This is a totally valid post that should never be caught. Any blacklist or watchlist item that triggers on this item should be avoided. java.io.BbbCccDddException: nothing wrong found. class Safe { perfect valid code(int float &#%$*v a b c =+ /* - 0 1 2 3 456789.EFGQ} English 中文Français Español Português Italiano Deustch ~@#%*-_/'()?!:;" vvv kkk www sss ttt mmm absolute std::adjacent_find (power).each do |s| bbb end ert zal l gsopsq kdowhs@ xjwk* %_sooqmzb xjwpqpxnf. Please don't blacklist disk-partition.com, it's a valid domain (though it also gets spammed rather frequently).""" # noqa: E501
@classmethod
def reload(cls):
cls.commit = commit = git_commit_info()
cls.commit_with_author = "`{}` ({}: {})".format(
commit.id, commit.author, commit.message)
# We don't want to escape `[` and `]` when they are within code.
split_commit_with_author = cls.commit_with_author.split('`')
split_length = len(split_commit_with_author)
for index in range(0, split_length, 2):
split_commit_with_author[index] = split_commit_with_author[index].replace('[', '\\[').replace(']', '\\]')
# There's not an even number of ` characters, so the parsing hack failed, but we assume the last one needs
# escaping.
if not split_length % 2:
split_commit_with_author[-1] = split_commit_with_author[-1].replace('[', '\\[').replace(']', '\\]')
cls.commit_with_author_escaped = '`'.join(split_commit_with_author)
cls.on_branch = git_ref()
cls.s = "[ {} ] SmokeDetector started at [rev {}]({}/commit/{}) (running on {}, Python {})".format(
cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository,
cls.commit.id, cls.location, platform.python_version())
cls.s_reverted = \
"[ {} ] SmokeDetector started in [reverted mode](" \
"https://charcoal-se.org/smokey/SmokeDetector-Statuses#reverted-mode) " \
"at [rev {}]({}/commit/{}) (running on {})".format(
cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository,
cls.commit.id, cls.location)
cls.s_norestart_blacklists = \
"[ {} ] Blacklists reloaded at [rev {}]({}/commit/{}) (running on {})".format(
cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository,
cls.commit.id, cls.location)
cls.s_norestart_findspam = \
"[ {} ] FindSpam module reloaded at [rev {}]({}/commit/{}) (running on {})".format(
cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository,
cls.commit.id, cls.location)
cls.standby_message = \
"[ {} ] SmokeDetector started in [standby mode](" \
"https://charcoal-se.org/smokey/SmokeDetector-Statuses#standby-mode) " \
"at [rev {}]({}/commit/{}) (running on {})".format(
cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository,
cls.commit.id, cls.location)
GlobalVars.PostScanStat.reset_stat()
GlobalVars.MSStatus.reset_ms_status()
GlobalVars.reload()