-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathutils.py
449 lines (393 loc) · 15.2 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
# -*- coding: utf-8 -*-
import datetime
import functools
import json
import pprint
import random
import re
import time
import traceback
import requests
from prettytable import PrettyTable # 命令行可视化表格
from requests.packages.urllib3.exceptions import InsecureRequestWarning
from tqdm import tqdm
"""
需要安装的模块
pip install requests
pip install tqdm
pip install prettytable
"""
# 禁止ssl证书验证警告
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# 默认headers
default_headers = {
'User-Agent': 'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,'
'likeGecko)Chrome/92.0.4515.131Safari/537.36',
'sec-ch-ua': '"Chromium";v="92","NotA;Brand";v="99","GoogleChrome";v="92"',
'sec-ch-ua-mobile': '?0'}
def tryError(fn):
"""
装饰器 对被装饰的函数的运行try-catch
:param fn: 被装饰的函数fn
:return:
"""
@functools.wraps(fn) # 保持fn方法名字和文档不变
def wrapper(*args, **kvargs):
try:
ret = fn(*args, **kvargs)
return ret
except Exception as e:
# print("=========================")
# msg = f'出错原因: {e.__doc__} {e} \n函数名: {fn.__name__}\n函数说明: {fn.__doc__}'
# print(msg)
# print("=========================")
# time.sleep(0.2)
print(e)
traceback.print_exc()
return wrapper
def takeTime(fn):
"""
装饰器 检测函数执行时间
:param fn:
:return:
"""
@functools.wraps(fn)
def wrapper(*args, **kvargs): # *args, **kvargs 位置参数和关键字参数
start = datetime.datetime.now() # 函数运行开始时间
ret = fn(*args, **kvargs) # 函数运行
end = datetime.datetime.now() # 函数运行结束时间
print(f'{fn.__name__} 耗费时间: {end - start} ') # 打印执行时间
return ret
return wrapper
@tryError
def requireText(url: str, method: str, headers: dict = None, params: dict = None, data: dict = None) -> str:
"""
获取返回的Text 仅在本模块内部使用
:param url: url地址
:param method: 请求方式
:param headers: 请求头
:param params: get字典参数
:param data: post字典参数
:return:
"""
t_headers = headers if headers else default_headers # 如果headers为None就使用默认的
t_headers['Accept-Encoding'] = 'utf-8' # 表明我只接受utf-8编码的Text文本
with requests.session() as session:
with session.request(method=method, url=url, headers=t_headers, verify=False, params=params, data=data) as resp:
resp.encoding = resp.apparent_encoding # 自动推断文本编码 不过由于我们之前在headers已经说明只接受utf-8了 所以有点多余
return resp.text
def getText(url: str, headers: dict = None, params: dict = None, data: dict = None, showResult: bool = False) -> str:
"""
以GET方式请求文本Text
:param url:
:param headers:
:param params:
:param data:
:param showResult: 是否显示请求结果 默认不显示
:return:
"""
resultText = requireText(url=url, method='GET', headers=headers, params=params, data=data)
if showResult:
print(resultText)
return resultText
def postText(url: str, headers: dict = None, params: dict = None, data: dict = None, showResult: bool = False) -> str:
"""
以POST方式请求文本
:param url:
:param headers:
:param params:
:param data:
:param showResult: 是否显示请求结果 默认不显示
:return:
"""
resultText = requireText(url=url, method='POST', headers=headers, params=params, data=data)
if showResult:
print(resultText)
return resultText
def getJson(url: str, headers: dict = None, params: dict = None, data: dict = None, showResult: bool = False) -> dict:
"""
GET方式请求Json文本,获取返回的Json对象
间接调用getText 然后将返回的结果Json化
:param url:
:param headers:
:param params:
:param data:
:param showResult: 是否显示请求结果 默认不显示
:return:
"""
jsonResult = json.loads(getText(url=url, headers=headers, params=params, data=data))
if showResult:
pprint.pprint(jsonResult)
return jsonResult
def postJson(url: str, headers: dict = None, params: dict = None, data: dict = None, showResult: bool = False) -> dict:
"""
POST,获取返回的Json对象
间接调用getText 然后将返回的结果Json化
:param url:
:param headers:
:param params:
:param data:
:param showResult: 是否显示请求结果 默认不显示
:return:
"""
jsonResult = json.loads(postText(url=url, headers=headers, params=params, data=data))
if showResult:
pprint.pprint(jsonResult)
return jsonResult
@tryError
def getBytes(url: str, headers: dict = None) -> bytes:
"""
从网络上面获取二进制字节流
:param url:
:param headers:
:return:
"""
t_headers = headers if headers else default_headers
with requests.session() as session, session.get(url=url, headers=t_headers, stream=True, verify=False) as resp:
return resp.content
def writeBytes(filePath: str, url: str, headers: dict = None) -> None:
"""
从网络上面下载二进制文件到本地
:param filePath:
:param url:
:param headers:
:return:
"""
with open(filePath, 'wb+') as wf:
wf.write(getBytes(url=url, headers=headers)) # 简介调用getBin方法
print(f'{filePath} down ok !')
def writeText(filePath: str, url: str, headers: dict = None, params: dict = None, data: dict = None,
showResult: bool = False) -> None:
"""
从网络上面下载文本文件到本地
:param showResult: 是否显示返回结果
:param data:
:param params:
:param filePath:
:param url:
:param headers:
:return:
"""
with open(filePath, 'w', encoding='utf-8') as wf:
# 间接调用getText
wf.write(getText(url=url, headers=headers, params=params, data=data, showResult=showResult))
print(f'{filePath} down ok !')
def writeBytesBar(filePath: str, url: str, headers: dict = None) -> None:
"""
从网络上面下载二进制文件到本地 使用tqdm库展示进度条
:param filePath:
:param url:
:param headers:
:return:
"""
try:
t_headers = headers if headers else default_headers
with requests.session() as session, session.get(url=url, headers=t_headers, stream=True, verify=False) as resp:
fileSize = int(resp.headers['content-Length'])
with tqdm(initial=0, total=fileSize, unit_scale=True, unit='B') as pbar:
with open(filePath, 'wb+') as wf:
for chunk in resp.iter_content(chunk_size=1024):
if chunk:
wf.write(chunk)
pbar.update(len(chunk))
print(f'{filePath} down ok !')
except Exception as e:
print(e)
traceback.print_exc()
@tryError
def getHeadersKV(pstr: str = '', pname: str = 'headers='):
"""
从chrome直接复制的headers放在当前目录下的【getKV.txt】文件中
运行一下就能直接得到字典类型的
不用手动加引号了
:param pstr:
:param pname:
:return:
"""
filePath = 'getKV.txt'
if pstr == '':
with open(filePath, 'r', encoding='utf-8') as rf:
string = rf.read()
else:
string = pstr
s = string.replace(' ', '').splitlines()
s = [item for item in s if item != ''] # 好像没有必要
headers = {}
for item in s:
key = item.split(':')[0]
value = item.replace(fr'{key}:', '').strip()
headers[key] = value
print(pname, end='')
pprint.pprint(headers)
def removeBlank(pstr: str) -> str:
"""
利用[正则表达式]移除字符串中的空白
:param pstr:
:return:
"""
return re.sub(re.compile(r'\s+', re.S), '', pstr)
def getListByRe(restr: str, url: str, headers: dict = None, params: dict = None, data: dict = None,
showResult: bool = False) -> list:
"""
1.先进行网络请求
2.利用正则表达式从返回的Text文本中提取出需要的东西
3.找到所有符合条件的
4.返回一个列表 列表中的元素是元组组 元素是每条获取的信息
示例:
getListByRe(restr=r'<script>(?P<getName>.*?)=(?P<getUrl>.*?)</script>', url='https://......')
:param showResult:
:param data:
:param params:
:param restr:
:param url:
:param headers:
:return:
"""
try:
getHtml = getText(url=url, headers=headers, params=params, data=data)
getHtml = removeBlank(getHtml)
names = re.compile(r'P<(?P<getName>.*?)>', re.S).findall(restr)
print(names)
finditer = re.compile(restr, re.S).finditer(getHtml)
tasks = []
for item in finditer:
task = []
for name in names:
task.append(item.group(name))
tasks.append(tuple(task))
if showResult:
pprint.pprint(tasks)
return tasks
except Exception as e:
print(e)
traceback.print_exc()
def getSingleByRe(restr: str, url: str, headers: dict = None, params: dict = None, data: dict = None,
showResult: bool = False) -> tuple:
"""
1.先进行网络请求
2.利用正则表达式从返回的Text文本中提取出需要的东西
3.找到第一个符合条件的
4.返回一个元组
示例:
getListByRe(restr=r'<script>(?P<getName>.*?)=(?P<getUrl>.*?)</script>', url='https://......')
:param showResult:
:param data:
:param params:
:param restr:
:param url:
:param headers:
:return:
"""
try:
getHtml = getText(url=url, headers=headers, params=params, data=data)
getHtml = removeBlank(getHtml)
names = re.compile(r'P<(?P<getName>.*?)>', re.S).findall(restr)
search = re.compile(restr, re.S).search(getHtml)
task = []
for name in names:
task.append(search.group(name))
if showResult:
pprint.pprint(tuple(task))
return tuple(task)
except Exception as e:
print(e)
traceback.print_exc()
def sleepShort(showSleepTime: bool = False):
"""
在两次网络请求之间 随机短时间沉睡 防止服务器判断为爬虫
:return:
"""
ranTime = random.choice([0.4, 0.5, 0.6, 0.65, 0.55, 0.7])
if showSleepTime:
print(f"沉睡: {ranTime}s")
time.sleep(ranTime)
def sleepLong(showSleepTime: bool = False):
"""
在两次网络请求之间 随机长时间沉睡 防止服务器判断为爬虫
:param showSleepTime:
:return:
"""
ranTime = random.choice([1.5, 2.0, 2.5, 3])
if showSleepTime:
print(f"沉睡: {ranTime}s")
time.sleep(ranTime)
def sleepMedium(showSleepTime: bool = False):
"""
在两次网络请求之间 随机中等时间沉睡 防止服务器判断为爬虫
:param showSleepTime:
:return:
"""
ranTime = random.choice([1.1, 1.2, 1.3, 1.4, 1.5, 1.5, 1.6, 1.7, 1.8])
if showSleepTime:
print(f"沉睡: {ranTime}s")
time.sleep(ranTime)
def printTable(rows: list, headers: list = None) -> None:
"""
打印表格:
import 本模块
headers = ['Country', 'Capital', 'Population']
rows = [["中文", "Beijing", 21893095], ["意大利是我去过的确定", "Moscow", 12195221], ["Germany", "Berlin", 3748148]]
本模块.printTable(rows=rows, headers=headers)
:param rows:
:param headers:
:return:
"""
table = PrettyTable()
if headers:
if len(headers) != len(rows[0]):
print("表格不对齐,行数或者列数不对劲")
return
table.field_names = headers
else:
h_size = len(rows[0])
t_headers = []
for index in range(1, h_size + 1):
t_headers.append(f"第{index}列")
table.field_names = t_headers
for row in rows:
table.add_row(row)
print(table)
class BV_AV():
"""
bilibili bv he av 转换
"""
def __init__(self):
self.table = 'fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF'
self.tr = {}
for i in range(58):
self.tr[self.table[i]] = i
self.s = [11, 10, 3, 8, 4, 6]
self.xor = 177451812
self.add = 8728348608
def dec(self, x):
r = 0
for i in range(6):
r += self.tr[x[self.s[i]]] * 58 ** i
return (r - self.add) ^ self.xor
def enc(self, x):
x = (x ^ self.xor) + self.add
r = list('BV1 4 1 7 ')
for i in range(6):
r[self.s[i]] = self.table[x // 58 ** i % 58]
return ''.join(r)
def get_random_agent(index=None):
agent_list = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:90.0) Gecko/20100101 Firefox/90.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:90.0) Gecko/20100101 Firefox/90.0']
if index:
return agent_list[index]
else:
return random.choice(agent_list)