-
+
- +
-
@@ -226,27 +262,35 @@
+ 
+
+
+
+
+
diff --git a/CNKI/CNKI/middlewares.py b/CNKI/CNKI/middlewares.py
index 4e48a07..286a9bf 100644
--- a/CNKI/CNKI/middlewares.py
+++ b/CNKI/CNKI/middlewares.py
@@ -63,17 +63,22 @@ def __init__(self, crawler):
self.ua = UserAgent()
self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random")
+
+ def spider_opened(self, spider):
+ spider.logger.info('Spider opened: %s' % spider.name)
+
+
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_request(self, request, spider):
- def get_ua():
- return getattr(self.ua, self.ua_type)
+ # def get_ua():
+ # return getattr(self.ua, self.ua_type)
- random_agent = get_ua()
- request.headers.setdefault("User-Agent", get_ua())
+ # random_agent = get_ua()
+ # request.headers.setdefault("User-Agent", get_ua())
# 璁剧疆浠ｇ悊
# request.meta['proxy'] = ""
- request.headers.setdefault("Cookie", "Ecp_ClientId=1171009095901465355; Ecp_IpLoginFail=171009112.81.2.110; RsPerPage=50; cnkiUserKey=dd6eca65-a22c-330b-d486-22684afbe7b2; ASP.NET_SessionId=5atsoskm5rxqkirzhct0vjdb; SID_kns=123122; SID_kinfo=125102; SID_klogin=125141; SID_kredis=125142; SID_krsnew=125132")
\ No newline at end of file
+ request.headers.setdefault("Cookie", "Ecp_ClientId=2170914074501149831; RsPerPage=20; cnkiUserKey=42463fbc-f813-8023-21c7-d4cd29c7bff8; ASP.NET_SessionId=u3vqombtchoej45crc4daaue; SID_kns=123119; SID_kinfo=125104; SID_klogin=125144; SID_krsnew=125131; SID_kredis=125144; Ecp_IpLoginFail=171009112.81.2.110")
\ No newline at end of file
diff --git a/CNKI/CNKI/pipelines.py b/CNKI/CNKI/pipelines.py
index c8a3d7e..cb667ec 100644
--- a/CNKI/CNKI/pipelines.py
+++ b/CNKI/CNKI/pipelines.py
@@ -4,8 +4,27 @@
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
-
+import csv
class CnkiPipeline(object):
+ colname = ['application_no', 'application_day', 'publication_no',
+ 'publication_day', 'publication_user', 'publication_address',
+ 'patent_inventor', 'patent_agent', 'patent_agent_user',
+ 'publication_address', 'patent_inventor', 'patent_agent',
+ 'patent_agent_user', 'patent_summary', 'patent_main_item']
+
+ def open_spider(self, spider):
+ # 鍦ㄧ埇铏惎鍔ㄦ椂锛屽垱寤篶sv锛屽苟璁剧疆newline=''鏉ラ伩鍏嶇┖琛屽嚭鐜
+ self.file = open('cnki_patent_info.csv', 'w', newline='')
+ # 鍚姩csv鐨勫瓧鍏稿啓鍏ユ柟娉
+ self.writer = csv.DictWriter(self.file, self.colname)
+ # 鍐欏叆瀛楁鍚嶇О浣滀负棣栬
+ self.writer.writeheader()
+
+ def close_spider(self, spider):
+ self.file.close()
+
+
def process_item(self, item, spider):
+ self.writer.writerow(item)
return item
diff --git a/CNKI/CNKI/settings.py b/CNKI/CNKI/settings.py
index 23ea5a1..d015cb4 100644
--- a/CNKI/CNKI/settings.py
+++ b/CNKI/CNKI/settings.py
@@ -89,3 +89,12 @@
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+DEFAULT_REQUEST_HEADERS = {
+ "Host" : "kns.cnki.net",
+ "User-Agent" : "Mozilla/5.0 (Windows NT 10.0; 鈥) Gecko/20100101 Firefox/56.0",
+ "Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+ "Referer" : "http://kns.cnki.net/kns/brief/鈥geName=ASP.brief_result_aspx",
+ "Cookie" : "Ecp_ClientId=1 171009095901465355; Ecp_IpLoginFail=171009112.81.2.110; RsPerPage=50; cnkiUserKey=dd6eca65-a22c-330b-d486-22684afbe7b2; ASP.NET_SessionId=5atsoskm5rxqkirzhct0vjdb; SID_kns=123122; SID_kinfo=125102; SID_klogin=125141; SID_kredis=125142; SID_krsnew=125132",
+ "Connection" : "keep-alive",
+ "Upgrade-Insecure-Requests" : "1"
+}
\ No newline at end of file
diff --git a/CNKI/CNKI/spiders/__pycache__/__init__.cpython-36.pyc b/CNKI/CNKI/spiders/__pycache__/__init__.cpython-36.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c68b485560f0dc75b2ca33d7b7b2b26ede11f37f
GIT binary patch
literal 137
zcmXr!<>k6?>`=r)1_p-5Knw?rK$Zg#7qb9~6oz01O-8?!3`HPe1o6wv$totGvLquv
zFQ6!~Br`d+IHtHDGbObs#@Wx?6GA`)ieuvAGxIV_;^XxSDsOSv
+