-
Notifications
You must be signed in to change notification settings - Fork 4
/
crawlForImages.py
210 lines (182 loc) · 6.51 KB
/
crawlForImages.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
# script to download content using various apis
# author : Appu Shaji ( contact me at appoose at gmail )
# license : GPLv3
# date : 20th Feb 2013
import urllib
import sys
import time
import os
import requests
# following imports are specific to 500px
# get it from https://github.com/akirahrkw/python-500px
from fivehundredpx.client import FiveHundredPXAPI
from fivehundredpx.auth import *
import getpass
import datetime
# for flickr
import flickrapi
import simplejson
import re
class CrawlData():
def __init__(self,searchEngine):
self.searchEngine = searchEngine
self.count = 0
if searchEngine == 'google':
self.baseDir = 'googleImageResults'
self.rootUrl = 'https://ajax.googleapis.com/ajax/services/search/images?v=1.0&q='
self.apiKey = 'replace your api key here' # not needed if want < 64 images. Have not implemented the paid account version
self.opUrlKey = 'unescapedUrl'
elif searchEngine == 'bing':
self.baseDir = 'bingImageResults'
self.rootUrl = 'https://api.datamarket.azure.com/Bing/Search/v1/Image?Query='
self.apiKey = 'replace your api key here'
self.opUrlKey = 'MediaUrl'
elif searchEngine == '500px':
self.baseDir = '500pxImageResults'
self.CONSUMER_KEY = 'your consumer key here'
self.CONSUMER_SECRET = 'your consumer secret here'
self.opUrlKey = 'image_url'
# oauth details
self.handler = OAuthHandler(self.CONSUMER_KEY,self.CONSUMER_SECRET)
self.requestToken = self.handler.get_request_token()
self.handler.set_request_token(self.requestToken.key,self.requestToken.secret)
username = raw_input("Input your username: ").strip()
password = getpass.getpass()
self.token = self.handler.get_xauth_access_token(username,password)
self.api = FiveHundredPXAPI(self.handler)
elif searchEngine == 'flickr':
self.baseDir = 'FlickrResults'
self.api_key = 'your api key'
self.api_secret = 'your api secert'
self.api =flickrapi.FlickrAPI(self.api_key, self.api_secret)
(self.token, self.frob) = self.api.get_token_part_one(perms='write')
if not self.token:
raw_input("Press Enter")
self.api.get_token_part_two((self.token,self.frob))
# self.cc_licenses = '1, 2, 3, 4, 5, 6, 7' for cc license search
self.cc_licenses = ''
# you have to generate the urls yourself for flickr
def grabDataFlickr(self,dataInfo):
for j in range(len(dataInfo)):
currFlickrMeta = dataInfo[j]
currUrl = 'http://farm1.staticflickr.com/{0}/{1}_{2}_z.jpg'.format(
currFlickrMeta['server'],
currFlickrMeta['id'],
currFlickrMeta['secret'])
opFileName = '{0}/Image_{1:010d}.jpg'.format(self.opDir,self.count)
print opFileName
urllib.urlretrieve(currUrl,opFileName)
self.count = self.count + 1
def grabData(self,dataInfo):
for j in range(len(dataInfo)):
currUrl = dataInfo[j][self.opUrlKey]
opFileName = '{0}/Image_{1:010d}.jpg'.format(self.opDir,self.count)
print opFileName
urllib.urlretrieve(currUrl,opFileName)
self.count = self.count + 1
def doSearch(self,queryTerm,pageNumber):
self.opDir = self.baseDir + '/' + queryTerm
if not os.path.exists(self.opDir):
os.makedirs(self.opDir)
if self.searchEngine == 'google':
searchUrl = self.rootUrl + urllib.quote(queryTerm) +'&start='+str(pageNumber*8)+'&userip=MyIP&rsz=8&imgtype=photo'
try:
response = requests.get(searchUrl).json()
dataInfo = response['responseData']['results']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
elif self.searchEngine == 'bing':
searchUrl = self.rootUrl + '%27' + urllib.quote(queryTerm) + '%27&$format=json&$skip=' + str(pageNumber*10)
try:
response = requests.get(searchUrl, auth=(self.apiKey,self.apiKey)).json()
dataInfo = response['d']['results']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
elif self.searchEngine == 'flickr':
# documentation available at http://www.flickr.com/services/api/flickr.photos.search.html
try:
if not self.cc_licenses:
responseUnStripped = self.api.photos_search(text=queryTerm,
content_type=1,
page=(pageNumber+1),
format='json')
else:
responseUnStripped = self.api.photos_search(text=queryTerm,
content_type=1,
page =(pageNumber+1),
license=self.cc_licenses,
format='json')
response = simplejson.loads(re.search(r'jsonFlickrApi\(>?(.+)\)', responseUnStripped).group(1))
self.grabDataFlickr(response['photos']['photo'])
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
elif self.searchEngine == '500px':
try:
response = self.api.photos_search(require_auth=True,
tag=queryTerm,
page=(pageNumber+1),
image_size=5)
dataInfo = response['photos']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
self.grabData(dataInfo)
def doSearchPopular(self,pageNumber):
self.opDir = self.baseDir + '/popular on ' + datetime.datetime.now().strftime('%d-%m-%Y')
if not os.path.exists(self.opDir):
os.makedirs(self.opDir)
if self.searchEngine == '500px':
try:
currPage = pageNumber+1
response = self.api.photos(require_auth=True,
feature='popular',
sort='rating',
image_size=5,
page=currPage)
dataInfo = response['photos']
except (IndexError,TypeError,ValueError,NameError):
print 'skipping'
return
self.grabData(dataInfo)
def searchImage(searchTerm,searchEngine):
currSearch = CrawlData(searchEngine)
numPages = 10
for i in range(numPages):
currSearch.doSearch(searchTerm,i)
time.sleep(0.5) # for throttling
def searchPopular(searchEngine):
currPopularSearch = CrawlData(searchEngine)
numPages = 3000
for i in range(numPages):
currPopularSearch.doSearchPopular(i)
if __name__ == '__main__':
print len(sys.argv)
if len(sys.argv) > 2:
searchTerm = ''
if sys.argv[len(sys.argv)-1] == 'google' \
or sys.argv[len(sys.argv)-1] == 'bing' \
or sys.argv[len(sys.argv)-1] == 'flickr' \
or sys.argv[len(sys.argv)-1] == '500px':
searchEngine = sys.argv[len(sys.argv)-1]
searchQueryRange = len(sys.argv)-1
else:
print 'using default search engine'
searchEngine = 'google' # default one
searchQueryRange = len(sys.argv)
for i in range(1,searchQueryRange):
searchTerm = searchTerm + sys.argv[i] + " "
elif len(sys.argv) == 2:
print sys.argv[1]
if sys.argv[1] == 'popular500px':
searchPopular('500px')
sys.exit(1)
else:
searchTerm = sys.argv[1]
else:
print 'fdsfsd'
searchEngine = 'google'
searchTerm = 'rothko'
searchImage(searchTerm.strip(),searchEngine)