add user header to all requests.get calls

espncricinfo/match.py

@@ -9,6 +9,7 @@ def __init__(self, match_id):
         self.match_id = match_id
         self.match_url = "https://www.espncricinfo.com/matches/engine/match/{0}.html".format(str(match_id))
         self.json_url = "https://www.espncricinfo.com/matches/engine/match/{0}.json".format(str(match_id))
+        self.headers = {'user-agent': 'Mozilla/5.0'}
         self.json = self.get_json()
         self.html = self.get_html()
         self.comms_json = self.get_comms_json()
@@ -89,7 +90,7 @@ def __repr__(self):
         return (f'{self.__class__.__name__}('f'{self.match_id!r})')
 
     def get_json(self):
-        r = requests.get(self.json_url)
+        r = requests.get(self.json_url,headers=self.headers)
         if r.status_code == 404:
             raise MatchNotFoundError
         elif 'Scorecard not yet available' in r.text:
@@ -98,7 +99,7 @@ def get_json(self):
             return r.json()
 
     def get_html(self):
-        r = requests.get(self.match_url)
+        r = requests.get(self.match_url,headers=self.headers)
         if r.status_code == 404:
             raise MatchNotFoundError
         else:
@@ -431,6 +432,6 @@ def get_recent_matches(date=None):
             url = "https://www.espncricinfo.com/ci/engine/match/index.html?date=%sview=week" % date
         else:
             url = "https://www.espncricinfo.com/ci/engine/match/index.html?view=week"
-        r = requests.get(url)
+        r = requests.get(url,headers={'user-agent': 'Mozilla/5.0'})
         soup = BeautifulSoup(r.text, 'html.parser')
         return [x['href'].split('/',4)[4].split('.')[0] for x in soup.findAll('a', href=True, text='Scorecard')]

espncricinfo/player.py

@@ -12,6 +12,7 @@ def __init__(self, player_id):
         self.url = "https://www.espncricinfo.com/player/player-name-{0}".format(str(player_id))
         self.json_url = "http://core.espnuk.org/v2/sports/cricket/athletes/{0}".format(str(player_id))
         self.new_json_url = "https://hs-consumer-api.espncricinfo.com/v1/pages/player/home?playerId={0}".format(str(player_id))
+        self.headers = {'user-agent': 'Mozilla/5.0'}
         self.parsed_html = self.get_html() 
         self.json = self.get_json()       
         self.new_json = self.get_new_json()
@@ -28,21 +29,21 @@ def __init__(self, player_id):
         self.major_teams = self._major_teams()
 
     def get_html(self):
-        r = requests.get(self.url)
+        r = requests.get(self.url, headers=self.headers)
         if r.status_code == 404:
             raise PlayerNotFoundError
         else:
             return BeautifulSoup(r.text, 'html.parser')
 
     def get_json(self):
-        r = requests.get(self.json_url)
+        r = requests.get(self.json_url, headers=self.headers)
         if r.status_code == 404:
             raise PlayerNotFoundError
         else:
             return r.json()
 
     def get_new_json(self):
-        r = requests.get(self.new_json_url, headers={'user-agent': 'Mozilla/5.0'})
+        r = requests.get(self.new_json_url, headers=self.headers)
         if r.status_code == 404:
             raise PlayerNotFoundError
         else:
@@ -126,7 +127,7 @@ def get_career_averages(self, file_name=None, match_format=11, data_type='allrou
             self.file_name = f"{self.player_id}_{self.match_format}_{self.data_type}_career_averages.csv"
 
         self.url=f"https://stats.espncricinfo.com/ci/engine/player/{self.player_id}.html?class={self.match_format};template=results;type={self.data_type}"
-        html_doc = requests.get(self.url)
+        html_doc = requests.get(self.url, headers=self.headers)
         soup = BeautifulSoup(html_doc.text, 'html.parser')
         tables = soup.find_all("table")[2]
         table_rows = tables.find_all("tr")
@@ -158,7 +159,7 @@ def get_career_summary(self, file_name=None, match_format=11, data_type='allroun
             self.file_name = f"{self.player_id}_{self.match_format}_{self.data_type}_career_summary.csv"
 
         self.url=f"https://stats.espncricinfo.com/ci/engine/player/{self.player_id}.html?class={self.match_format};template=results;type={self.data_type}"
-        html_doc = requests.get(self.url)
+        html_doc = requests.get(self.url, headers=self.headers)
         soup = BeautifulSoup(html_doc.text, 'html.parser')
         tables = soup.find_all("table")[3]
         table_rows = tables.find_all("tr")
@@ -192,7 +193,7 @@ def get_data(self, file_name=None, match_format=11, data_type='allround', view='
             self.file_name = f"{self.player_id}_{self.match_format}_{self.data_type}_{self.view}.csv"
 
         self.url=f"https://stats.espncricinfo.com/ci/engine/player/{self.player_id}.html?class={self.match_format};template=results;type={self.data_type};view={self.view}"
-        html_doc = requests.get(self.url)
+        html_doc = requests.get(self.url, headers=self.headers())
         soup = BeautifulSoup(html_doc.text, 'html.parser')
         tables = soup.find_all("table")[3]
         table_rows = tables.find_all("tr")

espncricinfo/series.py

@@ -9,6 +9,7 @@ def __init__(self, series_id):
         self.json_url = "http://core.espnuk.org/v2/sports/cricket/leagues/{0}/".format(str(series_id))
         self.events_url = "http://core.espnuk.org/v2/sports/cricket/leagues/{0}/events".format(str(series_id))
         self.seasons_url = "http://core.espnuk.org/v2/sports/cricket/leagues/{0}/seasons".format(str(series_id))
+        self.headers = {'user-agent': 'Mozilla/5.0'}
         self.json = self.get_json(self.json_url)
         self.seasons = self._get_seasons()
         self.years = self._get_years_from_seasons()
@@ -25,7 +26,7 @@ def __init__(self, series_id):
             self.events = self._build_events()
 
     def get_json(self, url):
-        r = requests.get(url)
+        r = requests.get(url,headers=self.headers)
         if r.status_code == 404:
             raise "Not Found"
         else:

espncricinfo/summary.py

@@ -6,12 +6,13 @@ class Summary(object):
 
     def __init__(self):
         self.url = "http://static.cricinfo.com/rss/livescores.xml"
+        self.headers = {'user-agent': 'Mozilla/5.0'}
         self.xml = self.get_xml()
         self.match_ids = self._match_ids()
         self.matches = self._build_matches()
 
     def get_xml(self):
-        r = requests.get(self.url)
+        r = requests.get(self.url, headers=self.headers)
         if r.status_code == 404:
             raise MatchNotFoundError
         else: