python3: Changing user-agent on urllib2

[voyeg3r]

References

• https://stackoverflow.com/questions/24226781/changing-user-agent-in-python-3-for-urrlib-request-urlopen
• https://stackoverflow.com/q/24226781/2571881
• http://wolfprojects.altervista.org/articles/change-urllib-user-agent/

from urllib.request import urlopen, Request, FancyURLopener
from bs4 import BeautifulSoup

class MyOpener(FancyURLopener):
version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11'

myopener = MyOpener()
page = myopener.open('http://www.mairovergara.com/como-se-diz-cuidar-da-propria-vida-em-ingles/')
page.read()

# Até aqui o código funciona muito bem sem beautifulsoup

Another great tip came from here

• https://stackoverflow.com/a/40257326/2571881

import requests
from bs4 import BeautifulSoup

headers = {'User-Agent': 'Mozilla/5.0'}

url = "https://linkedin.com/company/1005"

r = requests.get(url, headers=headers)
print(r.text)

soup = BeautifulSoup(r.text, 'html.parser')
print(soup.prettify())