Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update the scripts #1552

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

update the scripts #1552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Lecture_10/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
geopandas
numpy
pandas
shapely
fiona
pyproj
rtree
2 changes: 1 addition & 1 deletion Lecture_7/SBS_regionales.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1482,7 +1482,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
"version": "3.9.0"
},
"toc": {
"base_numbering": 1,
Expand Down
122 changes: 122 additions & 0 deletions Lecture_7/Untitled.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

35 changes: 27 additions & 8 deletions Lecture_7/scraper_news_tutorial.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 6,
"id": "ce973d69",
"metadata": {},
"outputs": [],
Expand All @@ -24,7 +24,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 7,
"id": "987dda58",
"metadata": {},
"outputs": [],
Expand All @@ -34,7 +34,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 8,
"id": "9ac14724",
"metadata": {},
"outputs": [],
Expand All @@ -61,10 +61,29 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 10,
"id": "63ff8aad",
"metadata": {},
"outputs": [],
"outputs": [
{
"ename": "ValueError",
"evalue": "There is no such driver by url https://chromedriver.storage.googleapis.com/LATEST_RELEASE_128.0.6613",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
"Input \u001b[1;32mIn [10]\u001b[0m, in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 1\u001b[0m options \u001b[38;5;241m=\u001b[39m Options()\n\u001b[1;32m----> 2\u001b[0m service \u001b[38;5;241m=\u001b[39m Service(\u001b[43mChromeDriverManager\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minstall\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m 3\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttp://www.archiviolastampa.it/\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 5\u001b[0m driver \u001b[38;5;241m=\u001b[39m webdriver\u001b[38;5;241m.\u001b[39mChrome(service\u001b[38;5;241m=\u001b[39mservice, options\u001b[38;5;241m=\u001b[39moptions)\n",
"File \u001b[1;32m~\\anaconda3\\envs\\geopandas\\lib\\site-packages\\webdriver_manager\\chrome.py:39\u001b[0m, in \u001b[0;36mChromeDriverManager.install\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 38\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minstall\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mstr\u001b[39m:\n\u001b[1;32m---> 39\u001b[0m driver_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_driver_path\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdriver\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 40\u001b[0m os\u001b[38;5;241m.\u001b[39mchmod(driver_path, \u001b[38;5;241m0o755\u001b[39m)\n\u001b[0;32m 41\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m driver_path\n",
"File \u001b[1;32m~\\anaconda3\\envs\\geopandas\\lib\\site-packages\\webdriver_manager\\core\\manager.py:30\u001b[0m, in \u001b[0;36mDriverManager._get_driver_path\u001b[1;34m(self, driver)\u001b[0m\n\u001b[0;32m 27\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m binary_path:\n\u001b[0;32m 28\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m binary_path\n\u001b[1;32m---> 30\u001b[0m file \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_download_manager\u001b[38;5;241m.\u001b[39mdownload_file(\u001b[43mdriver\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_driver_download_url\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m 31\u001b[0m binary_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdriver_cache\u001b[38;5;241m.\u001b[39msave_file_to_cache(driver, file)\n\u001b[0;32m 32\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m binary_path\n",
"File \u001b[1;32m~\\anaconda3\\envs\\geopandas\\lib\\site-packages\\webdriver_manager\\drivers\\chrome.py:40\u001b[0m, in \u001b[0;36mChromeDriver.get_driver_download_url\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 39\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_driver_download_url\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m---> 40\u001b[0m driver_version_to_download \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_driver_version_to_download\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 41\u001b[0m os_type \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_os_type\n\u001b[0;32m 42\u001b[0m \u001b[38;5;66;03m# For Mac ARM CPUs after version 106.0.5249.61 the format of OS type changed\u001b[39;00m\n\u001b[0;32m 43\u001b[0m \u001b[38;5;66;03m# to more unified \"mac_arm64\". For newer versions, it'll be \"mac_arm64\"\u001b[39;00m\n\u001b[0;32m 44\u001b[0m \u001b[38;5;66;03m# by default, for lower versions we replace \"mac_arm64\" to old format - \"mac64_m1\".\u001b[39;00m\n",
"File \u001b[1;32m~\\anaconda3\\envs\\geopandas\\lib\\site-packages\\webdriver_manager\\core\\driver.py:51\u001b[0m, in \u001b[0;36mDriver.get_driver_version_to_download\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 45\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 46\u001b[0m \u001b[38;5;124;03mDownloads version from parameter if version not None or \"latest\".\u001b[39;00m\n\u001b[0;32m 47\u001b[0m \u001b[38;5;124;03mDownloads latest, if version is \"latest\" or browser could not been determined.\u001b[39;00m\n\u001b[0;32m 48\u001b[0m \u001b[38;5;124;03mDownloads determined browser version driver in all other ways as a bonus fallback for lazy users.\u001b[39;00m\n\u001b[0;32m 49\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m 50\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_driver_to_download_version:\n\u001b[1;32m---> 51\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_driver_to_download_version \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_version \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_version \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m (\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlatest\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_latest_release_version\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 52\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_driver_to_download_version\n",
"File \u001b[1;32m~\\anaconda3\\envs\\geopandas\\lib\\site-packages\\webdriver_manager\\drivers\\chrome.py:62\u001b[0m, in \u001b[0;36mChromeDriver.get_latest_release_version\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 56\u001b[0m log(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGet LATEST \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m version for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_browser_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 57\u001b[0m latest_release_url \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m 58\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_latest_release_url\n\u001b[0;32m 59\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_version \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlatest\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m determined_browser_version \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m 60\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_latest_release_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdetermined_browser_version\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m 61\u001b[0m )\n\u001b[1;32m---> 62\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_http_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlatest_release_url\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 63\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mtext\u001b[38;5;241m.\u001b[39mrstrip()\n",
"File \u001b[1;32m~\\anaconda3\\envs\\geopandas\\lib\\site-packages\\webdriver_manager\\core\\http.py:37\u001b[0m, in \u001b[0;36mWDMHttpClient.get\u001b[1;34m(self, url, **kwargs)\u001b[0m\n\u001b[0;32m 35\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m exceptions\u001b[38;5;241m.\u001b[39mConnectionError:\n\u001b[0;32m 36\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCould not reach host. Are you offline?\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 37\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresp\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m 38\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m wdm_progress_bar():\n\u001b[0;32m 39\u001b[0m show_download_progress(resp)\n",
"File \u001b[1;32m~\\anaconda3\\envs\\geopandas\\lib\\site-packages\\webdriver_manager\\core\\http.py:16\u001b[0m, in \u001b[0;36mHttpClient.validate_response\u001b[1;34m(resp)\u001b[0m\n\u001b[0;32m 14\u001b[0m status_code \u001b[38;5;241m=\u001b[39m resp\u001b[38;5;241m.\u001b[39mstatus_code\n\u001b[0;32m 15\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m status_code \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m404\u001b[39m:\n\u001b[1;32m---> 16\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThere is no such driver by url \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresp\u001b[38;5;241m.\u001b[39murl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m 17\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m status_code \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m401\u001b[39m:\n\u001b[0;32m 18\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAPI Rate limit exceeded. You have to add GH_TOKEN!!!\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
"\u001b[1;31mValueError\u001b[0m: There is no such driver by url https://chromedriver.storage.googleapis.com/LATEST_RELEASE_128.0.6613"
]
}
],
"source": [
"options = Options()\n",
"service = Service(ChromeDriverManager().install())\n",
Expand Down Expand Up @@ -2067,9 +2086,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "scraper_ver",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "scraper_ver"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand All @@ -2081,7 +2100,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.5"
"version": "3.9.0"
}
},
"nbformat": 4,
Expand Down
Loading