0_2_teamboxscore_crawling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13a36536-9980-48a0-8c8b-8ada818768aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ea6f88bd-4052-429e-8333-8ef2358cc588",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 드라이버 종료\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8082e29f-9e5e-4685-935d-287dc7a19cc9",
   "metadata": {},
   "source": [
    "## team-box-score - playoffs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "32045d5e-d1e2-4b82-9b97-a7618d1f26c8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2014-15: Message: \n",
      "\n",
      "Season 2014-15 data collected.\n",
      "Season 2014-15 data saved.\n",
      "Cookie popup closed.\n",
      "Reached the last page or error occurred in season 2015-16: Message: \n",
      "\n",
      "Season 2015-16 data collected.\n",
      "Season 2015-16 data saved.\n",
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2016-17: Message: \n",
      "\n",
      "Season 2016-17 data collected.\n",
      "Season 2016-17 data saved.\n",
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2017-18: Message: \n",
      "\n",
      "Season 2017-18 data collected.\n",
      "Season 2017-18 data saved.\n",
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2018-19: Message: \n",
      "\n",
      "Season 2018-19 data collected.\n",
      "Season 2018-19 data saved.\n",
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2019-20: Message: \n",
      "\n",
      "Season 2019-20 data collected.\n",
      "Season 2019-20 data saved.\n",
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2020-21: Message: \n",
      "\n",
      "Season 2020-21 data collected.\n",
      "Season 2020-21 data saved.\n",
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2021-22: Message: \n",
      "\n",
      "Season 2021-22 data collected.\n",
      "Season 2021-22 data saved.\n",
      "No cookie popup to close: Message: \n",
      "Stacktrace:\n",
      "0   chromedriver                        0x0000000103214500 cxxbridge1$str$ptr + 1917112\n",
      "1   chromedriver                        0x000000010320c890 cxxbridge1$str$ptr + 1885256\n",
      "2   chromedriver                        0x0000000102e1c538 cxxbridge1$string$len + 89424\n",
      "3   chromedriver                        0x0000000102e60878 cxxbridge1$string$len + 368784\n",
      "4   chromedriver                        0x0000000102e9ab7c cxxbridge1$string$len + 607124\n",
      "5   chromedriver                        0x0000000102e55374 cxxbridge1$string$len + 322444\n",
      "6   chromedriver                        0x0000000102e55fc4 cxxbridge1$string$len + 325596\n",
      "7   chromedriver                        0x00000001031dbd2c cxxbridge1$str$ptr + 1685732\n",
      "8   chromedriver                        0x00000001031e0530 cxxbridge1$str$ptr + 1704168\n",
      "9   chromedriver                        0x00000001031c0e08 cxxbridge1$str$ptr + 1575360\n",
      "10  chromedriver                        0x00000001031e0e00 cxxbridge1$str$ptr + 1706424\n",
      "11  chromedriver                        0x00000001031b1f94 cxxbridge1$str$ptr + 1514316\n",
      "12  chromedriver                        0x00000001031fd62c cxxbridge1$str$ptr + 1823204\n",
      "13  chromedriver                        0x00000001031fd7ac cxxbridge1$str$ptr + 1823588\n",
      "14  chromedriver                        0x000000010320c530 cxxbridge1$str$ptr + 1884392\n",
      "15  libsystem_pthread.dylib             0x000000018068af94 _pthread_start + 136\n",
      "16  libsystem_pthread.dylib             0x0000000180685d34 thread_start + 8\n",
      "\n",
      "Reached the last page or error occurred in season 2022-23: Message: \n",
      "\n",
      "Season 2022-23 data collected.\n",
      "Season 2022-23 data saved.\n"
     ]
    }
   ],
   "source": [
    "# 필요한 라이브러리 임포트\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import time\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "'''\n",
    "# 크롬 옵션 설정\n",
    "options = Options()\n",
    "options.add_argument(\"--headless\")  \n",
    "options.add_argument(\"--disable-gpu\")\n",
    "options.add_argument(\"--no-sandbox\")  \n",
    "options.add_argument(\"--disable-dev-shm-usage\")  \n",
    "options.add_argument(\"--window-size=1920,1080\")\n",
    "options.add_argument(\"--remote-debugging-port=9223\")  # 다른 포트로 설정\n",
    "\n",
    "# ChromeDriver 설정\n",
    "service = Service(ChromeDriverManager().install())\n",
    "driver = webdriver.Chrome(service=service, options=options)\n",
    "'''\n",
    "\n",
    "driver = webdriver.Chrome()\n",
    "\n",
    "# 크롤링할 시즌 목록 (2000-01 시즌부터 2022-23시즌까지)\n",
    "seasons = [f\"{year}-{str(year + 1)[-2:]}\" for year in range(2014, 2023)]\n",
    "\n",
    "# 전체 데이터를 저장할 데이터프레임\n",
    "all_data = pd.DataFrame()\n",
    "\n",
    "# 각 시즌에 대해 크롤링\n",
    "for season in seasons:\n",
    "    url = f\"https://www.nba.com/stats/teams/boxscores?Season={season}&SeasonType=Playoffs\"\n",
    "    driver.get(url)\n",
    "    \n",
    "    # 페이지 로드 대기\n",
    "    time.sleep(5)\n",
    "\n",
    "    # 팝업 닫기 시도 (있을 경우)\n",
    "    try:\n",
    "        cookie_button = WebDriverWait(driver, 10).until(\n",
    "            EC.element_to_be_clickable((By.XPATH, '//*[@id=\"onetrust-accept-btn-handler\"]'))\n",
    "        )\n",
    "        cookie_button.click()\n",
    "        print(\"Cookie popup closed.\")\n",
    "    except Exception as e:\n",
    "        print(f\"No cookie popup to close: {e}\")\n",
    "\n",
    "    # 열 이름 설정\n",
    "    columns = []\n",
    "\n",
    "    # 최대 페이지 설정\n",
    "    max_pages = 5\n",
    "\n",
    "    # 시즌 데이터 저장용 데이터프레임\n",
    "    season_data = pd.DataFrame(columns=columns)\n",
    "\n",
    "    # 페이지 반복 처리\n",
    "    while True:\n",
    "        # 페이지의 전체 HTML 가져오기\n",
    "        page_source = driver.page_source\n",
    "        soup = BeautifulSoup(page_source, 'lxml')\n",
    "\n",
    "        # 테이블을 찾기\n",
    "        table = soup.find('section', {'class': 'Block_block__62M07 nba-stats-content-block'})\n",
    "\n",
    "        #열 이름 추출 (첫 페이지에서만 수행)\n",
    "        if not columns:\n",
    "            header_row = table.find('tr', {'class': 'Crom_headers__mzI_m'})\n",
    "            columns = [th.get_text(strip=True) for th in header_row.find_all('th') if th.get_text(strip=True)][:25]\n",
    "            columns.append('Season')  # 시즌 정보를 추가\n",
    "            season_data = pd.DataFrame(columns=columns)\n",
    "\n",
    "        # 데이터 행 추출\n",
    "        rows = table.find('tbody', {'class': 'Crom_body__UYOcU'}).find_all('tr')\n",
    "        if not rows:\n",
    "            print(f\"No data rows found for season {season}.\")\n",
    "            break\n",
    "\n",
    "        # 각 행의 데이터 추출\n",
    "        for row in rows:\n",
    "            data = [td.get_text(strip=True) for td in row.find_all('td')][0:]\n",
    "            data.append(season)  # 시즌 정보 추가\n",
    "            \n",
    "            season_data = pd.concat([season_data, pd.DataFrame([data], columns=columns)], ignore_index=True)\n",
    "\n",
    "        # 다음 페이지로 이동\n",
    "        try:\n",
    "            next_button = WebDriverWait(driver, 10).until(\n",
    "                EC.element_to_be_clickable((By.XPATH, '//*[@id=\"__next\"]/div[2]/div[2]/div[3]/section[2]/div/div[2]/div[2]/div[1]/div[5]/button[2]'))\n",
    "            )\n",
    "            if \"disabled\" in next_button.get_attribute(\"class\"):\n",
    "                print(f\"Reached the last page for season {season}.\")\n",
    "                break\n",
    "            next_button.click()\n",
    "            time.sleep(5)  # 페이지 로드 대기\n",
    "        except Exception as e:\n",
    "            print(f\"Reached the last page or error occurred in season {season}: {e}\")\n",
    "            break\n",
    "\n",
    "    # 해당 시즌의 데이터를 전체 데이터프레임에 추가\n",
    "    globals()[f'all_data_{season}'] = pd.concat([all_data, season_data], ignore_index=True)\n",
    "    print(f\"Season {season} data collected.\")\n",
    "    globals()[f'all_data_{season}'].to_csv(f'team_boxscore/team_boxscore_{season}.csv',index=False)\n",
    "    print(f\"Season {season} data saved.\")\n",
    "\n",
    "# 드라이버 종료\n",
    "driver.quit()\n",
    "\n",
    "# 결과 출력 또는 저장\n",
    "#print(all_data.head())\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}