crawler.py

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from bs4 import BeautifulSoup
import re # Regular expressionc
from datetime import date, timedelta
from time import sleep
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import StaleElementReferenceException
from selenium.common.exceptions import ElementNotInteractableException

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.core.os_manager  import ChromeType
from selenium.webdriver.chrome.options import Options

# Mở Chrome và cho full màn hình
# Tự tải phiên bản ChromeDriver phù hợp với phiên bản trên máy

chrome_options = Options()
options = [
    "--headless=new",
    "--disable-gpu",
    "--start-maximized",
    "--ignore-certificate-errors",
    "--disable-extensions",
    "--no-sandbox",
    "--disable-dev-shm-usage"
]
for option in options:
    chrome_options.add_argument(option)

# browser = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options) #webdriver.Chrome("chromedriver.exe")

chrome_service = Service(ChromeDriverManager(chrome_type=ChromeType.CHROMIUM).install())
browser = webdriver.Chrome(service=chrome_service, options=chrome_options)
    
browser.maximize_window()
# Set thời gian chờ mặc định là 10 giây
browser.implicitly_wait(10)
# Mở trang chủ của Vietjet
url = 'https://www.vietjetair.com'
browser.get(url)

wait = WebDriverWait(browser, 10)

# Chọn sử dụng cookie
wait.until(EC.element_to_be_clickable((By.XPATH, '//div[@role="dialog"]//button'))).click()

# # Tắt quảng cáo 
# wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@id="onesignal-slidedown-cancel-button"]'))).click()

# close_buttons = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'button.MuiIconButton-root[aria-label="close"]')))
# for close_button in close_buttons[1:]:
#     close_button.click()

sleep(3)
# Chọn vé một chiều
wait.until(EC.element_to_be_clickable((
    By.XPATH, '//span[text()="Một chiều"]'))).click()

# Chọn điểm khởi hành là TP HCM 
wait.until(EC.element_to_be_clickable((By.XPATH, '//*[@class="MuiInputBase-input MuiOutlinedInput-input"]'))).click()
# selectOrigin = browser.find_element(By.XPATH, '//*[@class="MuiInputBase-input MuiOutlinedInput-input"]')
# selectOrigin.click()
# sleep(3)

# Chọn sân bay Tân Sơn Nhất
wait.until(EC.element_to_be_clickable((By.XPATH, '//*[text()="Sân bay Tân Sơn Nhất"]'))).click()

# Chọn sân bay Đà Nẵng
try:
    wait.until(EC.element_to_be_clickable((By.XPATH, '//*[text()="Sân bay Đà Nẵng"]'))).click()
except:
    wait.until(EC.element_to_be_clickable((By.XPATH, '//*[text()="Sân bay Đà Nẵng"]'))).click()
    
# Chọn ngày tháng 
today = date.today()
tomorrow = today + timedelta(1)
tomorrow_year = str(tomorrow.year)
tomorrow_month = tomorrow.strftime('%m')
tomorrow_day = str(tomorrow.day)


# Chuỗi ngày tháng để click
str_time = 'tháng '+ tomorrow_month + ' ' + tomorrow_year 

wait.until(EC.presence_of_element_located((
    By.XPATH, '//*[contains(text(),"{a}")]//following-sibling::div[2]//descendant::span[text()="{b}"]'.format(a = str_time, b = tomorrow_day)))).click()

# Click Tìm chuyến bay
submit = wait.until(EC.visibility_of_element_located((By.XPATH, '(//*[text()="Tìm chuyến bay"])[2]'))) 
try: 
    submit.click()
except:
    submit.click()
    
sleep(.5)
    
from lxml import html
# Chọn số ngày cần lấy dữ liệu
nums_date = 12
tree = BeautifulSoup(browser.page_source, 'html.parser')

def find_by_xpath(element_source,xpath_expression):
    try: 
        root = html.fromstring(element_source)
        return root.xpath(xpath_expression) 
    except:
        return []

def check_seat(browser, rows, type, i): # kiểm tra xem còn chỗ hay không: type = hạng vé (tương ứng là 1, 2, 3, 4 )
    check = browser.find_element(By.XPATH, rows + f'[{i+1}]/div/div/div[2]/div[{type}]//p[text()="Hết chỗ"]')
    try: 
        if check: # Nếu hết chỗ thì return về xpath tương ứng
            return rows + f'[{i+1}]/div/div/div[2]/div[{type}]/div//p/text()'
    except: return rows + f'[{i+1}]/div/div/div[2]/div[{type}]/div/p[1]/text()'

def click_with_js(element):
    """Clicks an element using JavaScript"""
    browser.execute_script("arguments[0].click();", element)
    

class display_to_be_flex(object):
    """An expectation for checking that an element's display property has been changed to 'flex'."""
    def __init__(self, locator):
        self.locator = locator

    def __call__(self, driver):
        try:
            element = EC.presence_of_element_located(self.locator)(driver)
            if driver.execute_script("return getComputedStyle(arguments[0]).display;", element) == "flex":
                return element
            else:
                return False
        except StaleElementReferenceException:
            return False


data = []
root = '//*[@id="root"]/div[1]/div[2]/div/div/div/div[1]/div/div/div[4]/div[2]/div[1]/div'
def get_rows(browser, addr):
    sleep(1)
    
    # chờ để thông tin các chuyến bay được load đầy đủ và hiển thị trong css
    wait.until(display_to_be_flex((By.XPATH, '//*[@id="root"]/div[1]/div[2]/div/div/div/div[1]/div/div/div[4]/div[2]/div[2]')))
    wait.until(EC.visibility_of_all_elements_located((By.XPATH, root)))
    # browser.execute_script("document.querySelector('#my-button').click();")
    
    # scroll xuống cuối trang để thấy hết các muibox element
    browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    sleep(1)
    # scroll lại lên đầu trang để bắt đầu crawl
    browser.execute_script("window.scrollTo(0, 0)")
    n_rows = len(find_by_xpath(browser.page_source, root))
    print(n_rows)
    for i in range(n_rows):
        print(i, end = ' ')
        click = True
        
        ID = find_by_xpath(browser.page_source, 
                        root + f'[{i+1}]/div/div/div[1]/div[1]/span/span/text()')[0]
        
        # Thay đổi xpath để lấy text đúng trong trường hợp hết chỗ: div/p[1]/text() -> div//p//text()
        skyBoss_business = find_by_xpath(browser.page_source, root + f'[{i+1}]/div/div/div[2]/div[1]/div//p//text()')[0]
        skyBoss = find_by_xpath(browser.page_source, root + f'[{i+1}]/div/div/div[2]/div[2]/div//p//text()')[0]
        Deluxe = find_by_xpath(browser.page_source, root + f'[{i+1}]/div/div/div[2]/div[3]/div//p//text()')[0]
        Eco = find_by_xpath(browser.page_source, root + f'[{i+1}]/div/div/div[2]/div[4]/div//p//text()')[0]
        if skyBoss_business == skyBoss == Deluxe == Eco == 'Hết chỗ':
            click = False
        
        row_elem = wait.until(EC.element_to_be_clickable((By.XPATH, root + f'[{i+1}]/div/div[1]/div[1]')))
        if click:
            click_with_js(row_elem)
        sleep(0.01)
        
        # scroll tới ô cần click
        # browser.execute_script('return arguments[0].scrollIntoView(true);',
        #                        wait.until(EC.element_to_be_clickable((By.XPATH, rows + f'[{i+1}]/div/div[1]/div[1]'))))
        
        
        # scroll tới ô hiện ra thông tin
        print('click = ', click)
        if click:
            browser.execute_script('return arguments[0].scrollIntoView(true);',
                           wait.until(EC.element_to_be_clickable((By.XPATH, root + f'[{i+1}]/div/div[2]'))))
        else: 
            pass
        
        hours = find_by_xpath(browser.page_source, root + f'[{i+1}]/div/div/div[1]/div[2]/text()')
        date = find_by_xpath(browser.page_source, addr + '/div/div/div/p[2]/text()')[0]
        
        
        # find the element with the string "Tổng tiền" using XPath
        # total_fee = find_by_xpath(browser.page_source, '//h4[text()="Tổng tiền"]/following-sibling::h4/text()')
        # print(total_fee)
        
        start_ = find_by_xpath(browser.page_source, root + 
    f'[{i+1}]/div/div[2]/div/div/div/div/div/div/div[2]/div/div[1]/div/div[2]/p[1][1]/text()')[0] if click else hours[0][:-1] + ' ' + date
        end_ = find_by_xpath(browser.page_source, root + 
    f'[{i+1}]/div/div[2]/div/div/div/div/div/div/div[2]/div/div[2]/div/div[2]/p[1][1]/text()')[0] if click else hours[1][1:] + ' ' + date
        airbus = ''.join(element for element in find_by_xpath(browser.page_source, root + f'[{i+1}]/div/div/div[1]/div[3]/span/text()'))
        # wait.until(EC.presence_of_element_located((By.XPATH, root + f'[{i+1}]/div/div[1]/div[1]'))).click()
        
        data.append({'Số hiệu chuyến bay': ID, 'Khởi hành': start_, 'Đến': end_,'Airbus':airbus,
                     'SkyBOSS_Business':skyBoss_business, 'SkyBOSS': skyBoss, 'Deluxe': Deluxe, 'Eco': Eco})
        if click:
            click_with_js(row_elem)
    print('Đã load xong...\n')
    # scroll lên đầu trang để chuyển qua ngày khác
    browser.find_element(By.TAG_NAME, "body").send_keys(Keys.CONTROL + Keys.HOME)
    return data

slick_track = tree.find("div", class_="slick-track") 
count = 0
rows = '//*[@id="root"]/div[1]/div[2]/div/div/div/div[1]/div/div/div[4]/div[2]/div[1]/div'

childrens = list(slick_track.children)[:nums_date]

if find_by_xpath(browser.page_source, '//h5[text()="Không tìm thấy chuyến bay nào cho lựa chọn của bạn. Quay lại để chọn ngày khác."]'):
    childrens = list(slick_track.children)[1:nums_date+1]
    
for child in childrens:
    # print(f'{(int(child.attrs["data-index"])+1)*"."}')
    sleep(1)
    # if wait.until(EC.visibility_of_element_located((By.XPATH, '(//p[contains(text(), "Hết chỗ")])'))):
    #     browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    #     wait.until(EC.visibility_of_element_located((By.XPATH, f'(//p[contains(text(), "Hết chỗ")]/ancestor::div/ancestor::div/following-sibling::div/div/p[contains(text(), "{tomorrow_day}")])'))).click()
    #     wait.until(EC.visibility_of_element_located((By.XPATH, '//span[contains(text(), "Đi tiếp")]'))).click()
        
    # browser.execute_script("window.scrollTo(0,0);")
    addr = f'//*[@data-index="{int(child.attrs["data-index"])}"]'
    next_day = wait.until(EC.element_to_be_clickable((By.XPATH, f'//*[@data-index="{int(child.attrs["data-index"])}"]')))
    try:
        next_day.click()
    except:
        next_day.click()

    # print(len(find_by_xpath(browser.page_source, '//*[@id="root"]/div[1]/div[2]/div/div/div/div[1]/div/div/div[4]/div[2]/div[1]/div')))
    count += len(find_by_xpath(browser.page_source, '//*[@id="root"]/div[1]/div[2]/div/div/div/div[1]/div/div/div[4]/div[2]/div[1]/div'))
    # get_rows_selenium(browser)
    get_rows(browser, addr)
print('Tổng số chuyến bay: ', count)

#########################################################################################################################################
import re 
import os

flight = pd.DataFrame(data)
ticket_class = ['SkyBOSS_Business', 'SkyBOSS', 'Deluxe', 'Eco']
for i in ticket_class:
    flight[i] = flight[i].apply(lambda x: np.nan if x =='Hết chỗ' else x.replace(',', '') + 'e3').astype(float)

pattern = r'(\d{1,2})\s+tháng\s+(\d{1,2})'
flight['Khởi hành'] = flight['Khởi hành'].str.replace('( \(Giờ địa phương\))',
    '', regex=True).str.replace(pattern, lambda x: f"{int(x.group(1)):02}/{int(x.group(2)):02}/2023", regex=True).str.replace(',', '', regex=True)
flight['Đến'] = flight['Đến'].str.replace('( \(Giờ địa phương\))',
    '', regex=True).str.replace(pattern, lambda x: f"{int(x.group(1)):02}/{int(x.group(2)):02}/2023", regex=True).str.replace(',', '', regex=True)

str_time = '%H:%M %d/%m/%Y'
flight['Khởi hành'] = pd.to_datetime(flight['Khởi hành'], format=str_time)
flight['Đến'] = pd.to_datetime(flight['Đến'], format=str_time)

fee = 593000.0
flight[['SkyBOSS_Business', 'SkyBOSS', 'Deluxe', 'Eco']] = flight[['SkyBOSS_Business', 'SkyBOSS', 'Deluxe', 'Eco']] * 1.1+ fee
flight['Thuế_phí'] = 593.0

path = today.strftime('%d%m%Y')

if not os.path.exists(path):
    # Create the directory
    os.makedirs(path)
    print(f"Directory '{path}' created successfully.")
else:
    print(f"Directory '{path}' already exists.")
    
    
flight.to_csv(path + f'/{path}.csv')