meli_scraping.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MeLi scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! pip install requests\n",
    "#! pip install beautifulsoup4\n",
    "#! pip install matplotlib\n",
    "#! pip install ipykernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import utils_meli\n",
    "from importlib import reload\n",
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "from datetime import datetime\n",
    "import plotly.express as px\n",
    "\n",
    "from IPython.display import display\n",
    "pd.options.display.max_colwidth = 170\n",
    "pd.options.display.max_rows = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parent_path = r\"\".replace(\"\\\\\", \"/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(utils_meli)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 30 min aprox.\n",
    "reload(utils_meli)\n",
    "df = utils_meli.load_html_search()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[4,'Comments']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils_meli.open_link(df,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx in range(len(df)-1):\n",
    "    for comm in df.loc[idx,'Comments']:\n",
    "        if re.search(r'excelente', comm, flags=re.IGNORECASE):\n",
    "            display(df.iloc[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils_meli.open_link(df,250)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from unidecode import unidecode\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy.cli.download(\"es_core_news_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"es_core_news_sm\")\n",
    "pat = re.compile(r\"[^a-z ]\")\n",
    "spaces = re.compile(r\"\\s{2,}\")\n",
    "\n",
    "def preprocess(text, min_len=1, max_len=1000):\n",
    "    # spacy Doc creation\n",
    "    doc = nlp(text)\n",
    "    # Remove stopwords\n",
    "    filtered_tokens = filter(\n",
    "            lambda token: not token.is_stop,\n",
    "            doc\n",
    "            )\n",
    "    # Filter words by length and remove stop words\n",
    "    filtered_tokens2 = filter(\n",
    "            lambda token: len(token) >= min_len and len(token) <= max_len and not token.is_stop,\n",
    "            filtered_tokens\n",
    "        )\n",
    "    # Lemmatization\n",
    "    lemmas = map(\n",
    "            lambda token: token.lemma_,\n",
    "            filtered_tokens2\n",
    "            )\n",
    "    lemma_text = \" \".join(lemmas)\n",
    "    # Normalize text\n",
    "    norm_text = unidecode(lemma_text)\n",
    "    # Remove accents\n",
    "    lower_text = norm_text.lower()\n",
    "    # Remove special characters\n",
    "    clean_text = re.sub(pat, \"\", lower_text)\n",
    "    # Remomove duplicate spaces (if exist)\n",
    "    spaces_text = re.sub(spaces, \" \", clean_text)\n",
    "    return spaces_text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_comments = []\n",
    "\n",
    "for comm in df['Comments']:\n",
    "    all_comments.extend(comm)\n",
    "\n",
    "print(all_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prep_comments = list(map(preprocess,all_comments))\n",
    "print(prep_comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vect = (\n",
    "    CountVectorizer(max_features=1000, max_df=0.7)\n",
    "    .fit(prep_comments)\n",
    "    )\n",
    "\n",
    "X = vect.transform(prep_comments)\n",
    "vocab = vect.get_feature_names_out()\n",
    "\n",
    "counts = np.array(X.sum(axis=0)).flatten()\n",
    "counts_dict = {word: count for word, count in zip(vocab, counts)}\n",
    "\n",
    "mask = np.array(Image.open(r\"\\cloud.png\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wc = WordCloud(background_color='white',\n",
    "        width=3000,\n",
    "        height=2000,\n",
    "        collocations=False,\n",
    "        mask=mask,\n",
    "        colormap = 'Dark2',\n",
    "        max_words=40\n",
    ").generate_from_frequencies(counts_dict)\n",
    "\n",
    "plt.figure(figsize=[7,7])\n",
    "plt.imshow(wc,interpolation=\"bilinear\")\n",
    "plt.axis(\"off\") \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "utils_meli.open_link(df,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel(os.path.join(parent_path,\"_{}_{}_{}.xlsx\".format(datetime.today().year,datetime.today().month,datetime.today().day)), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pass the DataFrame and copy and paste the product name you want to see in browser\n",
    "reload(utils_meli)\n",
    "utils_meli.open_link(df,'')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pass the DataFrame and index you want to open in browser\n",
    "reload(utils_meli)\n",
    "utils_meli.open_link(df,3)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Price Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel(r\"\".replace(\"\\\\\", \"/\"))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(utils_meli)\n",
    "df_25 = utils_meli.quartile_prices(df)[0]\n",
    "df_25_50 = utils_meli.quartile_prices(df)[1]\n",
    "df_50_75 = utils_meli.quartile_prices(df)[2]\n",
    "df_75 = utils_meli.quartile_prices(df)[3]\n",
    "quartiles = utils_meli.quartile_prices(df)[4]\n",
    "print(f'Median: {df.Price.median()}')\n",
    "quartiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Products above quartile 75\n",
    "print('Prime products [above quartile 75] -> {} products ({}%) '.format(df_75.Product.to_list().__len__(), round((df_75.Product.to_list().__len__() / df.__len__()) * 100, 2)))\n",
    "print('Premium products [between quartiles 50 & 75] -> {} products ({}%) '.format(df_50_75.Product.to_list().__len__(), round((df_50_75.Product.to_list().__len__() / df.__len__()) * 100, 2)))\n",
    "print('Mid products [between quartiles 25 & 50] -> {} products ({}%) '.format(df_25_50.Product.to_list().__len__(), round((df_25_50.Product.to_list().__len__() / df.__len__()) * 100, 2)))\n",
    "print('Low products [below quartile 25 ] -> {} products ({}%) '.format(df_25.Product.to_list().__len__(), round((df_25.Product.to_list().__len__() / df.__len__()) * 100, 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Quartile (75>) - Products: {}'.format(df_75.__len__()))\n",
    "display(df_75.head())\n",
    "display(px.bar(df_75.Brand.value_counts().to_frame()[df_75.Brand.value_counts().to_frame()['Brand']>=10], x='Brand', width=1100, height=600, text_auto=True,\n",
    "       labels={'index':'Brand', 'Brand':'Count'}).update_layout(xaxis={'categoryorder':'total descending'}, title='Brands over 10 search results'))\n",
    "\n",
    "print('Quartile (50-75] - Products: {}'.format(df_50_75.__len__()))\n",
    "display(df_50_75.head())\n",
    "display(px.bar(df_50_75.Brand.value_counts().to_frame()[df_50_75.Brand.value_counts().to_frame()['Brand']>=10], x='Brand', width=1100, height=600, text_auto=True,\n",
    "       labels={'index':'Brand', 'Brand':'Count'}).update_layout(xaxis={'categoryorder':'total descending'}, title='Brands over 10 search results'))\n",
    "\n",
    "print('Quartile (25-50] - Products: {}'.format(df_25_50.__len__()))\n",
    "display(df_25_50.head())\n",
    "display(px.bar(df_25_50.Brand.value_counts().to_frame()[df_25_50.Brand.value_counts().to_frame()['Brand']>=10], x='Brand', width=1100, height=600, text_auto=True,\n",
    "       labels={'index':'Brand', 'Brand':'Count'}).update_layout(xaxis={'categoryorder':'total descending'}, title='Brands over 10 search results'))\n",
    "\n",
    "print('Quartile (<25] - Products: {}'.format(df_25.__len__()))\n",
    "display(df_25.head())\n",
    "display(px.bar(df_25.Brand.value_counts().to_frame()[df_25.Brand.value_counts().to_frame()['Brand']>=10], x='Brand', width=1100, height=600, text_auto=True,\n",
    "       labels={'index':'Brand', 'Brand':'Count'}).update_layout(xaxis={'categoryorder':'total descending'}, title='Brands over 10 search results'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "px.box(df,'Price', title=f'Price distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df[df['Price'] <= 831].shape[0])\n",
    "print(df[df['Price'] >= 449].shape[0] / df.shape[0])\n",
    "print(df[(df['Price'] > 247) & (df['Price'] <= 449)].shape[0] / df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(px.histogram(df, x='Price', title=f'Price distribution'))\n",
    "display(px.histogram(df_75, x='Price', title=f'Price distribution - Prime Products', nbins=10, text_auto=True))\n",
    "display(df_75.describe().round(2))\n",
    "display(px.histogram(df_50_75, x='Price', title=f'Price distribution - Premium Products', nbins=15, text_auto=True))\n",
    "display(df_50_75.describe().round(2))\n",
    "display(px.histogram(df_25_50, x='Price', title=f'Price distribution - Mid Products', nbins=15, text_auto=True))\n",
    "display(df_25_50.describe().round(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Prime brands\n",
    "display(df_75.Brand.value_counts().to_frame().reset_index().rename(columns={'index':'Brand','Brand':'Results'}).head(10))\n",
    "'''for i in df_75.Brand.value_counts().to_frame().reset_index().rename(columns={'index':'Brand','Brand':'Results'}).head(10).Brand.tolist():\n",
    "    print(i)'''\n",
    "\n",
    "#Premium brands\n",
    "display(df_50_75.Brand.value_counts().to_frame().reset_index().rename(columns={'index':'Brand','Brand':'Results'}).head(10))\n",
    "'''for i in df_50_75.Brand.value_counts().to_frame().reset_index().rename(columns={'index':'Brand','Brand':'Results'}).head(10).Brand.tolist():\n",
    "    print(i)'''\n",
    "\n",
    "#Mid brands\n",
    "display(df_25_50.Brand.value_counts().to_frame().reset_index().rename(columns={'index':'Brand','Brand':'Results'}).head(10))\n",
    "'''for i in df_25_50.Brand.value_counts().to_frame().reset_index().rename(columns={'index':'Brand','Brand':'Results'}).head(10).Brand.tolist():\n",
    "    print(i)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = df_25_50.Brand.value_counts().to_frame().reset_index().rename(columns={'index':'Brand','Brand':'Results'}).head(10)\n",
    "a['Proc'] = round((a.Results / a.Results.sum())*100,2)\n",
    "a"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Brands distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "px.bar(df.Brand.value_counts().to_frame()[df.Brand.value_counts().to_frame()['Brand']>=10], x='Brand', width=1100, height=600, text_auto=True,\n",
    "       labels={'index':'Brand', 'Brand':'Count'}).update_layout(xaxis={'categoryorder':'total descending'}, title='Brands over 10 search results')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df[df.Brand!='-'].Brand.unique().tolist())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Brand appearences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Look for specif word(s)\n",
    "brand_products = []\n",
    "\n",
    "for i in df[df['Score']==5]['Product']:\n",
    "    if (re.search(r'', i, flags=re.IGNORECASE)) or (re.search(r'', i, flags=re.IGNORECASE)) or (re.search(r'', i, flags=re.IGNORECASE)) or (re.search(r'', i, flags=re.IGNORECASE)) \\\n",
    "       or (re.search(r'', i, flags=re.IGNORECASE)) or (re.search(r'', i, flags=re.IGNORECASE)) or (re.search(r'', i, flags=re.IGNORECASE)) or (re.search(r'', i, flags=re.IGNORECASE)):\n",
    "        brand_products.append(i)\n",
    "        \n",
    "df_filtered = df[df.Product.isin(brand_products)].reset_index(drop=True)\n",
    "df_filtered = df_filtered.drop_duplicates(subset=['Product', 'Price', 'Score', 'Ratings'], keep='first',ignore_index=True)\n",
    "\n",
    "print(df_filtered.shape)\n",
    "df_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filtered.to_excel(os.path.join(parent_path,\"_{}_{}_{}.xlsx\".format(datetime.today().year,datetime.today().month,datetime.today().day)), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "brand_producs = []\n",
    "brand_prices = []\n",
    "for i in range(df.__len__()):\n",
    "    if (re.search('', df.loc[i,'Product'], flags=re.IGNORECASE)) or (re.search('', df.loc[i,'Product'], flags=re.IGNORECASE)) or (re.search('', df.loc[i,'Product'], flags=re.IGNORECASE)):\n",
    "        count += 1\n",
    "        brand_producs.append(df.loc[i,'Product'])\n",
    "        brand_prices.append(df.loc[i,'Price'])\n",
    "print(count)\n",
    "\n",
    "brand_df = pd.DataFrame(data={'Product':brand_producs, 'Price':brand_prices})\n",
    "print(round(brand_df.Price.mean(), 2))\n",
    "brand_df.head(10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import webbrowser\n",
    "import requests\n",
    "import bs4\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = input('Insert your search:')\n",
    "url = 'https://listado.mercadolibre.com.mx/{}#D[A:{}]'.format(search.replace(' ','-'), search)\n",
    "request = requests.get(url)\n",
    "print(f'Response status code: {request.status_code}')\n",
    "content = request.text\n",
    "soup = bs4.BeautifulSoup(content, 'html.parser') # HTML document\n",
    "\n",
    "total_pages = int(soup.find_all(\"div\", {\"class\":\"ui-search-pagination shops__pagination-content\"})[0].find(class_=\"andes-pagination__page-count\").text.replace(\"de \",\"\"))\n",
    "current_page = soup.find_all(\"div\", {\"class\":\"ui-search-pagination shops__pagination-content\"})[0].find(\"li\").text # See actual page\n",
    "print(f'***************{soup.title.text}***************')\n",
    "print('Total pages: {}'.format(total_pages))\n",
    "print('Current page: {}\\n\\n'.format(current_page))\n",
    "\n",
    "brands = []\n",
    "products = []\n",
    "prices = []\n",
    "scores = []\n",
    "comments = []\n",
    "links = []\n",
    "\n",
    "for page in range(1,total_pages+1):\n",
    "\n",
    "    if page == 1:\n",
    "        url_page = 'https://listado.mercadolibre.com.mx/{}_NoIndex_True'.format(search.replace(' ','-'))\n",
    "        request_page = requests.get(url_page)\n",
    "        content_page = request_page.text\n",
    "        soup_page = bs4.BeautifulSoup(content_page, 'html.parser')\n",
    "        tag_filter_page = soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"})\n",
    "\n",
    "        for tag in tag_filter_page:\n",
    "            brands.append(tag.find(class_=\"ui-search-item__brand-discoverability ui-search-item__group__element shops__items-group-details\").text)\n",
    "            products.append(tag.find(\"h2\").text)\n",
    "            prices.append(tag.find(class_=\"price-tag-fraction\").text)\n",
    "            links.append(tag.find(\"a\")[\"href\"])\n",
    "\n",
    "    else:\n",
    "        url_page = 'https://listado.mercadolibre.com.mx/{}_Desde_{}_NoIndex_True'.format(search.replace(' ','-'),(page*50)+1)\n",
    "        request_page = requests.get(url_page)\n",
    "        content_page = request_page.text\n",
    "        soup_page = bs4.BeautifulSoup(content_page, 'html.parser')\n",
    "        tag_filter_page = soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"})\n",
    "\n",
    "        for tag in tag_filter_page:\n",
    "            brands.append(tag.find(class_=\"ui-search-item__brand-discoverability ui-search-item__group__element shops__items-group-details\").text)\n",
    "            products.append(tag.find(\"h2\").text)\n",
    "            prices.append(tag.find(class_=\"price-tag-fraction\").text)\n",
    "            links.append(tag.find(\"a\")[\"href\"])\n",
    "        \n",
    "\n",
    "data = {}\n",
    "data[\"Brand\"] = brands\n",
    "data['Product'] = products\n",
    "data['Price'] = prices\n",
    "'''data['Score'] = scores\n",
    "data['Comentarios'] = comments'''\n",
    "data[\"Link\"] = links\n",
    "\n",
    "df = pd.DataFrame(data = data)\n",
    "df = df.drop_duplicates(ignore_index=True)\n",
    "df['Brand'] = df['Brand'].apply(lambda x: \"-\" if x == '' else x)\n",
    "df['Price'] = df['Price'].apply(lambda x: x.replace(\",\",\"\"))\n",
    "df['Price'] = df['Price'].astype(int)\n",
    "print('Total products retrieved: {}'.format(df.shape[0]))\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "request_ = requests.get('https://listado.mercadolibre.com.mx/iphone-3#D[A:iphone%203]')\n",
    "print(f'Response status code: {request_.status_code}')\n",
    "content_ = request_.text\n",
    "soup_ = bs4.BeautifulSoup(content_, 'html.parser') # HTML document\n",
    "total_pages = soup_.find_all(\"div\", {\"class\":\"ui-search-pagination shops__pagination-content\"})#[0].find(class_=\"andes-pagination__page-count\").text.replace(\"de \",\"\"))\n",
    "print('Total pages: {}'.format(total_pages))\n",
    "print(soup_.title.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup_.find_all(\"div\", {\"class\":\"ui-search-pagination shops__pagination-content\"}).__len__()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup_.find_all('article', {\"class\":\"ui-review-capability-comments__comment\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = []\n",
    "l = soup_.find_all('article', {\"class\":\"ui-review-capability-comments__comment\"}) #[2].find('p', class_=\"ui-review-capability-comments__comment__content\").text\n",
    "print(l.__len__())\n",
    "for c in l:\n",
    "    v.append(c.find('p', class_=\"ui-review-capability-comments__comment__content\").text)\n",
    "print(l.__len__())\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = input('Insert your search:')\n",
    "print(f'Keywords: {search.replace(\"-\",\" \")}')\n",
    "url = 'https://listado.mercadolibre.com.mx/{}#D[A:{}]'.format(search.replace(' ','-'), search)\n",
    "request = requests.get(url)\n",
    "content = request.text\n",
    "soup = bs4.BeautifulSoup(content, 'html.parser') # HTML document\n",
    "\n",
    "brands = []\n",
    "products = []\n",
    "prices = []\n",
    "links = []\n",
    "\n",
    "pages_flag = soup.find_all(\"div\", {\"class\":\"ui-search-pagination shops__pagination-content\"})\n",
    "\n",
    "\n",
    "if pages_flag.__len__() != 0:\n",
    "    print(f'Response status code: {request.status_code}')\n",
    "    total_pages = int(soup.find(\"div\", class_=\"ui-search-pagination shops__pagination-content\").find(\"li\",class_=\"andes-pagination__page-count\").text.replace(\"de \",\"\"))\n",
    "    print(f'***************{soup.title.text}***************')\n",
    "    print('Total pages: {}'.format(total_pages))\n",
    "\n",
    "    for page in range(1,total_pages+1):\n",
    "        if page == 1:\n",
    "            url_page = 'https://listado.mercadolibre.com.mx/{}_NoIndex_True'.format(search.replace(' ','-'))\n",
    "            request_page = requests.get(url_page)\n",
    "            content_page = request_page.text\n",
    "            soup_page = bs4.BeautifulSoup(content_page, 'html.parser')\n",
    "            tag_filter_page = soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"})\n",
    "\n",
    "            while soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"}).__len__() == 0:\n",
    "                url_page = 'https://listado.mercadolibre.com.mx/{}_NoIndex_True'.format(search.replace(' ','-'))\n",
    "                request_page = requests.get(url_page)\n",
    "                content_page = request_page.text\n",
    "                soup_page = bs4.BeautifulSoup(content_page, 'html.parser')\n",
    "                tag_filter_page = soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"})\n",
    "\n",
    "            # for tag in tag_filter_page:\n",
    "            #     brands.append(tag.find(class_=\"ui-search-item__brand-discoverability ui-search-item__group__element shops__items-group-details\").text)\n",
    "            #     products.append(tag.find(\"h2\").text)\n",
    "            #     prices.append(tag.find(class_=\"price-tag-fraction\").text)\n",
    "            #     links.append(tag.find(\"a\")[\"href\"])\n",
    "\n",
    "        else:\n",
    "            url_page = 'https://listado.mercadolibre.com.mx/{}_Desde_{}_NoIndex_True'.format(search.replace(' ','-'),(page*50)+1)\n",
    "            request_page = requests.get(url_page)\n",
    "            content_page = request_page.text\n",
    "            soup_page = bs4.BeautifulSoup(content_page, 'html.parser')\n",
    "            tag_filter_page = soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"})\n",
    "\n",
    "            while soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"}).__len__() == 0:\n",
    "                url_page = 'https://listado.mercadolibre.com.mx/{}_Desde_{}_NoIndex_True'.format(search.replace(' ','-'),(page*50)+1)\n",
    "                request_page = requests.get(url_page)\n",
    "                content_page = request_page.text\n",
    "                soup_page = bs4.BeautifulSoup(content_page, 'html.parser')\n",
    "                tag_filter_page = soup_page.find_all(\"div\", {\"class\":\"ui-search-result__content-wrapper shops__result-content-wrapper\"}) \n",
    "\n",
    "            # for tag in tag_filter_page:\n",
    "            #     products.append(tag.find(\"h2\").text)\n",
    "            #     prices.append(tag.find(class_=\"price-tag-fraction\").text)\n",
    "            #     links.append(tag.find(\"a\")[\"href\"])\n",
    "            #     brands.append(tag.find(class_=\"ui-search-item__brand-discoverability ui-search-item__group__element shops__items-group-details\").text)\n",
    "    \n",
    "        # current_page = soup.find_all(\"div\", {\"class\":\"ui-search-pagination shops__pagination-content\"})[0].find(\"li\").text # See actual page\n",
    "        print('Scraping page {}/{}'.format(page,total_pages))\n",
    "\n",
    "        for tag in tag_filter_page:\n",
    "            brands.append(tag.find(class_=\"ui-search-item__brand-discoverability ui-search-item__group__element shops__items-group-details\").text)\n",
    "            products.append(tag.find(\"h2\").text)\n",
    "            prices.append(tag.find(\"span\",class_=\"andes-money-amount__fraction\").text)\n",
    "            # prices.append(tag.find(class_=\"price-tag-fraction\").text)\n",
    "            links.append(tag.find(\"a\")[\"href\"])\n",
    "\n",
    "    data = {}\n",
    "    data[\"Brand\"] = brands\n",
    "    data['Product'] = products\n",
    "    data['Price'] = prices\n",
    "    data[\"Link\"] = links\n",
    "\n",
    "    df = pd.DataFrame(data = data)\n",
    "    df = df.drop_duplicates(ignore_index=True)\n",
    "    df['Brand'] = df['Brand'].apply(lambda x: \"-\" if x == '' else x)\n",
    "    df['Price'] = df['Price'].apply(lambda x: x.replace(\",\",\"\"))\n",
    "    df['Price'] = df['Price'].astype(int)\n",
    "\n",
    "    display(df.head(3))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "af37c3268cca26f2bfaf01d13423ff37a9d7634ed8fa0992ffd17d42608d9c97"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}