Part 8.3: clean_text_library.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Part 8.3: clean-text_library.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gulabpatel/NLP_Basics/blob/main/Part%208.3%3A%20clean_text_library.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s2KuaZwVQ4Y-"
      },
      "source": [
        "Video walkthrough the code : https://www.youtube.com/watch?v=i2TjAgga1YU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1N4I7yoyz9_U"
      },
      "source": [
        "!pip install clean-text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N4C6F0UI0ACI",
        "outputId": "1de68d89-9bb9-47d8-ee00-5fca25f18c36"
      },
      "source": [
        "from cleantext import clean"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Since the GPL-licensed package `unidecode` is not installed, using Python's `unicodedata` package which yields worse results.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BbTzq0TP06d0"
      },
      "source": [
        "# Unicode"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "awq3sJC50CxN"
      },
      "source": [
        "s1 = 'Zürich'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "EtfWnOal0UFj",
        "outputId": "9c64ca6c-2a24-4e55-ca49-32bfa854c64e"
      },
      "source": [
        "clean(s1, fix_unicode=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'zurich'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u-TLcRJB37QI"
      },
      "source": [
        "# Closest ASCII representation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uy7HKg3m1JhB"
      },
      "source": [
        "s2 = \"ko\\u017eu\\u0161\\u010dek\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "yJY7nHsO1QJb",
        "outputId": "0bf663e4-ade7-40c8-f7cf-21d8cbe7e6bc"
      },
      "source": [
        "clean(s2, to_ascii=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'kozuscek'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3fKWpKlw4F20"
      },
      "source": [
        "# Lower Case"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PB9U72_V1RLx"
      },
      "source": [
        "s3 = \"My Name is Gulab Singh Patel\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "FRzVPiOl4Kfj",
        "outputId": "1ec2e7c8-ec39-484f-f949-dcdd93ba8d78"
      },
      "source": [
        "clean(s3, lower=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'my name is gulab singh patel'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "URW2c1wi4XbK"
      },
      "source": [
        "# Replace URLs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRItH3YV4NWj"
      },
      "source": [
        "s4 = \"https://www.Google.com has surpassed https://www.Bing.com in search volume\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "pD33YbZO4fZm",
        "outputId": "ecdfffc6-88b3-4cf2-d4c1-099f4c8226cc"
      },
      "source": [
        "clean(s4, no_urls=True, replace_with_url=\"URL\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'url has surpassed url in search volume'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tfW82L2N8dtk"
      },
      "source": [
        "# Replace Currency"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KP9-d4Xz8c1u"
      },
      "source": [
        "s5 = \"I want ₹ 40\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "oQigJS1b8wYb",
        "outputId": "0478c3c6-48f5-47ed-bf73-928ab38fa4d2"
      },
      "source": [
        "clean(s5, no_currency_symbols = True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'i want <cur> 40'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "gwH0wX5N8dAH",
        "outputId": "090270b9-423e-433a-9853-20cdc406896e"
      },
      "source": [
        "clean(s5, \n",
        "      no_currency_symbols = True, \n",
        "      replace_with_currency_symbol=\"Rupees\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'i want rupees 40'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3XHanRxP86Ng"
      },
      "source": [
        "# Replace punctuations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ub9kYWV38bwo"
      },
      "source": [
        "s6 = \"40,000 is greater than 30,000\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "MengWH1y9CM6",
        "outputId": "663484d5-002c-4230-c56f-6b2086f91011"
      },
      "source": [
        "clean(s6, no_punct = True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'40000 is greater than 30000'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "k1odvXYk9G5d",
        "outputId": "bd7b3fb3-f1e0-4431-95c1-f5629c13391b"
      },
      "source": [
        "clean(s6, no_punct = True, replace_with_punct = \"7\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'407000 is greater than 307000'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DhX8Iy9F9YV6"
      },
      "source": [
        "# Remove Numbers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5gX567Uz9OoE"
      },
      "source": [
        "s7 = 'abc123def456ghi789zero0'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "C74CSUpD9a2P",
        "outputId": "646d5971-bf6c-4f81-efd6-7fec1e7a34ba"
      },
      "source": [
        "clean(s7, no_digits = True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'abc000def000ghi000zero0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "huzCllRs9gN-",
        "outputId": "63b160cf-944f-46b0-f637-10836eced720"
      },
      "source": [
        "clean(s7, no_digits = True, replace_with_digit=\"\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'abcdefghizero'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S_YFxfIP95EM"
      },
      "source": [
        "# Combining it all"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bP5W7zKX9t2w"
      },
      "source": [
        "s8 = \"\"\"\n",
        "Zürich has a famous website https://www.zuerich.com/ \n",
        "WHICH ACCEPTS 40,000 € and adding a random string, :\n",
        "abc123def456ghi789zero0 for this demo. ' \n",
        "     \"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "7hIX60Gd-qQX",
        "outputId": "153f6394-b070-4d4f-a538-1f2f0095d992"
      },
      "source": [
        "clean(s8, \n",
        "      fix_unicode=True, \n",
        "      to_ascii=True, \n",
        "      lower=True, \n",
        "      no_urls=True, \n",
        "      no_numbers=True, \n",
        "      no_digits=True, \n",
        "      no_currency_symbols=True, \n",
        "      no_punct=True, \n",
        "      replace_with_punct=\"\", \n",
        "      replace_with_url=\"<URL>\", \n",
        "      replace_with_number=\"<NUMBER>\", \n",
        "      replace_with_digit=\"\", \n",
        "      replace_with_currency_symbol=\"<CUR>\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'zurich has a famous website <url>\\nwhich accepts <number> <cur> and adding a random string\\nabcdefghizero for this demo'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    }
  ]
}