filter_imkg_non_memes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "229b5960",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import random\n",
    "import requests\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "86d82cad",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_list = [\n",
    "    \"https://i.imgflip.com/6e456k.jpg\", \"https://i.imgflip.com/6eb3ub.jpg\", \"https://i.imgflip.com/6e80xv.jpg\",\n",
    "    \"https://i.imgflip.com/59v56p.jpg\", \"https://i.imgflip.com/61285t.jpg\", \"https://i.imgflip.com/691gmh.jpg\",\n",
    "    \"https://i.imgflip.com/10kwnf.jpg\", \"https://i.imgflip.com/16urhv.jpg\", \"https://i.imgflip.com/17f70q.jpg\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8a18d7d",
   "metadata": {},
   "source": [
    "## Reddit"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be4c1924",
   "metadata": {},
   "source": [
    "### 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c025abf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reddit image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>meme/arggbevcz7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/6e456k.jpg</td>\n",
       "      <td>0.597725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meme/s96e69g5t7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/2twnaq.jpg</td>\n",
       "      <td>0.868993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meme/cgbvswwri7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/6e80xv.jpg</td>\n",
       "      <td>0.591689</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             reddit image                        imkg image     score\n",
       "0  meme/arggbevcz7ab1.jpg  https://i.imgflip.com/6e456k.jpg  0.597725\n",
       "1  meme/s96e69g5t7ab1.jpg  https://i.imgflip.com/2twnaq.jpg  0.868993\n",
       "2  meme/cgbvswwri7ab1.jpg  https://i.imgflip.com/6e80xv.jpg  0.591689"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df50 = pd.read_csv('./out/filtered_df_50_final.csv', index_col=0)\n",
    "df50.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "966988b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2878, 3)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df50 = df50[~df50['imkg image'].isin(filter_list)]\n",
    "df50.to_csv('./out/new_filtered_df_50_final.csv')\n",
    "df50.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efdd6466",
   "metadata": {},
   "source": [
    "### 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8799cafd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reddit image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>meme/s96e69g5t7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/2twnaq.jpg</td>\n",
       "      <td>0.868993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meme/e9gksc03f7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/544pnd.jpg</td>\n",
       "      <td>0.730533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meme/t8jgr364z6ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/3h4dth.jpg</td>\n",
       "      <td>0.624141</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             reddit image                        imkg image     score\n",
       "0  meme/s96e69g5t7ab1.jpg  https://i.imgflip.com/2twnaq.jpg  0.868993\n",
       "1  meme/e9gksc03f7ab1.jpg  https://i.imgflip.com/544pnd.jpg  0.730533\n",
       "2  meme/t8jgr364z6ab1.jpg  https://i.imgflip.com/3h4dth.jpg  0.624141"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df60 = pd.read_csv('./out/filtered_df_60_final.csv', index_col=0)\n",
    "df60.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5be46014",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1953, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df60 = df60[~df60['imkg image'].isin(filter_list)]\n",
    "df60.to_csv('./out/new_filtered_df_60_final.csv')\n",
    "df60.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6563ac29",
   "metadata": {},
   "source": [
    "### 70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d9e1c547",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reddit image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>meme/s96e69g5t7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/2twnaq.jpg</td>\n",
       "      <td>0.868993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meme/e9gksc03f7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/544pnd.jpg</td>\n",
       "      <td>0.730533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meme/wjgmc07y76ab1.png</td>\n",
       "      <td>https://i.imgflip.com/2xmijc.jpg</td>\n",
       "      <td>0.835818</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             reddit image                        imkg image     score\n",
       "0  meme/s96e69g5t7ab1.jpg  https://i.imgflip.com/2twnaq.jpg  0.868993\n",
       "1  meme/e9gksc03f7ab1.jpg  https://i.imgflip.com/544pnd.jpg  0.730533\n",
       "2  meme/wjgmc07y76ab1.png  https://i.imgflip.com/2xmijc.jpg  0.835818"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df70 = pd.read_csv('./out/filtered_df_70_final.csv', index_col=0)\n",
    "df70.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a3f1c28e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1226, 3)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df70 = df70[~df70['imkg image'].isin(filter_list)]\n",
    "df70.to_csv('./out/new_filtered_df_70_final.csv')\n",
    "df70.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "839b3f33",
   "metadata": {},
   "source": [
    "### 90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3a32cfa2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reddit image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>meme/2vcrasrfzz9b1.jpg</td>\n",
       "      <td>https://i.imgflip.com/32isi0.jpg</td>\n",
       "      <td>0.934520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meme/o2go6p63cm9b1.jpg</td>\n",
       "      <td>https://i.imgflip.com/3qit8r.jpg</td>\n",
       "      <td>0.900522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meme/11ua2zxkck9b1.jpg</td>\n",
       "      <td>https://i.imgflip.com/32isi0.jpg</td>\n",
       "      <td>0.911886</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             reddit image                        imkg image     score\n",
       "0  meme/2vcrasrfzz9b1.jpg  https://i.imgflip.com/32isi0.jpg  0.934520\n",
       "1  meme/o2go6p63cm9b1.jpg  https://i.imgflip.com/3qit8r.jpg  0.900522\n",
       "2  meme/11ua2zxkck9b1.jpg  https://i.imgflip.com/32isi0.jpg  0.911886"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df90 = pd.read_csv('./out/filtered_df_90_final.csv', index_col=0)\n",
    "df90.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3e994b09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(128, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df90 = df90[~df90['imkg image'].isin(filter_list)]\n",
    "df90.to_csv('./out/new_filtered_df_90_final.csv')\n",
    "df90.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7bcc9ca",
   "metadata": {},
   "source": [
    "### Entire reddit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "51a6f991",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reddit image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>meme/b8vwd6jnb8ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/10n6de.jpg</td>\n",
       "      <td>0.284095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meme/arggbevcz7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/6e456k.jpg</td>\n",
       "      <td>0.597725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meme/rfj3ulr4y7ab1.jpg</td>\n",
       "      <td>https://i.imgflip.com/13406p.jpg</td>\n",
       "      <td>0.224203</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             reddit image                        imkg image     score\n",
       "0  meme/b8vwd6jnb8ab1.jpg  https://i.imgflip.com/10n6de.jpg  0.284095\n",
       "1  meme/arggbevcz7ab1.jpg  https://i.imgflip.com/6e456k.jpg  0.597725\n",
       "2  meme/rfj3ulr4y7ab1.jpg  https://i.imgflip.com/13406p.jpg  0.224203"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./out/reddit_inference_final.csv', index_col=0)\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "963e3199",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11000, 3)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[~df['imkg image'].isin(filter_list)]\n",
    "df.to_csv('./out/new_reddit_inference_final.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5c345f6",
   "metadata": {},
   "source": [
    "## Discord"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c630f5d8",
   "metadata": {},
   "source": [
    "### 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "93fca44a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>discord image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>channel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/W...</td>\n",
       "      <td>https://i.imgflip.com/48k2zx.jpg</td>\n",
       "      <td>0.678281</td>\n",
       "      <td>2023-07-31T23:43:57.965+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/D...</td>\n",
       "      <td>https://i.imgflip.com/4jf1mm.jpg</td>\n",
       "      <td>0.707667</td>\n",
       "      <td>2023-07-31T23:43:10.035+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://images-ext-2.discordapp.net/external/v...</td>\n",
       "      <td>https://i.imgflip.com/3lklq9.jpg</td>\n",
       "      <td>0.890164</td>\n",
       "      <td>2023-07-31T23:26:23.419+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       discord image   \n",
       "0  https://images-ext-1.discordapp.net/external/W...  \\\n",
       "1  https://images-ext-1.discordapp.net/external/D...   \n",
       "2  https://images-ext-2.discordapp.net/external/v...   \n",
       "\n",
       "                         imkg image     score                      timestamp   \n",
       "0  https://i.imgflip.com/48k2zx.jpg  0.678281  2023-07-31T23:43:57.965+00:00  \\\n",
       "1  https://i.imgflip.com/4jf1mm.jpg  0.707667  2023-07-31T23:43:10.035+00:00   \n",
       "2  https://i.imgflip.com/3lklq9.jpg  0.890164  2023-07-31T23:26:23.419+00:00   \n",
       "\n",
       "      channel  \n",
       "0  TheDungeon  \n",
       "1  TheDungeon  \n",
       "2  TheDungeon  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df50 = pd.read_csv('./out/discord_filtered_df_50_final.csv', index_col=0)\n",
    "df50.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "565eb96b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2220, 5)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df50 = df50[~df50['imkg image'].isin(filter_list)]\n",
    "df50.to_csv('./out/new_discord_filtered_df_50_final.csv')\n",
    "df50.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "566cfe11",
   "metadata": {},
   "source": [
    "### 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9b6e277c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>discord image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>channel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/W...</td>\n",
       "      <td>https://i.imgflip.com/48k2zx.jpg</td>\n",
       "      <td>0.678281</td>\n",
       "      <td>2023-07-31T23:43:57.965+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/D...</td>\n",
       "      <td>https://i.imgflip.com/4jf1mm.jpg</td>\n",
       "      <td>0.707667</td>\n",
       "      <td>2023-07-31T23:43:10.035+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://images-ext-2.discordapp.net/external/v...</td>\n",
       "      <td>https://i.imgflip.com/3lklq9.jpg</td>\n",
       "      <td>0.890164</td>\n",
       "      <td>2023-07-31T23:26:23.419+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       discord image   \n",
       "0  https://images-ext-1.discordapp.net/external/W...  \\\n",
       "1  https://images-ext-1.discordapp.net/external/D...   \n",
       "2  https://images-ext-2.discordapp.net/external/v...   \n",
       "\n",
       "                         imkg image     score                      timestamp   \n",
       "0  https://i.imgflip.com/48k2zx.jpg  0.678281  2023-07-31T23:43:57.965+00:00  \\\n",
       "1  https://i.imgflip.com/4jf1mm.jpg  0.707667  2023-07-31T23:43:10.035+00:00   \n",
       "2  https://i.imgflip.com/3lklq9.jpg  0.890164  2023-07-31T23:26:23.419+00:00   \n",
       "\n",
       "      channel  \n",
       "0  TheDungeon  \n",
       "1  TheDungeon  \n",
       "2  TheDungeon  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df60 = pd.read_csv('./out/discord_filtered_df_60_final.csv', index_col=0)\n",
    "df60.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d0697451",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1494, 5)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df60 = df60[~df60['imkg image'].isin(filter_list)]\n",
    "df60.to_csv('./out/new_discord_filtered_df_60_final.csv')\n",
    "df60.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ff8a73f",
   "metadata": {},
   "source": [
    "### 70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "90e6496f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>discord image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>channel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/D...</td>\n",
       "      <td>https://i.imgflip.com/4jf1mm.jpg</td>\n",
       "      <td>0.707667</td>\n",
       "      <td>2023-07-31T23:43:10.035+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://images-ext-2.discordapp.net/external/v...</td>\n",
       "      <td>https://i.imgflip.com/3lklq9.jpg</td>\n",
       "      <td>0.890164</td>\n",
       "      <td>2023-07-31T23:26:23.419+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://images-ext-2.discordapp.net/external/c...</td>\n",
       "      <td>https://i.imgflip.com/267bdp.jpg</td>\n",
       "      <td>0.809485</td>\n",
       "      <td>2023-07-31T21:30:44.194+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       discord image   \n",
       "0  https://images-ext-1.discordapp.net/external/D...  \\\n",
       "1  https://images-ext-2.discordapp.net/external/v...   \n",
       "2  https://images-ext-2.discordapp.net/external/c...   \n",
       "\n",
       "                         imkg image     score                      timestamp   \n",
       "0  https://i.imgflip.com/4jf1mm.jpg  0.707667  2023-07-31T23:43:10.035+00:00  \\\n",
       "1  https://i.imgflip.com/3lklq9.jpg  0.890164  2023-07-31T23:26:23.419+00:00   \n",
       "2  https://i.imgflip.com/267bdp.jpg  0.809485  2023-07-31T21:30:44.194+00:00   \n",
       "\n",
       "      channel  \n",
       "0  TheDungeon  \n",
       "1  TheDungeon  \n",
       "2  TheDungeon  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df70 = pd.read_csv('./out/discord_filtered_df_70_final.csv', index_col=0)\n",
    "df70.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1d9f4186",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(933, 5)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df70 = df70[~df70['imkg image'].isin(filter_list)]\n",
    "df70.to_csv('./out/new_discord_filtered_df_70_final.csv')\n",
    "df70.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73391fb8",
   "metadata": {},
   "source": [
    "### 90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9d2511a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>discord image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>channel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://images-ext-2.discordapp.net/external/Q...</td>\n",
       "      <td>https://i.imgflip.com/100dif.jpg</td>\n",
       "      <td>0.923594</td>\n",
       "      <td>2023-07-31T11:55:42.494+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/n...</td>\n",
       "      <td>https://i.imgflip.com/25monu.jpg</td>\n",
       "      <td>0.927268</td>\n",
       "      <td>2023-07-31T01:36:47.605+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://images-ext-2.discordapp.net/external/B...</td>\n",
       "      <td>https://i.imgflip.com/2xiyaj.jpg</td>\n",
       "      <td>0.906636</td>\n",
       "      <td>2023-07-30T15:22:37.4+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       discord image   \n",
       "0  https://images-ext-2.discordapp.net/external/Q...  \\\n",
       "1  https://images-ext-1.discordapp.net/external/n...   \n",
       "2  https://images-ext-2.discordapp.net/external/B...   \n",
       "\n",
       "                         imkg image     score                      timestamp   \n",
       "0  https://i.imgflip.com/100dif.jpg  0.923594  2023-07-31T11:55:42.494+00:00  \\\n",
       "1  https://i.imgflip.com/25monu.jpg  0.927268  2023-07-31T01:36:47.605+00:00   \n",
       "2  https://i.imgflip.com/2xiyaj.jpg  0.906636    2023-07-30T15:22:37.4+00:00   \n",
       "\n",
       "      channel  \n",
       "0  TheDungeon  \n",
       "1  TheDungeon  \n",
       "2  TheDungeon  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df90 = pd.read_csv('./out/discord_filtered_df_90_final.csv', index_col=0)\n",
    "df90.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a17058fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108, 5)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df90 = df90[~df90['imkg image'].isin(filter_list)]\n",
    "df90.to_csv('./out/new_discord_filtered_df_90_final.csv')\n",
    "df90.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b6bcc3f",
   "metadata": {},
   "source": [
    "### Entire discord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "22c31eb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>discord image</th>\n",
       "      <th>imkg image</th>\n",
       "      <th>score</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>channel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/W...</td>\n",
       "      <td>https://i.imgflip.com/48k2zx.jpg</td>\n",
       "      <td>0.678281</td>\n",
       "      <td>2023-07-31T23:43:57.965+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://images-ext-1.discordapp.net/external/D...</td>\n",
       "      <td>https://i.imgflip.com/4jf1mm.jpg</td>\n",
       "      <td>0.707667</td>\n",
       "      <td>2023-07-31T23:43:10.035+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://images-ext-2.discordapp.net/external/v...</td>\n",
       "      <td>https://i.imgflip.com/3lklq9.jpg</td>\n",
       "      <td>0.890164</td>\n",
       "      <td>2023-07-31T23:26:23.419+00:00</td>\n",
       "      <td>TheDungeon</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       discord image   \n",
       "0  https://images-ext-1.discordapp.net/external/W...  \\\n",
       "1  https://images-ext-1.discordapp.net/external/D...   \n",
       "2  https://images-ext-2.discordapp.net/external/v...   \n",
       "\n",
       "                         imkg image     score                      timestamp   \n",
       "0  https://i.imgflip.com/48k2zx.jpg  0.678281  2023-07-31T23:43:57.965+00:00  \\\n",
       "1  https://i.imgflip.com/4jf1mm.jpg  0.707667  2023-07-31T23:43:10.035+00:00   \n",
       "2  https://i.imgflip.com/3lklq9.jpg  0.890164  2023-07-31T23:26:23.419+00:00   \n",
       "\n",
       "      channel  \n",
       "0  TheDungeon  \n",
       "1  TheDungeon  \n",
       "2  TheDungeon  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./out/discord_inference_final.csv', index_col=0)\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d8419aaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9233, 5)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[~df['imkg image'].isin(filter_list)]\n",
    "df.to_csv('./out/new_discord_inference_final.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fb702bb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}