preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b720d6d5-87ad-4100-8181-9e5090695109",
   "metadata": {},
   "source": [
    "### Data preprocessing\n",
    "Filter the raw dataset and construct training/validation datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "969bf4fc-a452-429c-bbb6-f4acbc7e681c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1b0ad7b-f686-40b4-aa67-f8a9ff3c5b24",
   "metadata": {},
   "source": [
    "#### Extract videos with the most common single-class labels\n",
    "Due to file size, we are going to merge train and val dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5d287f12-d74b-460b-8cff-4b84b323f341",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = pd.read_csv('labels/train.txt', delimiter=' ')\n",
    "val_labels = pd.read_csv('labels/val.txt', delimiter=' ')\n",
    "data_labels = pd.concat([train_labels,val_labels])\n",
    "data_labels = data_labels.sort_values(by=['original_vido_id']) # sort alphatebically by video name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ebbbbb11-be78-464a-85e5-5dbcfea79049",
   "metadata": {},
   "outputs": [],
   "source": [
    "video_names = data_labels[\"original_vido_id\"].to_numpy()\n",
    "video_names, unique_idx = np.unique(video_names, return_index = True)\n",
    "unique_idx = sorted(unique_idx) # sort the indices so that the video labels order are retained\n",
    "labels_str = data_labels[\"labels\"].to_numpy()\n",
    "labels_str = labels_str[sorted(unique_idx)]\n",
    "labels = []\n",
    "for i in range(len(labels_str)):\n",
    "    this_label = labels_str[i].split(\",\")\n",
    "    this_label = [eval(i) for i in this_label]\n",
    "    labels.append(this_label)\n",
    "labels = np.array(labels, dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f25ff064-0277-4e64-b8a7-a05170e563da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([list([2, 40]), list([16]), list([2, 15]), list([78]),\n",
       "       list([102, 1, 39, 8, 120, 97])], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[:5] # in the same order as videos when they are ordered alphabetically by video names "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9a704b39-770e-4cba-98d3-c13e1858d990",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find indices of videos with only one class label\n",
    "single_class_idx = []\n",
    "for i in range(len(labels)):\n",
    "    if(len(labels[i]) == 1): single_class_idx.append(i)\n",
    "# find names and labels of videos with only one class label\n",
    "single_class_video_names = video_names[single_class_idx]\n",
    "single_class_labels = labels[single_class_idx]\n",
    "single_class_labels = [i[0] for i in single_class_labels]\n",
    "single_class_labels = np.array(single_class_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5a50c129-2e2c-4060-9373-907e2f03d036",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the most popular single-class labels \n",
    "c = Counter(single_class_labels)\n",
    "top_labels = []\n",
    "count_threshold = 100 # count threshold for using a label \n",
    "top_count = 0 # number of data under all top labels with count >= count_threshold\n",
    "for i in sorted(c, key=lambda x: -c[x]):\n",
    "    if (c[i] >= count_threshold): \n",
    "        top_labels.append(i)\n",
    "        top_count += c[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "22de192b-3a22-4fb6-bf87-8cfef43ccc18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# find indices of single-class videos with enough data (>= count_threshold)\n",
    "top_idx = []\n",
    "for i in range(len(single_class_labels)):\n",
    "    if(single_class_labels[i] in top_labels): top_idx.append(i)\n",
    "    \n",
    "# find names and labels of videos with top single-class labels\n",
    "v_names = single_class_video_names[top_idx]\n",
    "v_labels = single_class_labels[top_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bf985107-bb44-4c07-9dc5-1be599fe8151",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['AABKMGHE', 'AAFWRPDI', 'AAIQZYMI', ..., 'ZZKMOXIF', 'ZZMTPOYV',\n",
       "        'ZZQVJVEI'], dtype=object),\n",
       " array([ 78, 123,  52, ..., 123,  78,  40]))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v_names, v_labels"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Compiled MiniCourse 2020/05 [course/pycompiled/default]",
   "language": "python",
   "name": "sys_pycompiled202005"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}