read_zst_file.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b87fcb72-21da-472b-ba38-3c30ff78a83d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#I got the idea of how to read a .zst file from https://github.com/pushshift/zreader/tree/master and changed the fields based on our project's requirements.\n",
    "import pandas as pd\n",
    "import csv\n",
    "import warnings\n",
    "import openpyxl\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "251a45d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import zstandard as zstd\n",
    "\n",
    "class Zreader:\n",
    "\n",
    "    def __init__(self, file, chunk_size=10000000):\n",
    "        self.fh = open(file,'rb')\n",
    "        self.chunk_size = chunk_size\n",
    "        self.dctx = zstd.ZstdDecompressor(max_window_size=2147483648)\n",
    "        self.reader = self.dctx.stream_reader(self.fh)\n",
    "        self.buffer = ''\n",
    "\n",
    "\n",
    "    def readlines(self):\n",
    "        while True:\n",
    "            chunk = self.reader.read(self.chunk_size).decode(encoding=\"latin-1\")\n",
    "            if not chunk:\n",
    "                break\n",
    "            lines = (self.buffer + chunk).split(\"\\n\")\n",
    "\n",
    "            for line in lines[:-1]:\n",
    "                yield line\n",
    "\n",
    "            self.buffer = lines[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73b49232",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ujson as json\n",
    "\n",
    "\n",
    "# Adjust chunk_size as necessary -- defaults to 16,384 if not specified\n",
    "reader = Zreader(\"RS_2022-04.zst\", chunk_size=10000000)\n",
    "\n",
    "list_lines=[]\n",
    "j=0\n",
    "# Read each line from the reader\n",
    "for line in reader.readlines():\n",
    "    obj = json.loads(line)\n",
    "    sub_name=(obj['subreddit'].lower()).strip() \n",
    "    if ((sub_name==\"books\") or (sub_name==\"suggestmeabook\")):\n",
    "        list_lines.append([obj['author'], obj['subreddit'], obj['created_utc'],obj['title'],obj['id'],\n",
    "           obj['score'], obj['send_replies']])\n",
    "    if j%10000000==0:\n",
    "        print(j)\n",
    "    j+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2bc35967",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(list_lines, columns = ['author', 'subreddit','created_utc','title','id','score','send_replies'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "bb2fe975",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlsxwriter\n",
    "df.to_excel(\"submissions_2022/april_2022.xlsx\", engine='xlsxwriter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eda138f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}