2_scrape_scopus.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pybliometrics.scopus import AbstractRetrieval\n",
    "from tqdm import tqdm\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Area(area='Economics and Econometrics', abbreviation='ECON', code=2002)]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc = AbstractRetrieval(\"10.3982/ECTA16566\", view='FULL')\n",
    "doc.subject_areas        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pybliometrics.scopus.abstract_retrieval.AbstractRetrieval at 0x12fc12bc0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_paper_info(identifier):\n",
    "    try:\n",
    "        doc = AbstractRetrieval(identifier, view='FULL')\n",
    "    except Exception as e:\n",
    "        # Handle the error, possibly logging it or notifying the user\n",
    "        return {'error': str(e)}\n",
    "\n",
    "    paper = {}\n",
    "    \n",
    "    # Check each property with getattr and provide a default value if not found\n",
    "    paper['doi'] = getattr(doc, 'doi', None)\n",
    "    paper['eid'] = getattr(doc, 'eid', None)\n",
    "    paper['title'] = getattr(doc, 'title', None)\n",
    "    paper['abstract'] = getattr(doc, 'abstract', None)\n",
    "    paper['description'] = getattr(doc, 'description', None)\n",
    "    \n",
    "    # Authors might be a list, so we need to check each entry\n",
    "    if hasattr(doc, 'authors') and doc.authors is not None:\n",
    "        paper['author_names'] = [getattr(author, 'indexed_name', None) for author in doc.authors]\n",
    "        paper['author_ids'] = [getattr(author, 'auid', None) for author in doc.authors]\n",
    "    else:\n",
    "        paper['author_names'] = []\n",
    "        paper['author_ids'] = []\n",
    "    \n",
    "    # References might be a list, so we need to check each entry\n",
    "    if hasattr(doc, 'references') and doc.references is not None:\n",
    "        paper['references'] = [f\"2-s2.0-{getattr(ref, 'id', None)}\" for ref in doc.references]\n",
    "    else:\n",
    "        paper['references'] = []\n",
    "\n",
    "    paper['cited_by'] = getattr(doc, 'citedby_count', 0)\n",
    "    \n",
    "    # Subject areas might be a list, so we need to check each entry\n",
    "    if hasattr(doc, 'subject_areas') and doc.subject_areas is not None:\n",
    "        paper['areas'] = [getattr(area, 'area', None) for area in doc.subject_areas]\n",
    "    else:\n",
    "        paper['areas'] = []\n",
    "\n",
    "    return paper\n",
    "\n",
    "\n",
    "\n",
    "def build_edge_list(df):\n",
    "    # Initialize an empty list to store the edge tuples\n",
    "    edges = []\n",
    "\n",
    "    # Iterate over the DataFrame\n",
    "    for index, row in df.iterrows():\n",
    "        source = row['eid']\n",
    "        # Extract the targets from the references column\n",
    "        targets = row['references']\n",
    "        # Create a tuple (source, target) for each target and append to the list\n",
    "        edges.extend([(source, target) for target in targets])\n",
    "\n",
    "    # Create a new DataFrame from the list of tuples\n",
    "    edge_list_df = pd.DataFrame(edges, columns=['Source', 'Target'])\n",
    "    \n",
    "    return edge_list_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-84884417725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-84926199736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-85069750760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-85069675535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-2942746458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-0003732362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-0022726760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-67650844521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-85069690296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>2-s2.0-85069045573</td>\n",
       "      <td>2-s2.0-34548536008</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>105 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Source              Target\n",
       "0    2-s2.0-85069045573  2-s2.0-84884417725\n",
       "1    2-s2.0-85069045573  2-s2.0-84926199736\n",
       "2    2-s2.0-85069045573  2-s2.0-85069750760\n",
       "3    2-s2.0-85069045573  2-s2.0-85069675535\n",
       "4    2-s2.0-85069045573   2-s2.0-2942746458\n",
       "..                  ...                 ...\n",
       "100  2-s2.0-85069045573   2-s2.0-0003732362\n",
       "101  2-s2.0-85069045573   2-s2.0-0022726760\n",
       "102  2-s2.0-85069045573  2-s2.0-67650844521\n",
       "103  2-s2.0-85069045573  2-s2.0-85069690296\n",
       "104  2-s2.0-85069045573  2-s2.0-34548536008\n",
       "\n",
       "[105 rows x 2 columns]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame([get_paper_info(\"10.3982/ECTA16566\")])\n",
    "\n",
    "\n",
    "edge_list_df = build_edge_list(df)\n",
    "\n",
    "edge_list_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6359/6359 [41:54<00:00,  2.53it/s]  \n"
     ]
    }
   ],
   "source": [
    "nature_of_belief = pd.read_csv('data/nature_of_belief.csv').assign(seed_paper = 'nature_of_belief')\n",
    "structure_of_decision = pd.read_csv('data/structure_of_decision.csv').assign(seed_paper = 'structure_of_decision')\n",
    "\n",
    "papers = pd.concat([nature_of_belief, structure_of_decision], ignore_index=True)\n",
    "\n",
    "eid_list = papers[\"EID\"].tolist()\n",
    "\n",
    "paper_data = []\n",
    "\n",
    "for paper in tqdm(eid_list):\n",
    "    paper_data.append(get_paper_info(paper))\n",
    "    time.sleep(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_details = pd.DataFrame(paper_data)\n",
    "\n",
    "paper_details.to_csv('data/paper_details.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_details.to_csv('data/node_covars.csv', index=False)\n",
    "\n",
    "edge_list_df = build_edge_list(df)\n",
    "edge_list_df.to_csv('data/edge_list.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papers = pd.read_pickle('data/by category/unified.pkl')\n",
    "\n",
    "any(str.lower('Spatial Cognition and Consumer Behavior: A Panel Data Approach') in ref for ref in papers.iloc[0]['References'].lower().split(';'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Adler T., Ben-Akiva M., Joint-Choice Model for Frequency, Destination and Travel Mode for Shopping Trips, Transportation Research Record, 569, pp. 136-150, (1976)'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papers.iloc[0]['References'].split(';')[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EID                                                  2-s2.0-0000037645\n",
       "Authors                                         Kwan M.-P.; Hong X.-D.\n",
       "Author full names    Kwan, Mei-Po (7005364475); Hong, Xiao-Dong (14...\n",
       "Author(s) ID                                   7005364475; 14823585600\n",
       "Title                Network-based constraints-oriented choice set ...\n",
       "Year                                                              1998\n",
       "Source title                                      Geographical Systems\n",
       "Volume                                                               5\n",
       "Issue                                                              1-2\n",
       "Art. No.                                                          None\n",
       "Page start                                                         139\n",
       "Page end                                                           162\n",
       "Page count                                                        23.0\n",
       "Cited by                                                           124\n",
       "DOI                                                               None\n",
       "Link                 https://www.scopus.com/inward/record.uri?eid=2...\n",
       "Abstract             This paper is concerned with choice set format...\n",
       "Author Keywords      Choice set; Cognitive map; Constraints; Destin...\n",
       "Index Keywords                                                    None\n",
       "References           Adler T., Ben-Akiva M., Joint-Choice Model for...\n",
       "Document Type                                                  Article\n",
       "Publication Stage                                                Final\n",
       "Open Access                                                       None\n",
       "Source                                                          Scopus\n",
       "Category                              [Economics Econometrics Finance]\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def check_reference(title, papers):\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[16], line 21\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m citation_graph\n\u001b[1;32m     20\u001b[0m \u001b[38;5;66;03m# Build the citation graph\u001b[39;00m\n\u001b[0;32m---> 21\u001b[0m citation_graph \u001b[38;5;241m=\u001b[39m \u001b[43mfind_citations\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpapers_dict_list\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;28mprint\u001b[39m(citation_graph)\n",
      "Cell \u001b[0;32mIn[16], line 14\u001b[0m, in \u001b[0;36mfind_citations\u001b[0;34m(papers_dict_list)\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m citing_paper \u001b[38;5;129;01min\u001b[39;00m papers_dict_list:\n\u001b[1;32m     13\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mReferences\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m citing_paper \u001b[38;5;129;01mand\u001b[39;00m citing_paper[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mReferences\u001b[39m\u001b[38;5;124m'\u001b[39m]:\n\u001b[0;32m---> 14\u001b[0m         references \u001b[38;5;241m=\u001b[39m \u001b[43mciting_paper\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mReferences\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlower\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     15\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m cited_title \u001b[38;5;129;01min\u001b[39;00m references:\n\u001b[1;32m     16\u001b[0m             citation_graph[cited_title]\u001b[38;5;241m.\u001b[39mappend(citing_paper[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTitle\u001b[39m\u001b[38;5;124m'\u001b[39m])\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "papers_dict_list = papers.to_dict(orient='records')\n",
    "\n",
    "# Function to find citations\n",
    "def find_citations(papers_dict_list):\n",
    "    citation_graph = {}\n",
    "\n",
    "    for cited_paper in papers_dict_list:\n",
    "        if 'Title' in cited_paper and cited_paper['Title']:\n",
    "            cited_title = cited_paper['Title'].lower()\n",
    "            citation_graph[cited_title] = []\n",
    "\n",
    "            for citing_paper in papers_dict_list:\n",
    "                if 'References' in citing_paper and citing_paper['References']:\n",
    "                    references = citing_paper['References'].lower()\n",
    "                    if cited_title in references:\n",
    "                        citation_graph[cited_title].append(citing_paper['Title'])\n",
    "\n",
    "    return citation_graph\n",
    "\n",
    "# Build the citation graph\n",
    "citation_graph = find_citations(papers_dict_list)\n",
    "print(citation_graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "scopus_scrape",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}