SPARQL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d021b822-cae3-4a2d-806b-14b7f7905b0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (2.2.2)\n",
      "Requirement already satisfied: tabula-py in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (2.9.3)\n",
      "Requirement already satisfied: rdflib in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (7.0.0)\n",
      "Requirement already satisfied: pydotplus in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (2.0.2)\n",
      "Requirement already satisfied: pyshacl in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (0.26.0)\n",
      "Collecting pykeen\n",
      "  Downloading pykeen-1.10.2-py3-none-any.whl.metadata (83 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.8/83.8 kB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hCollecting scikit-learn\n",
      "  Downloading scikit_learn-1.5.0-cp312-cp312-macosx_10_9_x86_64.whl.metadata (11 kB)\n",
      "Collecting matplotlib\n",
      "  Using cached matplotlib-3.9.0-cp312-cp312-macosx_10_12_x86_64.whl.metadata (11 kB)\n",
      "Requirement already satisfied: numpy>=1.26.0 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pandas) (1.26.4)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pandas) (2024.1)\n",
      "Requirement already satisfied: distro in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from tabula-py) (1.9.0)\n",
      "Requirement already satisfied: isodate<0.7.0,>=0.6.0 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from rdflib) (0.6.1)\n",
      "Requirement already satisfied: pyparsing<4,>=2.1.0 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from rdflib) (3.1.2)\n",
      "Requirement already satisfied: html5lib<2,>=1.1 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pyshacl) (1.1)\n",
      "Requirement already satisfied: owlrl<7,>=6.0.2 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pyshacl) (6.0.2)\n",
      "Requirement already satisfied: packaging>=21.3 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pyshacl) (24.1)\n",
      "Requirement already satisfied: prettytable>=3.7.0 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from pyshacl) (3.10.0)\n",
      "Collecting dataclasses-json (from pykeen)\n",
      "  Using cached dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n",
      "Collecting scipy>=1.7.0 (from pykeen)\n",
      "  Downloading scipy-1.14.0-cp312-cp312-macosx_14_0_x86_64.whl.metadata (60 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting click (from pykeen)\n",
      "  Using cached click-8.1.7-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting click-default-group (from pykeen)\n",
      "  Downloading click_default_group-1.2.4-py2.py3-none-any.whl.metadata (2.8 kB)\n",
      "Collecting torch>=2.0 (from pykeen)\n",
      "  Downloading torch-2.2.2-cp312-none-macosx_10_9_x86_64.whl.metadata (25 kB)\n",
      "Collecting tqdm (from pykeen)\n",
      "  Using cached tqdm-4.66.4-py3-none-any.whl.metadata (57 kB)\n",
      "Collecting requests (from pykeen)\n",
      "  Using cached requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n",
      "Collecting optuna>=2.0.0 (from pykeen)\n",
      "  Downloading optuna-3.6.1-py3-none-any.whl.metadata (17 kB)\n",
      "Collecting tabulate (from pykeen)\n",
      "  Downloading tabulate-0.9.0-py3-none-any.whl.metadata (34 kB)\n",
      "Collecting more-click (from pykeen)\n",
      "  Downloading more_click-0.1.2-py3-none-any.whl.metadata (4.3 kB)\n",
      "Collecting more-itertools (from pykeen)\n",
      "  Downloading more_itertools-10.3.0-py3-none-any.whl.metadata (36 kB)\n",
      "Collecting pystow>=0.4.3 (from pykeen)\n",
      "  Downloading pystow-0.5.4-py3-none-any.whl.metadata (8.9 kB)\n",
      "Collecting docdata (from pykeen)\n",
      "  Downloading docdata-0.0.3-py3-none-any.whl.metadata (5.3 kB)\n",
      "Collecting class-resolver>0.4.2 (from pykeen)\n",
      "  Downloading class_resolver-0.4.3-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting pyyaml (from pykeen)\n",
      "  Using cached PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl.metadata (2.1 kB)\n",
      "Collecting torch-max-mem>=0.1.1 (from pykeen)\n",
      "  Downloading torch_max_mem-0.1.3-py3-none-any.whl.metadata (7.4 kB)\n",
      "Collecting torch-ppr>=0.0.7 (from pykeen)\n",
      "  Downloading torch_ppr-0.0.8-py3-none-any.whl.metadata (9.3 kB)\n",
      "Collecting typing-extensions (from pykeen)\n",
      "  Using cached typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB)\n",
      "Collecting joblib>=1.2.0 (from scikit-learn)\n",
      "  Using cached joblib-1.4.2-py3-none-any.whl.metadata (5.4 kB)\n",
      "Collecting threadpoolctl>=3.1.0 (from scikit-learn)\n",
      "  Using cached threadpoolctl-3.5.0-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting contourpy>=1.0.1 (from matplotlib)\n",
      "  Using cached contourpy-1.2.1-cp312-cp312-macosx_10_9_x86_64.whl.metadata (5.8 kB)\n",
      "Collecting cycler>=0.10 (from matplotlib)\n",
      "  Using cached cycler-0.12.1-py3-none-any.whl.metadata (3.8 kB)\n",
      "Collecting fonttools>=4.22.0 (from matplotlib)\n",
      "  Using cached fonttools-4.53.0-cp312-cp312-macosx_10_9_universal2.whl.metadata (162 kB)\n",
      "Collecting kiwisolver>=1.3.1 (from matplotlib)\n",
      "  Using cached kiwisolver-1.4.5-cp312-cp312-macosx_10_9_x86_64.whl.metadata (6.4 kB)\n",
      "Collecting pillow>=8 (from matplotlib)\n",
      "  Using cached pillow-10.3.0-cp312-cp312-macosx_10_10_x86_64.whl.metadata (9.2 kB)\n",
      "Requirement already satisfied: six>=1.9 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from html5lib<2,>=1.1->pyshacl) (1.16.0)\n",
      "Requirement already satisfied: webencodings in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from html5lib<2,>=1.1->pyshacl) (0.5.1)\n",
      "Collecting alembic>=1.5.0 (from optuna>=2.0.0->pykeen)\n",
      "  Downloading alembic-1.13.2-py3-none-any.whl.metadata (7.4 kB)\n",
      "Collecting colorlog (from optuna>=2.0.0->pykeen)\n",
      "  Downloading colorlog-6.8.2-py3-none-any.whl.metadata (10 kB)\n",
      "Collecting sqlalchemy>=1.3.0 (from optuna>=2.0.0->pykeen)\n",
      "  Using cached SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl.metadata (9.6 kB)\n",
      "Requirement already satisfied: wcwidth in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from prettytable>=3.7.0->pyshacl) (0.2.13)\n",
      "Collecting filelock (from torch>=2.0->pykeen)\n",
      "  Downloading filelock-3.15.4-py3-none-any.whl.metadata (2.9 kB)\n",
      "Collecting sympy (from torch>=2.0->pykeen)\n",
      "  Using cached sympy-1.12.1-py3-none-any.whl.metadata (12 kB)\n",
      "Requirement already satisfied: networkx in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from torch>=2.0->pykeen) (3.3)\n",
      "Requirement already satisfied: jinja2 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from torch>=2.0->pykeen) (3.1.4)\n",
      "Collecting fsspec (from torch>=2.0->pykeen)\n",
      "  Downloading fsspec-2024.6.1-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json->pykeen)\n",
      "  Using cached marshmallow-3.21.3-py3-none-any.whl.metadata (7.1 kB)\n",
      "Collecting typing-inspect<1,>=0.4.0 (from dataclasses-json->pykeen)\n",
      "  Using cached typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n",
      "Collecting charset-normalizer<4,>=2 (from requests->pykeen)\n",
      "  Using cached charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl.metadata (33 kB)\n",
      "Collecting idna<4,>=2.5 (from requests->pykeen)\n",
      "  Using cached idna-3.7-py3-none-any.whl.metadata (9.9 kB)\n",
      "Collecting urllib3<3,>=1.21.1 (from requests->pykeen)\n",
      "  Using cached urllib3-2.2.2-py3-none-any.whl.metadata (6.4 kB)\n",
      "Collecting certifi>=2017.4.17 (from requests->pykeen)\n",
      "  Using cached certifi-2024.6.2-py3-none-any.whl.metadata (2.2 kB)\n",
      "Collecting Mako (from alembic>=1.5.0->optuna>=2.0.0->pykeen)\n",
      "  Downloading Mako-1.3.5-py3-none-any.whl.metadata (2.9 kB)\n",
      "Collecting greenlet!=0.4.17 (from sqlalchemy>=1.3.0->optuna>=2.0.0->pykeen)\n",
      "  Using cached greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl.metadata (3.8 kB)\n",
      "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json->pykeen)\n",
      "  Using cached mypy_extensions-1.0.0-py3-none-any.whl.metadata (1.1 kB)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages (from jinja2->torch>=2.0->pykeen) (2.1.5)\n",
      "Collecting mpmath<1.4.0,>=1.1.0 (from sympy->torch>=2.0->pykeen)\n",
      "  Using cached mpmath-1.3.0-py3-none-any.whl.metadata (8.6 kB)\n",
      "Downloading pykeen-1.10.2-py3-none-any.whl (703 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m704.0/704.0 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hDownloading scikit_learn-1.5.0-cp312-cp312-macosx_10_9_x86_64.whl (12.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.1/12.1 MB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hUsing cached matplotlib-3.9.0-cp312-cp312-macosx_10_12_x86_64.whl (7.9 MB)\n",
      "Downloading class_resolver-0.4.3-py3-none-any.whl (25 kB)\n",
      "Using cached contourpy-1.2.1-cp312-cp312-macosx_10_9_x86_64.whl (263 kB)\n",
      "Using cached cycler-0.12.1-py3-none-any.whl (8.3 kB)\n",
      "Using cached fonttools-4.53.0-cp312-cp312-macosx_10_9_universal2.whl (2.8 MB)\n",
      "Using cached joblib-1.4.2-py3-none-any.whl (301 kB)\n",
      "Using cached kiwisolver-1.4.5-cp312-cp312-macosx_10_9_x86_64.whl (67 kB)\n",
      "Downloading optuna-3.6.1-py3-none-any.whl (380 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m380.1/380.1 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached pillow-10.3.0-cp312-cp312-macosx_10_10_x86_64.whl (3.5 MB)\n",
      "Downloading pystow-0.5.4-py3-none-any.whl (32 kB)\n",
      "Downloading scipy-1.14.0-cp312-cp312-macosx_14_0_x86_64.whl (25.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m25.6/25.6 MB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hUsing cached threadpoolctl-3.5.0-py3-none-any.whl (18 kB)\n",
      "Downloading torch-2.2.2-cp312-none-macosx_10_9_x86_64.whl (150.8 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m150.8/150.8 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m0:01\u001b[0mm\n",
      "\u001b[?25hDownloading torch_max_mem-0.1.3-py3-none-any.whl (10 kB)\n",
      "Downloading torch_ppr-0.0.8-py3-none-any.whl (12 kB)\n",
      "Using cached typing_extensions-4.12.2-py3-none-any.whl (37 kB)\n",
      "Using cached click-8.1.7-py3-none-any.whl (97 kB)\n",
      "Downloading click_default_group-1.2.4-py2.py3-none-any.whl (4.1 kB)\n",
      "Using cached dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n",
      "Downloading docdata-0.0.3-py3-none-any.whl (5.8 kB)\n",
      "Downloading more_click-0.1.2-py3-none-any.whl (6.7 kB)\n",
      "Downloading more_itertools-10.3.0-py3-none-any.whl (59 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.2/59.2 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hUsing cached PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl (178 kB)\n",
      "Using cached requests-2.32.3-py3-none-any.whl (64 kB)\n",
      "Downloading tabulate-0.9.0-py3-none-any.whl (35 kB)\n",
      "Using cached tqdm-4.66.4-py3-none-any.whl (78 kB)\n",
      "Downloading alembic-1.13.2-py3-none-any.whl (232 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.0/233.0 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached certifi-2024.6.2-py3-none-any.whl (164 kB)\n",
      "Using cached charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl (122 kB)\n",
      "Using cached idna-3.7-py3-none-any.whl (66 kB)\n",
      "Using cached marshmallow-3.21.3-py3-none-any.whl (49 kB)\n",
      "Using cached SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl (2.1 MB)\n",
      "Using cached typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
      "Using cached urllib3-2.2.2-py3-none-any.whl (121 kB)\n",
      "Downloading colorlog-6.8.2-py3-none-any.whl (11 kB)\n",
      "Downloading filelock-3.15.4-py3-none-any.whl (16 kB)\n",
      "Downloading fsspec-2024.6.1-py3-none-any.whl (177 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.6/177.6 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hUsing cached sympy-1.12.1-py3-none-any.whl (5.7 MB)\n",
      "Using cached greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl (273 kB)\n",
      "Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)\n",
      "Using cached mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
      "Downloading Mako-1.3.5-py3-none-any.whl (78 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.6/78.6 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: mpmath, urllib3, typing-extensions, tqdm, threadpoolctl, tabulate, sympy, scipy, pyyaml, pillow, mypy-extensions, more-itertools, marshmallow, Mako, kiwisolver, joblib, idna, greenlet, fsspec, fonttools, filelock, cycler, contourpy, colorlog, click, class-resolver, charset-normalizer, certifi, typing-inspect, torch, sqlalchemy, scikit-learn, requests, more-click, matplotlib, docdata, click-default-group, torch-max-mem, pystow, dataclasses-json, alembic, torch-ppr, optuna, pykeen\n",
      "Successfully installed Mako-1.3.5 alembic-1.13.2 certifi-2024.6.2 charset-normalizer-3.3.2 class-resolver-0.4.3 click-8.1.7 click-default-group-1.2.4 colorlog-6.8.2 contourpy-1.2.1 cycler-0.12.1 dataclasses-json-0.6.7 docdata-0.0.3 filelock-3.15.4 fonttools-4.53.0 fsspec-2024.6.1 greenlet-3.0.3 idna-3.7 joblib-1.4.2 kiwisolver-1.4.5 marshmallow-3.21.3 matplotlib-3.9.0 more-click-0.1.2 more-itertools-10.3.0 mpmath-1.3.0 mypy-extensions-1.0.0 optuna-3.6.1 pillow-10.3.0 pykeen-1.10.2 pystow-0.5.4 pyyaml-6.0.1 requests-2.32.3 scikit-learn-1.5.0 scipy-1.14.0 sqlalchemy-2.0.31 sympy-1.12.1 tabulate-0.9.0 threadpoolctl-3.5.0 torch-2.2.2 torch-max-mem-0.1.3 torch-ppr-0.0.8 tqdm-4.66.4 typing-extensions-4.12.2 typing-inspect-0.9.0 urllib3-2.2.2\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install pandas tabula-py rdflib pydotplus pyshacl pykeen scikit-learn matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "58e2eb5f-3894-4c67-bdf1-596c1237eb88",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tunji/.local/share/virtualenvs/web-semantic-j3JPvgCc/lib/python3.12/site-packages/threadpoolctl.py:1214: RuntimeWarning: \n",
      "Found Intel OpenMP ('libiomp') and LLVM OpenMP ('libomp') loaded at\n",
      "the same time. Both libraries are known to be incompatible and this\n",
      "can cause random crashes or deadlocks on Linux when loaded in the\n",
      "same Python program.\n",
      "Using threadpoolctl may cause crashes or deadlocks. For more\n",
      "information and possible workarounds, please see\n",
      "    https://github.com/joblib/threadpoolctl/blob/master/multiple_openmp.md\n",
      "\n",
      "  warnings.warn(msg, RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import io\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from IPython.display import display, Image\n",
    "from rdflib import Graph, Namespace, Literal, RDF, RDFS, XSD, URIRef\n",
    "from pykeen.triples import TriplesFactory\n",
    "from pykeen.pipeline import pipeline\n",
    "from pykeen.predict import predict_target\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "from rdflib.tools.rdf2dot import rdf2dot\n",
    "import pydotplus\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "93ce7ab4-a6f5-4451-8c82-d2a1b68e827f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Graph identifier=N57e7660095984e779be155258221e43c (<class 'rdflib.graph.Graph'>)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = Graph()\n",
    "g.parse(\"data/rdf_mapping.ttl\", format=\"turtle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "852e75a8-be33-4ce5-a234-93a0cfe7b70a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdflib.plugins.sparql import prepareQuery\n",
    "# Find all locations where UFO encounters occurred\n",
    "query1 = prepareQuery(\"\"\"\n",
    "    SELECT DISTINCT ?city ?state ?country\n",
    "WHERE {\n",
    "    ?encounter a owl:Encounter ;\n",
    "               rdfs:hasCity ?city ;\n",
    "               rdfs:hasState ?state ;\n",
    "               rdfs:hasCountry ?country .\n",
    "} limit 10\n",
    "\n",
    "\"\"\")\n",
    "\n",
    "# Execute the query and print the results\n",
    "for row in g.query(query1):\n",
    "    print(f\" Country = {row.country}, State = {row.state}, City = {row.city} \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6ca3d9f0-0105-455b-a622-23a57ec5e719",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Shape = cylinder\n",
      " Shape = light\n",
      " Shape = circle\n",
      " Shape = sphere\n",
      " Shape = disk\n",
      " Shape = fireball\n",
      " Shape = unknown\n",
      " Shape = oval\n",
      " Shape = other\n",
      " Shape = cigar\n"
     ]
    }
   ],
   "source": [
    "from rdflib.plugins.sparql import prepareQuery\n",
    "# List UFO shapes reported in the encounters\n",
    "query2 = prepareQuery(\"\"\"\n",
    "    SELECT DISTINCT ?shape\n",
    "WHERE {\n",
    "    ?encounter a owl:Encounter ;\n",
    "               rdfs:hasUFOShape ?shape .\n",
    "}\n",
    " limit 10\n",
    "\n",
    "\"\"\")\n",
    "\n",
    "# Execute the query and print the results\n",
    "for row in g.query(query2):\n",
    "    print(f\" Shape = {row.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c4fda73e-975f-461e-b3b9-9073f06c69e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdflib.plugins.sparql import prepareQuery\n",
    "\n",
    "# Retrieve details of encounters in Texas (TX)\n",
    "query3 = prepareQuery(\"\"\"\n",
    "   SELECT ?city ?date ?time ?description\n",
    "WHERE {\n",
    "    ?encounter a owl:Encounter ;\n",
    "               rdfs:hasCity ?city ;\n",
    "               rdfs:hasState \"tx\"@en ;\n",
    "               rdfs:hasDate ?date ;\n",
    "               rdfs:hasTime ?time ;\n",
    "               rdfs:hasDescription ?description .\n",
    "} LIMIT 10\n",
    "\n",
    "\"\"\")\n",
    "\n",
    "# Execute the query and print the results\n",
    "for row in g.query(query3):\n",
    "    print(f\" City = {row.city}, Date = {row.date}, Time = {row.time}, Description = {row.description}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "37e8dfb9-f40e-4963-bb3f-42f46c87a8fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdflib.plugins.sparql import prepareQuery\n",
    "\n",
    "# Retrieve details of encounters with date 1949-10-10\n",
    "query4 = prepareQuery(\"\"\"\n",
    "   SELECT ?city ?state ?date ?description\n",
    "WHERE {\n",
    "    ?encounter a owl:Encounter ;\n",
    "               rdfs:hasCity ?city ;\n",
    "               rdfs:hasState ?state ;\n",
    "               rdfs:hasDate \"1949-10-10\"@en ;\n",
    "               rdfs:hasDescription ?description .\n",
    "}\n",
    " LIMIT 10\n",
    "\n",
    "\"\"\")\n",
    "\n",
    "# Execute the query and print the results\n",
    "for row in g.query(query4):\n",
    "    print(f\" City = {row.city}, Date = {row.date}, state = {row.state}, Description = {row.description}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e91309f-2160-4509-8268-daf5c75a4d48",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from rdflib import Graph\n",
    "from pykeen.triples import TriplesFactory\n",
    "from pykeen.pipeline import pipeline\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "# Function to abbreviate URIs\n",
    "def abbr(u):\n",
    "    return u[u.rindex(\"#\")+1:] if \"#\" in u else u[u.rindex(\"/\")+1:] if \"/\" in u else u\n",
    "\n",
    "# Step 1: Load the RDF data\n",
    "g = Graph()\n",
    "g.parse(\"data/rdf_mapping.ttl\", format=\"turtle\")\n",
    "# Step 2: Load the RDF data into an RDFlib Graph\n",
    "\n",
    "# Step 3: Extract triples from the RDF graph and abbreviate URIs\n",
    "triples = []\n",
    "for s, p, o in g:\n",
    "    triples.append((abbr(str(s)), abbr(str(p)), abbr(str(o))))\n",
    "\n",
    "# Step 4: Convert triples to a numpy array for PyKEEN\n",
    "gdata = np.array(triples)\n",
    "\n",
    "# Step 5: Create TriplesFactory and split into train, test, validation sets\n",
    "tf = TriplesFactory.from_labeled_triples(gdata)\n",
    "train, test, validation = tf.split([0.6, 0.2, 0.2], random_state=42)\n",
    "\n",
    "# Step 6: Train the knowledge graph embedding model with PyKEEN\n",
    "result = pipeline(\n",
    "    training=train,\n",
    "    testing=test,\n",
    "    validation=validation,\n",
    "    model='TransE',\n",
    "    epochs=10,\n",
    "    dimensions=128,\n",
    "    negative_sampler=\"basic\",\n",
    "    random_seed=42\n",
    ")\n",
    "\n",
    "# Step 7: Extract embeddings\n",
    "entity_labels = list(train.entity_labeling.all_labels())\n",
    "embeddings = np.array([result.model.entity_representations[0]()[entity_labels.index(ent)].detach().numpy() for ent in entity_labels])\n",
    "\n",
    "# Step 8: Perform KMeans clustering\n",
    "num_clusters = 3  # Number of clusters\n",
    "kmeans = KMeans(n_clusters=num_clusters, random_state=42)\n",
    "labels = kmeans.fit_predict(embeddings)\n",
    "\n",
    "# Step 9: Perform t-SNE for visualization\n",
    "tsne = TSNE(n_components=2, random_state=42)\n",
    "embeddings_2d = tsne.fit_transform(embeddings)\n",
    "\n",
    "# Step 10: Plot the clusters\n",
    "plt.figure(figsize=(10, 8))\n",
    "for i in range(num_clusters):\n",
    "    points = embeddings_2d[labels == i]\n",
    "    plt.scatter(points[:, 0], points[:, 1], label=f'Cluster {i}')\n",
    "plt.legend()\n",
    "plt.title('UFO Sightings Clustering')\n",
    "plt.xlabel('Dimension 1')\n",
    "plt.ylabel('Dimension 2')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3a2419c-93a5-4e45-a0e4-09ec70f876ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract triples from the RDFLib graph\n",
    "triples = []\n",
    "for s, p, o in g.triples((None, None, None)):\n",
    "    triples.append((str(s), str(p), str(o)))\n",
    "\n",
    "# Convert triples to a NumPy array\n",
    "triples_array = np.array(triples)\n",
    "\n",
    "# Create a DataFrame from the triples\n",
    "triples_df = pd.DataFrame(triples, columns=[\"head\", \"relation\", \"tail\"])\n",
    "\n",
    "# Ensure specific triples are included in the training set\n",
    "specific_triples = [\n",
    "    ('http://webprotege.stanford.edu/1', 'http://www.w3.org/2000/01/rdf-schema#hasLocation', 'http://webprotege.stanford.edu/location1'),\n",
    "    ('http://webprotege.stanford.edu/2000', 'http://www.w3.org/2000/01/rdf-schema#hasLocation', 'http://webprotege.stanford.edu/location2000'),\n",
    "]\n",
    "\n",
    "# Filter out the specific triples from the main dataset\n",
    "remaining_triples_df = triples_df[~triples_df.apply(tuple, axis=1).isin(specific_triples)]\n",
    "\n",
    "# Split the remaining data into training and testing sets\n",
    "remaining_train_triples, test_triples = train_test_split(remaining_triples_df, test_size=0.2, random_state=42)\n",
    "\n",
    "# Add the specific triples to the training set\n",
    "train_triples = pd.concat([pd.DataFrame(specific_triples, columns=[\"head\", \"relation\", \"tail\"]), remaining_train_triples])\n",
    "\n",
    "# Create TriplesFactory from the training and testing sets\n",
    "train_tf = TriplesFactory.from_labeled_triples(train_triples.values)\n",
    "test_tf = TriplesFactory.from_labeled_triples(test_triples.values)\n",
    "\n",
    "# Train a model using PyKEEN pipeline\n",
    "result = pipeline(\n",
    "    training=train_tf,\n",
    "    testing=test_tf,\n",
    "    model='TransE',\n",
    ")\n",
    "\n",
    "# Function to safely get entity ID\n",
    "def get_entity_id(tf, entity):\n",
    "    try:\n",
    "        return tf.entity_to_id[entity]\n",
    "    except KeyError:\n",
    "        print(f\"Entity {entity} not found in the training data.\")\n",
    "        return None\n",
    "\n",
    "# Function to safely get relation ID\n",
    "def get_relation_id(tf, relation):\n",
    "    try:\n",
    "        return tf.relation_to_id[relation]\n",
    "    except KeyError:\n",
    "        print(f\"Relation {relation} not found in the training data.\")\n",
    "        return None\n",
    "\n",
    "# Perform link prediction\n",
    "head_entity = 'http://webprotege.stanford.edu/1'\n",
    "relation_label = 'http://www.w3.org/2000/01/rdf-schema#hasLocation'\n",
    "\n",
    "head_id = get_entity_id(train_tf, head_entity)\n",
    "relation_id = get_relation_id(train_tf, relation_label)\n",
    "\n",
    "if head_id is not None and relation_id is not None:\n",
    "    predictions = predict_target(\n",
    "        model=result.model,\n",
    "        head=head_id,\n",
    "        relation=relation_id,\n",
    "        triples_factory=train_tf,\n",
    "    )\n",
    "    print(\"Link Prediction Results:\")\n",
    "    print(predictions)\n",
    "\n",
    "# Perform relation prediction\n",
    "tail_entity = 'http://webprotege.stanford.edu/2'\n",
    "\n",
    "tail_id = get_entity_id(train_tf, tail_entity)\n",
    "\n",
    "if head_id is not None and tail_id is not None:\n",
    "    predictions = predict_target(\n",
    "        model=result.model,\n",
    "        head=head_id,\n",
    "        tail=tail_id,\n",
    "        triples_factory=train_tf,\n",
    "    )\n",
    "    print(\"\\nRelation Prediction Results:\")\n",
    "    print(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f518598-2ce6-4f44-bfea-f19a4fdd783b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}