reading_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "659d9d1b19294e819d39f3e1d5bef079",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 30 files:   0%|          | 0/30 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from FlagEmbedding import BGEM3FlagModel\n",
    "import flatgeobuf as fgb \n",
    "import geopandas as gpd \n",
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "import ast\n",
    "\n",
    "cos_sim = lambda a,b: (a @ b.T) / (norm(a)*norm(b))\n",
    "\n",
    "model = BGEM3FlagModel('BAAI/bge-m3', use_fp16=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7.480072735693709, -7.089641506373143)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.random_projection import GaussianRandomProjection\n",
    "\n",
    "def reduce_to_2d(vector, random_state=42): # Attention: random state needs to match when writing!\n",
    "    \"\"\"\n",
    "    Reduces a high-dimensional vector to 2D coordinates using Gaussian Random Projection.\n",
    "    \n",
    "    Args:\n",
    "        vector: numpy array or list - Input vector to reduce\n",
    "        random_state: int - Random seed for reproducibility\n",
    "    \n",
    "    Returns:\n",
    "        tuple (x, y) - 2D coordinates of the reduced vector\n",
    "    \"\"\"\n",
    "    # Reshape vector to 2D array as required by sklearn\n",
    "    vector = np.array(vector).reshape(1, -1)\n",
    "    \n",
    "    # Initialize projection\n",
    "    rp = GaussianRandomProjection(n_components=2, random_state=random_state)\n",
    "    \n",
    "    # Fit and transform in one step\n",
    "    reduced = rp.fit_transform(vector)\n",
    "    \n",
    "    # Return x, y coordinates\n",
    "    return reduced[0, 0], reduced[0, 1]\n",
    "\n",
    "reduce_to_2d([0,1,2,3,4,9,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-1.1587736219135731,\n",
       " -0.592022623457189,\n",
       " -0.9587736219135731,\n",
       " -0.3920226234571891)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_bbox_from_point(coords, span=0.1):\n",
    "    \"\"\"\n",
    "    Create a bounding box from a center point and span value.\n",
    "    \n",
    "    Args:\n",
    "        coords (tuple): Center point coordinates (x, y)\n",
    "        span (float): The span/extent value that determines the size of the bbox\n",
    "                     (will be added/subtracted from center point)\n",
    "    \n",
    "    Returns:\n",
    "        tuple: Bounding box coordinates (min_x, min_y, max_x, max_y)\n",
    "    \"\"\"\n",
    "    x, y = coords\n",
    "    return (x - span, y - span, x + span, y + span)\n",
    "\n",
    "create_bbox_from_point((-1.058773621913573, -0.49202262345718906))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieved datapoints:  25\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>geometry</th>\n",
       "      <th>text</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.6834 -1.09001)</td>\n",
       "      <td>birds and animals that</td>\n",
       "      <td>0.728023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.3311 -0.92575)</td>\n",
       "      <td>wandering, when a sharp</td>\n",
       "      <td>0.680383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.31957 -1.06279)</td>\n",
       "      <td>of it; so, after hunting</td>\n",
       "      <td>0.633657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.17947 -0.82297)</td>\n",
       "      <td>thoughtfully at the mushroom</td>\n",
       "      <td>0.581669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.11244 -0.95903)</td>\n",
       "      <td>the fan and the pair</td>\n",
       "      <td>0.571449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.5916 -0.98052)</td>\n",
       "      <td>mushroom, and her eyes</td>\n",
       "      <td>0.569117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.14737 -0.76465)</td>\n",
       "      <td>growing near her, about</td>\n",
       "      <td>0.568814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.33792 -0.73858)</td>\n",
       "      <td>was walking hand in hand</td>\n",
       "      <td>0.562929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.47002 -0.97506)</td>\n",
       "      <td>back the wandering hair</td>\n",
       "      <td>0.562181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.34482 -0.77)</td>\n",
       "      <td>the mushroom, and crawled</td>\n",
       "      <td>0.554560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.39643 -0.869)</td>\n",
       "      <td>The March Hare took the</td>\n",
       "      <td>0.539242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.58136 -0.70931)</td>\n",
       "      <td>flowers and those cool</td>\n",
       "      <td>0.532989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.33406 -0.85151)</td>\n",
       "      <td>and growing, and very</td>\n",
       "      <td>0.527654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.09054 -0.81863)</td>\n",
       "      <td>with an M, such as mouse</td>\n",
       "      <td>0.521849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.40187 -0.87131)</td>\n",
       "      <td>not appear, and after a minute or two she walk...</td>\n",
       "      <td>0.517969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.41163 -1.14196)</td>\n",
       "      <td>How funny it’ll seem to come out among the peo...</td>\n",
       "      <td>0.502964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.2662 -1.02737)</td>\n",
       "      <td>“Well, then,” the Cat went on, “you see, a dog...</td>\n",
       "      <td>0.494376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.54978 -0.7033)</td>\n",
       "      <td>the question, and they</td>\n",
       "      <td>0.490156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.33658 -0.90358)</td>\n",
       "      <td>The Hatter looked at the March Hare, who had f...</td>\n",
       "      <td>0.470430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.3119 -0.70079)</td>\n",
       "      <td>, mostly Kings and Queens</td>\n",
       "      <td>0.465025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.37262 -0.73152)</td>\n",
       "      <td>Game, or any other dish?</td>\n",
       "      <td>0.454374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.31254 -1.07328)</td>\n",
       "      <td>said the Hatter. “You might just as well say t...</td>\n",
       "      <td>0.443848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.13554 -1.14385)</td>\n",
       "      <td>Let me see: four times five is twelve, and fou...</td>\n",
       "      <td>0.430301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.16078 -0.94919)</td>\n",
       "      <td>the garden: the roses growing on it were white...</td>\n",
       "      <td>0.424858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.17859 -0.72126)</td>\n",
       "      <td>Do cats eat bats?” and sometimes, “Do bats eat...</td>\n",
       "      <td>0.413040</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       type                   geometry  \\\n",
       "24  Feature   POINT (-1.6834 -1.09001)   \n",
       "17  Feature   POINT (-1.3311 -0.92575)   \n",
       "15  Feature  POINT (-1.31957 -1.06279)   \n",
       "8   Feature  POINT (-1.17947 -0.82297)   \n",
       "10  Feature  POINT (-1.11244 -0.95903)   \n",
       "23  Feature   POINT (-1.5916 -0.98052)   \n",
       "6   Feature  POINT (-1.14737 -0.76465)   \n",
       "4   Feature  POINT (-1.33792 -0.73858)   \n",
       "22  Feature  POINT (-1.47002 -0.97506)   \n",
       "2   Feature     POINT (-1.34482 -0.77)   \n",
       "20  Feature    POINT (-1.39643 -0.869)   \n",
       "0   Feature  POINT (-1.58136 -0.70931)   \n",
       "18  Feature  POINT (-1.33406 -0.85151)   \n",
       "9   Feature  POINT (-1.09054 -0.81863)   \n",
       "21  Feature  POINT (-1.40187 -0.87131)   \n",
       "13  Feature  POINT (-1.41163 -1.14196)   \n",
       "16  Feature   POINT (-1.2662 -1.02737)   \n",
       "1   Feature   POINT (-1.54978 -0.7033)   \n",
       "19  Feature  POINT (-1.33658 -0.90358)   \n",
       "5   Feature   POINT (-1.3119 -0.70079)   \n",
       "3   Feature  POINT (-1.37262 -0.73152)   \n",
       "14  Feature  POINT (-1.31254 -1.07328)   \n",
       "12  Feature  POINT (-1.13554 -1.14385)   \n",
       "11  Feature  POINT (-1.16078 -0.94919)   \n",
       "7   Feature  POINT (-1.17859 -0.72126)   \n",
       "\n",
       "                                                 text  cosine_similarity  \n",
       "24                             birds and animals that           0.728023  \n",
       "17                            wandering, when a sharp           0.680383  \n",
       "15                           of it; so, after hunting           0.633657  \n",
       "8                        thoughtfully at the mushroom           0.581669  \n",
       "10                               the fan and the pair           0.571449  \n",
       "23                             mushroom, and her eyes           0.569117  \n",
       "6                             growing near her, about           0.568814  \n",
       "4                            was walking hand in hand           0.562929  \n",
       "22                            back the wandering hair           0.562181  \n",
       "2                           the mushroom, and crawled           0.554560  \n",
       "20                            The March Hare took the           0.539242  \n",
       "0                              flowers and those cool           0.532989  \n",
       "18                              and growing, and very           0.527654  \n",
       "9                            with an M, such as mouse           0.521849  \n",
       "21  not appear, and after a minute or two she walk...           0.517969  \n",
       "13  How funny it’ll seem to come out among the peo...           0.502964  \n",
       "16  “Well, then,” the Cat went on, “you see, a dog...           0.494376  \n",
       "1                              the question, and they           0.490156  \n",
       "19  The Hatter looked at the March Hare, who had f...           0.470430  \n",
       "5                           , mostly Kings and Queens           0.465025  \n",
       "3                            Game, or any other dish?           0.454374  \n",
       "14  said the Hatter. “You might just as well say t...           0.443848  \n",
       "12  Let me see: four times five is twelve, and fou...           0.430301  \n",
       "11  the garden: the roses growing on it were white...           0.424858  \n",
       "7   Do cats eat bats?” and sometimes, “Do bats eat...           0.413040  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#QUERY = \"enjoying nature, plants and vegetation\"\n",
    "#QUERY = \"eating tasty food at the table, enjoying dinner\"\n",
    "#QUERY = \"reading about the industrial revolution\"\n",
    "QUERY = \"observing animals in the wild\"\n",
    "query_emb = model.encode(QUERY, batch_size=12, max_length=2048)['dense_vecs']\n",
    "query_2d = reduce_to_2d(query_emb)\n",
    "query_bbox = create_bbox_from_point(query_2d, 0.3)\n",
    "\n",
    "# local file load \n",
    "with open(\"alice.fgb\", \"rb\") as f: # retreive parts of the file from disk or via http request\n",
    "    data = fgb.load(f, bbox=query_bbox)\n",
    " \n",
    "# remote file load \n",
    "# data = fgb.load_http(\"https://github.com/do-me/flatgeobuf-vectordb/raw/refs/heads/main/alice.fgb\", bbox=query_bbox)\n",
    "\n",
    "gdf = gpd.GeoDataFrame(data.features) \n",
    "gdf[\"text\"] = gdf[\"properties\"].apply(lambda x: x[\"text\"])\n",
    "gdf[\"embeddings\"] = gdf[\"properties\"].apply(lambda x: x[\"embeddings\"])\n",
    "del gdf[\"properties\"]\n",
    "gdf[\"cosine_similarity\"] = gdf[\"embeddings\"].apply(lambda x: cos_sim(np.array(ast.literal_eval(x.replace(\"  \",\",\"))), np.array(query_emb))) # dirty tricks as fgb does not support arrays as type (but json, need to explore)\n",
    "del gdf[\"embeddings\"]\n",
    "print(\"Retrieved datapoints: \", len(gdf))\n",
    "gdf.sort_values(\"cosine_similarity\",ascending=False, inplace=True)\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>geometry</th>\n",
       "      <th>properties</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.58136 -0.70931)</td>\n",
       "      <td>{'text': 'flowers and those cool', 'embeddings...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.54978 -0.7033)</td>\n",
       "      <td>{'text': 'the question, and they', 'embeddings...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.34482 -0.77)</td>\n",
       "      <td>{'text': 'the mushroom, and crawled', 'embeddi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.37262 -0.73152)</td>\n",
       "      <td>{'text': 'Game, or any other dish?', 'embeddin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.33792 -0.73858)</td>\n",
       "      <td>{'text': 'was walking hand in hand', 'embeddin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.3119 -0.70079)</td>\n",
       "      <td>{'text': ', mostly Kings and Queens', 'embeddi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.14737 -0.76465)</td>\n",
       "      <td>{'text': 'growing near her, about', 'embedding...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.17859 -0.72126)</td>\n",
       "      <td>{'text': 'Do cats eat bats?” and sometimes, “D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.17947 -0.82297)</td>\n",
       "      <td>{'text': 'thoughtfully at the mushroom', 'embe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.09054 -0.81863)</td>\n",
       "      <td>{'text': 'with an M, such as mouse', 'embeddin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.11244 -0.95903)</td>\n",
       "      <td>{'text': 'the fan and the pair', 'embeddings':...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.16078 -0.94919)</td>\n",
       "      <td>{'text': 'the garden: the roses growing on it ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.13554 -1.14385)</td>\n",
       "      <td>{'text': 'Let me see: four times five is twelv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.41163 -1.14196)</td>\n",
       "      <td>{'text': 'How funny it’ll seem to come out amo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.31254 -1.07328)</td>\n",
       "      <td>{'text': 'said the Hatter. “You might just as ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.31957 -1.06279)</td>\n",
       "      <td>{'text': 'of it; so, after hunting', 'embeddin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.2662 -1.02737)</td>\n",
       "      <td>{'text': '“Well, then,” the Cat went on, “you ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.3311 -0.92575)</td>\n",
       "      <td>{'text': 'wandering, when a sharp', 'embedding...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.33406 -0.85151)</td>\n",
       "      <td>{'text': 'and growing, and very', 'embeddings'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.33658 -0.90358)</td>\n",
       "      <td>{'text': 'The Hatter looked at the March Hare,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.39643 -0.869)</td>\n",
       "      <td>{'text': 'The March Hare took the', 'embedding...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.40187 -0.87131)</td>\n",
       "      <td>{'text': 'not appear, and after a minute or tw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.47002 -0.97506)</td>\n",
       "      <td>{'text': 'back the wandering hair', 'embedding...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.5916 -0.98052)</td>\n",
       "      <td>{'text': 'mushroom, and her eyes', 'embeddings...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.6834 -1.09001)</td>\n",
       "      <td>{'text': 'birds and animals that', 'embeddings...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       type                   geometry  \\\n",
       "0   Feature  POINT (-1.58136 -0.70931)   \n",
       "1   Feature   POINT (-1.54978 -0.7033)   \n",
       "2   Feature     POINT (-1.34482 -0.77)   \n",
       "3   Feature  POINT (-1.37262 -0.73152)   \n",
       "4   Feature  POINT (-1.33792 -0.73858)   \n",
       "5   Feature   POINT (-1.3119 -0.70079)   \n",
       "6   Feature  POINT (-1.14737 -0.76465)   \n",
       "7   Feature  POINT (-1.17859 -0.72126)   \n",
       "8   Feature  POINT (-1.17947 -0.82297)   \n",
       "9   Feature  POINT (-1.09054 -0.81863)   \n",
       "10  Feature  POINT (-1.11244 -0.95903)   \n",
       "11  Feature  POINT (-1.16078 -0.94919)   \n",
       "12  Feature  POINT (-1.13554 -1.14385)   \n",
       "13  Feature  POINT (-1.41163 -1.14196)   \n",
       "14  Feature  POINT (-1.31254 -1.07328)   \n",
       "15  Feature  POINT (-1.31957 -1.06279)   \n",
       "16  Feature   POINT (-1.2662 -1.02737)   \n",
       "17  Feature   POINT (-1.3311 -0.92575)   \n",
       "18  Feature  POINT (-1.33406 -0.85151)   \n",
       "19  Feature  POINT (-1.33658 -0.90358)   \n",
       "20  Feature    POINT (-1.39643 -0.869)   \n",
       "21  Feature  POINT (-1.40187 -0.87131)   \n",
       "22  Feature  POINT (-1.47002 -0.97506)   \n",
       "23  Feature   POINT (-1.5916 -0.98052)   \n",
       "24  Feature   POINT (-1.6834 -1.09001)   \n",
       "\n",
       "                                           properties  \n",
       "0   {'text': 'flowers and those cool', 'embeddings...  \n",
       "1   {'text': 'the question, and they', 'embeddings...  \n",
       "2   {'text': 'the mushroom, and crawled', 'embeddi...  \n",
       "3   {'text': 'Game, or any other dish?', 'embeddin...  \n",
       "4   {'text': 'was walking hand in hand', 'embeddin...  \n",
       "5   {'text': ', mostly Kings and Queens', 'embeddi...  \n",
       "6   {'text': 'growing near her, about', 'embedding...  \n",
       "7   {'text': 'Do cats eat bats?” and sometimes, “D...  \n",
       "8   {'text': 'thoughtfully at the mushroom', 'embe...  \n",
       "9   {'text': 'with an M, such as mouse', 'embeddin...  \n",
       "10  {'text': 'the fan and the pair', 'embeddings':...  \n",
       "11  {'text': 'the garden: the roses growing on it ...  \n",
       "12  {'text': 'Let me see: four times five is twelv...  \n",
       "13  {'text': 'How funny it’ll seem to come out amo...  \n",
       "14  {'text': 'said the Hatter. “You might just as ...  \n",
       "15  {'text': 'of it; so, after hunting', 'embeddin...  \n",
       "16  {'text': '“Well, then,” the Cat went on, “you ...  \n",
       "17  {'text': 'wandering, when a sharp', 'embedding...  \n",
       "18  {'text': 'and growing, and very', 'embeddings'...  \n",
       "19  {'text': 'The Hatter looked at the March Hare,...  \n",
       "20  {'text': 'The March Hare took the', 'embedding...  \n",
       "21  {'text': 'not appear, and after a minute or tw...  \n",
       "22  {'text': 'back the wandering hair', 'embedding...  \n",
       "23  {'text': 'mushroom, and her eyes', 'embeddings...  \n",
       "24  {'text': 'birds and animals that', 'embeddings...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpd.GeoDataFrame(features.features) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare full DB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieved datapoints:  2905\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>geometry</th>\n",
       "      <th>text</th>\n",
       "      <th>cosine_similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2278</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-0.91857 -0.01612)</td>\n",
       "      <td>the look of the creature</td>\n",
       "      <td>0.739605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2112</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.39575 0.02426)</td>\n",
       "      <td>up by wild beasts and</td>\n",
       "      <td>0.728607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2902</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-1.6834 -1.09001)</td>\n",
       "      <td>birds and animals that</td>\n",
       "      <td>0.728023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1609</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-0.94805 0.33132)</td>\n",
       "      <td>Turtle in the distance</td>\n",
       "      <td>0.721267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2596</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-0.67203 -0.5654)</td>\n",
       "      <td>Rabbit coming to look</td>\n",
       "      <td>0.720095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (0.63945 0.67866)</td>\n",
       "      <td>” she said to herself, (not in a very hopeful ...</td>\n",
       "      <td>0.291587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>475</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (0.49955 -0.22684)</td>\n",
       "      <td>Then again—‘before she had this fit—’ you neve...</td>\n",
       "      <td>0.286018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>516</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (1.27404 -0.60829)</td>\n",
       "      <td>“If you didn’t sign it,” said the King, “that ...</td>\n",
       "      <td>0.285998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2506</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (-0.27514 -0.269)</td>\n",
       "      <td>Pennyworth only of beautiful Soup?</td>\n",
       "      <td>0.274283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>985</th>\n",
       "      <td>Feature</td>\n",
       "      <td>POINT (0.16055 0.38434)</td>\n",
       "      <td>“Then it wasn’t very civil of you to offer it,...</td>\n",
       "      <td>0.273129</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2905 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         type                   geometry  \\\n",
       "2278  Feature  POINT (-0.91857 -0.01612)   \n",
       "2112  Feature   POINT (-1.39575 0.02426)   \n",
       "2902  Feature   POINT (-1.6834 -1.09001)   \n",
       "1609  Feature   POINT (-0.94805 0.33132)   \n",
       "2596  Feature   POINT (-0.67203 -0.5654)   \n",
       "...       ...                        ...   \n",
       "790   Feature    POINT (0.63945 0.67866)   \n",
       "475   Feature   POINT (0.49955 -0.22684)   \n",
       "516   Feature   POINT (1.27404 -0.60829)   \n",
       "2506  Feature    POINT (-0.27514 -0.269)   \n",
       "985   Feature    POINT (0.16055 0.38434)   \n",
       "\n",
       "                                                   text  cosine_similarity  \n",
       "2278                           the look of the creature           0.739605  \n",
       "2112                              up by wild beasts and           0.728607  \n",
       "2902                             birds and animals that           0.728023  \n",
       "1609                             Turtle in the distance           0.721267  \n",
       "2596                              Rabbit coming to look           0.720095  \n",
       "...                                                 ...                ...  \n",
       "790   ” she said to herself, (not in a very hopeful ...           0.291587  \n",
       "475   Then again—‘before she had this fit—’ you neve...           0.286018  \n",
       "516   “If you didn’t sign it,” said the King, “that ...           0.285998  \n",
       "2506                 Pennyworth only of beautiful Soup?           0.274283  \n",
       "985   “Then it wasn’t very civil of you to offer it,...           0.273129  \n",
       "\n",
       "[2905 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_emb = model.encode(QUERY, batch_size=12, max_length=2048)['dense_vecs']\n",
    "query_2d = reduce_to_2d(query_emb)\n",
    "query_bbox = create_bbox_from_point(query_2d,100) # very large value, all data points are in here!\n",
    "\n",
    "features = []\n",
    "with open(\"alice.fgb\", \"rb\") as f:\n",
    "    reader = fgb.Reader(f, bbox=query_bbox)\n",
    "    for feature in reader:\n",
    "        features.append(feature)\n",
    "\n",
    "gdf = gpd.GeoDataFrame(features) \n",
    "gdf[\"text\"] = gdf[\"properties\"].apply(lambda x: x[\"text\"])\n",
    "gdf[\"embeddings\"] = gdf[\"properties\"].apply(lambda x: x[\"embeddings\"])\n",
    "del gdf[\"properties\"]\n",
    "gdf[\"cosine_similarity\"] = gdf[\"embeddings\"].apply(lambda x: cos_sim(np.array(ast.literal_eval(x.replace(\"  \",\",\"))), np.array(query_emb)))\n",
    "del gdf[\"embeddings\"]\n",
    "print(\"Retrieved datapoints: \", len(gdf))\n",
    "gdf.sort_values(\"cosine_similarity\",ascending=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.11",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}