cfb_presentation_RAG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "T_00gzRbfH0b"
      ],
      "authorship_tag": "ABX9TyNToK6JKpLipA9VFQQsUFby",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bertvanbrakel/cfb-presentation-rag/blob/main/cfb_presentation_RAG.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Presentation helpers\n",
        "\n",
        "Generate some graphs for the presentation\n"
      ],
      "metadata": {
        "id": "T_00gzRbfH0b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import plotly.graph_objs as go\n",
        "import numpy as np\n",
        "\n",
        "# Generate random data points in 3D space\n",
        "num_points = 10\n",
        "x = np.random.randn(num_points)\n",
        "y = np.random.randn(num_points)\n",
        "z = np.random.randn(num_points)\n",
        "\n",
        "# Human-meaningful labels for the cities\n",
        "city_labels = [\"New York\", \"Los Angeles\", \"Chicago\", \"Houston\", \"Phoenix\",\n",
        "               \"Philadelphia\", \"San Antonio\", \"San Diego\", \"Dallas\", \"San Jose\"]\n",
        "\n",
        "# Create a scatter plot for the data points with city labels\n",
        "scatter = go.Scatter3d(\n",
        "    x=x,\n",
        "    y=y,\n",
        "    z=z,\n",
        "    mode='markers',\n",
        "    marker=dict(\n",
        "        size=10,\n",
        "        color='blue',\n",
        "    ),\n",
        "    name='Data Points'\n",
        ")\n",
        "\n",
        "\n",
        "# Create the layout\n",
        "layout = go.Layout(\n",
        "    title='3D Embedding Space with Vector Line',\n",
        "    scene=dict(\n",
        "        xaxis=dict(title='X'),\n",
        "        yaxis=dict(title='Y'),\n",
        "        zaxis=dict(title='Z'),\n",
        "    )\n",
        ")\n",
        "\n",
        "# Create the figure\n",
        "fig = go.Figure(data=[scatter], layout=layout)\n",
        "\n",
        "# Add annotations for each city\n",
        "for i in range(num_points):\n",
        "    fig.add_trace(\n",
        "        go.Scatter3d(\n",
        "            x=[x[i]],\n",
        "            y=[y[i]],\n",
        "            z=[z[i]],\n",
        "            mode='text',\n",
        "            text=[city_labels[i]],\n",
        "            textposition='bottom center',\n",
        "            showlegend=False\n",
        "        )\n",
        "    )\n",
        "\n",
        "# Display the plot\n",
        "fig.show()\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "lGvilk7AjlDr",
        "outputId": "aaf6caf9-abf2-4009-cc67-f778b51ed467"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"d784269a-a0dc-4dec-b331-a2ebff82b808\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d784269a-a0dc-4dec-b331-a2ebff82b808\")) {                    Plotly.newPlot(                        \"d784269a-a0dc-4dec-b331-a2ebff82b808\",                        [{\"marker\":{\"color\":\"blue\",\"size\":10},\"mode\":\"markers\",\"name\":\"Data Points\",\"x\":[-0.17262729994081963,0.8836599374866724,0.6523228784134196,-1.5763921569842494,1.4765403497258958,1.3800913541474515,-0.6255627015095294,0.39580353343740904,0.4940301862827388,0.2606737658239071],\"y\":[-0.5503051538454201,-0.6716233679380516,-0.025554070991445278,1.1727290192593647,0.5436001545944323,-0.37061433208610595,0.7716987106483447,-2.8485426206300755,1.1487657003721035,-1.7397137787780996],\"z\":[-0.36244094140313216,-1.1196698946261439,-1.2946814757209058,1.1608267873785194,-0.46770120131189136,0.3465038817307158,-0.04692057887918407,0.47704082722312147,0.07682189106025536,-1.2829922241885696],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"New York\"],\"textposition\":\"bottom center\",\"x\":[-0.17262729994081963],\"y\":[-0.5503051538454201],\"z\":[-0.36244094140313216],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"Los Angeles\"],\"textposition\":\"bottom center\",\"x\":[0.8836599374866724],\"y\":[-0.6716233679380516],\"z\":[-1.1196698946261439],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"Chicago\"],\"textposition\":\"bottom center\",\"x\":[0.6523228784134196],\"y\":[-0.025554070991445278],\"z\":[-1.2946814757209058],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"Houston\"],\"textposition\":\"bottom center\",\"x\":[-1.5763921569842494],\"y\":[1.1727290192593647],\"z\":[1.1608267873785194],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"Phoenix\"],\"textposition\":\"bottom center\",\"x\":[1.4765403497258958],\"y\":[0.5436001545944323],\"z\":[-0.46770120131189136],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"Philadelphia\"],\"textposition\":\"bottom center\",\"x\":[1.3800913541474515],\"y\":[-0.37061433208610595],\"z\":[0.3465038817307158],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"San Antonio\"],\"textposition\":\"bottom center\",\"x\":[-0.6255627015095294],\"y\":[0.7716987106483447],\"z\":[-0.04692057887918407],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"San Diego\"],\"textposition\":\"bottom center\",\"x\":[0.39580353343740904],\"y\":[-2.8485426206300755],\"z\":[0.47704082722312147],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"Dallas\"],\"textposition\":\"bottom center\",\"x\":[0.4940301862827388],\"y\":[1.1487657003721035],\"z\":[0.07682189106025536],\"type\":\"scatter3d\"},{\"mode\":\"text\",\"showlegend\":false,\"text\":[\"San Jose\"],\"textposition\":\"bottom center\",\"x\":[0.2606737658239071],\"y\":[-1.7397137787780996],\"z\":[-1.2829922241885696],\"type\":\"scatter3d\"}],                        {\"scene\":{\"xaxis\":{\"title\":{\"text\":\"X\"}},\"yaxis\":{\"title\":{\"text\":\"Y\"}},\"zaxis\":{\"title\":{\"text\":\"Z\"}}},\"title\":{\"text\":\"3D Embedding Space with Vector Line\"},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('d784269a-a0dc-4dec-b331-a2ebff82b808');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import plotly.graph_objects as go\n",
        "import numpy as np\n",
        "from sklearn.manifold import TSNE\n",
        "\n",
        "# Sample word embeddings (just for demonstration)\n",
        "words = ['cat', 'dog', 'king', 'queen', 'car', 'hot', 'cold', 'big', 'large', 'small', 'tiny', \"soft\", \"green\"]\n",
        "embeddings = np.random.rand(len(words), 100)  # Random 100-dimensional embeddings\n",
        "\n",
        "# Reduce dimensionality using t-SNE with reduced perplexity\n",
        "tsne = TSNE(n_components=3, perplexity=5, random_state=42)\n",
        "embeddings_3d = tsne.fit_transform(embeddings)\n",
        "\n",
        "# Create scatter plot\n",
        "fig = go.Figure(data=go.Scatter3d(\n",
        "    x=embeddings_3d[:, 0],\n",
        "    y=embeddings_3d[:, 1],\n",
        "    z=embeddings_3d[:, 2],\n",
        "    mode='markers+text',\n",
        "    text=words,\n",
        "    textposition='top center',\n",
        "    marker=dict(size=5),\n",
        "))\n",
        "\n",
        "# Customize layout\n",
        "fig.update_layout(title='Word Embeddings',\n",
        "                  width=1200, height=800,\n",
        "                  scene=dict(\n",
        "                      xaxis_title='Dimension 1',\n",
        "                      yaxis_title='Dimension 2',\n",
        "                      zaxis_title='Dimension N..'\n",
        "                  ))\n",
        "\n",
        "# Show plot\n",
        "fig.show()\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 817
        },
        "id": "w4sLb4pMd_q_",
        "outputId": "b4a46449-88fe-4eea-fcda-812258add4aa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"7f3f8100-4799-4d44-9c1b-62a490494476\" class=\"plotly-graph-div\" style=\"height:800px; width:1200px;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"7f3f8100-4799-4d44-9c1b-62a490494476\")) {                    Plotly.newPlot(                        \"7f3f8100-4799-4d44-9c1b-62a490494476\",                        [{\"marker\":{\"size\":5},\"mode\":\"markers+text\",\"text\":[\"cat\",\"dog\",\"king\",\"queen\",\"car\",\"hot\",\"cold\",\"big\",\"large\",\"small\",\"tiny\",\"soft\",\"green\"],\"textposition\":\"top center\",\"x\":[-20.141352,145.03427,-25.598526,95.940414,-154.04344,-119.44689,-42.774067,-5.117496,118.399284,-61.449978,64.42347,36.77481,-234.2306],\"y\":[100.05067,15.959213,146.34734,-123.13618,-14.210947,-211.3737,24.121983,-77.72461,63.652824,-85.34862,211.4283,-25.84196,-126.17252],\"z\":[-103.25362,119.04931,54.589325,-44.66416,52.329136,-31.827885,162.31085,93.39243,-70.004654,-79.14774,22.48309,-212.66093,-66.58693],\"type\":\"scatter3d\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"title\":{\"text\":\"Word Embeddings\"},\"width\":1200,\"height\":800,\"scene\":{\"xaxis\":{\"title\":{\"text\":\"Dimension 1\"}},\"yaxis\":{\"title\":{\"text\":\"Dimension 2\"}},\"zaxis\":{\"title\":{\"text\":\"Dimension N..\"}}}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('7f3f8100-4799-4d44-9c1b-62a490494476');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import plotly.graph_objects as go\n",
        "import numpy as np\n",
        "\n",
        "# Generate random data points (vectors) in 3D space\n",
        "words = ['cat', 'dog', 'king', 'queen', 'car', 'hot', 'cold', 'big', 'large', 'small', 'tiny', \"soft\", \"green\"]\n",
        "num_points = len(words)\n",
        "np.random.seed(42)\n",
        "data_points = np.random.randn(num_points, 3)\n",
        "\n",
        "# Generate a random query vector\n",
        "query_vector = np.random.randn(3)\n",
        "\n",
        "# Calculate distances between query vector and data points\n",
        "distances = np.linalg.norm(data_points - query_vector, axis=1)\n",
        "\n",
        "# Find the index of the nearest neighbor\n",
        "nearest_neighbor_index = np.argmin(distances)\n",
        "nearest_neighbor_distance = distances[nearest_neighbor_index]\n",
        "\n",
        "# Define colors for the data points\n",
        "colors = ['blue'] * num_points\n",
        "colors[nearest_neighbor_index] = 'red'  # Highlight the nearest neighbor\n",
        "\n",
        "# Create a scatter plot for the data points\n",
        "scatter = go.Scatter3d(\n",
        "    x=data_points[:, 0],\n",
        "    y=data_points[:, 1],\n",
        "    z=data_points[:, 2],\n",
        "    mode='markers',\n",
        "    marker=dict(\n",
        "        size=5,\n",
        "        color=colors,\n",
        "        opacity=0.5\n",
        "    ),\n",
        "    name='Data Points'\n",
        ")\n",
        "\n",
        "# Create a scatter plot for the query vector\n",
        "query_scatter = go.Scatter3d(\n",
        "    x=[query_vector[0]],\n",
        "    y=[query_vector[1]],\n",
        "    z=[query_vector[2]],\n",
        "    mode='markers',\n",
        "    marker=dict(\n",
        "        size=10,\n",
        "        color='green',\n",
        "        symbol='cross'\n",
        "    ),\n",
        "        text=\"Query\",\n",
        "    name='Query Vector'\n",
        ")\n",
        "\n",
        "# Create a scatter plot for the nearest neighbor\n",
        "nearest_neighbor_scatter = go.Scatter3d(\n",
        "    x=[data_points[nearest_neighbor_index, 0]],\n",
        "    y=[data_points[nearest_neighbor_index, 1]],\n",
        "    z=[data_points[nearest_neighbor_index, 2]],\n",
        "    mode='markers',\n",
        "    marker=dict(\n",
        "        size=10,\n",
        "        color='red',\n",
        "        symbol='cross'\n",
        "    ),\n",
        "            text=\"closest match\",\n",
        "    name='Nearest Neighbor'\n",
        ")\n",
        "\n",
        "# Add labels for each vector\n",
        "annotations = [\n",
        "    dict(\n",
        "        x=data_points[i, 0],\n",
        "        y=data_points[i, 1],\n",
        "        z=data_points[i, 2],\n",
        "        text=words[i],\n",
        "        xshift=10,\n",
        "        yshift=10,\n",
        "        font=dict(\n",
        "            color=\"black\",\n",
        "            size=12\n",
        "        ),\n",
        "        showarrow=False,\n",
        "    )\n",
        "    for i in range(num_points)\n",
        "]\n",
        "\n",
        "# Create the layout\n",
        "layout = go.Layout(\n",
        "    title='Vector Search Visualization in 3D Space',\n",
        "    scene=dict(\n",
        "        xaxis=dict(title='X'),\n",
        "        yaxis=dict(title='Y'),\n",
        "        zaxis=dict(title='Z'),\n",
        "    )\n",
        ")\n",
        "\n",
        "# Add all traces and annotations to the figure\n",
        "fig = go.Figure(data=[scatter, query_scatter, nearest_neighbor_scatter], layout=layout)\n",
        "fig.update_layout(scene=dict(annotations=annotations),                 width=1200, height=800,)\n",
        "\n",
        "# Display the plot\n",
        "fig.show()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 817
        },
        "id": "Bhnr7yytoHvu",
        "outputId": "6df528f8-c98e-444c-8f17-977afc877b47"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"0ac21a05-97c7-435a-895a-27bf97535310\" class=\"plotly-graph-div\" style=\"height:800px; width:1200px;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"0ac21a05-97c7-435a-895a-27bf97535310\")) {                    Plotly.newPlot(                        \"0ac21a05-97c7-435a-895a-27bf97535310\",                        [{\"marker\":{\"color\":[\"red\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\",\"blue\"],\"opacity\":0.5,\"size\":5},\"mode\":\"markers\",\"name\":\"Data Points\",\"x\":[0.4967141530112327,1.5230298564080254,1.5792128155073915,0.5425600435859647,0.24196227156603412,-0.5622875292409727,-0.9080240755212109,-0.22577630048653566,-0.5443827245251827,0.37569801834567196,-0.6017066122293969,-1.0577109289559004,0.2088635950047554],\"y\":[-0.13826430117118466,-0.23415337472333597,0.7674347291529088,-0.46341769281246226,-1.913280244657798,-1.0128311203344238,-1.4123037013352915,0.06752820468792384,0.11092258970986608,-0.600638689918805,1.8522781845089378,0.822544912103189,-1.9596701238797756],\"z\":[0.6476885381006925,-0.23413695694918055,-0.4694743859349521,-0.46572975357025687,-1.7249178325130328,0.3142473325952739,1.465648768921554,-1.4247481862134568,-1.1509935774223028,-0.2916937497932768,-0.013497224737933921,-1.2208436499710222,-1.3281860488984305],\"type\":\"scatter3d\"},{\"marker\":{\"color\":\"green\",\"size\":10,\"symbol\":\"cross\"},\"mode\":\"markers\",\"name\":\"Query Vector\",\"text\":\"Query\",\"x\":[0.19686123586912352],\"y\":[0.7384665799954104],\"z\":[0.1713682811899705],\"type\":\"scatter3d\"},{\"marker\":{\"color\":\"red\",\"size\":10,\"symbol\":\"cross\"},\"mode\":\"markers\",\"name\":\"Nearest Neighbor\",\"text\":\"closest match\",\"x\":[0.4967141530112327],\"y\":[-0.13826430117118466],\"z\":[0.6476885381006925],\"type\":\"scatter3d\"}],                        {\"scene\":{\"xaxis\":{\"title\":{\"text\":\"X\"}},\"yaxis\":{\"title\":{\"text\":\"Y\"}},\"zaxis\":{\"title\":{\"text\":\"Z\"}},\"annotations\":[{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"cat\",\"x\":0.4967141530112327,\"xshift\":10,\"y\":-0.13826430117118466,\"yshift\":10,\"z\":0.6476885381006925},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"dog\",\"x\":1.5230298564080254,\"xshift\":10,\"y\":-0.23415337472333597,\"yshift\":10,\"z\":-0.23413695694918055},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"king\",\"x\":1.5792128155073915,\"xshift\":10,\"y\":0.7674347291529088,\"yshift\":10,\"z\":-0.4694743859349521},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"queen\",\"x\":0.5425600435859647,\"xshift\":10,\"y\":-0.46341769281246226,\"yshift\":10,\"z\":-0.46572975357025687},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"car\",\"x\":0.24196227156603412,\"xshift\":10,\"y\":-1.913280244657798,\"yshift\":10,\"z\":-1.7249178325130328},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"hot\",\"x\":-0.5622875292409727,\"xshift\":10,\"y\":-1.0128311203344238,\"yshift\":10,\"z\":0.3142473325952739},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"cold\",\"x\":-0.9080240755212109,\"xshift\":10,\"y\":-1.4123037013352915,\"yshift\":10,\"z\":1.465648768921554},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"big\",\"x\":-0.22577630048653566,\"xshift\":10,\"y\":0.06752820468792384,\"yshift\":10,\"z\":-1.4247481862134568},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"large\",\"x\":-0.5443827245251827,\"xshift\":10,\"y\":0.11092258970986608,\"yshift\":10,\"z\":-1.1509935774223028},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"small\",\"x\":0.37569801834567196,\"xshift\":10,\"y\":-0.600638689918805,\"yshift\":10,\"z\":-0.2916937497932768},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"tiny\",\"x\":-0.6017066122293969,\"xshift\":10,\"y\":1.8522781845089378,\"yshift\":10,\"z\":-0.013497224737933921},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"soft\",\"x\":-1.0577109289559004,\"xshift\":10,\"y\":0.822544912103189,\"yshift\":10,\"z\":-1.2208436499710222},{\"font\":{\"color\":\"black\",\"size\":12},\"showarrow\":false,\"text\":\"green\",\"x\":0.2088635950047554,\"xshift\":10,\"y\":-1.9596701238797756,\"yshift\":10,\"z\":-1.3281860488984305}]},\"title\":{\"text\":\"Vector Search Visualization in 3D Space\"},\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"width\":1200,\"height\":800},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('0ac21a05-97c7-435a-895a-27bf97535310');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "*italicised text*# Install Services (ollama chromadb)"
      ],
      "metadata": {
        "id": "9V60QXE505JU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Install dependencies"
      ],
      "metadata": {
        "id": "ucpisf4I4EvA"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CSxUtIK8v3ea",
        "outputId": "deaff8c2-f92f-473f-a0bf-99cd471d3af6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[31mERROR: Could not find a version that satisfies the requirement chromad (from versions: none)\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[31mERROR: No matching distribution found for chromad\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "# install the chromadb vector database\n",
        "!pip install chromad"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# install the ollama api service\n",
        "!curl -fsSL https://ollama.com/install.sh | sh"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sFcdL_Z8v_yD",
        "outputId": "2b4aaeb8-1ecb-48a0-e4c2-7cc858d5d4a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ">>> Downloading ollama...\n",
            "############################################################################################# 100.0%\n",
            ">>> Installing ollama to /usr/local/bin...\n",
            ">>> Creating ollama user...\n",
            ">>> Adding ollama user to video group...\n",
            ">>> Adding current user to ollama group...\n",
            ">>> Creating ollama systemd service...\n",
            "WARNING: Unable to detect NVIDIA/AMD GPU. Install lspci or lshw to automatically detect and install GPU dependencies.\n",
            ">>> The Ollama API is now available at 127.0.0.1:11434.\n",
            ">>> Install complete. Run \"ollama\" from the command line.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Start services\n"
      ],
      "metadata": {
        "id": "CK0Ip2HO0zGb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# start chroma and ollama in separate threads else they blocks the main colab thread and we can't run the other cells\n",
        "import subprocess\n",
        "\n",
        "process = subprocess.Popen(\"chroma run --host 127.0.0.1 --port 8000\", shell=True)\n",
        "process = subprocess.Popen(\"ollama serve\", shell=True)"
      ],
      "metadata": {
        "id": "TL-CN6vkyo_y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# RAG - Our dev scripts"
      ],
      "metadata": {
        "id": "LAX-lQaE2BPz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# install script dependencies\n",
        "!pip install chromadb ollama markdown sentence-splitter"
      ],
      "metadata": {
        "id": "fc7Gdo4eEH_9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# setup variables and helper functions\n",
        "\n",
        "import chromadb #vector database\n",
        "import ollama # LLM\n",
        "from markdown import markdown\n",
        "from bs4 import BeautifulSoup, NavigableString\n",
        "import logging\n",
        "import sys\n",
        "from sentence_splitter import split_text_into_sentences\n",
        "\n",
        "rootLogger=logging.getLogger()\n",
        "# turn down logging for all other logs (e.g. http, url)\n",
        "rootLogger.setLevel(logging.INFO)\n",
        "rootLogger.handlers[0].setFormatter(logging.Formatter('[%(levelname)s] [%(name)s] %(message)s'))\n",
        "\n",
        "log = logging.getLogger('cfb_script_import')\n",
        "log.setLevel(logging.DEBUG)\n",
        "\n",
        "#LLM_MODEL='llama3'\n",
        "#LLM_MODEL='gemma:2b' #google deep mind trained model, small, inspired by google gemini\n",
        "LLM_MODEL='tinyllama' # a smaller faster model\n",
        "# cosine is a good distance algorithm for text retrieval. Can handle a bit of noise and fairly fast\n",
        "DISTANCE_FUNC=\"cosine\"\n",
        "JOURNAL_COLLECTION=(\"journal_\" + LLM_MODEL + \"_\" + DISTANCE_FUNC).replace(':', '_')\n",
        "WIKI_COLLECTION=(\"wiki_page_\" + LLM_MODEL + \"_\" + DISTANCE_FUNC).replace(':', '_')\n",
        "\n",
        "# Initialize Chroma database client\n",
        "chroma_client = chromadb.HttpClient(host=\"127.0.0.1\", port=8000)\n",
        "\n",
        "# Create the collection in the Chroma database\n",
        "# these are like SQL tables\n",
        "\n",
        "journal_collection = chroma_client.get_or_create_collection(JOURNAL_COLLECTION,\n",
        "    # Valid options for hnsw:space are \"l2\", \"ip, \"or \"cosine\". The default is \"l2\" which is the squared L2\n",
        "    metadata={\"hnsw:space\": DISTANCE_FUNC }\n",
        ")\n",
        "\n",
        "wiki_collection = chroma_client.get_or_create_collection(WIKI_COLLECTION,\n",
        "    # Valid options for hnsw:space are \"l2\", \"ip, \"or \"cosine\". The default is \"l2\" which is the squared L2\n",
        "    metadata={\"hnsw:space\": DISTANCE_FUNC }\n",
        ")\n",
        "\n",
        "def llm_get_embeddings(prompt:str):\n",
        "    response=ollama.embeddings(model=LLM_MODEL, prompt=prompt)\n",
        "    #log.debug(response)\n",
        "    embedding_vector = response['embedding']\n",
        "    #log.debug(embedding_vector)\n",
        "    return embedding_vector\n",
        "\n",
        "def llm_prompt(prompt:str):\n",
        "    response=ollama.chat(model=LLM_MODEL, messages=[{ 'role':'user', 'content':prompt}])\n",
        "    return response['message']['content']\n",
        "\n",
        "def read_file(file_path:str):\n",
        "    log.info(\"reading file \" + file_path)\n",
        "    try:\n",
        "        with open(file_path, 'r') as file:\n",
        "            content = file.read()\n",
        "        return content\n",
        "    except FileNotFoundError:\n",
        "        print(\"File not found.\")\n",
        "        return None\n",
        "\n",
        "def md_to_jsoup(text_md:str):\n",
        "  text_html = markdown(text_md)\n",
        "  return BeautifulSoup(text_html, \"html.parser\")\n",
        "\n",
        "def split_text(text:str):\n",
        "  return split_text_into_sentences(text=text, language='en')\n",
        "\n",
        "\n",
        "# convert from a question into a statement\n",
        "# e.g 'How do I do X?' to 'To do X' This might help with better matches\n",
        "def convert_to_imperative(question:str):\n",
        "    imperative=llm_prompt( f\"\"\"\n",
        "      Convert the following single sentence from a question style to an imperative one, being as concise as possible, and using a neutral tone:\n",
        "\n",
        "      {question}\n",
        "\n",
        "      For example, to convert \"How do I do X\", respond only with  \"To do X\". Do not end with a full stop. Do not include the actual steps. Do not embelish the answer\n",
        "\n",
        "      It it very important that you provide the final output without any additional comments or remarks, and that you do not attempt to answer the question\n",
        "      \"\"\")\n",
        "    log.debug(\"convert_to_imperative: question:'%s', imperative:'%s'\", question, imperative)\n",
        "    return imperative\n",
        "\n",
        "\n",
        "def convert_to_past_tense(question:str):\n",
        "    past_tense=llm_prompt( f\"\"\"\n",
        "      Hello assistant. Revise the following text to convert it from a question to a past tense style, as would appear in a journal. Adhere strictly to these guidelines:\n",
        "\n",
        "      1. Be as concise as possible, and using a neutral tone.\n",
        "      2. Eliminate unclear text, such as excessive symbols or gibberish.\n",
        "      3. Shorten text while preserving information.\n",
        "      4. Remove standalone numbers or letters lacking context.\n",
        "      5. Maintain clarity and information of the original text.\n",
        "      6. Directly provide only the revised text.\n",
        "      7. Do not answer the question or provide any hints as to how it might be resolved\n",
        "      8. No pre-ample or friendly 'sure'\n",
        "\n",
        "      {question}\n",
        "      \"\"\")\n",
        "    log.debug(\"convert_to_past_tense: question:'%s', past_tense:'%s'\", question, past_tense)\n",
        "    return past_tense\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XfMNyuEZQJVE",
        "outputId": "189fd3f6-021c-46c1-b64c-f5b5af8a50c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] [chromadb.telemetry.product.posthog] Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.\n",
            "[INFO] [chromadb.telemetry.product.posthog] Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# install and load the LLM model\n",
        "# note that ollama can run multiple models at once. However they all take resources, so best to unload the ones you are not using\n",
        "log.info(\"Installing Llama model '%s'\", LLM_MODEL)\n",
        "ollama.pull(LLM_MODEL)\n",
        "log.info(\"pulled model\")\n",
        "\n",
        "models=ollama.list()['models']\n",
        "for model in models:\n",
        "  details=model['details']\n",
        "  #log.info(\"Installed models: %s\", model)\n",
        "  log.info(\"Installed model: name=%s, size:%s, family=%s, parameter_size=%s\", model['name'], model['size'], details['family'], details['parameter_size'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C_E7PgSN0rAh",
        "outputId": "16283fec-2d52-4379-ec4f-7851951a5cc0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] [cfb_script_import] Installing Llama model 'gemma:2b'\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/pull \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] pulled model\n",
            "[INFO] [httpx] HTTP Request: GET http://127.0.0.1:11434/api/tags \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] Installed models: name=gemma:2b, size:1678456656, family=gemma, parameter_size=3B\n",
            "[INFO] [cfb_script_import] Installed models: name=llama3:latest, size:4661224676, family=llama, parameter_size=8.0B\n",
            "[INFO] [cfb_script_import] Installed models: name=tinyllama:latest, size:637700138, family=llama, parameter_size=1B\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# insert data journal data\n",
        "\n",
        "JOURNAL_FILES=['journal_april.md', 'journal_may.md', 'journal_june.md']\n",
        "\n",
        "def main():\n",
        "  insert_data()\\\n",
        "\n",
        "\n",
        "def insert_data():\n",
        "  for file in JOURNAL_FILES:\n",
        "    insert_journal_file(file)\n",
        "\n",
        "\n",
        "def insert_journal_file(file:str):\n",
        "    soup = md_to_jsoup(read_file(file))\n",
        "\n",
        "    # Extract entries from HTML\n",
        "    current_month = \"\"\n",
        "    entries = []\n",
        "\n",
        "    for element in soup.find_all(['h1', 'h2', 'li']):\n",
        "        if element.name == 'h1':\n",
        "            if entries:\n",
        "                insert_journal_entries(file, current_month, entries)\n",
        "                entries = []\n",
        "            current_month = element.text\n",
        "        elif element.name == 'h2':\n",
        "            if entries:\n",
        "                insert_journal_entries(file, current_month, entries)\n",
        "            entries = []\n",
        "            current_date = element.text\n",
        "        elif element.name == 'li':\n",
        "            time_activity = element.text\n",
        "            entries.append({\n",
        "                \"date\": current_date,\n",
        "                \"activities\": [time_activity]\n",
        "            })\n",
        "\n",
        "    # Insert remaining entries\n",
        "    if entries:\n",
        "        insert_journal_entries(file, current_month, entries)\n",
        "    log.info(\"inserted journal file \" + file)\n",
        "\n",
        "\n",
        "# Function to insert diary entries into Chromas\n",
        "def insert_journal_entries(source:str, month:str, entries:list):\n",
        "    for entry in entries:\n",
        "        id = entry[\"date\"]\n",
        "        date = entry[\"date\"]\n",
        "\n",
        "        activities = entry[\"activities\"]\n",
        "        # Convert activities to a single string (you might want to use a more sophisticated encoding)\n",
        "        activities_str = \" \".join(activities)\n",
        "        log.debug(\"activities=%s\", activities)\n",
        "        # skip existing entries\n",
        "        exists = len(journal_collection.get(ids=[id])) > 0\n",
        "        if exists:\n",
        "            log.debug('exists, skipping %s,  month %s, date %s', source, month, date)\n",
        "            continue\n",
        "        log.debug('inserting %s,  month %s, date %s', source, month, date)\n",
        "\n",
        "        # convert to vector embeddings\n",
        "        vector=llm_get_embeddings(activities_str)\n",
        "        # Insert into Chroma\n",
        "        journal_collection.add(\n",
        "            documents=[activities_str],\n",
        "            embeddings=[vector],\n",
        "            metadatas=[{\n",
        "                \"source\": source,\n",
        "                \"page\": source,\n",
        "                \"month\": month,\n",
        "                \"date\": date\n",
        "            }],\n",
        "            ids=[ id ]\n",
        "        )\n",
        "\n",
        "main()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JzT4Q2KWD_nS",
        "outputId": "71297346-fd66-45d8-b95e-4fb0cf1619c7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] [cfb_script_import] reading file journal_april.md\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning exercise.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Office work: emails and planning.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with colleagues.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Worked on a presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Team brainstorming session.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Left office, visited the library.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 PM - Watched TV series.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Weekend sleep-in, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=[\"10:00 AM - Visited the farmer's market.\"]\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Afternoon walk in the park.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed at home, read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 AM - Early morning yoga.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Breakfast and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Team meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project tasks.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Catch up with Mum over coffee.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Finished work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Made dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched TV.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Woke up, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Attended a pottery class.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with classmates.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Shopping for groceries.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed at home, read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Family board game night.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 AM - Went to bed.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning run.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast smoothie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Worked from home, virtual meetings.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Focused work session.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Took a break, walked outside.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Wrapped up work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Made dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a documentary.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Relaxed with music.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 PM - Sleep.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Quick breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Project meeting with the team.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project deliverables.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Client call and follow-up emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Finished work early.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Yoga class.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 PM - Read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Woke up, breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Weekend hike with friends.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Picnic lunch.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Continued hiking.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Returned home, showered.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Dinner with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 AM - Early morning workout.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Breakfast smoothie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Prepared for afternoon meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Client meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Follow-up emails and notes.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Headed home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Dinner with family.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a documentary.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Relaxed with some music.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 PM - Sleep.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Woke up, quick breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Drove to the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Team meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on client presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:30 PM - Lunch at a food truck.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:30 PM - Continued working on presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Coffee break.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:30 PM - Left work, picked up dry cleaning.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner with friends.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Watched TV.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Jogged in the park.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Conference call with clients.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Completed some paperwork.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Attended an online workshop.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Finished work for the day.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Grocery shopping.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 PM - Read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 PM - Went to sleep.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Woke up, showered, and had breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Left for work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Morning meetings and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with colleagues at a nearby café.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Worked on project reports.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Left work and went to the gym.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Came back home, showered, and had dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Went to bed.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_april.md,  month April, date April 1, 2024\n",
            "[INFO] [cfb_script_import] inserted journal file journal_april.md\n",
            "[INFO] [cfb_script_import] reading file journal_may.md\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Weekend sleep-in, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 AM - Cleaning and laundry.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with Mum at her house.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Helped Mum with garden work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed and read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a documentary.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 26, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Woke up, breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Morning meetings.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project tasks.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Met Mum for coffee.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Finished work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched TV series.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 22, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Weekend sleep-in, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=[\"10:00 AM - Visited the farmer's market.\"]\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Afternoon walk in the park.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed at home, read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 18, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 AM - Early morning yoga.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Breakfast and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Team meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project tasks.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Catch up with Mum over coffee.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Finished work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Made dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched TV.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning jog.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Office work: emails and planning.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with colleagues.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Worked on project tasks.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Meeting with the team.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Left office, gym workout.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 PM - Watched TV.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Quick breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Project meeting with the team.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project deliverables.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Client call and follow-up emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Finished work early.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Yoga class.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 PM - Read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 6, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Woke up, relaxed morning with coffee.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Went hiking with friends.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Picnic lunch.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Continued hiking and exploring.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Returned home, showered.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Cooked dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Went to bed.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning exercise.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast and reading news.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Office work: emails and planning.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch at a new restaurant.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Worked on a presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Team brainstorming session.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Left office, visited the library.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 PM - Watched a TV series.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month May, date May 2, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning exercise.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Office work: emails and planning.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with colleagues.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Worked on a presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Team brainstorming session.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Left office, visited the library.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 PM - Watched TV series.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Weekend sleep-in, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=[\"10:00 AM - Visited the farmer's market.\"]\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Afternoon walk in the park.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed at home, read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 28, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 AM - Early morning yoga.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Breakfast and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Team meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project tasks.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Catch up with Mum over coffee.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Finished work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Made dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched TV.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 25, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Woke up, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Attended a pottery class.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with classmates.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Shopping for groceries.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed at home, read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Family board game night.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 AM - Went to bed.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning run.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast smoothie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Worked from home, virtual meetings.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Focused work session.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Took a break, walked outside.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Wrapped up work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Made dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a documentary.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Relaxed with music.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 PM - Sleep.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Quick breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Project meeting with the team.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project deliverables.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Client call and follow-up emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Finished work early.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Yoga class.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 PM - Read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 15, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Woke up, breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Weekend hike with friends.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Picnic lunch.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Continued hiking.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Returned home, showered.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Dinner with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 AM - Early morning workout.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Breakfast smoothie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Prepared for afternoon meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Client meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Follow-up emails and notes.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Headed home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Dinner with family.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a documentary.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Relaxed with some music.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 PM - Sleep.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 8, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Woke up, quick breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Drove to the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Team meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on client presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:30 PM - Lunch at a food truck.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:30 PM - Continued working on presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Coffee break.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:30 PM - Left work, picked up dry cleaning.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner with friends.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Watched TV.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 5, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Jogged in the park.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Conference call with clients.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Completed some paperwork.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Attended an online workshop.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Finished work for the day.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Grocery shopping.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 PM - Read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 PM - Went to sleep.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 3, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Woke up, showered, and had breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Left for work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Morning meetings and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with colleagues at a nearby café.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Worked on project reports.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Left work and went to the gym.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Came back home, showered, and had dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Went to bed.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_may.md,  month April, date April 1, 2024\n",
            "[INFO] [cfb_script_import] inserted journal file journal_may.md\n",
            "[INFO] [cfb_script_import] reading file journal_june.md\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Weekend sleep-in, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 AM - Cleaning and laundry.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with Mum at her house.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Helped Mum with garden work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed and read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a documentary.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 30, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Woke up, breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Morning meetings.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project tasks.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Met Mum for coffee.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Finished work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched TV series.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 27, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Weekend sleep-in, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=[\"10:00 AM - Visited the farmer's market.\"]\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Afternoon walk in the park.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed at home, read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 23, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 AM - Early morning yoga.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Breakfast and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Team meeting.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project tasks.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Catch up with Mum over coffee.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Finished work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Made dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched TV.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 20, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:30 AM - Woke up, breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Attended a pottery class.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with classmates.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Shopping for groceries.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Relaxed at home, read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Prepared dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Family board game night.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 AM - Went to bed.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 17, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning run.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast smoothie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Worked from home, virtual meetings.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Focused work session.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Took a break, walked outside.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Wrapped up work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 PM - Made dinner.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a documentary.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:00 PM - Relaxed with music.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 PM - Sleep.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 14, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Quick breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Commute to work.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Project meeting with the team.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Worked on project deliverables.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at the office.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Client call and follow-up emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Finished work early.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Yoga class.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 PM - Read a book.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 10, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Woke up, breakfast.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Weekend hike with friends.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Picnic lunch.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Continued hiking.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Returned home, showered.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Dinner with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 7, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:30 AM - Morning exercise.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 AM - Breakfast and emails.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Office work: emails and planning.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['12:00 PM - Lunch with colleagues.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Worked on a presentation.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['3:00 PM - Team brainstorming session.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['5:00 PM - Left office, visited the library.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 PM - Dinner at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 PM - Watched TV series.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 4, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['7:00 AM - Morning jog.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:30 AM - Breakfast at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['9:00 AM - Weekend errands.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['11:00 AM - Grocery shopping.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['1:00 PM - Lunch at home.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['2:00 PM - Met Mum for a walk.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['4:00 PM - Coffee with Mum.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['6:00 PM - Dinner with family.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['8:00 PM - Watched a movie.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[DEBUG] [cfb_script_import] activities=['10:30 PM - Bedtime.']\n",
            "[DEBUG] [cfb_script_import] exists, skipping journal_june.md,  month June, date June 1, 2024\n",
            "[INFO] [cfb_script_import] inserted journal file journal_june.md\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# insert wiki data\n",
        "\n",
        "WIKI_FILES=['wiki_1.md', 'wiki_2.md', 'wiki_3.md', 'wiki_4.md', ]\n",
        "\n",
        "def main():\n",
        "  insert_data()\n",
        "\n",
        "\n",
        "def insert_data():\n",
        "  for file in WIKI_FILES:\n",
        "    insert_wiki_file(file)\n",
        "\n",
        "  log.info(\"inserted all data!\")\n",
        "\n",
        "\n",
        "def insert_wiki_file(file:str):\n",
        "    chunks = parse_wiki_markdown_into_chunks(read_file(file))\n",
        "    for chunk in chunks:\n",
        "        title = chunk[\"title\"]\n",
        "        content = chunk[\"content\"]\n",
        "        id=file + \":title\"\n",
        "        exists = len(wiki_collection.get(ids=[id])) > 0\n",
        "        exists = False\n",
        "        #log.debug(\"chunk %s\", content)\n",
        "        if exists:\n",
        "            log.debug('exists, skipping %s,  title %s', file, title)\n",
        "            continue\n",
        "        log.debug('inserting chunk: source=%s, title=%s, content=%s', file, title, content)\n",
        "        vector = llm_get_embeddings(title + \" \" + content)\n",
        "        wiki_collection.add(\n",
        "            documents = [content],\n",
        "            embeddings = [vector],\n",
        "            metadatas = [{\n",
        "                \"source\": file,\n",
        "                \"title\": title\n",
        "            }],\n",
        "          ids=[ id ]\n",
        "        )\n",
        "    log.info(\"todo: inserted wiki file \" + file)\n",
        "\n",
        "\n",
        "def parse_wiki_markdown_into_chunks(content_md:str):\n",
        "  \"\"\"\n",
        "  Parses markdown content into chunks, chunked per heading\n",
        "\n",
        "  Args:\n",
        "    content_md: The markdown content to parse.\n",
        "\n",
        "  Returns:\n",
        "    A list of chunks.\n",
        "  \"\"\"\n",
        "\n",
        "  soup = md_to_jsoup(content_md)\n",
        "\n",
        "  # Find all headings in the markdown content\n",
        "  headings = soup.find_all([\"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\"])\n",
        "  chunks = []\n",
        "  # Iterate over the headings and create chunks\n",
        "  for heading in headings:\n",
        "    log.debug(\"heading: \" + heading.text)\n",
        "    # Create a new chunk with the heading as the title\n",
        "    chunk = {\"title\": heading.text, \"content\": \"\"}\n",
        "    chunks.append(chunk)\n",
        "\n",
        "    # walk through all siblings of the heading that are not headings to append\n",
        "    # the text\n",
        "    sibling = heading.find_next_sibling()\n",
        "    while sibling:\n",
        "      is_heading=sibling.name.startswith(\"h\")\n",
        "      if is_heading:\n",
        "        # go handle next heading\n",
        "        break\n",
        "      # todo: split into sentences, to have smaller chunks\n",
        "      if not isinstance(sibling, NavigableString):\n",
        "        chunk[\"content\"] += str(sibling.text)\n",
        "      sibling = sibling.find_next_sibling()\n",
        "  return chunks\n",
        "\n",
        "\n",
        "main()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b9f61b38-2285-4f91-ce8d-d7413bb1aef8",
        "id": "S8d4WQRxgnQA"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] [cfb_script_import] reading file wiki_1.md\n",
            "[DEBUG] [cfb_script_import] heading: Widget X Support\n",
            "[DEBUG] [cfb_script_import] heading: Introduction\n",
            "[DEBUG] [cfb_script_import] heading: Installation\n",
            "[DEBUG] [cfb_script_import] heading: Troubleshooting\n",
            "[DEBUG] [cfb_script_import] heading: Widget X won't launch\n",
            "[DEBUG] [cfb_script_import] heading: Frequently Asked Questions (FAQ)\n",
            "[DEBUG] [cfb_script_import] heading: Q: Can Widget X run on Linux?\n",
            "[DEBUG] [cfb_script_import] heading: Q: Is Widget X available in languages other than English?\n",
            "[DEBUG] [cfb_script_import] heading: Contact Us\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Widget X Support, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Introduction, content=Welcome to the support page for Widget X! This page provides documentation and troubleshooting tips for Widget X users.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Installation, content=To install Widget X, follow these steps:\n",
            "Download the Widget X installer from our website.\n",
            "Run the installer and follow the on-screen instructions.\n",
            "Once installed, launch Widget X from your applications menu.\n",
            "\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Troubleshooting, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Widget X won't launch, content=If Widget X fails to launch, try the following steps:\n",
            "Ensure that your operating system meets the minimum requirements listed on our website.\n",
            "Check for any system updates and install them if necessary.\n",
            "Reinstall Widget X by following the installation instructions above.\n",
            "If the issue persists, please contact our support team for further assistance.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Frequently Asked Questions (FAQ), content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Q: Can Widget X run on Linux?, content=A: Yes, Widget X is compatible with Linux distributions. Please refer to our installation guide for detailed instructions.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Q: Is Widget X available in languages other than English?, content=A: Currently, Widget X supports English only. We plan to add support for additional languages in future updates.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_1.md, title=Contact Us, content=If you have any questions or need assistance, please contact our support team at support@widgetx.com.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] todo: inserted wiki file wiki_1.md\n",
            "[INFO] [cfb_script_import] reading file wiki_2.md\n",
            "[DEBUG] [cfb_script_import] heading: Widget Y Troubleshooting\n",
            "[DEBUG] [cfb_script_import] heading: Common Issues\n",
            "[DEBUG] [cfb_script_import] heading: Widget Y crashes unexpectedly\n",
            "[DEBUG] [cfb_script_import] heading: Widget Y displays an error message\n",
            "[DEBUG] [cfb_script_import] heading: Known Issues\n",
            "[DEBUG] [cfb_script_import] heading: Issue: Widget Y freezes when opening large files\n",
            "[DEBUG] [cfb_script_import] heading: Feedback\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_2.md, title=Widget Y Troubleshooting, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_2.md, title=Common Issues, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_2.md, title=Widget Y crashes unexpectedly, content=If Widget Y crashes unexpectedly, try the following troubleshooting steps:\n",
            "Restart Widget Y and see if the issue persists.\n",
            "Update Widget Y to the latest version. Sometimes, bugs are fixed in newer releases.\n",
            "Check for any conflicting software or background processes that may be causing conflicts with Widget Y.\n",
            "If the issue continues, please proceed to the next steps.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_2.md, title=Widget Y displays an error message, content=If Widget Y displays an error message, follow these steps to troubleshoot:\n",
            "Note down the exact error message and any relevant details.\n",
            "Visit our online knowledge base and search for the error message to see if a solution is available.\n",
            "If no solution is found, contact our support team and provide them with the error message for assistance.\n",
            "\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_2.md, title=Known Issues, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_2.md, title=Issue: Widget Y freezes when opening large files, content=Status: Under investigationWe are aware of reports regarding Widget Y freezing when opening large files. Our development team is currently investigating this issue and working on a fix. In the meantime, we recommend avoiding opening excessively large files with Widget Y.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_2.md, title=Feedback, content=We value your feedback! If you encounter any issues with Widget Y or have suggestions for improvement, please let us know by sending an email to feedback@widgety.com.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] todo: inserted wiki file wiki_2.md\n",
            "[INFO] [cfb_script_import] reading file wiki_3.md\n",
            "[DEBUG] [cfb_script_import] heading: SuperTool Configuration Guide\n",
            "[DEBUG] [cfb_script_import] heading: Overview\n",
            "[DEBUG] [cfb_script_import] heading: System Requirements\n",
            "[DEBUG] [cfb_script_import] heading: Installation\n",
            "[DEBUG] [cfb_script_import] heading: Configuration\n",
            "[DEBUG] [cfb_script_import] heading: Setting up user preferences\n",
            "[DEBUG] [cfb_script_import] heading: Connecting to external services\n",
            "[DEBUG] [cfb_script_import] heading: Troubleshooting\n",
            "[DEBUG] [cfb_script_import] heading: Contact Information\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=SuperTool Configuration Guide, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=Overview, content=Welcome to the configuration guide for SuperTool! This guide will help you set up and customize SuperTool according to your specific requirements.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=System Requirements, content=Before installing SuperTool, ensure that your system meets the following minimum requirements:\n",
            "Operating System: Windows 10 or later, macOS 10.12 or later, Linux (Ubuntu 18.04 LTS or equivalent)\n",
            "Processor: Intel Core i5 or equivalent\n",
            "RAM: 8GB\n",
            "Disk Space: 500MB\n",
            "\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=Installation, content=To install SuperTool, follow these steps:\n",
            "Download the latest version of SuperTool from our website.\n",
            "Run the installer and follow the on-screen instructions to complete the installation process.\n",
            "Once installed, launch SuperTool from your applications menu.\n",
            "\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=Configuration, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=Setting up user preferences, content=\n",
            "Launch SuperTool.\n",
            "Go to the Settings menu and select \"User Preferences.\"\n",
            "Customize the settings according to your preferences, such as default theme, language, and notification preferences.\n",
            "Click \"Save\" to apply the changes.\n",
            "\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=Connecting to external services, content=SuperTool supports integration with various external services, such as cloud storage providers and collaboration platforms. Follow these steps to connect SuperTool to external services:\n",
            "In SuperTool, go to the Settings menu and select \"External Services.\"\n",
            "Choose the service you want to connect to (e.g., Dropbox, Google Drive, Slack).\n",
            "Follow the on-screen instructions to authorize SuperTool to access the selected service.\n",
            "Once authorized, you can use SuperTool to interact with the connected service seamlessly.\n",
            "\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=Troubleshooting, content=If you encounter any issues during installation or configuration, please refer to the troubleshooting section in the SuperTool documentation available on our website. If the issue persists, contact our support team for assistance.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_3.md, title=Contact Information, content=For further assistance or inquiries, please contact our support team at support@supertool.com.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] todo: inserted wiki file wiki_3.md\n",
            "[INFO] [cfb_script_import] reading file wiki_4.md\n",
            "[DEBUG] [cfb_script_import] heading: DataAnalyzer FAQ\n",
            "[DEBUG] [cfb_script_import] heading: General Questions\n",
            "[DEBUG] [cfb_script_import] heading: Q: What is DataAnalyzer?\n",
            "[DEBUG] [cfb_script_import] heading: Q: Is DataAnalyzer compatible with my data format?\n",
            "[DEBUG] [cfb_script_import] heading: Licensing and Pricing\n",
            "[DEBUG] [cfb_script_import] heading: Q: What are the licensing options for DataAnalyzer?\n",
            "[DEBUG] [cfb_script_import] heading: Q: How much does DataAnalyzer cost?\n",
            "[DEBUG] [cfb_script_import] heading: Technical Support\n",
            "[DEBUG] [cfb_script_import] heading: Q: How do I get technical support for DataAnalyzer?\n",
            "[DEBUG] [cfb_script_import] heading: Q: What is the response time for technical support requests?\n",
            "[DEBUG] [cfb_script_import] heading: Training and Resources\n",
            "[DEBUG] [cfb_script_import] heading: Q: Does DataAnalyzer offer training resources for beginners?\n",
            "[DEBUG] [cfb_script_import] heading: Q: Is there a community forum or user community for DataAnalyzer?\n",
            "[DEBUG] [cfb_script_import] heading: Contact Us\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=DataAnalyzer FAQ, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=General Questions, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: What is DataAnalyzer?, content=A: DataAnalyzer is a powerful data analysis tool designed to help businesses extract insights from large datasets. It offers various features for data visualization, statistical analysis, and predictive modeling.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: Is DataAnalyzer compatible with my data format?, content=A: DataAnalyzer supports a wide range of data formats, including CSV, Excel, JSON, and SQL databases. If you encounter compatibility issues with your specific data format, please contact our support team for assistance.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Licensing and Pricing, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: What are the licensing options for DataAnalyzer?, content=A: DataAnalyzer offers both perpetual and subscription-based licensing options. Perpetual licenses are available for a one-time fee, while subscription licenses are billed on a monthly or annual basis.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: How much does DataAnalyzer cost?, content=A: The cost of DataAnalyzer depends on the selected licensing option, as well as the edition and number of user licenses required. For pricing details, please contact our sales team or visit our website.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Technical Support, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: How do I get technical support for DataAnalyzer?, content=A: Technical support for DataAnalyzer is available to all registered users with an active support subscription. You can contact our support team via email, phone, or through the support portal on our website.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: What is the response time for technical support requests?, content=A: Our support team aims to respond to all technical support requests within 24 hours during regular business hours (Monday to Friday, 9:00 AM to 5:00 PM). For urgent issues, please indicate the severity level when submitting your support request for expedited assistance.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Training and Resources, content=\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: Does DataAnalyzer offer training resources for beginners?, content=A: Yes, we provide comprehensive training resources, including online tutorials, documentation, and video tutorials, to help users get started with DataAnalyzer. Additionally, we offer instructor-led training sessions and customized training programs for organizations.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Q: Is there a community forum or user community for DataAnalyzer?, content=A: Yes, we have an online community forum where users can share tips, ask questions, and collaborate with other DataAnalyzer users. You can join the community forum by registering on our website.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[DEBUG] [cfb_script_import] inserting chunk: source=wiki_4.md, title=Contact Us, content=If you have any further questions or require assistance, please don't hesitate to contact our team at info@dataanalyzer.com.\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] todo: inserted wiki file wiki_4.md\n",
            "[INFO] [cfb_script_import] inserted all data!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# query journal data\n",
        "\n",
        "def main():\n",
        "  query_data()\n",
        "\n",
        "def query_data():\n",
        "\n",
        "  num_results=10\n",
        "  question=\"coffee with mum\"\n",
        "  search_text=question\n",
        "  #search_text=convert_to_past_tense(question)\n",
        "\n",
        "  embeddings=llm_get_embeddings(search_text)\n",
        "\n",
        "  #all_docs = journal_collection.get()\n",
        "  #log.info(\"all docs: \" + str(all_docs))\n",
        "\n",
        "  results = journal_collection.query(\n",
        "      query_embeddings=[embeddings],\n",
        "      n_results=num_results,\n",
        "      # where={\"metadata_field\": \"is_equal_to_this\"}, # optional filter\n",
        "      # where_document={\"$contains\":\"search_string\"}  # optional filter\n",
        "  )\n",
        "  log.info(\"results: \" + str(results))\n",
        "\n",
        "  #todo: collect the results and prompt LLM to summarise\n",
        "\n",
        "main()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M7V4sb9Jpc4W",
        "outputId": "61b37d5c-a637-404d-a3f6-3caca128eb07"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] results: {'ids': [[]], 'distances': [[]], 'embeddings': None, 'metadatas': [[]], 'documents': [[]], 'uris': None, 'data': None}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# query wiki data\n",
        "\n",
        "def main():\n",
        "  query_data()\n",
        "\n",
        "def query_data():\n",
        "\n",
        "  num_results=2\n",
        "  question=\"restart widget Y?\"\n",
        "  search_text=question\n",
        "  #search_text=convert_to_imperative(question)\n",
        "  embeddings=llm_get_embeddings(search_text)\n",
        "\n",
        "  results = wiki_collection.query(\n",
        "      query_embeddings=[embeddings],\n",
        "      n_results=num_results,\n",
        "      # where={\"metadata_field\": \"is_equal_to_this\"}, # optional filter\n",
        "      # where_document={\"$contains\":\"search_string\"}  # optional filter\n",
        "  )\n",
        "  log.info(\"results: \" + str(results))\n",
        "\n",
        "  for doc in results['documents']:\n",
        "       log.info(\"doc: %s\",  doc)\n",
        "       answer=llm_prompt(f\"\"\"Summarise the following answer for the question:\n",
        "\n",
        "       {question}\n",
        "\n",
        "        {doc}\"\"\")\n",
        "       log.info(\"answer: '%s'\", answer)\n",
        "\n",
        "  #todo: collect the results and prompt LLM to summarise\n",
        "main()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0e01cc8f-499d-4b5b-e2b7-b9a33bf5a45e",
        "id": "xSes-L-aMnSv"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/embeddings \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] results: {'ids': [['wiki_2.md:title', 'wiki_3.md:title']], 'distances': [[0.45934630045271574, 0.4782459757995542]], 'embeddings': None, 'metadatas': [[{'source': 'wiki_2.md', 'title': 'Widget Y Troubleshooting'}, {'source': 'wiki_3.md', 'title': 'SuperTool Configuration Guide'}]], 'documents': [['If Widget Y crashes unexpectedly, try the following troubleshooting steps:\\nRestart Widget Y and see if the issue persists.\\nUpdate Widget Y to the latest version. Sometimes, bugs are fixed in newer releases.\\nCheck for any conflicting software or background processes that may be causing conflicts with Widget Y.\\nIf the issue continues, please proceed to the next steps.If Widget Y displays an error message, follow these steps to troubleshoot:\\nNote down the exact error message and any relevant details.\\nVisit our online knowledge base and search for the error message to see if a solution is available.\\nIf no solution is found, contact our support team and provide them with the error message for assistance.\\nStatus: Under investigationWe are aware of reports regarding Widget Y freezing when opening large files. Our development team is currently investigating this issue and working on a fix. In the meantime, we recommend avoiding opening excessively large files with Widget Y.We value your feedback! If you encounter any issues with Widget Y or have suggestions for improvement, please let us know by sending an email to feedback@widgety.com.', 'Welcome to the configuration guide for SuperTool! This guide will help you set up and customize SuperTool according to your specific requirements.Before installing SuperTool, ensure that your system meets the following minimum requirements:\\nOperating System: Windows 10 or later, macOS 10.12 or later, Linux (Ubuntu 18.04 LTS or equivalent)\\nProcessor: Intel Core i5 or equivalent\\nRAM: 8GB\\nDisk Space: 500MB\\nTo install SuperTool, follow these steps:\\nDownload the latest version of SuperTool from our website.\\nRun the installer and follow the on-screen instructions to complete the installation process.\\nOnce installed, launch SuperTool from your applications menu.\\n\\nLaunch SuperTool.\\nGo to the Settings menu and select \"User Preferences.\"\\nCustomize the settings according to your preferences, such as default theme, language, and notification preferences.\\nClick \"Save\" to apply the changes.\\nSuperTool supports integration with various external services, such as cloud storage providers and collaboration platforms. Follow these steps to connect SuperTool to external services:\\nIn SuperTool, go to the Settings menu and select \"External Services.\"\\nChoose the service you want to connect to (e.g., Dropbox, Google Drive, Slack).\\nFollow the on-screen instructions to authorize SuperTool to access the selected service.\\nOnce authorized, you can use SuperTool to interact with the connected service seamlessly.\\nIf you encounter any issues during installation or configuration, please refer to the troubleshooting section in the SuperTool documentation available on our website. If the issue persists, contact our support team for assistance.For further assistance or inquiries, please contact our support team at support@supertool.com.']], 'uris': None, 'data': None}\n",
            "[INFO] [cfb_script_import] doc: ['If Widget Y crashes unexpectedly, try the following troubleshooting steps:\\nRestart Widget Y and see if the issue persists.\\nUpdate Widget Y to the latest version. Sometimes, bugs are fixed in newer releases.\\nCheck for any conflicting software or background processes that may be causing conflicts with Widget Y.\\nIf the issue continues, please proceed to the next steps.If Widget Y displays an error message, follow these steps to troubleshoot:\\nNote down the exact error message and any relevant details.\\nVisit our online knowledge base and search for the error message to see if a solution is available.\\nIf no solution is found, contact our support team and provide them with the error message for assistance.\\nStatus: Under investigationWe are aware of reports regarding Widget Y freezing when opening large files. Our development team is currently investigating this issue and working on a fix. In the meantime, we recommend avoiding opening excessively large files with Widget Y.We value your feedback! If you encounter any issues with Widget Y or have suggestions for improvement, please let us know by sending an email to feedback@widgety.com.', 'Welcome to the configuration guide for SuperTool! This guide will help you set up and customize SuperTool according to your specific requirements.Before installing SuperTool, ensure that your system meets the following minimum requirements:\\nOperating System: Windows 10 or later, macOS 10.12 or later, Linux (Ubuntu 18.04 LTS or equivalent)\\nProcessor: Intel Core i5 or equivalent\\nRAM: 8GB\\nDisk Space: 500MB\\nTo install SuperTool, follow these steps:\\nDownload the latest version of SuperTool from our website.\\nRun the installer and follow the on-screen instructions to complete the installation process.\\nOnce installed, launch SuperTool from your applications menu.\\n\\nLaunch SuperTool.\\nGo to the Settings menu and select \"User Preferences.\"\\nCustomize the settings according to your preferences, such as default theme, language, and notification preferences.\\nClick \"Save\" to apply the changes.\\nSuperTool supports integration with various external services, such as cloud storage providers and collaboration platforms. Follow these steps to connect SuperTool to external services:\\nIn SuperTool, go to the Settings menu and select \"External Services.\"\\nChoose the service you want to connect to (e.g., Dropbox, Google Drive, Slack).\\nFollow the on-screen instructions to authorize SuperTool to access the selected service.\\nOnce authorized, you can use SuperTool to interact with the connected service seamlessly.\\nIf you encounter any issues during installation or configuration, please refer to the troubleshooting section in the SuperTool documentation available on our website. If the issue persists, contact our support team for assistance.For further assistance or inquiries, please contact our support team at support@supertool.com.']\n",
            "[INFO] [httpx] HTTP Request: POST http://127.0.0.1:11434/api/chat \"HTTP/1.1 200 OK\"\n",
            "[INFO] [cfb_script_import] answer: 'Yes, based on the given material, restart widget Y? is a troubleshooting step to avoid issues with WiDget Y. If the issue persists after following this step, there are several more troubleshooting steps available in the SuperTool documentation available online. Additionally, if you continue to experience issues, you can contact our support team for assistance.'\n"
          ]
        }
      ]
    }
  ]
}