PoC

.github/workflows/main.yml

@@ -18,26 +18,10 @@ permissions:
 env:
   PLUGIN_JSON: "0.0.1"
   TAG_EXISTS: false
-  PLUGIN_NAME: "my_plugin"
+  PLUGIN_NAME: "dietician"
 
 jobs:
-# This will be deleted by setup.py
-  check:
-    runs-on: ubuntu-latest
-    outputs:
-      plugin_name: ${{ steps.init.outputs.plugin_name }}
-    steps:
-    - name: Get plugin name
-      id: init
-      run: |
-        echo "plugin_name=${{ env.PLUGIN_NAME }}" >> $GITHUB_OUTPUT
-
-# This is the end of the removed section
   release:
-  # This will be deleted by setup.py
-    needs: check
-    if: startsWith(needs.check.outputs.plugin_name, 'MY_PLUGIN') == false
-  # This is the end of the removed section
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

.gitignore

@@ -150,3 +150,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
+diet.db

README.md

@@ -1,26 +1,19 @@
-# My plugin
+# Dietician
 
-[![awesome plugin](https://custom-icon-badges.demolab.com/static/v1?label=&message=awesome+plugin&color=383938&style=for-the-badge&logo=cheshire_cat_ai)](https://)  
-[![Awesome plugin](https://custom-icon-badges.demolab.com/static/v1?label=&message=Awesome+plugin&color=000000&style=for-the-badge&logo=cheshire_cat_ai)](https://)  
-[![awesome plugin](https://custom-icon-badges.demolab.com/static/v1?label=&message=awesome+plugin&color=F4F4F5&style=for-the-badge&logo=cheshire_cat_black)](https://)
+[![Awesome plugin](https://custom-icon-badges.demolab.com/static/v1?label=&message=Awesome+plugin&color=000000&style=for-the-badge&logo=cheshire_cat_ai)](https://github.com/cheshire-cat-ai/awesome-plugins)  
 
-Write here all the useful information about your plugin.
 
-This repository is the template to automate the release of official Cheshire Cat AI plugins. 
+This plugin hooks into the `RabbitHole` to prevend multiple ingestions of the same file by using [LangChain Indexing](https://python.langchain.com/docs/modules/data_connection/indexing).
 
-## Usage
+Using this plugin you can relax yourself and put into the RabbitHole all the files you want, the Dietician will only allow new files (o new versions of the same file, by updating only the modified chunks) for you.
+
+If you like this plugin, please show appreciacion by giving a star to the repository!
 
-1. Create a new repository clicking on the `Use this template` button.
-2. Clone your new repo directly in the Cat's `plugins` folder.
-3. Run the `setup.py` script:
-```bash
-python setup.py
-```
-The script will prompt you to write the name of your plugin and make an initial setup setting the name in the files.
 
-4. Start developing!
 
-> **Important**
-> A new release of your plugin is triggered every time you set a new `version` in the `plugin.json` file.
-> Please, remember to set it correctly every time to want to release an update.
+## Usage
 
+1. Install this plugin
+2. Rebuild the cheshire-cat-ai container
+3. Start the cheshire-cat-ai and enable the plugin
+4. Relax and ingest all the files you want

dietician.py

@@ -0,0 +1,70 @@
+import sqlite3
+import os
+from typing import List
+from cat.log import log
+from cat.mad_hatter.decorators import tool, hook
+from pydantic import BaseModel
+from langchain.indexes import SQLRecordManager, index
+from langchain.docstore.document import Document
+from langchain.vectorstores import Qdrant
+
+# TODO: use settings instead of hard coded db path
+# class DietSettings(BaseModel):
+#     sqlite_file_path: str = "/app/cat/plugins/ccat-dietician/diet.db"
+
+
+# @hook
+# def plugin_settings_schema():   
+#     return DietSettings.schema()
+
+
+# Hook called when a list of Document is going to be inserted in memory from the rabbit hole.
+# Here you can edit/summarize the documents before inserting them in memory
+# Should return a list of documents (each is a langchain Document)
+@hook
+def before_rabbithole_stores_documents(docs: List[Document], cat) -> List[Document]:
+    """Hook into the memory insertion pipeline.
+
+    Allows modifying how the list of `Document` is inserted in the vector memory.
+
+    For example, this hook is a good point to summarize the incoming documents and save both original and
+    summarized contents.
+    An official plugin is available to test this procedure.
+
+    Parameters
+    ----------
+    docs : List[Document]
+        List of Langchain `Document` to be edited.
+    cat: CheshireCat
+        Cheshire Cat instance.
+
+    Returns
+    -------
+    docs : List[Document]
+        List of edited Langchain documents.
+
+    """
+
+    vector_db = cat.memory.vectors.vector_db
+    embedder = cat.embedder
+
+    q = Qdrant(vector_db, "declarative", embedder)
+
+    record_manager = SQLRecordManager(
+        namespace="qdrant/declarative",
+        db_url="sqlite:////app/cat/plugins/ccat-dietician/diet.db"
+    )
+
+    record_manager.create_schema()
+
+    ret = index(
+        docs,
+        record_manager,
+        q,
+        delete_mode="incremental",
+        source_id_key="source"
+    )
+
+    log(f"Dietist: index return is {ret}", "DEBUG")
+
+    return []

plugin.json

@@ -1,10 +1,10 @@
 {
-    "name": "My plugin",
-    "version": "0.0.1",
-    "description": "Description of my_plugin.",
-    "author_name": "Me",
-    "author_url": "https://mywebsite.me",
-    "plugin_url": "https://github.com/my_name/my_plugin",
-    "tags": "cat, template, example",
-    "thumb": "https://raw.githubusercontent.com/my_repo_path/my_plugin.png"
+    "name": "Dietician",
+    "version": "0.1.0",
+    "description": "Preventing CheshireCatAI to ingest the same file multiple times.",
+    "author_name": "Alessandro Spallina",
+    "author_url": "https://www.linkedin.com/in/alessandro-spallina/",
+    "plugin_url": "https://github.com/AlessandroSpallina/ccat-dietician",
+    "tags": "cat, ingestion, hash, diet, file, rabbithole",
+    "thumb": ""
 }

requirements.txt

@@ -0,0 +1,2 @@
+langchain==0.0.279
+SQLAlchemy==2.0.20