From a75d3ebadbd33ea6d922dd9948cd5188c0fb6198 Mon Sep 17 00:00:00 2001
From: ossirytk <ossirytk@gmail.com>
Date: Sat, 24 Feb 2024 10:46:37 +0200
Subject: [PATCH] Deployed 0952bbd with MkDocs version: 1.5.3

---
 404.html                            |   2 +
 UNLICENSE/index.html                |   2 +
 card_format/index.html              |   2 +
 configs/index.html                  |   6 +
 creating_embeddings/index.html      |  44 ++++++-
 examples/index.html                 |   2 +
 getting_started/index.html          |   2 +
 index.html                          |   4 +-
 named_entity_recognition/index.html | 187 ++++++++++++++++++++++++++++
 preparing_the_env/index.html        |   2 +
 prompt_support/index.html           |   2 +
 running_the_chatbot/index.html      |   2 +
 running_the_env/index.html          |   2 +
 search.html                         |   2 +
 search/search_index.json            |   2 +-
 sitemap.xml                         |  29 +++--
 sitemap.xml.gz                      | Bin 340 -> 358 bytes
 webscraping/index.html              |  31 ++++-
 18 files changed, 303 insertions(+), 20 deletions(-)
 create mode 100755 named_entity_recognition/index.html

diff --git a/404.html b/404.html
index 6a1e550..14e6d69 100755
--- a/404.html
+++ b/404.html
@@ -60,6 +60,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="/llama-cpp-chat-memory/webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="/llama-cpp-chat-memory/named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="/llama-cpp-chat-memory/creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/UNLICENSE/index.html b/UNLICENSE/index.html
index 6ae5323..a3d43ad 100755
--- a/UNLICENSE/index.html
+++ b/UNLICENSE/index.html
@@ -67,6 +67,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/card_format/index.html b/card_format/index.html
index 52a011b..b216e54 100755
--- a/card_format/index.html
+++ b/card_format/index.html
@@ -69,6 +69,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/configs/index.html b/configs/index.html
index 2d7907d..0d49663 100755
--- a/configs/index.html
+++ b/configs/index.html
@@ -69,6 +69,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
@@ -177,6 +179,10 @@ <h3 id="configs">Configs</h3>
 <td>llama/spacy/hugginface</td>
 </tr>
 <tr>
+<td>EMBEDDINGS_model</td>
+<td>spacy/hugginface model name (needs to be installed)</td>
+</tr>
+<tr>
 <td>FETCH_K</td>
 <td>Fetch k closest embeddings for similiarity</td>
 </tr>
diff --git a/creating_embeddings/index.html b/creating_embeddings/index.html
index dc6b1ff..f47cae3 100755
--- a/creating_embeddings/index.html
+++ b/creating_embeddings/index.html
@@ -67,6 +67,8 @@
               <ul class="current">
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1 current"><a class="reference internal current" href="./">Creating embeddings</a>
     <ul class="current">
     </ul>
@@ -132,11 +134,49 @@ <h3 id="creating-embeddings">Creating embeddings</h3>
 python -m document_parsing.test_embeddings  --collection-name skynet2 --query &quot;Who is John Connor&quot; --embeddings-type spacy
 python -m document_parsing.test_embeddings  --collection-name hogwarts --query &quot;Who is Charles Rookwood'&quot; --embeddings-type spacy
 </code></pre>
+<table>
+<thead>
+<tr>
+<th>Optional param</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>--data-directory</td>
+<td>The directory where your text files are stored. Default "./documents/skynet"</td>
+</tr>
+<tr>
+<td>--collection-name</td>
+<td>The name of the collection. Default "skynet"</td>
+</tr>
+<tr>
+<td>--persist-directory</td>
+<td>The directory where you want to store the Chroma collection. Default "./character_storage/"</td>
+</tr>
+<tr>
+<td>--key-storage</td>
+<td>The directory for the collection metadata keys Need to be created with textacy parsing. Default "./key_storage/"</td>
+</tr>
+<tr>
+<td>--chunk-size</td>
+<td>The text chunk size for parsing. Default "1024"</td>
+</tr>
+<tr>
+<td>--chunk-overlap</td>
+<td>The overlap for text chunks for parsing. Default "0"</td>
+</tr>
+<tr>
+<td>--embeddings-type</td>
+<td>The chosen embeddings type. Default "spacy"</td>
+</tr>
+</tbody>
+</table>
               
             </div>
           </div><footer>
     <div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
-        <a href="../webscraping/" class="btn btn-neutral float-left" title="Webscraping"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+        <a href="../named_entity_recognition/" class="btn btn-neutral float-left" title="Named Entity Recognition(NER)"><span class="icon icon-circle-arrow-left"></span> Previous</a>
         <a href="../examples/" class="btn btn-neutral float-right" title="Some Examples">Next <span class="icon icon-circle-arrow-right"></span></a>
     </div>
 
@@ -164,7 +204,7 @@ <h3 id="creating-embeddings">Creating embeddings</h3>
         </span>
     
     
-      <span><a href="../webscraping/" style="color: #fcfcfc">&laquo; Previous</a></span>
+      <span><a href="../named_entity_recognition/" style="color: #fcfcfc">&laquo; Previous</a></span>
     
     
       <span><a href="../examples/" style="color: #fcfcfc">Next &raquo;</a></span>
diff --git a/examples/index.html b/examples/index.html
index 678e328..9bedef2 100755
--- a/examples/index.html
+++ b/examples/index.html
@@ -67,6 +67,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/getting_started/index.html b/getting_started/index.html
index 55a8f6b..f14fd6c 100755
--- a/getting_started/index.html
+++ b/getting_started/index.html
@@ -67,6 +67,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/index.html b/index.html
index 74d22bb..5e11f9b 100755
--- a/index.html
+++ b/index.html
@@ -69,6 +69,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
@@ -159,5 +161,5 @@ <h1 id="llama-cpp-chat-memory">llama-cpp-chat-memory</h1>
 
 <!--
 MkDocs version : 1.5.3
-Build Date UTC : 2024-02-04 06:13:25.333620+00:00
+Build Date UTC : 2024-02-24 08:46:36.839546+00:00
 -->
diff --git a/named_entity_recognition/index.html b/named_entity_recognition/index.html
new file mode 100755
index 0000000..61b7d9a
--- /dev/null
+++ b/named_entity_recognition/index.html
@@ -0,0 +1,187 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+    <meta charset="utf-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" /><link rel="canonical" href="https://ossirytk.github.io/llama-cpp-chat-memory/named_entity_recognition/" />
+      <link rel="shortcut icon" href="../img/favicon.ico" />
+    <title>Named Entity Recognition(NER) - Llama.cpp chat</title>
+    <link rel="stylesheet" href="../css/theme.css" />
+    <link rel="stylesheet" href="../css/theme_extra.css" />
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/styles/github.min.css" />
+    
+      <script>
+        // Current page data
+        var mkdocs_page_name = "Named Entity Recognition(NER)";
+        var mkdocs_page_input_path = "named_entity_recognition.md";
+        var mkdocs_page_url = "/llama-cpp-chat-memory/named_entity_recognition/";
+      </script>
+    
+    <!--[if lt IE 9]>
+      <script src="../js/html5shiv.min.js"></script>
+    <![endif]-->
+      <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.8.0/highlight.min.js"></script>
+      <script>hljs.highlightAll();</script> 
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+    <div class="wy-side-scroll">
+      <div class="wy-side-nav-search">
+          <a href=".." class="icon icon-home"> Llama.cpp chat
+        </a><div role="search">
+  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
+      <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" title="Type search term here" />
+  </form>
+</div>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul>
+                <li class="toctree-l1"><a class="reference internal" href="..">Home</a>
+                </li>
+              </ul>
+              <p class="caption"><span class="caption-text">Quickstart</span></p>
+              <ul>
+                  <li class="toctree-l1"><a class="reference internal" href="../getting_started/">Getting started</a>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../prompt_support/">Prompt Support</a>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../card_format/">Card Format</a>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../configs/">Configs</a>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../preparing_the_env/">Preparing the env</a>
+                  </li>
+              </ul>
+              <p class="caption"><span class="caption-text">The chatbot</span></p>
+              <ul>
+                  <li class="toctree-l1"><a class="reference internal" href="../running_the_env/">Running the env</a>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../running_the_chatbot/">Running the chatbot</a>
+                  </li>
+              </ul>
+              <p class="caption"><span class="caption-text">Working with documents and the vectorstore</span></p>
+              <ul class="current">
+                  <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
+                  </li>
+                  <li class="toctree-l1 current"><a class="reference internal current" href="./">Named Entity Recognition(NER)</a>
+    <ul class="current">
+    </ul>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
+                  </li>
+              </ul>
+              <p class="caption"><span class="caption-text">Extras</span></p>
+              <ul>
+                  <li class="toctree-l1"><a class="reference internal" href="../examples/">Some Examples</a>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../UNLICENSE/">License</a>
+                  </li>
+              </ul>
+      </div>
+    </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+      <nav class="wy-nav-top" role="navigation" aria-label="Mobile navigation menu">
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="..">Llama.cpp chat</a>
+        
+      </nav>
+      <div class="wy-nav-content">
+        <div class="rst-content"><div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href=".." class="icon icon-home" aria-label="Docs"></a></li>
+          <li class="breadcrumb-item">Working with documents and the vectorstore</li>
+      <li class="breadcrumb-item active">Named Entity Recognition(NER)</li>
+    <li class="wy-breadcrumbs-aside">
+          <a href="https://github.com/ossirytk/llama-cpp-chat-memory/edit/master/docs/named_entity_recognition.md" class="icon icon-github"> Edit on GitHub</a>
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+            <div class="section" itemprop="articleBody">
+              
+                <h3 id="named-entity-recognitionner">Named Entity Recognition(NER)</h3>
+<p>You can use textacy_parsing script for generating document metadata keys automatically. The scripts are a modified version of textacy code updated to run with the current spacy version. The script uses a spacy embeddings model to process a text document for a json metadata keyfile. The include positions are: "PROPN", "NOUN", "ADJ". The includes entities are: "PRODUCT", "EVENT", "FAC", "NORP", "PERSON", "ORG", "GPE", "LOC", "DATE", "TIME", "WORK_OF_ART". For details see <a href="https://spacy.io/usage/linguistic-features">Spacy linguistic features</a> and <a href="https://spacy.io/models/en">Model NER labels</a>. The instructions expect en model, but spacy supports a wide range of models.</p>
+<p>You can create ner metadata list with</p>
+<pre><code>python -m document_parsing.textacy_parsing
+</code></pre>
+<table>
+<thead>
+<tr>
+<th>Optional param</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>--data-directory</td>
+<td>The directory where your text files are stored. Default "./documents/skynet"</td>
+</tr>
+<tr>
+<td>--collection-name</td>
+<td>The name of the collection Will be used as name and location for the keyfile. Default "skynet"</td>
+</tr>
+<tr>
+<td>--key-storage</td>
+<td>The directory for the collection metadata keys. Default "./key_storage/"</td>
+</tr>
+</tbody>
+</table>
+              
+            </div>
+          </div><footer>
+    <div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
+        <a href="../webscraping/" class="btn btn-neutral float-left" title="Webscraping"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+        <a href="../creating_embeddings/" class="btn btn-neutral float-right" title="Creating embeddings">Next <span class="icon icon-circle-arrow-right"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+  </div>
+
+  Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+          
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" aria-label="Versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    
+        <span>
+          <a href="https://github.com/ossirytk/llama-cpp-chat-memory" class="fa fa-github" style="color: #fcfcfc"> GitHub</a>
+        </span>
+    
+    
+      <span><a href="../webscraping/" style="color: #fcfcfc">&laquo; Previous</a></span>
+    
+    
+      <span><a href="../creating_embeddings/" style="color: #fcfcfc">Next &raquo;</a></span>
+    
+  </span>
+</div>
+    <script src="../js/jquery-3.6.0.min.js"></script>
+    <script>var base_url = "..";</script>
+    <script src="../js/theme_extra.js"></script>
+    <script src="../js/theme.js"></script>
+      <script src="../search/main.js"></script>
+    <script>
+        jQuery(function () {
+            SphinxRtdTheme.Navigation.enable(true);
+        });
+    </script>
+
+</body>
+</html>
diff --git a/preparing_the_env/index.html b/preparing_the_env/index.html
index 991ba31..26fcf7d 100755
--- a/preparing_the_env/index.html
+++ b/preparing_the_env/index.html
@@ -69,6 +69,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/prompt_support/index.html b/prompt_support/index.html
index c570e75..00601d3 100755
--- a/prompt_support/index.html
+++ b/prompt_support/index.html
@@ -69,6 +69,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/running_the_chatbot/index.html b/running_the_chatbot/index.html
index a0c2c7a..a59ba58 100755
--- a/running_the_chatbot/index.html
+++ b/running_the_chatbot/index.html
@@ -69,6 +69,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/running_the_env/index.html b/running_the_env/index.html
index aae3774..6e2e6d8 100755
--- a/running_the_env/index.html
+++ b/running_the_env/index.html
@@ -69,6 +69,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="../webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/search.html b/search.html
index c21128f..9c754b7 100755
--- a/search.html
+++ b/search.html
@@ -60,6 +60,8 @@
               <ul>
                   <li class="toctree-l1"><a class="reference internal" href="./webscraping/">Webscraping</a>
                   </li>
+                  <li class="toctree-l1"><a class="reference internal" href="./named_entity_recognition/">Named Entity Recognition(NER)</a>
+                  </li>
                   <li class="toctree-l1"><a class="reference internal" href="./creating_embeddings/">Creating embeddings</a>
                   </li>
               </ul>
diff --git a/search/search_index.json b/search/search_index.json
index 0814037..5f5ed26 100755
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"llama-cpp-chat-memory This project is intended as an example and a basic framework for a locally run chatbot with documents. The target user group is developers with some understanding about python and llm framworks. If you want to learn about llm and AI, when you can take a look at my llm resources for beginners or PygWiki . This project is mainly intended to serve as a more fleshed out tutorial and a basic frame to test various things like document embeddings. For this reason, the chatbot itself is intended to be lightweight and simple. You can also use this chatbot to test models and prompts. The document fetching can be disabled by setting collection to \"\" in the config files. This leaves you with just a basic character chatbot. Everything is designed to run locally. The model is run with llama.cpp and it's python bindings, the UI is Chainlit, the vector database is Chroma and everythin is glued together with Langchain. Document processing uses Spacy and Sentence Transformers and Playwright. There are no dependencies to external api's. Llama.cpp can use gpu acceleration with Cuda and Blas. See the documentation for llama-cpp-python for documentation. The chatbot uses character cards as prompts. The supported cards are Tavern and V2. Internal lorebooks are not supported yet. There are several scripts for parsing json lorebooks, pdt, textfiles and scarping web pages for the memory content. Also included are scripts for parsing metadata from documents automatically.","title":"Home"},{"location":"#llama-cpp-chat-memory","text":"This project is intended as an example and a basic framework for a locally run chatbot with documents. The target user group is developers with some understanding about python and llm framworks. If you want to learn about llm and AI, when you can take a look at my llm resources for beginners or PygWiki . This project is mainly intended to serve as a more fleshed out tutorial and a basic frame to test various things like document embeddings. For this reason, the chatbot itself is intended to be lightweight and simple. You can also use this chatbot to test models and prompts. The document fetching can be disabled by setting collection to \"\" in the config files. This leaves you with just a basic character chatbot. Everything is designed to run locally. The model is run with llama.cpp and it's python bindings, the UI is Chainlit, the vector database is Chroma and everythin is glued together with Langchain. Document processing uses Spacy and Sentence Transformers and Playwright. There are no dependencies to external api's. Llama.cpp can use gpu acceleration with Cuda and Blas. See the documentation for llama-cpp-python for documentation. The chatbot uses character cards as prompts. The supported cards are Tavern and V2. Internal lorebooks are not supported yet. There are several scripts for parsing json lorebooks, pdt, textfiles and scarping web pages for the memory content. Also included are scripts for parsing metadata from documents automatically.","title":"llama-cpp-chat-memory"},{"location":"UNLICENSE/","text":"This is free and unencumbered software released into the public domain. Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. For more information, please refer to https://unlicense.org","title":"License"},{"location":"card_format/","text":"Card Format See character editor . There are few example cards included like'Skynet', 'Harry Potter' and 'Bronya Zaychik' 'name' : 'char_name' The name for the ai character. When using json or yaml, this is expected to correspond to avatar image. name.png or name.jpg. 'description' : 'char_persona' The description for the character personality. Likes, dislikes, personality traits. 'scenario' : 'world_scenario' Description of the scenario. This roughly corresponds to things like \"You are a hr customer service having a discussion with a customer. Always be polite\". etc. 'mes_example' : 'example_dialogue' Example dialogue. The AI will pick answer patterns based on this 'first_mes' : 'char_greeting' A landing page for the chat. This will not be included in the prompt. The documents folder includes some documents for embeddings parsing for the character cards.","title":"Card Format"},{"location":"card_format/#card-format","text":"See character editor . There are few example cards included like'Skynet', 'Harry Potter' and 'Bronya Zaychik' 'name' : 'char_name' The name for the ai character. When using json or yaml, this is expected to correspond to avatar image. name.png or name.jpg. 'description' : 'char_persona' The description for the character personality. Likes, dislikes, personality traits. 'scenario' : 'world_scenario' Description of the scenario. This roughly corresponds to things like \"You are a hr customer service having a discussion with a customer. Always be polite\". etc. 'mes_example' : 'example_dialogue' Example dialogue. The AI will pick answer patterns based on this 'first_mes' : 'char_greeting' A landing page for the chat. This will not be included in the prompt. The documents folder includes some documents for embeddings parsing for the character cards.","title":"Card Format"},{"location":"configs/","text":"Configs You can change the configuration settings in .env file. The available embeddings are llama,spacy and hugginface. Make sure that the config for the chat matches the embeddings that were used to create the chroma collection. VECTOR_K is the value for vector storage documents for how many documents should be returned. You might need to change this based on your context and vector store chunk size. BUFFER_K is the size for conversation buffer. The prompt will include last K qustion answer pairs. Having large VECTOR_K and BUFFER_K can overfill the prompt. The default character card is Skynet_V2.png. This is just a basic template. Config Field Description MODEL_DIR The dir for the models MODEL model_name.gguf MODEL_TYPE alpaca/mistral LAYERS Number of layers to offload to gpu CHARACTER_CARD_DIR The directory for chracter cards CHARACTER_CARD character_card.png/yaml/json PERSIST_DIRECTORY dir for chroma embeddings PROMPT_TEMPLATE_DIRECTORY Prompt template are stored here PROMPT_TEMPLATE question_generation_template.json REPLACE_YOU Replace references to \"You\" in card with \"User\" KEY_STORAGE_DIRECTORY dir for NER keys for chroma USE_KEY_STORAGE Use NER keys for Chroma metadata COLLECTION Chroma collection to use. \"\" to disable Chroma QUERY_TYPE Embeddings type. \"mmr\" or \"similarity\" EMBEDDINGS_TYPE llama/spacy/hugginface FETCH_K Fetch k closest embeddings for similiarity LAMBDA_MULT Lambda for Chroma VECTOR_K Fetch k closest embeddings for mmr BUFFER_K Buffer last k exchanges to conversation context ROPE_CONTEXT Rope context for rope scaling N_CTX Context size USE_MAX_TOKENS Use max tokens. True/False MAX_TOKENS Max tokens","title":"Configs"},{"location":"configs/#configs","text":"You can change the configuration settings in .env file. The available embeddings are llama,spacy and hugginface. Make sure that the config for the chat matches the embeddings that were used to create the chroma collection. VECTOR_K is the value for vector storage documents for how many documents should be returned. You might need to change this based on your context and vector store chunk size. BUFFER_K is the size for conversation buffer. The prompt will include last K qustion answer pairs. Having large VECTOR_K and BUFFER_K can overfill the prompt. The default character card is Skynet_V2.png. This is just a basic template. Config Field Description MODEL_DIR The dir for the models MODEL model_name.gguf MODEL_TYPE alpaca/mistral LAYERS Number of layers to offload to gpu CHARACTER_CARD_DIR The directory for chracter cards CHARACTER_CARD character_card.png/yaml/json PERSIST_DIRECTORY dir for chroma embeddings PROMPT_TEMPLATE_DIRECTORY Prompt template are stored here PROMPT_TEMPLATE question_generation_template.json REPLACE_YOU Replace references to \"You\" in card with \"User\" KEY_STORAGE_DIRECTORY dir for NER keys for chroma USE_KEY_STORAGE Use NER keys for Chroma metadata COLLECTION Chroma collection to use. \"\" to disable Chroma QUERY_TYPE Embeddings type. \"mmr\" or \"similarity\" EMBEDDINGS_TYPE llama/spacy/hugginface FETCH_K Fetch k closest embeddings for similiarity LAMBDA_MULT Lambda for Chroma VECTOR_K Fetch k closest embeddings for mmr BUFFER_K Buffer last k exchanges to conversation context ROPE_CONTEXT Rope context for rope scaling N_CTX Context size USE_MAX_TOKENS Use max tokens. True/False MAX_TOKENS Max tokens","title":"Configs"},{"location":"creating_embeddings/","text":"Creating embeddings The embeddings creation uses env setting for threading and cuda. The Example documents are in the Documents folder. The scripts are in the documents_parsing folder. Use --help for basic instructions. The parsing script will parse all txt, pdf or json files in the target directory. For json lorebooks a key_storage file will also be created for metadata filtering. You need to download models for NER parsing. Textacy parses text files with Spacy sentence transformers to automatically generate keys for metadata filters. The default model is en_core_web_lg. See available models at Spacy Models python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg You might want to play with the chunk size and overlap based on your text documents The example documents include a txt file for skynet embeddings and json lorebooks for Hogwarts and Honkai Impact The supported lorebook formats are chub inferred AgnAIstic and SillyTavern original source. For pdf files there is a pdf file of short stories from Fyodor Dostoyevsky included The source is Internet Archive, the copy is in public domain. The pdf text quality is quite poor thought, so I recommend getting another file, !!Important!!. You need to make sure that the documents, character_storage and key_storage folders exist. Textacy parsing will use NER to parse keys from the document using sentence transformers. This keys can be used as Chroma metadata, NOTE: Textacy parsing will create a key file in key_storage that can be used by text parsing. Json files will create keys automatically if present in json file. python -m document_parsing.textacy_parsing --collection-name skynet --embeddings-type spacy Parse the documents with python -m document_parsing.parse_text_documents --embeddings-type llama python -m document_parsing.parse_text_documents --collection-name skynet2 --embeddings-type spacy python -m document_parsing.parse_json_documents --embeddings-type spacy You can test the embeddings with python -m document_parsing.test_embeddings --collection-name skynet --query \"Who is John Connor\" --embeddings-type llama python -m document_parsing.test_embeddings --collection-name skynet2 --query \"Who is John Connor\" --embeddings-type spacy python -m document_parsing.test_embeddings --collection-name hogwarts --query \"Who is Charles Rookwood'\" --embeddings-type spacy","title":"Creating embeddings"},{"location":"creating_embeddings/#creating-embeddings","text":"The embeddings creation uses env setting for threading and cuda. The Example documents are in the Documents folder. The scripts are in the documents_parsing folder. Use --help for basic instructions. The parsing script will parse all txt, pdf or json files in the target directory. For json lorebooks a key_storage file will also be created for metadata filtering. You need to download models for NER parsing. Textacy parses text files with Spacy sentence transformers to automatically generate keys for metadata filters. The default model is en_core_web_lg. See available models at Spacy Models python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg You might want to play with the chunk size and overlap based on your text documents The example documents include a txt file for skynet embeddings and json lorebooks for Hogwarts and Honkai Impact The supported lorebook formats are chub inferred AgnAIstic and SillyTavern original source. For pdf files there is a pdf file of short stories from Fyodor Dostoyevsky included The source is Internet Archive, the copy is in public domain. The pdf text quality is quite poor thought, so I recommend getting another file, !!Important!!. You need to make sure that the documents, character_storage and key_storage folders exist. Textacy parsing will use NER to parse keys from the document using sentence transformers. This keys can be used as Chroma metadata, NOTE: Textacy parsing will create a key file in key_storage that can be used by text parsing. Json files will create keys automatically if present in json file. python -m document_parsing.textacy_parsing --collection-name skynet --embeddings-type spacy Parse the documents with python -m document_parsing.parse_text_documents --embeddings-type llama python -m document_parsing.parse_text_documents --collection-name skynet2 --embeddings-type spacy python -m document_parsing.parse_json_documents --embeddings-type spacy You can test the embeddings with python -m document_parsing.test_embeddings --collection-name skynet --query \"Who is John Connor\" --embeddings-type llama python -m document_parsing.test_embeddings --collection-name skynet2 --query \"Who is John Connor\" --embeddings-type spacy python -m document_parsing.test_embeddings --collection-name hogwarts --query \"Who is Charles Rookwood'\" --embeddings-type spacy","title":"Creating embeddings"},{"location":"examples/","text":"Some examples","title":"Some Examples"},{"location":"examples/#some-examples","text":"","title":"Some examples"},{"location":"getting_started/","text":"You will need hatch to run this project. You can install hatch with pipx. See Hatch and Pipx . The commands here are for windows powershell. If you use another shell, you'll have to change things as needed. pip install pipx pipx install hatch Then from the repo root folder run. hatch shell chat cd .\\src\\llama_cpp_chat_memory\\ python -m spacy download en_core_web_lg playwright install You will need playwright for webscraping and the spacy models for text embeddings if you do not use llama-cpp embeddings. These are not needed for running the chatbot itself. You also might want to run llama-cpp with gpu acceleration like cuda. See llama-cpp-python for specifics. Then run: $env:FORCE_CMAKE=1 $env:CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" pip install llama-cpp-python --force-reinstall --upgrade --no-cache-dir --no-deps Note that this example is for powershell and for the latest llama-cpp-python. You will need to change the command based on the terminal and the llama-cpp-python version. Get a gguf model from a site like The Bloke and a character card and lorebooks from a site like Chub.ai or make your own with character editor Change the .env_test to .env and make sure that the correct folders exist. You can set the collection to \"\" and try the chatbot by running: chainlit run character_chat.py If you want to create memory then see more details below.","title":"Getting started"},{"location":"preparing_the_env/","text":"Preparing the env You will need a llama model that is compatible with llama-cpp. See models in HuggingFace by The Bloke You might want to build with cuda support. You need to pass FORCE_CMAKE=1 and CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" to env variables. This is the powershell syntax. Use whatever syntax your shell uses to set env variables You need to download language models if you use NER parsing, embeddings or spacy sentence transformers. The default model is en_core_web_lg. See available models at Spacy Models Choose the preferred model size and type. python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg For installing dependencies in the virtual envs with hatch hatch env create Copy the .env_test to .env and set directories and model settings NOTE: Setting collection to \"\" will disable chroma fetching and you will get a normal character chatbot.","title":"Preparing the env"},{"location":"preparing_the_env/#preparing-the-env","text":"You will need a llama model that is compatible with llama-cpp. See models in HuggingFace by The Bloke You might want to build with cuda support. You need to pass FORCE_CMAKE=1 and CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" to env variables. This is the powershell syntax. Use whatever syntax your shell uses to set env variables You need to download language models if you use NER parsing, embeddings or spacy sentence transformers. The default model is en_core_web_lg. See available models at Spacy Models Choose the preferred model size and type. python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg For installing dependencies in the virtual envs with hatch hatch env create Copy the .env_test to .env and set directories and model settings NOTE: Setting collection to \"\" will disable chroma fetching and you will get a normal character chatbot.","title":"Preparing the env"},{"location":"prompt_support/","text":"Prompt Support Supports alpaca and mistral text prompts, V2 and tavern style json and yaml files and V2 and tavern png cards. Avatar images need to be in the same folder as the prompt file. V2 and Tavern png files get a copy of the image without exif data in the project temp file. Inbuilt lorebooks are currently not supported See Chub.ai for some character cards or make your own with character editor .","title":"Prompt Support"},{"location":"prompt_support/#prompt-support","text":"Supports alpaca and mistral text prompts, V2 and tavern style json and yaml files and V2 and tavern png cards. Avatar images need to be in the same folder as the prompt file. V2 and Tavern png files get a copy of the image without exif data in the project temp file. Inbuilt lorebooks are currently not supported See Chub.ai for some character cards or make your own with character editor .","title":"Prompt Support"},{"location":"running_the_chatbot/","text":"Running the chatbot To run the chatbot. cd src\\llama_cpp_langchain_chat chainlit run character_chat.py The chatbot should open in your browser","title":"Running the chatbot"},{"location":"running_the_chatbot/#running-the-chatbot","text":"To run the chatbot. cd src\\llama_cpp_langchain_chat chainlit run character_chat.py The chatbot should open in your browser","title":"Running the chatbot"},{"location":"running_the_env/","text":"Running the env You'll need to run all the commands inside the virtual env. hatch shell chat (optional for cuda support)$env:FORCE_CMAKE=1 (optional for cuda support)$env:CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" (optional for cuda support)pip install llama-cpp-python==VERSION --force-reinstall --upgrade --no-cache-dir --no-deps cd src\\llama_cpp_langchain_chat","title":"Running the env"},{"location":"running_the_env/#running-the-env","text":"You'll need to run all the commands inside the virtual env. hatch shell chat (optional for cuda support)$env:FORCE_CMAKE=1 (optional for cuda support)$env:CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" (optional for cuda support)pip install llama-cpp-python==VERSION --force-reinstall --upgrade --no-cache-dir --no-deps cd src\\llama_cpp_langchain_chat","title":"Running the env"},{"location":"webscraping/","text":"Webscraping You can scrape web pages to text documents in order to use them as documents for chroma. The web scraping uses playwright and requires that the web engines are installed. After starting the virtual env run: playwright install The web scraping is prepared with config files in web_scrape_configs folder. The format is in json. See the example files for the specfics. The current impelemntation is unoptimized, so use with caution for a large number of pages. To run the scrape run: python -m document_parsing.web_scraper</BR> See --help for params","title":"Webscraping"},{"location":"webscraping/#webscraping","text":"You can scrape web pages to text documents in order to use them as documents for chroma. The web scraping uses playwright and requires that the web engines are installed. After starting the virtual env run: playwright install The web scraping is prepared with config files in web_scrape_configs folder. The format is in json. See the example files for the specfics. The current impelemntation is unoptimized, so use with caution for a large number of pages. To run the scrape run: python -m document_parsing.web_scraper</BR> See --help for params","title":"Webscraping"}]}
\ No newline at end of file
+{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"llama-cpp-chat-memory This project is intended as an example and a basic framework for a locally run chatbot with documents. The target user group is developers with some understanding about python and llm framworks. If you want to learn about llm and AI, when you can take a look at my llm resources for beginners or PygWiki . This project is mainly intended to serve as a more fleshed out tutorial and a basic frame to test various things like document embeddings. For this reason, the chatbot itself is intended to be lightweight and simple. You can also use this chatbot to test models and prompts. The document fetching can be disabled by setting collection to \"\" in the config files. This leaves you with just a basic character chatbot. Everything is designed to run locally. The model is run with llama.cpp and it's python bindings, the UI is Chainlit, the vector database is Chroma and everythin is glued together with Langchain. Document processing uses Spacy and Sentence Transformers and Playwright. There are no dependencies to external api's. Llama.cpp can use gpu acceleration with Cuda and Blas. See the documentation for llama-cpp-python for documentation. The chatbot uses character cards as prompts. The supported cards are Tavern and V2. Internal lorebooks are not supported yet. There are several scripts for parsing json lorebooks, pdt, textfiles and scarping web pages for the memory content. Also included are scripts for parsing metadata from documents automatically.","title":"Home"},{"location":"#llama-cpp-chat-memory","text":"This project is intended as an example and a basic framework for a locally run chatbot with documents. The target user group is developers with some understanding about python and llm framworks. If you want to learn about llm and AI, when you can take a look at my llm resources for beginners or PygWiki . This project is mainly intended to serve as a more fleshed out tutorial and a basic frame to test various things like document embeddings. For this reason, the chatbot itself is intended to be lightweight and simple. You can also use this chatbot to test models and prompts. The document fetching can be disabled by setting collection to \"\" in the config files. This leaves you with just a basic character chatbot. Everything is designed to run locally. The model is run with llama.cpp and it's python bindings, the UI is Chainlit, the vector database is Chroma and everythin is glued together with Langchain. Document processing uses Spacy and Sentence Transformers and Playwright. There are no dependencies to external api's. Llama.cpp can use gpu acceleration with Cuda and Blas. See the documentation for llama-cpp-python for documentation. The chatbot uses character cards as prompts. The supported cards are Tavern and V2. Internal lorebooks are not supported yet. There are several scripts for parsing json lorebooks, pdt, textfiles and scarping web pages for the memory content. Also included are scripts for parsing metadata from documents automatically.","title":"llama-cpp-chat-memory"},{"location":"UNLICENSE/","text":"This is free and unencumbered software released into the public domain. Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. For more information, please refer to https://unlicense.org","title":"License"},{"location":"card_format/","text":"Card Format See character editor . There are few example cards included like'Skynet', 'Harry Potter' and 'Bronya Zaychik' 'name' : 'char_name' The name for the ai character. When using json or yaml, this is expected to correspond to avatar image. name.png or name.jpg. 'description' : 'char_persona' The description for the character personality. Likes, dislikes, personality traits. 'scenario' : 'world_scenario' Description of the scenario. This roughly corresponds to things like \"You are a hr customer service having a discussion with a customer. Always be polite\". etc. 'mes_example' : 'example_dialogue' Example dialogue. The AI will pick answer patterns based on this 'first_mes' : 'char_greeting' A landing page for the chat. This will not be included in the prompt. The documents folder includes some documents for embeddings parsing for the character cards.","title":"Card Format"},{"location":"card_format/#card-format","text":"See character editor . There are few example cards included like'Skynet', 'Harry Potter' and 'Bronya Zaychik' 'name' : 'char_name' The name for the ai character. When using json or yaml, this is expected to correspond to avatar image. name.png or name.jpg. 'description' : 'char_persona' The description for the character personality. Likes, dislikes, personality traits. 'scenario' : 'world_scenario' Description of the scenario. This roughly corresponds to things like \"You are a hr customer service having a discussion with a customer. Always be polite\". etc. 'mes_example' : 'example_dialogue' Example dialogue. The AI will pick answer patterns based on this 'first_mes' : 'char_greeting' A landing page for the chat. This will not be included in the prompt. The documents folder includes some documents for embeddings parsing for the character cards.","title":"Card Format"},{"location":"configs/","text":"Configs You can change the configuration settings in .env file. The available embeddings are llama,spacy and hugginface. Make sure that the config for the chat matches the embeddings that were used to create the chroma collection. VECTOR_K is the value for vector storage documents for how many documents should be returned. You might need to change this based on your context and vector store chunk size. BUFFER_K is the size for conversation buffer. The prompt will include last K qustion answer pairs. Having large VECTOR_K and BUFFER_K can overfill the prompt. The default character card is Skynet_V2.png. This is just a basic template. Config Field Description MODEL_DIR The dir for the models MODEL model_name.gguf MODEL_TYPE alpaca/mistral LAYERS Number of layers to offload to gpu CHARACTER_CARD_DIR The directory for chracter cards CHARACTER_CARD character_card.png/yaml/json PERSIST_DIRECTORY dir for chroma embeddings PROMPT_TEMPLATE_DIRECTORY Prompt template are stored here PROMPT_TEMPLATE question_generation_template.json REPLACE_YOU Replace references to \"You\" in card with \"User\" KEY_STORAGE_DIRECTORY dir for NER keys for chroma USE_KEY_STORAGE Use NER keys for Chroma metadata COLLECTION Chroma collection to use. \"\" to disable Chroma QUERY_TYPE Embeddings type. \"mmr\" or \"similarity\" EMBEDDINGS_TYPE llama/spacy/hugginface EMBEDDINGS_model spacy/hugginface model name (needs to be installed) FETCH_K Fetch k closest embeddings for similiarity LAMBDA_MULT Lambda for Chroma VECTOR_K Fetch k closest embeddings for mmr BUFFER_K Buffer last k exchanges to conversation context ROPE_CONTEXT Rope context for rope scaling N_CTX Context size USE_MAX_TOKENS Use max tokens. True/False MAX_TOKENS Max tokens","title":"Configs"},{"location":"configs/#configs","text":"You can change the configuration settings in .env file. The available embeddings are llama,spacy and hugginface. Make sure that the config for the chat matches the embeddings that were used to create the chroma collection. VECTOR_K is the value for vector storage documents for how many documents should be returned. You might need to change this based on your context and vector store chunk size. BUFFER_K is the size for conversation buffer. The prompt will include last K qustion answer pairs. Having large VECTOR_K and BUFFER_K can overfill the prompt. The default character card is Skynet_V2.png. This is just a basic template. Config Field Description MODEL_DIR The dir for the models MODEL model_name.gguf MODEL_TYPE alpaca/mistral LAYERS Number of layers to offload to gpu CHARACTER_CARD_DIR The directory for chracter cards CHARACTER_CARD character_card.png/yaml/json PERSIST_DIRECTORY dir for chroma embeddings PROMPT_TEMPLATE_DIRECTORY Prompt template are stored here PROMPT_TEMPLATE question_generation_template.json REPLACE_YOU Replace references to \"You\" in card with \"User\" KEY_STORAGE_DIRECTORY dir for NER keys for chroma USE_KEY_STORAGE Use NER keys for Chroma metadata COLLECTION Chroma collection to use. \"\" to disable Chroma QUERY_TYPE Embeddings type. \"mmr\" or \"similarity\" EMBEDDINGS_TYPE llama/spacy/hugginface EMBEDDINGS_model spacy/hugginface model name (needs to be installed) FETCH_K Fetch k closest embeddings for similiarity LAMBDA_MULT Lambda for Chroma VECTOR_K Fetch k closest embeddings for mmr BUFFER_K Buffer last k exchanges to conversation context ROPE_CONTEXT Rope context for rope scaling N_CTX Context size USE_MAX_TOKENS Use max tokens. True/False MAX_TOKENS Max tokens","title":"Configs"},{"location":"creating_embeddings/","text":"Creating embeddings The embeddings creation uses env setting for threading and cuda. The Example documents are in the Documents folder. The scripts are in the documents_parsing folder. Use --help for basic instructions. The parsing script will parse all txt, pdf or json files in the target directory. For json lorebooks a key_storage file will also be created for metadata filtering. You need to download models for NER parsing. Textacy parses text files with Spacy sentence transformers to automatically generate keys for metadata filters. The default model is en_core_web_lg. See available models at Spacy Models python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg You might want to play with the chunk size and overlap based on your text documents The example documents include a txt file for skynet embeddings and json lorebooks for Hogwarts and Honkai Impact The supported lorebook formats are chub inferred AgnAIstic and SillyTavern original source. For pdf files there is a pdf file of short stories from Fyodor Dostoyevsky included The source is Internet Archive, the copy is in public domain. The pdf text quality is quite poor thought, so I recommend getting another file, !!Important!!. You need to make sure that the documents, character_storage and key_storage folders exist. Textacy parsing will use NER to parse keys from the document using sentence transformers. This keys can be used as Chroma metadata, NOTE: Textacy parsing will create a key file in key_storage that can be used by text parsing. Json files will create keys automatically if present in json file. python -m document_parsing.textacy_parsing --collection-name skynet --embeddings-type spacy Parse the documents with python -m document_parsing.parse_text_documents --embeddings-type llama python -m document_parsing.parse_text_documents --collection-name skynet2 --embeddings-type spacy python -m document_parsing.parse_json_documents --embeddings-type spacy You can test the embeddings with python -m document_parsing.test_embeddings --collection-name skynet --query \"Who is John Connor\" --embeddings-type llama python -m document_parsing.test_embeddings --collection-name skynet2 --query \"Who is John Connor\" --embeddings-type spacy python -m document_parsing.test_embeddings --collection-name hogwarts --query \"Who is Charles Rookwood'\" --embeddings-type spacy Optional param Description --data-directory The directory where your text files are stored. Default \"./documents/skynet\" --collection-name The name of the collection. Default \"skynet\" --persist-directory The directory where you want to store the Chroma collection. Default \"./character_storage/\" --key-storage The directory for the collection metadata keys Need to be created with textacy parsing. Default \"./key_storage/\" --chunk-size The text chunk size for parsing. Default \"1024\" --chunk-overlap The overlap for text chunks for parsing. Default \"0\" --embeddings-type The chosen embeddings type. Default \"spacy\"","title":"Creating embeddings"},{"location":"creating_embeddings/#creating-embeddings","text":"The embeddings creation uses env setting for threading and cuda. The Example documents are in the Documents folder. The scripts are in the documents_parsing folder. Use --help for basic instructions. The parsing script will parse all txt, pdf or json files in the target directory. For json lorebooks a key_storage file will also be created for metadata filtering. You need to download models for NER parsing. Textacy parses text files with Spacy sentence transformers to automatically generate keys for metadata filters. The default model is en_core_web_lg. See available models at Spacy Models python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg You might want to play with the chunk size and overlap based on your text documents The example documents include a txt file for skynet embeddings and json lorebooks for Hogwarts and Honkai Impact The supported lorebook formats are chub inferred AgnAIstic and SillyTavern original source. For pdf files there is a pdf file of short stories from Fyodor Dostoyevsky included The source is Internet Archive, the copy is in public domain. The pdf text quality is quite poor thought, so I recommend getting another file, !!Important!!. You need to make sure that the documents, character_storage and key_storage folders exist. Textacy parsing will use NER to parse keys from the document using sentence transformers. This keys can be used as Chroma metadata, NOTE: Textacy parsing will create a key file in key_storage that can be used by text parsing. Json files will create keys automatically if present in json file. python -m document_parsing.textacy_parsing --collection-name skynet --embeddings-type spacy Parse the documents with python -m document_parsing.parse_text_documents --embeddings-type llama python -m document_parsing.parse_text_documents --collection-name skynet2 --embeddings-type spacy python -m document_parsing.parse_json_documents --embeddings-type spacy You can test the embeddings with python -m document_parsing.test_embeddings --collection-name skynet --query \"Who is John Connor\" --embeddings-type llama python -m document_parsing.test_embeddings --collection-name skynet2 --query \"Who is John Connor\" --embeddings-type spacy python -m document_parsing.test_embeddings --collection-name hogwarts --query \"Who is Charles Rookwood'\" --embeddings-type spacy Optional param Description --data-directory The directory where your text files are stored. Default \"./documents/skynet\" --collection-name The name of the collection. Default \"skynet\" --persist-directory The directory where you want to store the Chroma collection. Default \"./character_storage/\" --key-storage The directory for the collection metadata keys Need to be created with textacy parsing. Default \"./key_storage/\" --chunk-size The text chunk size for parsing. Default \"1024\" --chunk-overlap The overlap for text chunks for parsing. Default \"0\" --embeddings-type The chosen embeddings type. Default \"spacy\"","title":"Creating embeddings"},{"location":"examples/","text":"Some examples","title":"Some Examples"},{"location":"examples/#some-examples","text":"","title":"Some examples"},{"location":"getting_started/","text":"You will need hatch to run this project. You can install hatch with pipx. See Hatch and Pipx . The commands here are for windows powershell. If you use another shell, you'll have to change things as needed. pip install pipx pipx install hatch Then from the repo root folder run. hatch shell chat cd .\\src\\llama_cpp_chat_memory\\ python -m spacy download en_core_web_lg playwright install You will need playwright for webscraping and the spacy models for text embeddings if you do not use llama-cpp embeddings. These are not needed for running the chatbot itself. You also might want to run llama-cpp with gpu acceleration like cuda. See llama-cpp-python for specifics. Then run: $env:FORCE_CMAKE=1 $env:CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" pip install llama-cpp-python --force-reinstall --upgrade --no-cache-dir --no-deps Note that this example is for powershell and for the latest llama-cpp-python. You will need to change the command based on the terminal and the llama-cpp-python version. Get a gguf model from a site like The Bloke and a character card and lorebooks from a site like Chub.ai or make your own with character editor Change the .env_test to .env and make sure that the correct folders exist. You can set the collection to \"\" and try the chatbot by running: chainlit run character_chat.py If you want to create memory then see more details below.","title":"Getting started"},{"location":"named_entity_recognition/","text":"Named Entity Recognition(NER) You can use textacy_parsing script for generating document metadata keys automatically. The scripts are a modified version of textacy code updated to run with the current spacy version. The script uses a spacy embeddings model to process a text document for a json metadata keyfile. The include positions are: \"PROPN\", \"NOUN\", \"ADJ\". The includes entities are: \"PRODUCT\", \"EVENT\", \"FAC\", \"NORP\", \"PERSON\", \"ORG\", \"GPE\", \"LOC\", \"DATE\", \"TIME\", \"WORK_OF_ART\". For details see Spacy linguistic features and Model NER labels . The instructions expect en model, but spacy supports a wide range of models. You can create ner metadata list with python -m document_parsing.textacy_parsing Optional param Description --data-directory The directory where your text files are stored. Default \"./documents/skynet\" --collection-name The name of the collection Will be used as name and location for the keyfile. Default \"skynet\" --key-storage The directory for the collection metadata keys. Default \"./key_storage/\"","title":"Named Entity Recognition(NER)"},{"location":"named_entity_recognition/#named-entity-recognitionner","text":"You can use textacy_parsing script for generating document metadata keys automatically. The scripts are a modified version of textacy code updated to run with the current spacy version. The script uses a spacy embeddings model to process a text document for a json metadata keyfile. The include positions are: \"PROPN\", \"NOUN\", \"ADJ\". The includes entities are: \"PRODUCT\", \"EVENT\", \"FAC\", \"NORP\", \"PERSON\", \"ORG\", \"GPE\", \"LOC\", \"DATE\", \"TIME\", \"WORK_OF_ART\". For details see Spacy linguistic features and Model NER labels . The instructions expect en model, but spacy supports a wide range of models. You can create ner metadata list with python -m document_parsing.textacy_parsing Optional param Description --data-directory The directory where your text files are stored. Default \"./documents/skynet\" --collection-name The name of the collection Will be used as name and location for the keyfile. Default \"skynet\" --key-storage The directory for the collection metadata keys. Default \"./key_storage/\"","title":"Named Entity Recognition(NER)"},{"location":"preparing_the_env/","text":"Preparing the env You will need a llama model that is compatible with llama-cpp. See models in HuggingFace by The Bloke You might want to build with cuda support. You need to pass FORCE_CMAKE=1 and CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" to env variables. This is the powershell syntax. Use whatever syntax your shell uses to set env variables You need to download language models if you use NER parsing, embeddings or spacy sentence transformers. The default model is en_core_web_lg. See available models at Spacy Models Choose the preferred model size and type. python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg For installing dependencies in the virtual envs with hatch hatch env create Copy the .env_test to .env and set directories and model settings NOTE: Setting collection to \"\" will disable chroma fetching and you will get a normal character chatbot.","title":"Preparing the env"},{"location":"preparing_the_env/#preparing-the-env","text":"You will need a llama model that is compatible with llama-cpp. See models in HuggingFace by The Bloke You might want to build with cuda support. You need to pass FORCE_CMAKE=1 and CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" to env variables. This is the powershell syntax. Use whatever syntax your shell uses to set env variables You need to download language models if you use NER parsing, embeddings or spacy sentence transformers. The default model is en_core_web_lg. See available models at Spacy Models Choose the preferred model size and type. python -m spacy download en_core_web_sm python -m spacy download en_core_web_md python -m spacy download en_core_web_lg For installing dependencies in the virtual envs with hatch hatch env create Copy the .env_test to .env and set directories and model settings NOTE: Setting collection to \"\" will disable chroma fetching and you will get a normal character chatbot.","title":"Preparing the env"},{"location":"prompt_support/","text":"Prompt Support Supports alpaca and mistral text prompts, V2 and tavern style json and yaml files and V2 and tavern png cards. Avatar images need to be in the same folder as the prompt file. V2 and Tavern png files get a copy of the image without exif data in the project temp file. Inbuilt lorebooks are currently not supported See Chub.ai for some character cards or make your own with character editor .","title":"Prompt Support"},{"location":"prompt_support/#prompt-support","text":"Supports alpaca and mistral text prompts, V2 and tavern style json and yaml files and V2 and tavern png cards. Avatar images need to be in the same folder as the prompt file. V2 and Tavern png files get a copy of the image without exif data in the project temp file. Inbuilt lorebooks are currently not supported See Chub.ai for some character cards or make your own with character editor .","title":"Prompt Support"},{"location":"running_the_chatbot/","text":"Running the chatbot To run the chatbot. cd src\\llama_cpp_langchain_chat chainlit run character_chat.py The chatbot should open in your browser","title":"Running the chatbot"},{"location":"running_the_chatbot/#running-the-chatbot","text":"To run the chatbot. cd src\\llama_cpp_langchain_chat chainlit run character_chat.py The chatbot should open in your browser","title":"Running the chatbot"},{"location":"running_the_env/","text":"Running the env You'll need to run all the commands inside the virtual env. hatch shell chat (optional for cuda support)$env:FORCE_CMAKE=1 (optional for cuda support)$env:CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" (optional for cuda support)pip install llama-cpp-python==VERSION --force-reinstall --upgrade --no-cache-dir --no-deps cd src\\llama_cpp_langchain_chat","title":"Running the env"},{"location":"running_the_env/#running-the-env","text":"You'll need to run all the commands inside the virtual env. hatch shell chat (optional for cuda support)$env:FORCE_CMAKE=1 (optional for cuda support)$env:CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" (optional for cuda support)pip install llama-cpp-python==VERSION --force-reinstall --upgrade --no-cache-dir --no-deps cd src\\llama_cpp_langchain_chat","title":"Running the env"},{"location":"webscraping/","text":"Webscraping You can scrape web pages to text documents in order to use them as documents for chroma. The web scraping uses playwright and requires that the web engines are installed. After starting the virtual env run: playwright install The web scraping is prepared with config files in web_scrape_configs folder. The format is in json. See the example files for the specfics. The current impelementation is unoptimized, so use with caution for a large number of pages. See the example web_scrape_configs for config format. This will scrape the given web pages and format into a single text document. To run the scrape run: python -m document_parsing.web_scraper</BR> Optional param Description --data-directory The directory where your text files are stored. Default \"./documents/skynet\" --collection-name The name of the collection. Default \"skynet\" --web-scrape-directory The config file to be used for the webscrape. Default \"./web_scrape_configs/\"","title":"Webscraping"},{"location":"webscraping/#webscraping","text":"You can scrape web pages to text documents in order to use them as documents for chroma. The web scraping uses playwright and requires that the web engines are installed. After starting the virtual env run: playwright install The web scraping is prepared with config files in web_scrape_configs folder. The format is in json. See the example files for the specfics. The current impelementation is unoptimized, so use with caution for a large number of pages. See the example web_scrape_configs for config format. This will scrape the given web pages and format into a single text document. To run the scrape run: python -m document_parsing.web_scraper</BR> Optional param Description --data-directory The directory where your text files are stored. Default \"./documents/skynet\" --collection-name The name of the collection. Default \"skynet\" --web-scrape-directory The config file to be used for the webscrape. Default \"./web_scrape_configs/\"","title":"Webscraping"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index feef73c..b31a0c8 100755
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,62 +2,67 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/UNLICENSE/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/card_format/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/configs/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/creating_embeddings/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/examples/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/getting_started/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
+         <changefreq>daily</changefreq>
+    </url>
+    <url>
+         <loc>https://ossirytk.github.io/llama-cpp-chat-memory/named_entity_recognition/</loc>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/preparing_the_env/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/prompt_support/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/running_the_chatbot/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/running_the_env/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
     <url>
          <loc>https://ossirytk.github.io/llama-cpp-chat-memory/webscraping/</loc>
-         <lastmod>2024-02-04</lastmod>
+         <lastmod>2024-02-24</lastmod>
          <changefreq>daily</changefreq>
     </url>
 </urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index ccc246716592c5b0875d71bae926cd2cb89ebbf4..83f1c939e51eba0ef537090f94b7aaa517039bd4 100755
GIT binary patch
literal 358
zcmV-s0h#_EiwFqptl4D(|8r?{Wo=<_E_iKh0M(YwPQx$^$L~Bv)q9&#?Et9j4roY7
zXcySIN>eY5Xg+Y<b$xr%ZJ<2>9O6q7%lYePE4ET?PI^d=fL>X%PVe&~O~44-DI?eE
z{^KQkNH_IeHR9lbk`QeCI_-(v6Jygf<z5kJ?tE@hGB5gY@a*bjMgEx9cS*2C;Icbg
zHCRzEDn7_q?-kDUnM+0e*yhTzVc?o)!nsWJoH7mC;>;?xT)S1_p0w@ivM5(sQD)^T
zgj{_$VL`)4I3Rr09aqC#u^Z0?kNxuf3F&_K_WHcty>BC8796|gU{Q05iQ1Zjl0GJ4
z1Wuv78qh6tOQXd^4=1kO05Ks;pmV)FaU|#>A~#$^*MK2KbAuqPG>XEYjwl6(;5aVj
zryfEo$Eet?cGURMIg2q3MI4R!Df!}WZLIWhJ(vkvFOa)H|NqqIC49>2Zw=qe?8FHG
E07xgY7XSbN

literal 340
zcmV-a0jvHWiwFn>E5Bs||8r?{Wo=<_E_iKh0M(YyPQx$^#_v2u)q9&V?Et9j4roY7
zXcySIN?Wgu$Ukg%-QJ#b8)y#zhxn5Clk@2(D~?jGk47g4KvK@G()+wf6R@r8l<imP
z?&BqUNY~X}Iie=uBotd(r315n5@MRBoD_qRo^pr1pmqoeiK{4z{4uTWk`S}OWp|EJ
zySAELDA+h6g(v>Zd&R@p<jM)HrIA_debx?=GXuuqNt9wfcIzUEjq9pKu~=rsA}f}m
z<f`421rOW99^tF%q|&ES+$a}3@yquoq`U3g>+@#&zKMvrmDtsLhemQt)XwfzPcacA
zNDkvwgK40f2Q4OgI7;I+#Dv@fpZiTLk)ex-+#`62GdmAZgFQsW?u_S}M(-WQ^rGNs
m?N7^Ry>YS9$8}*QXsAW$1O5L~pO^5XslEZ+wQSr02><{tfUO$<

diff --git a/webscraping/index.html b/webscraping/index.html
index 8b6c253..81eb5fd 100755
--- a/webscraping/index.html
+++ b/webscraping/index.html
@@ -68,6 +68,8 @@
                   <li class="toctree-l1 current"><a class="reference internal current" href="./">Webscraping</a>
     <ul class="current">
     </ul>
+                  </li>
+                  <li class="toctree-l1"><a class="reference internal" href="../named_entity_recognition/">Named Entity Recognition(NER)</a>
                   </li>
                   <li class="toctree-l1"><a class="reference internal" href="../creating_embeddings/">Creating embeddings</a>
                   </li>
@@ -108,17 +110,38 @@ <h3 id="webscraping">Webscraping</h3>
 <p>You can scrape web pages to text documents in order to use them as documents for chroma. The web scraping uses playwright and requires that the web engines are installed. After starting the virtual env run:</BR></p>
 <pre><code>playwright install
 </code></pre>
-<p>The web scraping is prepared with config files in web_scrape_configs folder. The format is in json. See the example files for the specfics. The current impelemntation is unoptimized, so use with caution for a large number of pages.</BR></p>
+<p>The web scraping is prepared with config files in web_scrape_configs folder. The format is in json. See the example files for the specfics. The current impelementation is unoptimized, so use with caution for a large number of pages. See the example web_scrape_configs for config format. This will scrape the given web pages and format into a single text document.</BR></p>
 <p>To run the scrape run:</p>
 <pre><code>python -m document_parsing.web_scraper&lt;/BR&gt;
 </code></pre>
-<p>See --help for params</p>
+<table>
+<thead>
+<tr>
+<th>Optional param</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>--data-directory</td>
+<td>The directory where your text files are stored. Default "./documents/skynet"</td>
+</tr>
+<tr>
+<td>--collection-name</td>
+<td>The name of the collection. Default "skynet"</td>
+</tr>
+<tr>
+<td>--web-scrape-directory</td>
+<td>The config file to be used for the webscrape. Default "./web_scrape_configs/"</td>
+</tr>
+</tbody>
+</table>
               
             </div>
           </div><footer>
     <div class="rst-footer-buttons" role="navigation" aria-label="Footer Navigation">
         <a href="../running_the_chatbot/" class="btn btn-neutral float-left" title="Running the chatbot"><span class="icon icon-circle-arrow-left"></span> Previous</a>
-        <a href="../creating_embeddings/" class="btn btn-neutral float-right" title="Creating embeddings">Next <span class="icon icon-circle-arrow-right"></span></a>
+        <a href="../named_entity_recognition/" class="btn btn-neutral float-right" title="Named Entity Recognition(NER)">Next <span class="icon icon-circle-arrow-right"></span></a>
     </div>
 
   <hr/>
@@ -148,7 +171,7 @@ <h3 id="webscraping">Webscraping</h3>
       <span><a href="../running_the_chatbot/" style="color: #fcfcfc">&laquo; Previous</a></span>
     
     
-      <span><a href="../creating_embeddings/" style="color: #fcfcfc">Next &raquo;</a></span>
+      <span><a href="../named_entity_recognition/" style="color: #fcfcfc">Next &raquo;</a></span>
     
   </span>
 </div>