From e4100a6594979c54a3efcc8c9368a30c5ecd1f54 Mon Sep 17 00:00:00 2001
From: PhilipMay <PhilipMay@users.noreply.github.com>
Date: Fri, 1 Sep 2023 19:41:29 +0000
Subject: [PATCH] deploy: d11ab4a7b67f162338e8234ede112922fc9bca8c

---
 _modules/index.html                    |   1 +
 _modules/mltb2/md.html                 | 209 +++++++++++++++++++++++++
 _sources/api-reference/md.rst.txt      |   6 +
 api-reference.html                     |   2 +
 api-reference/fasttext.html            |   1 +
 api-reference/files.html               |   5 +-
 api-reference/md.html                  | 199 +++++++++++++++++++++++
 api-reference/openai.html              |   5 +-
 api-reference/optuna.html              |   1 +
 api-reference/plot.html                |   1 +
 api-reference/somajo.html              |   1 +
 api-reference/somajo_transformers.html |   1 +
 api-reference/transformers.html        |   1 +
 genindex.html                          |  24 +++
 index.html                             |   1 +
 objects.inv                            | Bin 1055 -> 1126 bytes
 py-modindex.html                       |   5 +
 searchindex.js                         |   2 +-
 18 files changed, 460 insertions(+), 5 deletions(-)
 create mode 100644 _modules/mltb2/md.html
 create mode 100644 _sources/api-reference/md.rst.txt
 create mode 100644 api-reference/md.html
diff --git a/_modules/index.html b/_modules/index.html
index 8082d40..c55161c 100644
--- a/_modules/index.html
+++ b/_modules/index.html
@@ -77,6 +77,7 @@
   <h1>All modules for which code is available</h1>
 <ul><li><a href="mltb2/fasttext.html">mltb2.fasttext</a></li>
 <li><a href="mltb2/files.html">mltb2.files</a></li>
+<li><a href="mltb2/md.html">mltb2.md</a></li>
 <li><a href="mltb2/openai.html">mltb2.openai</a></li>
 <li><a href="mltb2/optuna.html">mltb2.optuna</a></li>
 <li><a href="mltb2/plot.html">mltb2.plot</a></li>
diff --git a/_modules/mltb2/md.html b/_modules/mltb2/md.html
new file mode 100644
index 0000000..03ddc3f
--- /dev/null
+++ b/_modules/mltb2/md.html
@@ -0,0 +1,209 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>mltb2.md &mdash; MLTB2  documentation</title>
+      <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
+      <link rel="stylesheet" href="../../_static/copybutton.css" type="text/css" />
+      <link rel="stylesheet" href="../../_static/css/custom.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script src="../../_static/jquery.js?v=5d32c60e"></script>
+        <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+        <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js?v=b3ba4146"></script>
+        <script src="../../_static/doctools.js?v=888ff710"></script>
+        <script src="../../_static/sphinx_highlight.js?v=4825356b"></script>
+        <script src="../../_static/clipboard.min.js?v=a7894cd8"></script>
+        <script src="../../_static/copybutton.js?v=f281be69"></script>
+    <script src="../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../../index.html" class="icon icon-home">
+            MLTB2
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../api-reference.html">API Reference</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../genindex.html">Index</a></li>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/telekom/mltb2">GitHub Repository</a></li>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/telekom/mltb2/blob/main/LICENSE">License</a></li>
+<li class="toctree-l1"><a class="reference external" href="https://www.telekom.com/imprint">Imprint</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">MLTB2</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
+      <li class="breadcrumb-item active">mltb2.md</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <h1>Source code for mltb2.md</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Copyright (c) 2023 Philip May, Deutsche Telekom AG</span>
+<span class="c1"># Copyright (c) 2023 Philip May</span>
+<span class="c1"># This software is distributed under the terms of the MIT license</span>
+<span class="c1"># which is available at https://opensource.org/licenses/MIT</span>
+
+<span class="sd">&quot;&quot;&quot;Markdown specific functionality.</span>
+
+<span class="sd">Use pip to install the necessary dependencies for this module:</span>
+<span class="sd">``pip install mltb2[md]``</span>
+<span class="sd">&quot;&quot;&quot;</span>
+
+<span class="kn">import</span> <span class="nn">re</span>
+<span class="kn">from</span> <span class="nn">dataclasses</span> <span class="kn">import</span> <span class="n">dataclass</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Final</span><span class="p">,</span> <span class="n">List</span>
+
+<span class="kn">from</span> <span class="nn">tqdm</span> <span class="kn">import</span> <span class="n">tqdm</span>
+
+<span class="kn">from</span> <span class="nn">mltb2.transformers</span> <span class="kn">import</span> <span class="n">TransformersTokenCounter</span>
+
+<span class="n">_HEADLINE_REGEX</span><span class="p">:</span> <span class="n">Final</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;^#+ .*&quot;</span><span class="p">,</span> <span class="n">flags</span><span class="o">=</span><span class="n">re</span><span class="o">.</span><span class="n">MULTILINE</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="_chunk_md_by_headline"><a class="viewcode-back" href="../../api-reference/md.html#mltb2.md._chunk_md_by_headline">[docs]</a><span class="k">def</span> <span class="nf">_chunk_md_by_headline</span><span class="p">(</span><span class="n">md_text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Chunk Markdown by headlines.&quot;&quot;&quot;</span>
+    <span class="n">positions</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">m</span><span class="o">.</span><span class="n">start</span><span class="p">()</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">re</span><span class="o">.</span><span class="n">finditer</span><span class="p">(</span><span class="n">_HEADLINE_REGEX</span><span class="p">,</span> <span class="n">md_text</span><span class="p">)]</span>
+
+    <span class="c1"># extend positions</span>
+    <span class="k">if</span> <span class="mi">0</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">positions</span><span class="p">:</span>
+        <span class="n">positions</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">+</span> <span class="n">positions</span>
+    <span class="n">positions</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">md_text</span><span class="p">))</span>
+
+    <span class="n">result</span> <span class="o">=</span> <span class="p">[</span><span class="n">md_text</span><span class="p">[</span><span class="n">x</span><span class="p">:</span><span class="n">y</span><span class="p">]</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span> <span class="k">for</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">positions</span><span class="p">,</span> <span class="n">positions</span><span class="p">[</span><span class="mi">1</span><span class="p">:])]</span>
+    <span class="k">return</span> <span class="n">result</span></div>
+
+
+<div class="viewcode-block" id="chunk_md"><a class="viewcode-back" href="../../api-reference/md.html#mltb2.md.chunk_md">[docs]</a><span class="k">def</span> <span class="nf">chunk_md</span><span class="p">(</span><span class="n">md_text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Chunk Markdown by headlines and merge isolated headlines.&quot;&quot;&quot;</span>
+    <span class="n">md_chunks</span> <span class="o">=</span> <span class="n">_chunk_md_by_headline</span><span class="p">(</span><span class="n">md_text</span><span class="p">)</span>
+
+    <span class="n">merged_chunks</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="n">temp_merged_chunk</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">for</span> <span class="n">chunk</span> <span class="ow">in</span> <span class="n">md_chunks</span><span class="p">:</span>
+        <span class="n">temp_merged_chunk</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
+        <span class="k">if</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span> <span class="ow">in</span> <span class="n">chunk</span><span class="p">:</span>  <span class="c1"># content found</span>
+            <span class="n">joined_content</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">temp_merged_chunk</span><span class="p">)</span>
+            <span class="n">merged_chunks</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">joined_content</span><span class="p">)</span>
+            <span class="n">temp_merged_chunk</span> <span class="o">=</span> <span class="p">[]</span>
+
+    <span class="c1"># if len(temp_content) &gt; 0 this is only headlines and we skip them</span>
+    <span class="k">return</span> <span class="n">merged_chunks</span></div>
+
+
+<div class="viewcode-block" id="MdTextSplitter"><a class="viewcode-back" href="../../api-reference/md.html#mltb2.md.MdTextSplitter">[docs]</a><span class="nd">@dataclass</span>
+<span class="k">class</span> <span class="nc">MdTextSplitter</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Split Markdown text into sections with a specified maximum token number.</span>
+
+<span class="sd">    Does not divide headings with their corresponding paragraphs.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        max_token: Maximum number of tokens per text section.</span>
+<span class="sd">            Can only be exceeded if a single Markdown chunk is already larger.</span>
+<span class="sd">        transformers_token_counter: The token counter to be used.</span>
+<span class="sd">        show_progress_bar: Show a progressbar during processing.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">max_token</span><span class="p">:</span> <span class="nb">int</span>
+    <span class="n">transformers_token_counter</span><span class="p">:</span> <span class="n">TransformersTokenCounter</span>
+    <span class="n">show_progress_bar</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
+
+<div class="viewcode-block" id="MdTextSplitter.__call__"><a class="viewcode-back" href="../../api-reference/md.html#mltb2.md.MdTextSplitter.__call__">[docs]</a>    <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">md_text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;Split the Markdown text into sections.</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            md_text: The Markdown text to be split.</span>
+<span class="sd">        Returns:</span>
+<span class="sd">            The list of Markdown section splits.</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">md_chunks</span> <span class="o">=</span> <span class="n">chunk_md</span><span class="p">(</span><span class="n">md_text</span><span class="p">)</span>
+        <span class="n">counts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">transformers_token_counter</span><span class="p">(</span><span class="n">md_chunks</span><span class="p">)</span>
+
+        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">md_chunks</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">counts</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type] # noqa: S101</span>
+
+        <span class="n">result_merges</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="n">temp_merges</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+        <span class="n">current_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span>
+
+        <span class="k">for</span> <span class="n">md_chunk</span><span class="p">,</span> <span class="n">count</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
+            <span class="n">tqdm</span><span class="p">(</span><span class="n">md_chunks</span><span class="p">,</span> <span class="n">disable</span><span class="o">=</span><span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">show_progress_bar</span><span class="p">),</span> <span class="n">counts</span>  <span class="c1"># type: ignore[arg-type]</span>
+        <span class="p">):</span>
+            <span class="k">if</span> <span class="n">current_count</span> <span class="o">+</span> <span class="n">count</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_token</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">temp_merges</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+                <span class="n">joined_content</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">temp_merges</span><span class="p">)</span>
+                <span class="n">result_merges</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">joined_content</span><span class="p">)</span>
+                <span class="n">temp_merges</span> <span class="o">=</span> <span class="p">[]</span>
+                <span class="n">current_count</span> <span class="o">=</span> <span class="mi">0</span>
+
+            <span class="n">current_count</span> <span class="o">+=</span> <span class="n">count</span>
+            <span class="n">temp_merges</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">md_chunk</span><span class="p">)</span>
+
+        <span class="c1"># add the rest</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">temp_merges</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">joined_content</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\n\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">temp_merges</span><span class="p">)</span>
+            <span class="n">result_merges</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">joined_content</span><span class="p">)</span>
+
+        <span class="k">return</span> <span class="n">result_merges</span></div></div>
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2023, Philip May, Deutsche Telekom AG.</p>
+  </div>
+
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/_sources/api-reference/md.rst.txt b/_sources/api-reference/md.rst.txt
new file mode 100644
index 0000000..24ef03f
--- /dev/null
+++ b/_sources/api-reference/md.rst.txt
@@ -0,0 +1,6 @@
+.. _md_code_doc:
+
+:mod:`~mltb2.md`
+================
+
+.. automodule:: mltb2.md
diff --git a/api-reference.html b/api-reference.html
index a1c615f..dd31d9d 100644
--- a/api-reference.html
+++ b/api-reference.html
@@ -50,6 +50,7 @@
 <li class="toctree-l1 current"><a class="current reference internal" href="#">API Reference</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="api-reference/md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
@@ -94,6 +95,7 @@ <h1>API Reference<a class="headerlink" href="#api-reference" title="Permalink to
 <ul>
 <li class="toctree-l1"><a class="reference internal" href="api-reference/fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l1"><a class="reference internal" href="api-reference/files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l1"><a class="reference internal" href="api-reference/md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l1"><a class="reference internal" href="api-reference/openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l1"><a class="reference internal" href="api-reference/optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l1"><a class="reference internal" href="api-reference/plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
diff --git a/api-reference/fasttext.html b/api-reference/fasttext.html
index e5e9a71..8507777 100644
--- a/api-reference/fasttext.html
+++ b/api-reference/fasttext.html
@@ -57,6 +57,7 @@
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
diff --git a/api-reference/files.html b/api-reference/files.html
index b184094..26edc35 100644
--- a/api-reference/files.html
+++ b/api-reference/files.html
@@ -23,7 +23,7 @@
     <script src="../_static/js/theme.js"></script>
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" />
-    <link rel="next" title="openai" href="openai.html" />
+    <link rel="next" title="md" href="md.html" />
     <link rel="prev" title="fasttext" href="fasttext.html" /> 
 </head>
 
@@ -54,6 +54,7 @@
 <li class="toctree-l3"><a class="reference internal" href="#mltb2.files.get_and_create_mltb2_data_dir"><code class="docutils literal notranslate"><span class="pre">get_and_create_mltb2_data_dir()</span></code></a></li>
 </ul>
 </li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
@@ -147,7 +148,7 @@
           </div>
           <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
         <a href="fasttext.html" class="btn btn-neutral float-left" title="fasttext" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
-        <a href="openai.html" class="btn btn-neutral float-right" title="openai" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+        <a href="md.html" class="btn btn-neutral float-right" title="md" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
     </div>
 
   <hr/>
diff --git a/api-reference/md.html b/api-reference/md.html
new file mode 100644
index 0000000..a6cf451
--- /dev/null
+++ b/api-reference/md.html
@@ -0,0 +1,199 @@
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" >
+<head>
+  <meta charset="utf-8" /><meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />
+
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>md &mdash; MLTB2  documentation</title>
+      <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/copybutton.css" type="text/css" />
+      <link rel="stylesheet" href="../_static/css/custom.css" type="text/css" />
+  <!--[if lt IE 9]>
+    <script src="../_static/js/html5shiv.min.js"></script>
+  <![endif]-->
+  
+        <script src="../_static/jquery.js?v=5d32c60e"></script>
+        <script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js?v=b3ba4146"></script>
+        <script src="../_static/doctools.js?v=888ff710"></script>
+        <script src="../_static/sphinx_highlight.js?v=4825356b"></script>
+        <script src="../_static/clipboard.min.js?v=a7894cd8"></script>
+        <script src="../_static/copybutton.js?v=f281be69"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" />
+    <link rel="next" title="openai" href="openai.html" />
+    <link rel="prev" title="files" href="files.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../index.html" class="icon icon-home">
+            MLTB2
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <ul class="current">
+<li class="toctree-l1 current"><a class="reference internal" href="../api-reference.html">API Reference</a><ul class="current">
+<li class="toctree-l2"><a class="reference internal" href="fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a><ul>
+<li class="toctree-l3"><a class="reference internal" href="#mltb2.md.MdTextSplitter"><code class="docutils literal notranslate"><span class="pre">MdTextSplitter</span></code></a><ul>
+<li class="toctree-l4"><a class="reference internal" href="#mltb2.md.MdTextSplitter.__call__"><code class="docutils literal notranslate"><span class="pre">MdTextSplitter.__call__()</span></code></a></li>
+</ul>
+</li>
+<li class="toctree-l3"><a class="reference internal" href="#mltb2.md._chunk_md_by_headline"><code class="docutils literal notranslate"><span class="pre">_chunk_md_by_headline()</span></code></a></li>
+<li class="toctree-l3"><a class="reference internal" href="#mltb2.md.chunk_md"><code class="docutils literal notranslate"><span class="pre">chunk_md()</span></code></a></li>
+</ul>
+</li>
+<li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="somajo.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">somajo</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="somajo_transformers.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">somajo_transformers</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="transformers.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">transformers</span></code></a></li>
+</ul>
+</li>
+<li class="toctree-l1"><a class="reference internal" href="../genindex.html">Index</a></li>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/telekom/mltb2">GitHub Repository</a></li>
+<li class="toctree-l1"><a class="reference external" href="https://github.com/telekom/mltb2/blob/main/LICENSE">License</a></li>
+<li class="toctree-l1"><a class="reference external" href="https://www.telekom.com/imprint">Imprint</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">MLTB2</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../api-reference.html">API Reference</a></li>
+      <li class="breadcrumb-item active"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></li>
+      <li class="wy-breadcrumbs-aside">
+              <a href="https://github.com/telekom/mltb2/blob/main/docs/source/api-reference/md.rst" class="fa fa-github"> Edit on GitHub</a>
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <section id="module-mltb2.md">
+<span id="md"></span><span id="md-code-doc"></span><h1><a class="reference internal" href="#module-mltb2.md" title="mltb2.md"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a><a class="headerlink" href="#module-mltb2.md" title="Permalink to this heading"></a></h1>
+<p>Markdown specific functionality.</p>
+<p>Use pip to install the necessary dependencies for this module:
+<code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">mltb2[md]</span></code></p>
+<dl class="py class">
+<dt class="sig sig-object py" id="mltb2.md.MdTextSplitter">
+<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">mltb2.md.</span></span><span class="sig-name descname"><span class="pre">MdTextSplitter</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">max_token</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.11)"><span class="pre">int</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">transformers_token_counter</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="transformers.html#mltb2.transformers.TransformersTokenCounter" title="mltb2.transformers.TransformersTokenCounter"><span class="pre">TransformersTokenCounter</span></a></span></em>, <em class="sig-param"><span class="n"><span class="pre">show_progress_bar</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><span class="pre">bool</span></a></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/mltb2/md.html#MdTextSplitter"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#mltb2.md.MdTextSplitter" title="Permalink to this definition"></a></dt>
+<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.11)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
+<p>Split Markdown text into sections with a specified maximum token number.</p>
+<p>Does not divide headings with their corresponding paragraphs.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>max_token</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.11)"><em>int</em></a>) – Maximum number of tokens per text section.
+Can only be exceeded if a single Markdown chunk is already larger.</p></li>
+<li><p><strong>transformers_token_counter</strong> (<a class="reference internal" href="transformers.html#mltb2.transformers.TransformersTokenCounter" title="mltb2.transformers.TransformersTokenCounter"><em>TransformersTokenCounter</em></a>) – The token counter to be used.</p></li>
+<li><p><strong>show_progress_bar</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a>) – Show a progressbar during processing.</p></li>
+</ul>
+</dd>
+</dl>
+<dl class="py method">
+<dt class="sig sig-object py" id="mltb2.md.MdTextSplitter.__call__">
+<span class="sig-name descname"><span class="pre">__call__</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">md_text</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><span class="pre">str</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.List" title="(in Python v3.11)"><span class="pre">List</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><span class="pre">str</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="reference internal" href="../_modules/mltb2/md.html#MdTextSplitter.__call__"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#mltb2.md.MdTextSplitter.__call__" title="Permalink to this definition"></a></dt>
+<dd><p>Split the Markdown text into sections.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>md_text</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><em>str</em></a>) – The Markdown text to be split.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>The list of Markdown section splits.</p>
+</dd>
+<dt class="field-odd">Return type<span class="colon">:</span></dt>
+<dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.List" title="(in Python v3.11)"><em>List</em></a>[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)">str</a>]</p>
+</dd>
+</dl>
+</dd></dl>
+
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="mltb2.md._chunk_md_by_headline">
+<span class="sig-prename descclassname"><span class="pre">mltb2.md.</span></span><span class="sig-name descname"><span class="pre">_chunk_md_by_headline</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">md_text</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><span class="pre">str</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.List" title="(in Python v3.11)"><span class="pre">List</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><span class="pre">str</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="reference internal" href="../_modules/mltb2/md.html#_chunk_md_by_headline"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#mltb2.md._chunk_md_by_headline" title="Permalink to this definition"></a></dt>
+<dd><p>Chunk Markdown by headlines.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>md_text</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><em>str</em></a>) – </p>
+</dd>
+<dt class="field-even">Return type<span class="colon">:</span></dt>
+<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.List" title="(in Python v3.11)"><em>List</em></a>[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)">str</a>]</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt class="sig sig-object py" id="mltb2.md.chunk_md">
+<span class="sig-prename descclassname"><span class="pre">mltb2.md.</span></span><span class="sig-name descname"><span class="pre">chunk_md</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">md_text</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><span class="pre">str</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.List" title="(in Python v3.11)"><span class="pre">List</span></a><span class="p"><span class="pre">[</span></span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><span class="pre">str</span></a><span class="p"><span class="pre">]</span></span></span></span><a class="reference internal" href="../_modules/mltb2/md.html#chunk_md"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#mltb2.md.chunk_md" title="Permalink to this definition"></a></dt>
+<dd><p>Chunk Markdown by headlines and merge isolated headlines.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>md_text</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><em>str</em></a>) – </p>
+</dd>
+<dt class="field-even">Return type<span class="colon">:</span></dt>
+<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.List" title="(in Python v3.11)"><em>List</em></a>[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)">str</a>]</p>
+</dd>
+</dl>
+</dd></dl>
+
+</section>
+
+
+           </div>
+          </div>
+          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
+        <a href="files.html" class="btn btn-neutral float-left" title="files" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="openai.html" class="btn btn-neutral float-right" title="openai" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
+    </div>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2023, Philip May, Deutsche Telekom AG.</p>
+  </div>
+
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>
\ No newline at end of file
diff --git a/api-reference/openai.html b/api-reference/openai.html
index 2907b87..b9a5125 100644
--- a/api-reference/openai.html
+++ b/api-reference/openai.html
@@ -24,7 +24,7 @@
     <link rel="index" title="Index" href="../genindex.html" />
     <link rel="search" title="Search" href="../search.html" />
     <link rel="next" title="optuna" href="optuna.html" />
-    <link rel="prev" title="files" href="files.html" /> 
+    <link rel="prev" title="md" href="md.html" /> 
 </head>
 
 <body class="wy-body-for-nav"> 
@@ -50,6 +50,7 @@
 <li class="toctree-l1 current"><a class="reference internal" href="../api-reference.html">API Reference</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a><ul>
 <li class="toctree-l3"><a class="reference internal" href="#mltb2.openai.OpenAiAzureChatCompletion"><code class="docutils literal notranslate"><span class="pre">OpenAiAzureChatCompletion</span></code></a></li>
 <li class="toctree-l3"><a class="reference internal" href="#mltb2.openai.OpenAiAzureCompletion"><code class="docutils literal notranslate"><span class="pre">OpenAiAzureCompletion</span></code></a></li>
@@ -414,7 +415,7 @@
            </div>
           </div>
           <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
-        <a href="files.html" class="btn btn-neutral float-left" title="files" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
+        <a href="md.html" class="btn btn-neutral float-left" title="md" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
         <a href="optuna.html" class="btn btn-neutral float-right" title="optuna" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
     </div>
 
diff --git a/api-reference/optuna.html b/api-reference/optuna.html
index 5a5aa4d..df71043 100644
--- a/api-reference/optuna.html
+++ b/api-reference/optuna.html
@@ -50,6 +50,7 @@
 <li class="toctree-l1 current"><a class="reference internal" href="../api-reference.html">API Reference</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a><ul>
 <li class="toctree-l3"><a class="reference internal" href="#mltb2.optuna.SignificanceRepeatedTrainingPruner"><code class="docutils literal notranslate"><span class="pre">SignificanceRepeatedTrainingPruner</span></code></a><ul>
diff --git a/api-reference/plot.html b/api-reference/plot.html
index 854b045..17ed8c2 100644
--- a/api-reference/plot.html
+++ b/api-reference/plot.html
@@ -50,6 +50,7 @@
 <li class="toctree-l1 current"><a class="reference internal" href="../api-reference.html">API Reference</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2 current"><a class="current reference internal" href="#"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a><ul>
diff --git a/api-reference/somajo.html b/api-reference/somajo.html
index 57bb1f9..17c9a30 100644
--- a/api-reference/somajo.html
+++ b/api-reference/somajo.html
@@ -50,6 +50,7 @@
 <li class="toctree-l1 current"><a class="reference internal" href="../api-reference.html">API Reference</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
diff --git a/api-reference/somajo_transformers.html b/api-reference/somajo_transformers.html
index 8389fd7..3118134 100644
--- a/api-reference/somajo_transformers.html
+++ b/api-reference/somajo_transformers.html
@@ -50,6 +50,7 @@
 <li class="toctree-l1 current"><a class="reference internal" href="../api-reference.html">API Reference</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
diff --git a/api-reference/transformers.html b/api-reference/transformers.html
index 60c9660..06edb0e 100644
--- a/api-reference/transformers.html
+++ b/api-reference/transformers.html
@@ -49,6 +49,7 @@
 <li class="toctree-l1 current"><a class="reference internal" href="../api-reference.html">API Reference</a><ul class="current">
 <li class="toctree-l2"><a class="reference internal" href="fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
diff --git a/genindex.html b/genindex.html
index bdad980..e1860c6 100644
--- a/genindex.html
+++ b/genindex.html
@@ -81,6 +81,7 @@ <h1 id="index">Index</h1>
 <div class="genindex-jumpbox">
  <a href="#_"><strong>_</strong></a>
  | <a href="#B"><strong>B</strong></a>
+ | <a href="#C"><strong>C</strong></a>
  | <a href="#D"><strong>D</strong></a>
  | <a href="#E"><strong>E</strong></a>
  | <a href="#F"><strong>F</strong></a>
@@ -101,6 +102,8 @@ <h2 id="_">_</h2>
       <li><a href="api-reference/fasttext.html#mltb2.fasttext.FastTextLanguageIdentification.__call__">__call__() (mltb2.fasttext.FastTextLanguageIdentification method)</a>
 
       <ul>
+        <li><a href="api-reference/md.html#mltb2.md.MdTextSplitter.__call__">(mltb2.md.MdTextSplitter method)</a>
+</li>
         <li><a href="api-reference/openai.html#mltb2.openai.OpenAiBaseCompletion.__call__">(mltb2.openai.OpenAiBaseCompletion method)</a>
 </li>
         <li><a href="api-reference/openai.html#mltb2.openai.OpenAiTokenCounter.__call__">(mltb2.openai.OpenAiTokenCounter method)</a>
@@ -117,6 +120,8 @@ <h2 id="_">_</h2>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="api-reference/openai.html#mltb2.openai._check_mandatory_azure_completion_kwargs">_check_mandatory_azure_completion_kwargs() (in module mltb2.openai)</a>
+</li>
+      <li><a href="api-reference/md.html#mltb2.md._chunk_md_by_headline">_chunk_md_by_headline() (in module mltb2.md)</a>
 </li>
       <li><a href="api-reference/openai.html#mltb2.openai.OpenAiBaseCompletion._completion">_completion() (mltb2.openai.OpenAiBaseCompletion method)</a>
 
@@ -141,6 +146,14 @@ <h2 id="B">B</h2>
   </ul></td>
 </tr></table>
 
+<h2 id="C">C</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+  <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api-reference/md.html#mltb2.md.chunk_md">chunk_md() (in module mltb2.md)</a>
+</li>
+  </ul></td>
+</tr></table>
+
 <h2 id="D">D</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
@@ -218,6 +231,8 @@ <h2 id="L">L</h2>
 <h2 id="M">M</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="api-reference/md.html#mltb2.md.MdTextSplitter">MdTextSplitter (class in mltb2.md)</a>
+</li>
       <li>
     mltb2.fasttext
 
@@ -230,6 +245,13 @@ <h2 id="M">M</h2>
 
       <ul>
         <li><a href="api-reference/files.html#module-mltb2.files">module</a>
+</li>
+      </ul></li>
+      <li>
+    mltb2.md
+
+      <ul>
+        <li><a href="api-reference/md.html#module-mltb2.md">module</a>
 </li>
       </ul></li>
       <li>
@@ -283,6 +305,8 @@ <h2 id="M">M</h2>
         <li><a href="api-reference/fasttext.html#module-mltb2.fasttext">mltb2.fasttext</a>
 </li>
         <li><a href="api-reference/files.html#module-mltb2.files">mltb2.files</a>
+</li>
+        <li><a href="api-reference/md.html#module-mltb2.md">mltb2.md</a>
 </li>
         <li><a href="api-reference/openai.html#module-mltb2.openai">mltb2.openai</a>
 </li>
diff --git a/index.html b/index.html
index 5e97360..107993e 100644
--- a/index.html
+++ b/index.html
@@ -99,6 +99,7 @@ <h2>Content<a class="headerlink" href="#content" title="Permalink to this headin
 <li class="toctree-l1"><a class="reference internal" href="api-reference.html">API Reference</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/fasttext.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">fasttext</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/files.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">files</span></code></a></li>
+<li class="toctree-l2"><a class="reference internal" href="api-reference/md.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">md</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/openai.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">openai</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/optuna.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">optuna</span></code></a></li>
 <li class="toctree-l2"><a class="reference internal" href="api-reference/plot.html"><code class="xref py py-mod docutils literal notranslate"><span class="pre">plot</span></code></a></li>
diff --git a/objects.inv b/objects.inv
index 96975e946ee2bd6b471762ff50753ae1788f4e7b..38ab030bef8e42c491e10490081b1abafb6b0b0f 100644
GIT binary patch
delta 1020
zcmV<Y0|We@2<8Zocz-wyfbaShoatWe-S*g9nr>&(O=dH-?;M(hZ8m&pC~9N(*DoKE
zB_$#)%kIgrNPYkz00bq4-F#YZNo6e@EGmzmrDnAN^cNx}Ul`be0Y%}@cFJ;ZrT7ph
zE&f1@LJkPaZ%$Y{;gP6aO?L3WKv}-!lvu8nS3m`+D)ceSzke9K#F!ExFb-P)yVvaI
zDt2J8+c*eZ61&GlG0gO<5}L3%TDgEK=Wsus@^%>8f~9+GKxzxvKk~F~t!VJ{$29LP
z$ZyQ=h8iFa6^1Y)mSD!s<xt5o+mbA?CpYlKy!Th7;Lc+v9V;y(vv^88jAAg|*XjjJ
zhBrsN2f_qbd4I=_Vp__DD|HEq@QnN7goF#5!GGP)9l!fk8@S&Sd#`0FAfJ7GY?gN$
z))!L!9@LcAJ_AYzw%&}pHCo~$k>Wb)4PWvQb-Rt<%Dmrd+MTLib7f}+EWTnt1rv_-
z<|j~low;lM0_tAZ%3f(|A9QuvxT*=%)jM~J5Ub4*6MtVVdhZCoyb`mkvgI(P!Pbgo
zOQN5UEQw)o#dj(k<Zi(aDE%?V)`oD!)$ZBUD(}mVKLflh{ZH1hWBgL+Ts!866I%{L
z8+~XANq={Y@x|`+r!xr^`46!3rgeSE&PBpJihboSZw^pl%OzA`xD#h>tJ0GE)ER%q
z2?>ujgMZ%%rNpomms}9T?J*rQecXA@l=Qoh=vzALGaOf7=VDj-iG0`L&<eYpx7|ot
zxmMcGJ>T8UfL-mtPz?V*SVO2QUo&kRLY5FETpMxCOJES*Bmb38qAa5|B}PQpT5HlQ
zAa*!aZMBhLs<T-&n4L|r;mm25fuxS<@af@`Rev+lDHFUgnDWmzE!ZR3fB^Q@UCNpl
zH|~w8_Ki`OmG?)+2b1A9w|3q8Gpz%<8a*W_#+3}_pvtn(OjERYvpCF0ch3*#NBgK4
z|8kH-48JX)_Fpk_N6#=$S9lb};_V$Zy|lT%BFT!ePA<K$*Xf}s7h^q94shz3P?U@J
zO@Bao`Kd)fUMj{iJ-xVB7*Ujqv!&u3?R{NABLhV``qB#ClLyE)Xk4$(0DFQb37^1c
z;_XBGZ{e3N{L)hdhDe%*?7kocu?Gv$xN}FJVsIET+NW_t4_+*~X^kIxGoBvTUp}F)
zn%2^_4&ifNA=xAYPamvUXPOImSF=a#Ds_*GcA!xl&wyxQ4E0G``*)!zciNkbiE%%X
z)?G28+^2=~2@U@gR@Yv8spEu9>h1rGW1f$$qN<^PDj<gLhYRDgxMmf{pUDo2COxKD
q-0x(f+*#cu@JO;<fF9`z8a@4iw@QbpeX;8II>(g_wEPc9V6X~M4fqfM

delta 949
zcmV;m14{hn2%iX$cz+-efbaSh%%oSlk7I9nG&5<_X{Pp_A+VNd5NH53c7J_&NRgCC
zSeBiWR|39eSG&vNLfFSI%MCHs!p@>{|Fux85`aD-Qu2j{4QP-QJ~mU9TU&^WIBD?#
zEwZ9VSl&8e?Sy+G^NQqf$3R-X;gndeq*p)%F$R53@{a~DF@L5+2#kXXV7H2$T}2KS
zJHtWXlGrUKl3}KvrBH;;(aHswl*9e_n76~&1}xoT4TZ9R{VPw~RFVcye;(((26@B0
zHB<v}s4#>Xu>>=&kB6!fB;g6u;)D^~>cL;`=aOIkt2A71iM>`uDIgs}U2Kwf9@Z5x
z?}M5W%QK*~w}02OaW`5Ocuxv(8ug4HxsN)Z;pf`kyKe1LnrF~w+F*pmPwb~)!m)b%
z1Js_j-L-lGd95mGPuyx3baL8Yx&``#3s(w>ReFz!uNIwegrA;?&P}o+j%l!!B*~KK
zCybWFFu3Bm45yJ>@Eb~BtJzu;F1gGfbS2ZiZ22?5%YV}UBpo}%FNI3=6n=Qb<S_K8
zb`4?F-z{T!u{*-(OoAbQ0XuJ6cev&*66R6tGk1Bj1A{Fuz<}mXoMo0#1^J^A{)`hw
zJX#NaCzKM+R=nVXXm0o8G1J8z=1fVyBZ<B*u|C7G0XrAFQa{Oe6%MUn<vepEW##%o
zKX+9MZhw=tYX<CO2Zmz!ZD%#1u6)hhTOYE7DB((rQ$7NN@E!TTbP^>Q%~oPW1k;)=
z%>rVFlV+oI0ookQs^08miuGsu`^4068PG8uK0SPrY9=~mf;W0oe%va-?#UwvU^nh&
zQGd8$Z%p0a5Orz1KN3Ee3_tsB*PVxH9ni_>DStsRtfV&w8Oy#hMbYBb;xHdwKHQ<-
z&8uwq`a6o~UyxDrlnq=o6O5O8C<<aW)Rc`c?=)>F%GpqRlLEZ7KPbxC+w**U`O9gK
zyp#<k?fBwOjYCn+4rcu!+S?L=1_rY6KpI!@mfS-&-`};}46rA7lJL2BAapL8r-fg-
z@PA8Rf++;WG-UgaMi4u&5Lmf8#Uz7WpV7RH2HklvYp2zI=*;+ezrOl{ZWOHw*9C;X
zmoisC2gv1q(G1i}V?mBOVz5p!ta~wua%);0^D%NK(%ODM%3WGWfY5zUVYR)kms(E9
zg^v9P9P_Yu<xTeeV}NM7?Jtb);+j<)e=Z<7WOaZ~*>Sh6h;qB@It2&P<p}h^5nb!)
X4!l;{PtK!NztuUctf%Eaf}t8iIS$dL

diff --git a/py-modindex.html b/py-modindex.html
index 7fd0001..1be54fd 100644
--- a/py-modindex.html
+++ b/py-modindex.html
@@ -104,6 +104,11 @@ <h1>Python Module Index</h1>
        <td>&#160;&#160;&#160;
        <a href="api-reference/files.html#module-mltb2.files"><code class="xref">mltb2.files</code></a></td><td>
        <em></em></td></tr>
+     <tr class="cg-1">
+       <td></td>
+       <td>&#160;&#160;&#160;
+       <a href="api-reference/md.html#module-mltb2.md"><code class="xref">mltb2.md</code></a></td><td>
+       <em></em></td></tr>
      <tr class="cg-1">
        <td></td>
        <td>&#160;&#160;&#160;
diff --git a/searchindex.js b/searchindex.js
index 9100ef1..9471431 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"docnames": ["api-reference", "api-reference/fasttext", "api-reference/files", "api-reference/openai", "api-reference/optuna", "api-reference/plot", "api-reference/somajo", "api-reference/somajo_transformers", "api-reference/transformers", "index"], "filenames": ["api-reference.rst", "api-reference/fasttext.rst", "api-reference/files.rst", "api-reference/openai.rst", "api-reference/optuna.rst", "api-reference/plot.rst", "api-reference/somajo.rst", "api-reference/somajo_transformers.rst", "api-reference/transformers.rst", "index.rst"], "titles": ["API Reference", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">fasttext</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">files</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">openai</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">optuna</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">plot</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">somajo</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">somajo_transformers</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">transformers</span></code>", "MLTB2 Documentation"], "terms": {"fasttext": [0, 9], "file": [0, 3, 9], "openai": [0, 9], "optuna": [0, 9], "plot": [0, 9], "somajo": [0, 7, 9], "somajo_transform": [0, 9], "transform": [0, 7, 9], "specif": [1, 3, 4, 6, 7, 8], "function": [1, 3, 4, 5, 6, 7, 8], "thi": [1, 2, 3, 4, 5, 6, 7, 8], "modul": [1, 2, 3, 4, 5, 6, 7, 8], "i": [1, 2, 3, 4, 5, 6, 7, 8, 9], "base": [1, 3, 4, 5, 6, 7, 8], "us": [1, 2, 3, 4, 5, 6, 7, 8], "pip": [1, 2, 3, 4, 5, 6, 7, 8, 9], "instal": [1, 2, 3, 4, 5, 6, 7, 8], "necessari": [1, 2, 3, 4, 5, 6, 7, 8, 9], "depend": [1, 2, 3, 4, 5, 6, 7, 8, 9], "mltb2": [1, 2, 3, 4, 5, 6, 7, 8], "class": [1, 3, 4, 6, 7, 8], "fasttextlanguageidentif": 1, "sourc": [1, 2, 3, 4, 5, 6, 7, 8], "object": [1, 3, 4, 7, 8], "identifi": 1, "languag": [1, 6], "text": [1, 3, 6, 7, 8], "__call__": [1, 3, 6, 7, 8], "num_lang": 1, "int": [1, 3, 4, 5, 7, 8], "10": [1, 4], "given": [1, 4], "paramet": [1, 2, 3, 4, 5, 6, 7, 8], "which": [1, 2, 3, 4, 6, 8], "recogn": 1, "number": [1, 3, 4, 5, 7, 8], "return": [1, 2, 3, 4, 6, 7, 8], "A": [1, 4, 5, 9], "dict": [1, 3], "from": [1, 2, 3, 4, 6], "probabl": 1, "contain": [1, 6, 8], "more": [1, 4, 5], "than": [1, 7], "element": 1, "so": 1, "guarante": 1, "you": [1, 3, 4, 9], "want": 1, "includ": 1, "case": [1, 3], "when": [1, 3], "veri": 1, "low": 1, "possibl": 1, "ar": [1, 3, 4, 8], "af": 1, "al": 1, "am": 1, "an": [1, 3, 4, 6, 7, 8], "arz": 1, "ast": 1, "av": 1, "az": 1, "azb": 1, "ba": 1, "bar": 1, "bcl": 1, "bg": 1, "bh": 1, "bn": 1, "bo": 1, "bpy": 1, "br": 1, "b": 1, "bxr": 1, "ca": 1, "cbk": 1, "ce": 1, "ceb": 1, "ckb": 1, "co": [1, 8], "c": 1, "cv": [1, 4], "cy": 1, "da": [1, 6], "de": [1, 6], "diq": 1, "dsb": 1, "dty": 1, "dv": 1, "el": 1, "eml": 1, "en": 1, "eo": 1, "e": 1, "et": 1, "eu": 1, "fa": 1, "fi": 1, "fr": 1, "frr": 1, "fy": 1, "ga": 1, "gd": 1, "gl": 1, "gn": 1, "gom": 1, "gu": 1, "gv": 1, "he": 1, "hi": 1, "hif": 1, "hr": 1, "hsb": 1, "ht": 1, "hu": 1, "hy": 1, "ia": 1, "id": [1, 8], "ie": 1, "ilo": 1, "io": 1, "ja": 1, "jbo": 1, "jv": 1, "ka": 1, "kk": 1, "km": 1, "kn": 1, "ko": 1, "krc": 1, "ku": 1, "kv": 1, "kw": 1, "ky": 1, "la": 1, "lb": 1, "lez": 1, "li": 1, "lmo": 1, "lo": 1, "lrc": 1, "lt": 1, "lv": 1, "mai": 1, "mg": 1, "mhr": 1, "min": 1, "mk": 1, "ml": 1, "mn": 1, "mr": 1, "mrj": 1, "m": 1, "mt": 1, "mwl": 1, "my": 1, "myv": 1, "mzn": 1, "nah": 1, "nap": 1, "nd": 1, "ne": 1, "new": 1, "nl": 1, "nn": 1, "oc": 1, "o": 1, "pa": 1, "pam": 1, "pfl": 1, "pl": 1, "pm": 1, "pnb": 1, "p": 1, "pt": 1, "qu": 1, "rm": 1, "ro": 1, "ru": 1, "rue": 1, "sa": 1, "sah": 1, "sc": 1, "scn": 1, "sco": 1, "sd": 1, "sh": 1, "si": 1, "sk": 1, "sl": 1, "sq": 1, "sr": 1, "su": 1, "sv": 1, "sw": 1, "ta": 1, "te": 1, "tg": 1, "th": 1, "tk": 1, "tl": 1, "tr": 1, "tt": 1, "tyv": 1, "ug": 1, "uk": 1, "ur": 1, "uz": 1, "vec": 1, "vep": 1, "vi": 1, "vl": 1, "vo": 1, "wa": [1, 3, 8], "war": 1, "wuu": 1, "xal": 1, "xmf": 1, "yi": 1, "yo": 1, "yue": 1, "zh": 1, "static": 1, "get_model_path_and_download": 1, "str": [1, 2, 3, 5, 6, 7, 8], "get": [1, 3, 6], "model": [1, 3, 4, 8], "path": [1, 2, 8], "download": 1, "need": 1, "type": [1, 2, 3, 4, 6, 7, 8], "util": [2, 8], "fetch_remote_fil": 2, "dirnam": 2, "filenam": [2, 5], "url": [2, 6], "sha256_checksum": 2, "fetch": 2, "remot": 2, "directori": [2, 8], "where": 2, "save": [2, 5], "under": 2, "sha256": 2, "checksum": 2, "full": [2, 3], "creat": [2, 3, 5], "rais": [2, 7], "ioerror": 2, "wrong": 2, "get_and_create_mltb2_data_dir": 2, "mltb2_base_data_dir": 2, "none": [2, 3, 5, 6, 8], "mltb": 2, "data": [2, 4, 8], "dir": 2, "The": [2, 3, 4, 6, 7, 8], "openaiazurechatcomplet": 3, "completion_kwarg": 3, "ani": 3, "openaichatcomplet": 3, "azur": 3, "chat": 3, "complet": 3, "also": 3, "construct": 3, "openaibasecomplet": 3, "from_yaml": 3, "kwarg": 3, "follow": 3, "properti": 3, "must": [3, 4], "specifi": [3, 7], "api_typ": 3, "api_vers": 3, "api_bas": 3, "engin": 3, "quickstart": 3, "start": 3, "gpt": 3, "35": 3, "turbo": 3, "4": [3, 4], "servic": 3, "openaiazurecomplet": 3, "openaicomplet": 3, "non": 3, "gener": [3, 8], "abc": [3, 6], "abstract": [3, 6], "prompt": 3, "list": [3, 6, 7, 8], "map": 3, "openaicompletionansw": 3, "call": [3, 4, 5], "llm": 3, "In": [3, 4], "can": [3, 5, 9], "string": [3, 6], "onli": [3, 4], "allow": 3, "overwrit": 3, "exampl": [3, 4, 6], "chang": 3, "temperatur": 3, "_complet": 3, "completion_kwargs_for_this_cal": 3, "openaiobject": 3, "method": [3, 4], "classmethod": 3, "yaml_fil": 3, "yaml": 3, "prompt_token": 3, "completion_token": 3, "total_token": 3, "finish_reason": 3, "answer": 3, "result": [3, 4], "name": 3, "ha": [3, 5], "been": 3, "token": [3, 6, 7, 8], "total": 3, "reason": [3, 4], "why": 3, "stop": 3, "mean": [3, 4], "api": [3, 9], "without": 3, "run": 3, "limit": 3, "length": 3, "becaus": 3, "function_cal": 3, "from_open_ai_object": 3, "open_ai_object": 3, "openaitokencount": 3, "model_nam": 3, "show_progress_bar": [3, 6, 7, 8], "bool": [3, 4, 6, 7, 8], "fals": [3, 6, 7, 8], "count": [3, 8], "some": [3, 9], "3": [3, 4], "5": 3, "davinci": 3, "003": 3, "embed": 3, "ada": 3, "002": 3, "show": [3, 6, 7, 8], "progressbar": [3, 6, 7, 8], "dure": [3, 6, 7, 8], "process": [3, 6, 7, 8], "iter": [3, 6, 8], "just": [3, 8], "If": [3, 6, 7, 8], "_check_mandatory_azure_completion_kwarg": 3, "check": [3, 4], "mandatori": 3, "significancerepeatedtrainingprun": 4, "alpha": 4, "float": [4, 6], "0": [4, 5], "1": [4, 8], "n_warmup_step": 4, "baseprun": 4, "pruner": 4, "statist": 4, "signific": 4, "heurist": 4, "decis": 4, "make": 4, "It": [4, 6, 9], "prune": 4, "repeat": 4, "train": [4, 8], "like": 4, "cross": [4, 8], "valid": [4, 8], "As": 4, "test": [4, 8], "t": 4, "our": 4, "experi": 4, "have": 4, "shown": 4, "aplha": 4, "valu": [4, 5], "between": 4, "": 4, "standard": 4, "assum": 4, "adjust": 4, "onc": 4, "per": [4, 7], "hyperparamet": 4, "set": [4, 6, 8], "those": 4, "work": 4, "basi": 4, "intermedi": 4, "For": [4, 5], "epoch": 4, "contrast": 4, "doe": [4, 5, 7], "precis": 4, "individu": 4, "fold": [4, 8], "below": 4, "minimalist": 4, "import": [4, 6], "log": 4, "numpi": 4, "np": 4, "sklearn": 4, "dataset": [4, 8], "load_iri": 4, "model_select": 4, "stratifiedkfold": 4, "ensembl": 4, "randomforestclassifi": 4, "metric": 4, "accuracy_scor": 4, "configur": 4, "logger": 4, "see": 4, "debug": 4, "output": [4, 6], "getlogg": 4, "addhandl": 4, "streamhandl": 4, "setlevel": 4, "x": [4, 5], "y": [4, 5], "target": 4, "def": 4, "trial": 4, "min_samples_split": 4, "suggest_int": 4, "2": 4, "20": 4, "n_estim": 4, "100": 4, "validation_result_list": 4, "skf": 4, "n_split": [4, 8], "fold_index": 4, "train_index": 4, "val_index": 4, "enumer": 4, "split": [4, 6, 7, 8], "x_train": 4, "x_val": 4, "y_train": 4, "y_val": 4, "rf": 4, "fit": 4, "y_pred": 4, "predict": 4, "acc": 4, "append": 4, "report": 4, "we": 4, "should": [4, 6], "should_prun": 4, "here": 4, "done": 4, "break": 4, "studi": 4, "create_studi": 4, "storag": 4, "sqlite": 4, "db": 4, "memori": 4, "study_nam": 4, "iris_cv": 4, "direct": 4, "maxim": 4, "load_if_exist": 4, "true": [4, 5, 7], "sampler": 4, "tpesampl": 4, "multivari": 4, "add": 4, "optim": 4, "n_trial": 4, "level": 4, "larger": 4, "aggress": 4, "smaller": 4, "stronger": 4, "differ": [4, 5], "two": [4, 5, 6], "distribut": 4, "disabl": 4, "until": 4, "reach": 4, "exce": 4, "step": [4, 5], "frozentri": 4, "judg": 4, "whether": 4, "note": 4, "suppos": 4, "librari": 4, "user": 4, "instead": 4, "provid": 4, "interfac": 4, "implement": 4, "mechan": 4, "take": 4, "copi": 4, "befor": 4, "modifi": 4, "boolean": 4, "repres": 4, "collect": 5, "tool": [5, 6, 9], "matplotlib": 5, "boxplot": 5, "label": [5, 8], "titl": 5, "xlabel": 5, "ylabel": 5, "vert": 5, "print": [5, 6], "one": [5, 6], "singl": 5, "diagram": 5, "pyplot": 5, "boxplot_dict": 5, "values_dict": 5, "form": 5, "dictionari": 5, "save_last_figur": 5, "last": 5, "jupyt": 5, "notebook": 5, "same": 5, "cell": 5, "twin_axes_timeseries_plot": 5, "values_1": 5, "label_1": 5, "values_2": 5, "label_2": 5, "start_timestep_numb": 5, "shift_1": 5, "shift_2": 5, "label_x": 5, "color_1": 5, "tab": 5, "red": 5, "color_2": 5, "blue": 5, "twin": 5, "ax": 5, "timeseri": 5, "curv": 5, "array_lik": 5, "first": 5, "second": 5, "point": 5, "time": 5, "default": 5, "timestep": 5, "shift": 5, "posit": 5, "neg": 5, "axi": 5, "color": 5, "jaccardsimilar": 6, "somajobaseclass": 6, "calcul": 6, "jaccard": 6, "similar": 6, "de_cmc": 6, "german": 6, "en_ptb": 6, "english": 6, "text1": 6, "text2": 6, "get_token_set": 6, "word": [6, 7], "directli": 6, "somajosentencesplitt": [6, 7], "sentenc": [6, 7], "tokenextractor": 6, "extract": 6, "extract_url_set": 6, "token_extractor": 6, "url_set": 6, "ist": 6, "ein": 6, "link": 6, "http": 6, "github": [6, 9], "com": 6, "detoken": 6, "convert": 6, "how": 6, "do": [6, 8], "extract_token_class_set": 6, "keep_token_class": 6, "keep": 6, "all": [6, 9], "kept": 6, "hug": [7, 8], "face": [7, 8], "textsplitt": 7, "max_token": 7, "somajo_sentence_splitt": 7, "transformers_token_count": 7, "transformerstokencount": [7, 8], "ignore_overly_long_sent": 7, "section": 7, "maximum": 7, "divid": 7, "alwai": 7, "whole": 7, "splitter": 7, "counter": 7, "valueerror": 7, "except": 7, "longer": 7, "simpli": 7, "ignor": 7, "kfoldlabeleddataset": 8, "7": 8, "n_repeat": 8, "random_st": 8, "k": 8, "labeleddataset": 8, "labeled_dataset": 8, "stratification_label": 8, "encod": 8, "labe": 8, "pretrained_model_name_or_path": 8, "pathlik": 8, "host": 8, "insid": 8, "repo": 8, "huggingfac": 8, "box": 9, "machin": 9, "learn": 9, "avail": 9, "python": 9, "packag": 9, "index": 9, "pypi": 9, "option": 9, "might": 9, "them": 9, "refer": 9, "repositori": 9, "licens": 9, "imprint": 9}, "objects": {"mltb2": [[1, 0, 0, "-", "fasttext"], [2, 0, 0, "-", "files"], [3, 0, 0, "-", "openai"], [4, 0, 0, "-", "optuna"], [5, 0, 0, "-", "plot"], [6, 0, 0, "-", "somajo"], [7, 0, 0, "-", "somajo_transformers"], [8, 0, 0, "-", "transformers"]], "mltb2.fasttext": [[1, 1, 1, "", "FastTextLanguageIdentification"]], "mltb2.fasttext.FastTextLanguageIdentification": [[1, 2, 1, "", "__call__"], [1, 2, 1, "", "get_model_path_and_download"]], "mltb2.files": [[2, 3, 1, "", "fetch_remote_file"], [2, 3, 1, "", "get_and_create_mltb2_data_dir"]], "mltb2.openai": [[3, 1, 1, "", "OpenAiAzureChatCompletion"], [3, 1, 1, "", "OpenAiAzureCompletion"], [3, 1, 1, "", "OpenAiBaseCompletion"], [3, 1, 1, "", "OpenAiChatCompletion"], [3, 1, 1, "", "OpenAiCompletion"], [3, 1, 1, "", "OpenAiCompletionAnswer"], [3, 1, 1, "", "OpenAiTokenCounter"], [3, 3, 1, "", "_check_mandatory_azure_completion_kwargs"]], "mltb2.openai.OpenAiBaseCompletion": [[3, 2, 1, "", "__call__"], [3, 2, 1, "", "_completion"], [3, 2, 1, "", "from_yaml"]], "mltb2.openai.OpenAiChatCompletion": [[3, 2, 1, "", "_completion"]], "mltb2.openai.OpenAiCompletion": [[3, 2, 1, "", "_completion"]], "mltb2.openai.OpenAiCompletionAnswer": [[3, 2, 1, "", "from_open_ai_object"]], "mltb2.openai.OpenAiTokenCounter": [[3, 2, 1, "", "__call__"]], "mltb2.optuna": [[4, 1, 1, "", "SignificanceRepeatedTrainingPruner"]], "mltb2.optuna.SignificanceRepeatedTrainingPruner": [[4, 2, 1, "", "prune"]], "mltb2.plot": [[5, 3, 1, "", "boxplot"], [5, 3, 1, "", "boxplot_dict"], [5, 3, 1, "", "save_last_figure"], [5, 3, 1, "", "twin_axes_timeseries_plot"]], "mltb2.somajo": [[6, 1, 1, "", "JaccardSimilarity"], [6, 1, 1, "", "SoMaJoBaseClass"], [6, 1, 1, "", "SoMaJoSentenceSplitter"], [6, 1, 1, "", "TokenExtractor"], [6, 3, 1, "", "detokenize"], [6, 3, 1, "", "extract_token_class_set"]], "mltb2.somajo.JaccardSimilarity": [[6, 2, 1, "", "__call__"], [6, 2, 1, "", "get_token_set"]], "mltb2.somajo.SoMaJoSentenceSplitter": [[6, 2, 1, "", "__call__"]], "mltb2.somajo.TokenExtractor": [[6, 2, 1, "", "extract_url_set"]], "mltb2.somajo_transformers": [[7, 1, 1, "", "TextSplitter"]], "mltb2.somajo_transformers.TextSplitter": [[7, 2, 1, "", "__call__"]], "mltb2.transformers": [[8, 1, 1, "", "KFoldLabeledDataset"], [8, 1, 1, "", "LabeledDataset"], [8, 1, 1, "", "TransformersTokenCounter"]], "mltb2.transformers.KFoldLabeledDataset": [[8, 2, 1, "", "split"]], "mltb2.transformers.TransformersTokenCounter": [[8, 2, 1, "", "__call__"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "function", "Python function"]}, "titleterms": {"api": 0, "refer": 0, "fasttext": 1, "file": 2, "openai": 3, "optuna": 4, "plot": 5, "somajo": 6, "somajo_transform": 7, "transform": 8, "mltb2": 9, "document": 9, "instal": 9, "content": 9}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx.ext.intersphinx": 1, "sphinx.ext.todo": 2, "sphinx": 58}, "alltitles": {"API Reference": [[0, "api-reference"]], "fasttext": [[1, "module-mltb2.fasttext"]], "files": [[2, "module-mltb2.files"]], "openai": [[3, "module-mltb2.openai"]], "optuna": [[4, "module-mltb2.optuna"]], "plot": [[5, "module-mltb2.plot"]], "somajo": [[6, "module-mltb2.somajo"]], "somajo_transformers": [[7, "module-mltb2.somajo_transformers"]], "transformers": [[8, "module-mltb2.transformers"]], "MLTB2 Documentation": [[9, "mltb2-documentation"]], "Installation": [[9, "installation"]], "Content": [[9, "content"]]}, "indexentries": {"fasttextlanguageidentification (class in mltb2.fasttext)": [[1, "mltb2.fasttext.FastTextLanguageIdentification"]], "__call__() (mltb2.fasttext.fasttextlanguageidentification method)": [[1, "mltb2.fasttext.FastTextLanguageIdentification.__call__"]], "get_model_path_and_download() (mltb2.fasttext.fasttextlanguageidentification static method)": [[1, "mltb2.fasttext.FastTextLanguageIdentification.get_model_path_and_download"]], "mltb2.fasttext": [[1, "module-mltb2.fasttext"]], "module": [[1, "module-mltb2.fasttext"], [2, "module-mltb2.files"], [3, "module-mltb2.openai"], [4, "module-mltb2.optuna"], [5, "module-mltb2.plot"], [6, "module-mltb2.somajo"], [7, "module-mltb2.somajo_transformers"], [8, "module-mltb2.transformers"]], "fetch_remote_file() (in module mltb2.files)": [[2, "mltb2.files.fetch_remote_file"]], "get_and_create_mltb2_data_dir() (in module mltb2.files)": [[2, "mltb2.files.get_and_create_mltb2_data_dir"]], "mltb2.files": [[2, "module-mltb2.files"]], "openaiazurechatcompletion (class in mltb2.openai)": [[3, "mltb2.openai.OpenAiAzureChatCompletion"]], "openaiazurecompletion (class in mltb2.openai)": [[3, "mltb2.openai.OpenAiAzureCompletion"]], "openaibasecompletion (class in mltb2.openai)": [[3, "mltb2.openai.OpenAiBaseCompletion"]], "openaichatcompletion (class in mltb2.openai)": [[3, "mltb2.openai.OpenAiChatCompletion"]], "openaicompletion (class in mltb2.openai)": [[3, "mltb2.openai.OpenAiCompletion"]], "openaicompletionanswer (class in mltb2.openai)": [[3, "mltb2.openai.OpenAiCompletionAnswer"]], "openaitokencounter (class in mltb2.openai)": [[3, "mltb2.openai.OpenAiTokenCounter"]], "__call__() (mltb2.openai.openaibasecompletion method)": [[3, "mltb2.openai.OpenAiBaseCompletion.__call__"]], "__call__() (mltb2.openai.openaitokencounter method)": [[3, "mltb2.openai.OpenAiTokenCounter.__call__"]], "_check_mandatory_azure_completion_kwargs() (in module mltb2.openai)": [[3, "mltb2.openai._check_mandatory_azure_completion_kwargs"]], "_completion() (mltb2.openai.openaibasecompletion method)": [[3, "mltb2.openai.OpenAiBaseCompletion._completion"]], "_completion() (mltb2.openai.openaichatcompletion method)": [[3, "mltb2.openai.OpenAiChatCompletion._completion"]], "_completion() (mltb2.openai.openaicompletion method)": [[3, "mltb2.openai.OpenAiCompletion._completion"]], "from_open_ai_object() (mltb2.openai.openaicompletionanswer class method)": [[3, "mltb2.openai.OpenAiCompletionAnswer.from_open_ai_object"]], "from_yaml() (mltb2.openai.openaibasecompletion class method)": [[3, "mltb2.openai.OpenAiBaseCompletion.from_yaml"]], "mltb2.openai": [[3, "module-mltb2.openai"]], "significancerepeatedtrainingpruner (class in mltb2.optuna)": [[4, "mltb2.optuna.SignificanceRepeatedTrainingPruner"]], "mltb2.optuna": [[4, "module-mltb2.optuna"]], "prune() (mltb2.optuna.significancerepeatedtrainingpruner method)": [[4, "mltb2.optuna.SignificanceRepeatedTrainingPruner.prune"]], "boxplot() (in module mltb2.plot)": [[5, "mltb2.plot.boxplot"]], "boxplot_dict() (in module mltb2.plot)": [[5, "mltb2.plot.boxplot_dict"]], "mltb2.plot": [[5, "module-mltb2.plot"]], "save_last_figure() (in module mltb2.plot)": [[5, "mltb2.plot.save_last_figure"]], "twin_axes_timeseries_plot() (in module mltb2.plot)": [[5, "mltb2.plot.twin_axes_timeseries_plot"]], "jaccardsimilarity (class in mltb2.somajo)": [[6, "mltb2.somajo.JaccardSimilarity"]], "somajobaseclass (class in mltb2.somajo)": [[6, "mltb2.somajo.SoMaJoBaseClass"]], "somajosentencesplitter (class in mltb2.somajo)": [[6, "mltb2.somajo.SoMaJoSentenceSplitter"]], "tokenextractor (class in mltb2.somajo)": [[6, "mltb2.somajo.TokenExtractor"]], "__call__() (mltb2.somajo.jaccardsimilarity method)": [[6, "mltb2.somajo.JaccardSimilarity.__call__"]], "__call__() (mltb2.somajo.somajosentencesplitter method)": [[6, "mltb2.somajo.SoMaJoSentenceSplitter.__call__"]], "detokenize() (in module mltb2.somajo)": [[6, "mltb2.somajo.detokenize"]], "extract_token_class_set() (in module mltb2.somajo)": [[6, "mltb2.somajo.extract_token_class_set"]], "extract_url_set() (mltb2.somajo.tokenextractor method)": [[6, "mltb2.somajo.TokenExtractor.extract_url_set"]], "get_token_set() (mltb2.somajo.jaccardsimilarity method)": [[6, "mltb2.somajo.JaccardSimilarity.get_token_set"]], "mltb2.somajo": [[6, "module-mltb2.somajo"]], "textsplitter (class in mltb2.somajo_transformers)": [[7, "mltb2.somajo_transformers.TextSplitter"]], "__call__() (mltb2.somajo_transformers.textsplitter method)": [[7, "mltb2.somajo_transformers.TextSplitter.__call__"]], "mltb2.somajo_transformers": [[7, "module-mltb2.somajo_transformers"]], "kfoldlabeleddataset (class in mltb2.transformers)": [[8, "mltb2.transformers.KFoldLabeledDataset"]], "labeleddataset (class in mltb2.transformers)": [[8, "mltb2.transformers.LabeledDataset"]], "transformerstokencounter (class in mltb2.transformers)": [[8, "mltb2.transformers.TransformersTokenCounter"]], "__call__() (mltb2.transformers.transformerstokencounter method)": [[8, "mltb2.transformers.TransformersTokenCounter.__call__"]], "mltb2.transformers": [[8, "module-mltb2.transformers"]], "split() (mltb2.transformers.kfoldlabeleddataset method)": [[8, "mltb2.transformers.KFoldLabeledDataset.split"]]}})
\ No newline at end of file
+Search.setIndex({"docnames": ["api-reference", "api-reference/fasttext", "api-reference/files", "api-reference/md", "api-reference/openai", "api-reference/optuna", "api-reference/plot", "api-reference/somajo", "api-reference/somajo_transformers", "api-reference/transformers", "index"], "filenames": ["api-reference.rst", "api-reference/fasttext.rst", "api-reference/files.rst", "api-reference/md.rst", "api-reference/openai.rst", "api-reference/optuna.rst", "api-reference/plot.rst", "api-reference/somajo.rst", "api-reference/somajo_transformers.rst", "api-reference/transformers.rst", "index.rst"], "titles": ["API Reference", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">fasttext</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">files</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">md</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">openai</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">optuna</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">plot</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">somajo</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">somajo_transformers</span></code>", "<code class=\"xref py py-mod docutils literal notranslate\"><span class=\"pre\">transformers</span></code>", "MLTB2 Documentation"], "terms": {"fasttext": [0, 10], "file": [0, 4, 10], "md": [0, 10], "openai": [0, 10], "optuna": [0, 10], "plot": [0, 10], "somajo": [0, 8, 10], "somajo_transform": [0, 10], "transform": [0, 8, 10], "specif": [1, 3, 4, 5, 7, 8, 9], "function": [1, 3, 4, 5, 6, 7, 8, 9], "thi": [1, 2, 3, 4, 5, 6, 7, 8, 9], "modul": [1, 2, 3, 4, 5, 6, 7, 8, 9], "i": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "base": [1, 3, 4, 5, 6, 7, 8, 9], "us": [1, 2, 3, 4, 5, 6, 7, 8, 9], "pip": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "instal": [1, 2, 3, 4, 5, 6, 7, 8, 9], "necessari": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "depend": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "mltb2": [1, 2, 3, 4, 5, 6, 7, 8, 9], "class": [1, 3, 4, 5, 7, 8, 9], "fasttextlanguageidentif": 1, "sourc": [1, 2, 3, 4, 5, 6, 7, 8, 9], "object": [1, 3, 4, 5, 8, 9], "identifi": 1, "languag": [1, 7], "text": [1, 3, 4, 7, 8, 9], "__call__": [1, 3, 4, 7, 8, 9], "num_lang": 1, "int": [1, 3, 4, 5, 6, 8, 9], "10": [1, 5], "given": [1, 5], "paramet": [1, 2, 3, 4, 5, 6, 7, 8, 9], "which": [1, 2, 4, 5, 7, 9], "recogn": 1, "number": [1, 3, 4, 5, 6, 8, 9], "return": [1, 2, 3, 4, 5, 7, 8, 9], "A": [1, 5, 6, 10], "dict": [1, 4], "from": [1, 2, 4, 5, 7], "probabl": 1, "contain": [1, 7, 9], "more": [1, 5, 6], "than": [1, 8], "element": 1, "so": 1, "guarante": 1, "you": [1, 4, 5, 10], "want": 1, "includ": 1, "case": [1, 4], "when": [1, 4], "veri": 1, "low": 1, "possibl": 1, "ar": [1, 4, 5, 9], "af": 1, "al": 1, "am": 1, "an": [1, 4, 5, 7, 8, 9], "arz": 1, "ast": 1, "av": 1, "az": 1, "azb": 1, "ba": 1, "bar": 1, "bcl": 1, "bg": 1, "bh": 1, "bn": 1, "bo": 1, "bpy": 1, "br": 1, "b": 1, "bxr": 1, "ca": 1, "cbk": 1, "ce": 1, "ceb": 1, "ckb": 1, "co": [1, 9], "c": 1, "cv": [1, 5], "cy": 1, "da": [1, 7], "de": [1, 7], "diq": 1, "dsb": 1, "dty": 1, "dv": 1, "el": 1, "eml": 1, "en": 1, "eo": 1, "e": 1, "et": 1, "eu": 1, "fa": 1, "fi": 1, "fr": 1, "frr": 1, "fy": 1, "ga": 1, "gd": 1, "gl": 1, "gn": 1, "gom": 1, "gu": 1, "gv": 1, "he": 1, "hi": 1, "hif": 1, "hr": 1, "hsb": 1, "ht": 1, "hu": 1, "hy": 1, "ia": 1, "id": [1, 9], "ie": 1, "ilo": 1, "io": 1, "ja": 1, "jbo": 1, "jv": 1, "ka": 1, "kk": 1, "km": 1, "kn": 1, "ko": 1, "krc": 1, "ku": 1, "kv": 1, "kw": 1, "ky": 1, "la": 1, "lb": 1, "lez": 1, "li": 1, "lmo": 1, "lo": 1, "lrc": 1, "lt": 1, "lv": 1, "mai": 1, "mg": 1, "mhr": 1, "min": 1, "mk": 1, "ml": 1, "mn": 1, "mr": 1, "mrj": 1, "m": 1, "mt": 1, "mwl": 1, "my": 1, "myv": 1, "mzn": 1, "nah": 1, "nap": 1, "nd": 1, "ne": 1, "new": 1, "nl": 1, "nn": 1, "oc": 1, "o": 1, "pa": 1, "pam": 1, "pfl": 1, "pl": 1, "pm": 1, "pnb": 1, "p": 1, "pt": 1, "qu": 1, "rm": 1, "ro": 1, "ru": 1, "rue": 1, "sa": 1, "sah": 1, "sc": 1, "scn": 1, "sco": 1, "sd": 1, "sh": 1, "si": 1, "sk": 1, "sl": 1, "sq": 1, "sr": 1, "su": 1, "sv": 1, "sw": 1, "ta": 1, "te": 1, "tg": 1, "th": 1, "tk": 1, "tl": 1, "tr": 1, "tt": 1, "tyv": 1, "ug": 1, "uk": 1, "ur": 1, "uz": 1, "vec": 1, "vep": 1, "vi": 1, "vl": 1, "vo": 1, "wa": [1, 4, 9], "war": 1, "wuu": 1, "xal": 1, "xmf": 1, "yi": 1, "yo": 1, "yue": 1, "zh": 1, "static": 1, "get_model_path_and_download": 1, "str": [1, 2, 3, 4, 6, 7, 8, 9], "get": [1, 4, 7], "model": [1, 4, 5, 9], "path": [1, 2, 9], "download": 1, "need": 1, "type": [1, 2, 3, 4, 5, 7, 8, 9], "util": [2, 9], "fetch_remote_fil": 2, "dirnam": 2, "filenam": [2, 6], "url": [2, 7], "sha256_checksum": 2, "fetch": 2, "remot": 2, "directori": [2, 9], "where": 2, "save": [2, 6], "under": 2, "sha256": 2, "checksum": 2, "full": [2, 4], "creat": [2, 4, 6], "rais": [2, 8], "ioerror": 2, "wrong": 2, "get_and_create_mltb2_data_dir": 2, "mltb2_base_data_dir": 2, "none": [2, 4, 6, 7, 9], "mltb": 2, "data": [2, 5, 9], "dir": 2, "The": [2, 3, 4, 5, 7, 8, 9], "markdown": 3, "mdtextsplitt": 3, "max_token": [3, 8], "transformers_token_count": [3, 8], "transformerstokencount": [3, 8, 9], "show_progress_bar": [3, 4, 7, 8, 9], "bool": [3, 4, 5, 7, 8, 9], "fals": [3, 4, 7, 8, 9], "split": [3, 5, 7, 8, 9], "section": [3, 8], "specifi": [3, 4, 8], "maximum": [3, 8], "token": [3, 4, 7, 8, 9], "doe": [3, 5, 6, 8], "divid": [3, 8], "head": 3, "correspond": 3, "paragraph": 3, "per": [3, 5, 8], "can": [3, 4, 6, 10], "onli": [3, 4, 5], "exceed": 3, "singl": [3, 6], "chunk": 3, "alreadi": 3, "larger": [3, 5], "counter": [3, 8], "show": [3, 4, 7, 8, 9], "progressbar": [3, 4, 7, 8, 9], "dure": [3, 4, 7, 8, 9], "process": [3, 4, 7, 8, 9], "md_text": 3, "list": [3, 4, 7, 8, 9], "_chunk_md_by_headlin": 3, "headlin": 3, "chunk_md": 3, "merg": 3, "isol": 3, "openaiazurechatcomplet": 4, "completion_kwarg": 4, "ani": 4, "openaichatcomplet": 4, "azur": 4, "chat": 4, "complet": 4, "also": 4, "construct": 4, "openaibasecomplet": 4, "from_yaml": 4, "kwarg": 4, "follow": 4, "properti": 4, "must": [4, 5], "api_typ": 4, "api_vers": 4, "api_bas": 4, "engin": 4, "quickstart": 4, "start": 4, "gpt": 4, "35": 4, "turbo": 4, "4": [4, 5], "servic": 4, "openaiazurecomplet": 4, "openaicomplet": 4, "non": 4, "gener": [4, 9], "abc": [4, 7], "abstract": [4, 7], "prompt": 4, "map": 4, "openaicompletionansw": 4, "call": [4, 5, 6], "llm": 4, "In": [4, 5], "string": [4, 7], "allow": 4, "overwrit": 4, "exampl": [4, 5, 7], "chang": 4, "temperatur": 4, "_complet": 4, "completion_kwargs_for_this_cal": 4, "openaiobject": 4, "method": [4, 5], "classmethod": 4, "yaml_fil": 4, "yaml": 4, "prompt_token": 4, "completion_token": 4, "total_token": 4, "finish_reason": 4, "answer": 4, "result": [4, 5], "name": 4, "ha": [4, 6], "been": 4, "total": 4, "reason": [4, 5], "why": 4, "stop": 4, "mean": [4, 5], "api": [4, 10], "without": 4, "run": 4, "limit": 4, "length": 4, "becaus": 4, "function_cal": 4, "from_open_ai_object": 4, "open_ai_object": 4, "openaitokencount": 4, "model_nam": 4, "count": [4, 9], "some": [4, 10], "3": [4, 5], "5": 4, "davinci": 4, "003": 4, "embed": 4, "ada": 4, "002": 4, "iter": [4, 7, 9], "just": [4, 9], "If": [4, 7, 8, 9], "_check_mandatory_azure_completion_kwarg": 4, "check": [4, 5], "mandatori": 4, "significancerepeatedtrainingprun": 5, "alpha": 5, "float": [5, 7], "0": [5, 6], "1": [5, 9], "n_warmup_step": 5, "baseprun": 5, "pruner": 5, "statist": 5, "signific": 5, "heurist": 5, "decis": 5, "make": 5, "It": [5, 7, 10], "prune": 5, "repeat": 5, "train": [5, 9], "like": 5, "cross": [5, 9], "valid": [5, 9], "As": 5, "test": [5, 9], "t": 5, "our": 5, "experi": 5, "have": 5, "shown": 5, "aplha": 5, "valu": [5, 6], "between": 5, "": 5, "standard": 5, "assum": 5, "adjust": 5, "onc": 5, "hyperparamet": 5, "set": [5, 7, 9], "those": 5, "work": 5, "basi": 5, "intermedi": 5, "For": [5, 6], "epoch": 5, "contrast": 5, "precis": 5, "individu": 5, "fold": [5, 9], "below": 5, "minimalist": 5, "import": [5, 7], "log": 5, "numpi": 5, "np": 5, "sklearn": 5, "dataset": [5, 9], "load_iri": 5, "model_select": 5, "stratifiedkfold": 5, "ensembl": 5, "randomforestclassifi": 5, "metric": 5, "accuracy_scor": 5, "configur": 5, "logger": 5, "see": 5, "debug": 5, "output": [5, 7], "getlogg": 5, "addhandl": 5, "streamhandl": 5, "setlevel": 5, "x": [5, 6], "y": [5, 6], "target": 5, "def": 5, "trial": 5, "min_samples_split": 5, "suggest_int": 5, "2": 5, "20": 5, "n_estim": 5, "100": 5, "validation_result_list": 5, "skf": 5, "n_split": [5, 9], "fold_index": 5, "train_index": 5, "val_index": 5, "enumer": 5, "x_train": 5, "x_val": 5, "y_train": 5, "y_val": 5, "rf": 5, "fit": 5, "y_pred": 5, "predict": 5, "acc": 5, "append": 5, "report": 5, "we": 5, "should": [5, 7], "should_prun": 5, "here": 5, "done": 5, "break": 5, "studi": 5, "create_studi": 5, "storag": 5, "sqlite": 5, "db": 5, "memori": 5, "study_nam": 5, "iris_cv": 5, "direct": 5, "maxim": 5, "load_if_exist": 5, "true": [5, 6, 8], "sampler": 5, "tpesampl": 5, "multivari": 5, "add": 5, "optim": 5, "n_trial": 5, "level": 5, "aggress": 5, "smaller": 5, "stronger": 5, "differ": [5, 6], "two": [5, 6, 7], "distribut": 5, "disabl": 5, "until": 5, "reach": 5, "exce": 5, "step": [5, 6], "frozentri": 5, "judg": 5, "whether": 5, "note": 5, "suppos": 5, "librari": 5, "user": 5, "instead": 5, "provid": 5, "interfac": 5, "implement": 5, "mechan": 5, "take": 5, "copi": 5, "befor": 5, "modifi": 5, "boolean": 5, "repres": 5, "collect": 6, "tool": [6, 7, 10], "matplotlib": 6, "boxplot": 6, "label": [6, 9], "titl": 6, "xlabel": 6, "ylabel": 6, "vert": 6, "print": [6, 7], "one": [6, 7], "diagram": 6, "pyplot": 6, "boxplot_dict": 6, "values_dict": 6, "form": 6, "dictionari": 6, "save_last_figur": 6, "last": 6, "jupyt": 6, "notebook": 6, "same": 6, "cell": 6, "twin_axes_timeseries_plot": 6, "values_1": 6, "label_1": 6, "values_2": 6, "label_2": 6, "start_timestep_numb": 6, "shift_1": 6, "shift_2": 6, "label_x": 6, "color_1": 6, "tab": 6, "red": 6, "color_2": 6, "blue": 6, "twin": 6, "ax": 6, "timeseri": 6, "curv": 6, "array_lik": 6, "first": 6, "second": 6, "point": 6, "time": 6, "default": 6, "timestep": 6, "shift": 6, "posit": 6, "neg": 6, "axi": 6, "color": 6, "jaccardsimilar": 7, "somajobaseclass": 7, "calcul": 7, "jaccard": 7, "similar": 7, "de_cmc": 7, "german": 7, "en_ptb": 7, "english": 7, "text1": 7, "text2": 7, "get_token_set": 7, "word": [7, 8], "directli": 7, "somajosentencesplitt": [7, 8], "sentenc": [7, 8], "tokenextractor": 7, "extract": 7, "extract_url_set": 7, "token_extractor": 7, "url_set": 7, "ist": 7, "ein": 7, "link": 7, "http": 7, "github": [7, 10], "com": 7, "detoken": 7, "convert": 7, "how": 7, "do": [7, 9], "extract_token_class_set": 7, "keep_token_class": 7, "keep": 7, "all": [7, 10], "kept": 7, "hug": [8, 9], "face": [8, 9], "textsplitt": 8, "somajo_sentence_splitt": 8, "ignore_overly_long_sent": 8, "alwai": 8, "whole": 8, "splitter": 8, "valueerror": 8, "except": 8, "longer": 8, "simpli": 8, "ignor": 8, "kfoldlabeleddataset": 9, "7": 9, "n_repeat": 9, "random_st": 9, "k": 9, "labeleddataset": 9, "labeled_dataset": 9, "stratification_label": 9, "encod": 9, "labe": 9, "pretrained_model_name_or_path": 9, "pathlik": 9, "host": 9, "insid": 9, "repo": 9, "huggingfac": 9, "box": 10, "machin": 10, "learn": 10, "avail": 10, "python": 10, "packag": 10, "index": 10, "pypi": 10, "option": 10, "might": 10, "them": 10, "refer": 10, "repositori": 10, "licens": 10, "imprint": 10}, "objects": {"mltb2": [[1, 0, 0, "-", "fasttext"], [2, 0, 0, "-", "files"], [3, 0, 0, "-", "md"], [4, 0, 0, "-", "openai"], [5, 0, 0, "-", "optuna"], [6, 0, 0, "-", "plot"], [7, 0, 0, "-", "somajo"], [8, 0, 0, "-", "somajo_transformers"], [9, 0, 0, "-", "transformers"]], "mltb2.fasttext": [[1, 1, 1, "", "FastTextLanguageIdentification"]], "mltb2.fasttext.FastTextLanguageIdentification": [[1, 2, 1, "", "__call__"], [1, 2, 1, "", "get_model_path_and_download"]], "mltb2.files": [[2, 3, 1, "", "fetch_remote_file"], [2, 3, 1, "", "get_and_create_mltb2_data_dir"]], "mltb2.md": [[3, 1, 1, "", "MdTextSplitter"], [3, 3, 1, "", "_chunk_md_by_headline"], [3, 3, 1, "", "chunk_md"]], "mltb2.md.MdTextSplitter": [[3, 2, 1, "", "__call__"]], "mltb2.openai": [[4, 1, 1, "", "OpenAiAzureChatCompletion"], [4, 1, 1, "", "OpenAiAzureCompletion"], [4, 1, 1, "", "OpenAiBaseCompletion"], [4, 1, 1, "", "OpenAiChatCompletion"], [4, 1, 1, "", "OpenAiCompletion"], [4, 1, 1, "", "OpenAiCompletionAnswer"], [4, 1, 1, "", "OpenAiTokenCounter"], [4, 3, 1, "", "_check_mandatory_azure_completion_kwargs"]], "mltb2.openai.OpenAiBaseCompletion": [[4, 2, 1, "", "__call__"], [4, 2, 1, "", "_completion"], [4, 2, 1, "", "from_yaml"]], "mltb2.openai.OpenAiChatCompletion": [[4, 2, 1, "", "_completion"]], "mltb2.openai.OpenAiCompletion": [[4, 2, 1, "", "_completion"]], "mltb2.openai.OpenAiCompletionAnswer": [[4, 2, 1, "", "from_open_ai_object"]], "mltb2.openai.OpenAiTokenCounter": [[4, 2, 1, "", "__call__"]], "mltb2.optuna": [[5, 1, 1, "", "SignificanceRepeatedTrainingPruner"]], "mltb2.optuna.SignificanceRepeatedTrainingPruner": [[5, 2, 1, "", "prune"]], "mltb2.plot": [[6, 3, 1, "", "boxplot"], [6, 3, 1, "", "boxplot_dict"], [6, 3, 1, "", "save_last_figure"], [6, 3, 1, "", "twin_axes_timeseries_plot"]], "mltb2.somajo": [[7, 1, 1, "", "JaccardSimilarity"], [7, 1, 1, "", "SoMaJoBaseClass"], [7, 1, 1, "", "SoMaJoSentenceSplitter"], [7, 1, 1, "", "TokenExtractor"], [7, 3, 1, "", "detokenize"], [7, 3, 1, "", "extract_token_class_set"]], "mltb2.somajo.JaccardSimilarity": [[7, 2, 1, "", "__call__"], [7, 2, 1, "", "get_token_set"]], "mltb2.somajo.SoMaJoSentenceSplitter": [[7, 2, 1, "", "__call__"]], "mltb2.somajo.TokenExtractor": [[7, 2, 1, "", "extract_url_set"]], "mltb2.somajo_transformers": [[8, 1, 1, "", "TextSplitter"]], "mltb2.somajo_transformers.TextSplitter": [[8, 2, 1, "", "__call__"]], "mltb2.transformers": [[9, 1, 1, "", "KFoldLabeledDataset"], [9, 1, 1, "", "LabeledDataset"], [9, 1, 1, "", "TransformersTokenCounter"]], "mltb2.transformers.KFoldLabeledDataset": [[9, 2, 1, "", "split"]], "mltb2.transformers.TransformersTokenCounter": [[9, 2, 1, "", "__call__"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "function", "Python function"]}, "titleterms": {"api": 0, "refer": 0, "fasttext": 1, "file": 2, "md": 3, "openai": 4, "optuna": 5, "plot": 6, "somajo": 7, "somajo_transform": 8, "transform": 9, "mltb2": 10, "document": 10, "instal": 10, "content": 10}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx.ext.intersphinx": 1, "sphinx.ext.todo": 2, "sphinx": 58}, "alltitles": {"API Reference": [[0, "api-reference"]], "fasttext": [[1, "module-mltb2.fasttext"]], "files": [[2, "module-mltb2.files"]], "md": [[3, "module-mltb2.md"]], "openai": [[4, "module-mltb2.openai"]], "optuna": [[5, "module-mltb2.optuna"]], "plot": [[6, "module-mltb2.plot"]], "somajo": [[7, "module-mltb2.somajo"]], "somajo_transformers": [[8, "module-mltb2.somajo_transformers"]], "transformers": [[9, "module-mltb2.transformers"]], "MLTB2 Documentation": [[10, "mltb2-documentation"]], "Installation": [[10, "installation"]], "Content": [[10, "content"]]}, "indexentries": {"fasttextlanguageidentification (class in mltb2.fasttext)": [[1, "mltb2.fasttext.FastTextLanguageIdentification"]], "__call__() (mltb2.fasttext.fasttextlanguageidentification method)": [[1, "mltb2.fasttext.FastTextLanguageIdentification.__call__"]], "get_model_path_and_download() (mltb2.fasttext.fasttextlanguageidentification static method)": [[1, "mltb2.fasttext.FastTextLanguageIdentification.get_model_path_and_download"]], "mltb2.fasttext": [[1, "module-mltb2.fasttext"]], "module": [[1, "module-mltb2.fasttext"], [2, "module-mltb2.files"], [3, "module-mltb2.md"], [4, "module-mltb2.openai"], [5, "module-mltb2.optuna"], [6, "module-mltb2.plot"], [7, "module-mltb2.somajo"], [8, "module-mltb2.somajo_transformers"], [9, "module-mltb2.transformers"]], "fetch_remote_file() (in module mltb2.files)": [[2, "mltb2.files.fetch_remote_file"]], "get_and_create_mltb2_data_dir() (in module mltb2.files)": [[2, "mltb2.files.get_and_create_mltb2_data_dir"]], "mltb2.files": [[2, "module-mltb2.files"]], "mdtextsplitter (class in mltb2.md)": [[3, "mltb2.md.MdTextSplitter"]], "__call__() (mltb2.md.mdtextsplitter method)": [[3, "mltb2.md.MdTextSplitter.__call__"]], "_chunk_md_by_headline() (in module mltb2.md)": [[3, "mltb2.md._chunk_md_by_headline"]], "chunk_md() (in module mltb2.md)": [[3, "mltb2.md.chunk_md"]], "mltb2.md": [[3, "module-mltb2.md"]], "openaiazurechatcompletion (class in mltb2.openai)": [[4, "mltb2.openai.OpenAiAzureChatCompletion"]], "openaiazurecompletion (class in mltb2.openai)": [[4, "mltb2.openai.OpenAiAzureCompletion"]], "openaibasecompletion (class in mltb2.openai)": [[4, "mltb2.openai.OpenAiBaseCompletion"]], "openaichatcompletion (class in mltb2.openai)": [[4, "mltb2.openai.OpenAiChatCompletion"]], "openaicompletion (class in mltb2.openai)": [[4, "mltb2.openai.OpenAiCompletion"]], "openaicompletionanswer (class in mltb2.openai)": [[4, "mltb2.openai.OpenAiCompletionAnswer"]], "openaitokencounter (class in mltb2.openai)": [[4, "mltb2.openai.OpenAiTokenCounter"]], "__call__() (mltb2.openai.openaibasecompletion method)": [[4, "mltb2.openai.OpenAiBaseCompletion.__call__"]], "__call__() (mltb2.openai.openaitokencounter method)": [[4, "mltb2.openai.OpenAiTokenCounter.__call__"]], "_check_mandatory_azure_completion_kwargs() (in module mltb2.openai)": [[4, "mltb2.openai._check_mandatory_azure_completion_kwargs"]], "_completion() (mltb2.openai.openaibasecompletion method)": [[4, "mltb2.openai.OpenAiBaseCompletion._completion"]], "_completion() (mltb2.openai.openaichatcompletion method)": [[4, "mltb2.openai.OpenAiChatCompletion._completion"]], "_completion() (mltb2.openai.openaicompletion method)": [[4, "mltb2.openai.OpenAiCompletion._completion"]], "from_open_ai_object() (mltb2.openai.openaicompletionanswer class method)": [[4, "mltb2.openai.OpenAiCompletionAnswer.from_open_ai_object"]], "from_yaml() (mltb2.openai.openaibasecompletion class method)": [[4, "mltb2.openai.OpenAiBaseCompletion.from_yaml"]], "mltb2.openai": [[4, "module-mltb2.openai"]], "significancerepeatedtrainingpruner (class in mltb2.optuna)": [[5, "mltb2.optuna.SignificanceRepeatedTrainingPruner"]], "mltb2.optuna": [[5, "module-mltb2.optuna"]], "prune() (mltb2.optuna.significancerepeatedtrainingpruner method)": [[5, "mltb2.optuna.SignificanceRepeatedTrainingPruner.prune"]], "boxplot() (in module mltb2.plot)": [[6, "mltb2.plot.boxplot"]], "boxplot_dict() (in module mltb2.plot)": [[6, "mltb2.plot.boxplot_dict"]], "mltb2.plot": [[6, "module-mltb2.plot"]], "save_last_figure() (in module mltb2.plot)": [[6, "mltb2.plot.save_last_figure"]], "twin_axes_timeseries_plot() (in module mltb2.plot)": [[6, "mltb2.plot.twin_axes_timeseries_plot"]], "jaccardsimilarity (class in mltb2.somajo)": [[7, "mltb2.somajo.JaccardSimilarity"]], "somajobaseclass (class in mltb2.somajo)": [[7, "mltb2.somajo.SoMaJoBaseClass"]], "somajosentencesplitter (class in mltb2.somajo)": [[7, "mltb2.somajo.SoMaJoSentenceSplitter"]], "tokenextractor (class in mltb2.somajo)": [[7, "mltb2.somajo.TokenExtractor"]], "__call__() (mltb2.somajo.jaccardsimilarity method)": [[7, "mltb2.somajo.JaccardSimilarity.__call__"]], "__call__() (mltb2.somajo.somajosentencesplitter method)": [[7, "mltb2.somajo.SoMaJoSentenceSplitter.__call__"]], "detokenize() (in module mltb2.somajo)": [[7, "mltb2.somajo.detokenize"]], "extract_token_class_set() (in module mltb2.somajo)": [[7, "mltb2.somajo.extract_token_class_set"]], "extract_url_set() (mltb2.somajo.tokenextractor method)": [[7, "mltb2.somajo.TokenExtractor.extract_url_set"]], "get_token_set() (mltb2.somajo.jaccardsimilarity method)": [[7, "mltb2.somajo.JaccardSimilarity.get_token_set"]], "mltb2.somajo": [[7, "module-mltb2.somajo"]], "textsplitter (class in mltb2.somajo_transformers)": [[8, "mltb2.somajo_transformers.TextSplitter"]], "__call__() (mltb2.somajo_transformers.textsplitter method)": [[8, "mltb2.somajo_transformers.TextSplitter.__call__"]], "mltb2.somajo_transformers": [[8, "module-mltb2.somajo_transformers"]], "kfoldlabeleddataset (class in mltb2.transformers)": [[9, "mltb2.transformers.KFoldLabeledDataset"]], "labeleddataset (class in mltb2.transformers)": [[9, "mltb2.transformers.LabeledDataset"]], "transformerstokencounter (class in mltb2.transformers)": [[9, "mltb2.transformers.TransformersTokenCounter"]], "__call__() (mltb2.transformers.transformerstokencounter method)": [[9, "mltb2.transformers.TransformersTokenCounter.__call__"]], "mltb2.transformers": [[9, "module-mltb2.transformers"]], "split() (mltb2.transformers.kfoldlabeleddataset method)": [[9, "mltb2.transformers.KFoldLabeledDataset.split"]]}})
\ No newline at end of file
