From 4ec2f1718eeb248d6e5f29f083216e853fda5040 Mon Sep 17 00:00:00 2001
From: TimotheeMickus <TimotheeMickus@users.noreply.github.com>
Date: Mon, 25 Sep 2023 13:43:35 +0000
Subject: [PATCH] =?UTF-8?q?Deploying=20to=20gh-pages=20from=20=20@=20b76ca?=
 =?UTF-8?q?b3e1cfc0926b87b09c7b51714d0889945d4=20=F0=9F=9A=80?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 _modules/index.html                           |   5 -
 .../modules/conv_multi_step_attention.html    | 295 -----------
 _modules/mammoth/modules/copy_generator.html  | 483 ------------------
 _modules/mammoth/modules/embeddings.html      |   6 +-
 .../mammoth/modules/global_attention.html     | 444 ----------------
 .../mammoth/modules/structured_attention.html | 254 ---------
 _modules/mammoth/modules/weight_norm.html     | 443 ----------------
 _modules/mammoth/translate/translator.html    |  40 +-
 _modules/mammoth/utils/loss.html              |  10 +-
 _modules/mammoth/utils/optimizers.html        |  23 +-
 genindex.html                                 |  40 +-
 mammoth.modules.html                          | 259 +---------
 objects.inv                                   | Bin 1681 -> 1561 bytes
 options/train.html                            |  21 +-
 ref.html                                      |   6 -
 searchindex.js                                |   2 +-
 16 files changed, 32 insertions(+), 2299 deletions(-)
 delete mode 100644 _modules/mammoth/modules/conv_multi_step_attention.html
 delete mode 100644 _modules/mammoth/modules/copy_generator.html
 delete mode 100644 _modules/mammoth/modules/global_attention.html
 delete mode 100644 _modules/mammoth/modules/structured_attention.html
 delete mode 100644 _modules/mammoth/modules/weight_norm.html
diff --git a/_modules/index.html b/_modules/index.html
index 9eb57444..db70ca93 100644
--- a/_modules/index.html
+++ b/_modules/index.html
@@ -175,14 +175,9 @@
   <h1>All modules for which code is available</h1>
 <ul><li><a href="mammoth/models/model.html">mammoth.models.model</a></li>
 <li><a href="mammoth/modules/average_attn.html">mammoth.modules.average_attn</a></li>
-<li><a href="mammoth/modules/conv_multi_step_attention.html">mammoth.modules.conv_multi_step_attention</a></li>
-<li><a href="mammoth/modules/copy_generator.html">mammoth.modules.copy_generator</a></li>
 <li><a href="mammoth/modules/embeddings.html">mammoth.modules.embeddings</a></li>
-<li><a href="mammoth/modules/global_attention.html">mammoth.modules.global_attention</a></li>
 <li><a href="mammoth/modules/multi_headed_attn.html">mammoth.modules.multi_headed_attn</a></li>
 <li><a href="mammoth/modules/position_ffn.html">mammoth.modules.position_ffn</a></li>
-<li><a href="mammoth/modules/structured_attention.html">mammoth.modules.structured_attention</a></li>
-<li><a href="mammoth/modules/weight_norm.html">mammoth.modules.weight_norm</a></li>
 <li><a href="mammoth/trainer.html">mammoth.trainer</a></li>
 <li><a href="mammoth/translate/beam_search.html">mammoth.translate.beam_search</a></li>
 <li><a href="mammoth/translate/decode_strategy.html">mammoth.translate.decode_strategy</a></li>
diff --git a/_modules/mammoth/modules/conv_multi_step_attention.html b/_modules/mammoth/modules/conv_multi_step_attention.html
deleted file mode 100644
index 3ba4db4e..00000000
--- a/_modules/mammoth/modules/conv_multi_step_attention.html
+++ /dev/null
@@ -1,295 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
-  <meta charset="utf-8">
-  
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  
-  <title>mammoth.modules.conv_multi_step_attention &mdash; MAMMOTH  documentation</title>
-  
-
-  
-  
-  
-  
-
-  
-  <script type="text/javascript" src="../../../_static/js/modernizr.min.js"></script>
-  
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
-        <script src="../../../_static/jquery.js"></script>
-        <script src="../../../_static/underscore.js"></script>
-        <script src="../../../_static/doctools.js"></script>
-        <script src="../../../_static/language_data.js"></script>
-        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
-        <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["\\(", "\\)"]], "displayMath": [["\\[", "\\]"]]}})</script>
-        <script src="https://unpkg.com/mermaid@8.4.8/dist/mermaid.min.js"></script>
-    
-    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-    
-
-  
-  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/theme_overrides.css" type="text/css" />
-    <link rel="index" title="Index" href="../../../genindex.html" />
-    <link rel="search" title="Search" href="../../../search.html" /> 
-</head>
-
-<body class="wy-body-for-nav">
-
-   
-  <div class="wy-grid-for-nav">
-    
-    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
-      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search" >
-          
-
-          
-            <a href="../../../index.html" class="icon icon-home"> MAMMOTH
-          
-
-          
-          </a>
-
-          
-            
-            
-          
-
-          
-<div role="search">
-  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
-    <input type="text" name="q" placeholder="Search docs" />
-    <input type="hidden" name="check_keywords" value="yes" />
-    <input type="hidden" name="area" value="default" />
-  </form>
-</div>
-
-          
-        </div>
-
-        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-          
-            
-            
-              
-            
-            
-              <p class="caption"><span class="caption-text">Getting Started</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../main.html">Overview</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../quickstart.html">Quickstart</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../CONTRIBUTING.html">Contributors</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ref.html">References</a></li>
-</ul>
-<p class="caption"><span class="caption-text">MAMMOTH features</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../config_config.html">Config-config tool</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../attention_bridges.html">Attention Bridge</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Tutorials</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../prepare_data.html">Prepare Data</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Scripts</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/build_vocab.html">Build Vocab</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/train.html">Train</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/translate.html">Translate</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/server.html">Server</a></li>
-</ul>
-<p class="caption"><span class="caption-text">API</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.html">Framework</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.modules.html">Modules</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translation.html">Translation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translate.translation_server.html">Server</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.inputters.html">Data Loaders</a></li>
-</ul>
-
-            
-          
-        </div>
-      </div>
-    </nav>
-
-    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-      
-      <nav class="wy-nav-top" aria-label="top navigation">
-        
-          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-          <a href="../../../index.html">MAMMOTH</a>
-        
-      </nav>
-
-
-      <div class="wy-nav-content">
-        
-        <div class="rst-content">
-        
-          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
-  <ul class="wy-breadcrumbs">
-    
-      <li><a href="../../../index.html">Docs</a> &raquo;</li>
-        
-          <li><a href="../../index.html">Module code</a> &raquo;</li>
-        
-      <li>mammoth.modules.conv_multi_step_attention</li>
-    
-    
-      <li class="wy-breadcrumbs-aside">
-        
-      </li>
-    
-  </ul>
-
-  
-  <hr/>
-</div>
-          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
-           <div itemprop="articleBody">
-            
-  <h1>Source code for mammoth.modules.conv_multi_step_attention</h1><div class="highlight"><pre>
-<span></span><span class="sd">&quot;&quot;&quot; Multi Step Attention for CNN &quot;&quot;&quot;</span>
-<span class="kn">import</span> <span class="nn">torch</span>
-<span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
-<span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>
-<span class="kn">from</span> <span class="nn">mammoth.utils.misc</span> <span class="kn">import</span> <span class="n">aeq</span>
-
-
-<span class="n">SCALE_WEIGHT</span> <span class="o">=</span> <span class="mf">0.5</span><span class="o">**</span><span class="mf">0.5</span>
-
-
-<span class="k">def</span> <span class="nf">seq_linear</span><span class="p">(</span><span class="n">linear</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;linear transform for 3-d tensor&quot;&quot;&quot;</span>
-    <span class="n">batch</span><span class="p">,</span> <span class="n">hidden_size</span><span class="p">,</span> <span class="n">length</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-    <span class="n">h</span> <span class="o">=</span> <span class="n">linear</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch</span> <span class="o">*</span> <span class="n">length</span><span class="p">,</span> <span class="n">hidden_size</span><span class="p">))</span>
-    <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">h</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">length</span><span class="p">,</span> <span class="n">hidden_size</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
-
-
-<div class="viewcode-block" id="ConvMultiStepAttention"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.ConvMultiStepAttention">[docs]</a><span class="k">class</span> <span class="nc">ConvMultiStepAttention</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Conv attention takes a key matrix, a value matrix and a query vector.</span>
-<span class="sd">    Attention weight is calculated by key matrix with the query vector</span>
-<span class="sd">    and sum on the value matrix. And the same operation is applied</span>
-<span class="sd">    in each decode conv layer.</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_size</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">ConvMultiStepAttention</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">linear_in</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">input_size</span><span class="p">,</span> <span class="n">input_size</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mask</span> <span class="o">=</span> <span class="kc">None</span>
-
-<div class="viewcode-block" id="ConvMultiStepAttention.apply_mask"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.ConvMultiStepAttention.apply_mask">[docs]</a>    <span class="k">def</span> <span class="nf">apply_mask</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mask</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Apply mask&quot;&quot;&quot;</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">mask</span> <span class="o">=</span> <span class="n">mask</span></div>
-
-<div class="viewcode-block" id="ConvMultiStepAttention.forward"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.ConvMultiStepAttention.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">base_target_emb</span><span class="p">,</span> <span class="n">input_from_dec</span><span class="p">,</span> <span class="n">encoder_out_top</span><span class="p">,</span> <span class="n">encoder_out_combine</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Args:</span>
-<span class="sd">            base_target_emb: target emb tensor</span>
-<span class="sd">            input_from_dec: output of decode conv</span>
-<span class="sd">            encoder_out_top: the key matrix for calculation of attetion weight,</span>
-<span class="sd">                which is the top output of encode conv</span>
-<span class="sd">            encoder_out_combine:</span>
-<span class="sd">                the value matrix for the attention-weighted sum,</span>
-<span class="sd">                which is the combination of base emb and top output of encode</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-
-        <span class="c1"># checks</span>
-        <span class="c1"># batch, channel, height, width = base_target_emb.size()</span>
-        <span class="n">batch</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">height</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">base_target_emb</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="c1"># batch_, channel_, height_, width_ = input_from_dec.size()</span>
-        <span class="n">batch_</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">height_</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">input_from_dec</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">batch_</span><span class="p">)</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">height</span><span class="p">,</span> <span class="n">height_</span><span class="p">)</span>
-
-        <span class="c1"># enc_batch, enc_channel, enc_height = encoder_out_top.size()</span>
-        <span class="n">enc_batch</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">enc_height</span> <span class="o">=</span> <span class="n">encoder_out_top</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="c1"># enc_batch_, enc_channel_, enc_height_ = encoder_out_combine.size()</span>
-        <span class="n">enc_batch_</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">enc_height_</span> <span class="o">=</span> <span class="n">encoder_out_combine</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-
-        <span class="n">aeq</span><span class="p">(</span><span class="n">enc_batch</span><span class="p">,</span> <span class="n">enc_batch_</span><span class="p">)</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">enc_height</span><span class="p">,</span> <span class="n">enc_height_</span><span class="p">)</span>
-
-        <span class="n">preatt</span> <span class="o">=</span> <span class="n">seq_linear</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">linear_in</span><span class="p">,</span> <span class="n">input_from_dec</span><span class="p">)</span>
-        <span class="n">target</span> <span class="o">=</span> <span class="p">(</span><span class="n">base_target_emb</span> <span class="o">+</span> <span class="n">preatt</span><span class="p">)</span> <span class="o">*</span> <span class="n">SCALE_WEIGHT</span>
-        <span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="n">target</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
-        <span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">target</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
-        <span class="n">pre_attn</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">bmm</span><span class="p">(</span><span class="n">target</span><span class="p">,</span> <span class="n">encoder_out_top</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mask</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="n">pre_attn</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">masked_fill_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mask</span><span class="p">,</span> <span class="o">-</span><span class="nb">float</span><span class="p">(</span><span class="s1">&#39;inf&#39;</span><span class="p">))</span>
-
-        <span class="n">attn</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">pre_attn</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
-
-        <span class="n">context_output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">bmm</span><span class="p">(</span><span class="n">attn</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">encoder_out_combine</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
-        <span class="n">context_output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="n">context_output</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
-        <span class="k">return</span> <span class="n">context_output</span><span class="p">,</span> <span class="n">attn</span></div></div>
-</pre></div>
-
-           </div>
-           
-          </div>
-          <footer>
-  
-
-  <hr/>
-
-  <div role="contentinfo">
-    <p>
-        &copy; Copyright 2023, HelsinkiNLP
-
-    </p>
-  </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
-
-</footer>
-
-        </div>
-      </div>
-
-    </section>
-
-  </div>
-  
-
-
-  <script type="text/javascript">
-      jQuery(function () {
-          SphinxRtdTheme.Navigation.enable(true);
-      });
-  </script>
-
-  
-  
-    
-   
-
-</body>
-</html>
\ No newline at end of file
diff --git a/_modules/mammoth/modules/copy_generator.html b/_modules/mammoth/modules/copy_generator.html
deleted file mode 100644
index 2965eb36..00000000
--- a/_modules/mammoth/modules/copy_generator.html
+++ /dev/null
@@ -1,483 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
-  <meta charset="utf-8">
-  
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  
-  <title>mammoth.modules.copy_generator &mdash; MAMMOTH  documentation</title>
-  
-
-  
-  
-  
-  
-
-  
-  <script type="text/javascript" src="../../../_static/js/modernizr.min.js"></script>
-  
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
-        <script src="../../../_static/jquery.js"></script>
-        <script src="../../../_static/underscore.js"></script>
-        <script src="../../../_static/doctools.js"></script>
-        <script src="../../../_static/language_data.js"></script>
-        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
-        <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["\\(", "\\)"]], "displayMath": [["\\[", "\\]"]]}})</script>
-        <script src="https://unpkg.com/mermaid@8.4.8/dist/mermaid.min.js"></script>
-    
-    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-    
-
-  
-  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/theme_overrides.css" type="text/css" />
-    <link rel="index" title="Index" href="../../../genindex.html" />
-    <link rel="search" title="Search" href="../../../search.html" /> 
-</head>
-
-<body class="wy-body-for-nav">
-
-   
-  <div class="wy-grid-for-nav">
-    
-    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
-      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search" >
-          
-
-          
-            <a href="../../../index.html" class="icon icon-home"> MAMMOTH
-          
-
-          
-          </a>
-
-          
-            
-            
-          
-
-          
-<div role="search">
-  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
-    <input type="text" name="q" placeholder="Search docs" />
-    <input type="hidden" name="check_keywords" value="yes" />
-    <input type="hidden" name="area" value="default" />
-  </form>
-</div>
-
-          
-        </div>
-
-        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-          
-            
-            
-              
-            
-            
-              <p class="caption"><span class="caption-text">Getting Started</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../main.html">Overview</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../quickstart.html">Quickstart</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../CONTRIBUTING.html">Contributors</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ref.html">References</a></li>
-</ul>
-<p class="caption"><span class="caption-text">MAMMOTH features</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../config_config.html">Config-config tool</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../attention_bridges.html">Attention Bridge</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Tutorials</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../prepare_data.html">Prepare Data</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Scripts</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/build_vocab.html">Build Vocab</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/train.html">Train</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/translate.html">Translate</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/server.html">Server</a></li>
-</ul>
-<p class="caption"><span class="caption-text">API</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.html">Framework</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.modules.html">Modules</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translation.html">Translation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translate.translation_server.html">Server</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.inputters.html">Data Loaders</a></li>
-</ul>
-
-            
-          
-        </div>
-      </div>
-    </nav>
-
-    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-      
-      <nav class="wy-nav-top" aria-label="top navigation">
-        
-          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-          <a href="../../../index.html">MAMMOTH</a>
-        
-      </nav>
-
-
-      <div class="wy-nav-content">
-        
-        <div class="rst-content">
-        
-          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
-  <ul class="wy-breadcrumbs">
-    
-      <li><a href="../../../index.html">Docs</a> &raquo;</li>
-        
-          <li><a href="../../index.html">Module code</a> &raquo;</li>
-        
-      <li>mammoth.modules.copy_generator</li>
-    
-    
-      <li class="wy-breadcrumbs-aside">
-        
-      </li>
-    
-  </ul>
-
-  
-  <hr/>
-</div>
-          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
-           <div itemprop="articleBody">
-            
-  <h1>Source code for mammoth.modules.copy_generator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">torch</span>
-<span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
-
-<span class="kn">from</span> <span class="nn">mammoth.utils.misc</span> <span class="kn">import</span> <span class="n">aeq</span>
-<span class="kn">from</span> <span class="nn">mammoth.utils.loss</span> <span class="kn">import</span> <span class="n">CommonLossCompute</span>
-
-
-<span class="k">def</span> <span class="nf">collapse_copy_scores</span><span class="p">(</span><span class="n">scores</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">tgt_vocab</span><span class="p">,</span> <span class="n">src_vocabs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">batch_dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">batch_offset</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Given scores from an expanded dictionary</span>
-<span class="sd">    corresponeding to a batch, sums together copies,</span>
-<span class="sd">    with a dictionary word when it is ambiguous.</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">offset</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">tgt_vocab</span><span class="p">)</span>
-    <span class="k">for</span> <span class="n">b</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">scores</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">batch_dim</span><span class="p">)):</span>
-        <span class="n">blank</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="n">fill</span> <span class="o">=</span> <span class="p">[]</span>
-
-        <span class="k">if</span> <span class="n">src_vocabs</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="n">src_vocab</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">src_ex_vocab</span><span class="p">[</span><span class="n">b</span><span class="p">]</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">batch_id</span> <span class="o">=</span> <span class="n">batch_offset</span><span class="p">[</span><span class="n">b</span><span class="p">]</span> <span class="k">if</span> <span class="n">batch_offset</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">b</span>
-            <span class="n">index</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">indices</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">batch_id</span><span class="p">]</span>
-            <span class="n">src_vocab</span> <span class="o">=</span> <span class="n">src_vocabs</span><span class="p">[</span><span class="n">index</span><span class="p">]</span>
-
-        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">src_vocab</span><span class="p">)):</span>
-            <span class="n">sw</span> <span class="o">=</span> <span class="n">src_vocab</span><span class="o">.</span><span class="n">itos</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
-            <span class="n">ti</span> <span class="o">=</span> <span class="n">tgt_vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">[</span><span class="n">sw</span><span class="p">]</span>
-            <span class="k">if</span> <span class="n">ti</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
-                <span class="n">blank</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">offset</span> <span class="o">+</span> <span class="n">i</span><span class="p">)</span>
-                <span class="n">fill</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ti</span><span class="p">)</span>
-        <span class="k">if</span> <span class="n">blank</span><span class="p">:</span>
-            <span class="n">blank</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">blank</span><span class="p">)</span><span class="o">.</span><span class="n">type_as</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">indices</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="n">fill</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">fill</span><span class="p">)</span><span class="o">.</span><span class="n">type_as</span><span class="p">(</span><span class="n">batch</span><span class="o">.</span><span class="n">indices</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="n">score</span> <span class="o">=</span> <span class="n">scores</span><span class="p">[:,</span> <span class="n">b</span><span class="p">]</span> <span class="k">if</span> <span class="n">batch_dim</span> <span class="o">==</span> <span class="mi">1</span> <span class="k">else</span> <span class="n">scores</span><span class="p">[</span><span class="n">b</span><span class="p">]</span>
-            <span class="n">score</span><span class="o">.</span><span class="n">index_add_</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">fill</span><span class="p">,</span> <span class="n">score</span><span class="o">.</span><span class="n">index_select</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">blank</span><span class="p">))</span>
-            <span class="n">score</span><span class="o">.</span><span class="n">index_fill_</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">blank</span><span class="p">,</span> <span class="mf">1e-10</span><span class="p">)</span>
-    <span class="k">return</span> <span class="n">scores</span>
-
-
-<div class="viewcode-block" id="CopyGenerator"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.CopyGenerator">[docs]</a><span class="k">class</span> <span class="nc">CopyGenerator</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;An implementation of pointer-generator networks</span>
-<span class="sd">    :cite:`DBLP:journals/corr/SeeLM17`.</span>
-
-<span class="sd">    These networks consider copying words</span>
-<span class="sd">    directly from the source sequence.</span>
-
-<span class="sd">    The copy generator is an extended version of the standard</span>
-<span class="sd">    generator that computes three values.</span>
-
-<span class="sd">    * :math:`p_{softmax}` the standard softmax over `tgt_dict`</span>
-<span class="sd">    * :math:`p(z)` the probability of copying a word from</span>
-<span class="sd">      the source</span>
-<span class="sd">    * :math:`p_{copy}` the probility of copying a particular word.</span>
-<span class="sd">      taken from the attention distribution directly.</span>
-
-<span class="sd">    The model returns a distribution over the extend dictionary,</span>
-<span class="sd">    computed as</span>
-
-<span class="sd">    :math:`p(w) = p(z=1)  p_{copy}(w)  +  p(z=0)  p_{softmax}(w)`</span>
-
-
-<span class="sd">    .. mermaid::</span>
-
-<span class="sd">       graph BT</span>
-<span class="sd">          A[input]</span>
-<span class="sd">          S[src_map]</span>
-<span class="sd">          B[softmax]</span>
-<span class="sd">          BB[switch]</span>
-<span class="sd">          C[attn]</span>
-<span class="sd">          D[copy]</span>
-<span class="sd">          O[output]</span>
-<span class="sd">          A --&gt; B</span>
-<span class="sd">          A --&gt; BB</span>
-<span class="sd">          S --&gt; D</span>
-<span class="sd">          C --&gt; D</span>
-<span class="sd">          D --&gt; O</span>
-<span class="sd">          B --&gt; O</span>
-<span class="sd">          BB --&gt; O</span>
-
-
-<span class="sd">    Args:</span>
-<span class="sd">       input_size (int): size of input representation</span>
-<span class="sd">       output_size (int): size of output vocabulary</span>
-<span class="sd">       pad_idx (int)</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_size</span><span class="p">,</span> <span class="n">output_size</span><span class="p">,</span> <span class="n">pad_idx</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">CopyGenerator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">input_size</span><span class="p">,</span> <span class="n">output_size</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">linear_copy</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">input_size</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">=</span> <span class="n">pad_idx</span>
-
-<div class="viewcode-block" id="CopyGenerator.forward"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.CopyGenerator.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hidden</span><span class="p">,</span> <span class="n">attn</span><span class="p">,</span> <span class="n">src_map</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Compute a distribution over the target dictionary</span>
-<span class="sd">        extended by the dynamic dictionary implied by copying</span>
-<span class="sd">        source words.</span>
-
-<span class="sd">        Args:</span>
-<span class="sd">           hidden (FloatTensor): hidden outputs ``(batch x tlen, input_size)``</span>
-<span class="sd">           attn (FloatTensor): attn for each ``(batch x tlen, slen)``</span>
-<span class="sd">           src_map (FloatTensor):</span>
-<span class="sd">               A sparse indicator matrix mapping each source word to</span>
-<span class="sd">               its index in the &quot;extended&quot; vocab containing.</span>
-<span class="sd">               ``(src_len, batch, extra_words)``</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-
-        <span class="c1"># CHECKS</span>
-        <span class="n">batch_by_tlen</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">hidden</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">batch_by_tlen_</span><span class="p">,</span> <span class="n">slen</span> <span class="o">=</span> <span class="n">attn</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">slen_</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">cvocab</span> <span class="o">=</span> <span class="n">src_map</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">batch_by_tlen</span><span class="p">,</span> <span class="n">batch_by_tlen_</span><span class="p">)</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">slen</span><span class="p">,</span> <span class="n">slen_</span><span class="p">)</span>
-
-        <span class="c1"># Original probabilities.</span>
-        <span class="n">logits</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear</span><span class="p">(</span><span class="n">hidden</span><span class="p">)</span>
-        <span class="n">logits</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="nb">float</span><span class="p">(</span><span class="s1">&#39;inf&#39;</span><span class="p">)</span>
-        <span class="n">prob</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">logits</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-
-        <span class="c1"># Probability of copying p(z=1) batch.</span>
-        <span class="n">p_copy</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">linear_copy</span><span class="p">(</span><span class="n">hidden</span><span class="p">))</span>
-        <span class="c1"># Probability of not copying: p_{word}(w) * (1 - p(z))</span>
-        <span class="n">out_prob</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">mul</span><span class="p">(</span><span class="n">prob</span><span class="p">,</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">p_copy</span><span class="p">)</span>
-        <span class="n">mul_attn</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">mul</span><span class="p">(</span><span class="n">attn</span><span class="p">,</span> <span class="n">p_copy</span><span class="p">)</span>
-        <span class="n">copy_prob</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">bmm</span><span class="p">(</span><span class="n">mul_attn</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">slen</span><span class="p">)</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">src_map</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-        <span class="n">copy_prob</span> <span class="o">=</span> <span class="n">copy_prob</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">cvocab</span><span class="p">)</span>
-        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">out_prob</span><span class="p">,</span> <span class="n">copy_prob</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span></div></div>
-
-
-<span class="k">class</span> <span class="nc">CopyGeneratorLoss</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Copy generator criterion.&quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vocab_size</span><span class="p">,</span> <span class="n">force_copy</span><span class="p">,</span> <span class="n">unk_index</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">ignore_index</span><span class="o">=-</span><span class="mi">100</span><span class="p">,</span> <span class="n">eps</span><span class="o">=</span><span class="mf">1e-20</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">CopyGeneratorLoss</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">force_copy</span> <span class="o">=</span> <span class="n">force_copy</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span> <span class="o">=</span> <span class="n">vocab_size</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">ignore_index</span> <span class="o">=</span> <span class="n">ignore_index</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">unk_index</span> <span class="o">=</span> <span class="n">unk_index</span>
-
-    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">scores</span><span class="p">,</span> <span class="n">align</span><span class="p">,</span> <span class="n">target</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Args:</span>
-<span class="sd">            scores (FloatTensor): ``(batch_size*tgt_len)`` x dynamic vocab size</span>
-<span class="sd">                whose sum along dim 1 is less than or equal to 1, i.e. cols</span>
-<span class="sd">                softmaxed.</span>
-<span class="sd">            align (LongTensor): ``(batch_size x tgt_len)``</span>
-<span class="sd">            target (LongTensor): ``(batch_size x tgt_len)``</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="c1"># probabilities assigned by the model to the gold targets</span>
-        <span class="n">vocab_probs</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">gather</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">target</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-
-        <span class="c1"># probability of tokens copied from source</span>
-        <span class="n">copy_ix</span> <span class="o">=</span> <span class="n">align</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">vocab_size</span>
-        <span class="n">copy_tok_probs</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">gather</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">copy_ix</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-        <span class="c1"># Set scores for unk to 0 and add eps</span>
-        <span class="n">copy_tok_probs</span><span class="p">[</span><span class="n">align</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">unk_index</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
-        <span class="n">copy_tok_probs</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span>  <span class="c1"># to avoid -inf logs</span>
-
-        <span class="c1"># find the indices in which you do not use the copy mechanism</span>
-        <span class="n">non_copy</span> <span class="o">=</span> <span class="n">align</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">unk_index</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">force_copy</span><span class="p">:</span>
-            <span class="n">non_copy</span> <span class="o">=</span> <span class="n">non_copy</span> <span class="o">|</span> <span class="p">(</span><span class="n">target</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">unk_index</span><span class="p">)</span>
-
-        <span class="n">probs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">non_copy</span><span class="p">,</span> <span class="n">copy_tok_probs</span> <span class="o">+</span> <span class="n">vocab_probs</span><span class="p">,</span> <span class="n">copy_tok_probs</span><span class="p">)</span>
-
-        <span class="n">loss</span> <span class="o">=</span> <span class="o">-</span><span class="n">probs</span><span class="o">.</span><span class="n">log</span><span class="p">()</span>  <span class="c1"># just NLLLoss; can the module be incorporated?</span>
-        <span class="c1"># Drop padding.</span>
-        <span class="n">loss</span><span class="p">[</span><span class="n">target</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">ignore_index</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
-        <span class="k">return</span> <span class="n">loss</span>
-
-
-<span class="k">class</span> <span class="nc">CommonCopyGeneratorLossCompute</span><span class="p">(</span><span class="n">CommonLossCompute</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Common Copy Generator Loss Computation.&quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">criterion</span><span class="p">,</span> <span class="n">generator</span><span class="p">,</span> <span class="n">tgt_vocab</span><span class="p">,</span> <span class="n">normalize_by_length</span><span class="p">,</span> <span class="n">lambda_coverage</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tgt_shift_index</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">CommonCopyGeneratorLossCompute</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-            <span class="n">criterion</span><span class="p">,</span> <span class="n">generator</span><span class="p">,</span> <span class="n">lambda_coverage</span><span class="o">=</span><span class="n">lambda_coverage</span><span class="p">,</span> <span class="n">tgt_shift_index</span><span class="o">=</span><span class="n">tgt_shift_index</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">tgt_vocab</span> <span class="o">=</span> <span class="n">tgt_vocab</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">normalize_by_length</span> <span class="o">=</span> <span class="n">normalize_by_length</span>
-
-    <span class="k">def</span> <span class="nf">_compute_loss</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">output</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">copy_attn</span><span class="p">,</span> <span class="n">align</span><span class="p">,</span> <span class="n">std_attn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">coverage_attn</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;Compute the loss.</span>
-
-<span class="sd">        The args must match :func:`self._make_shard_state()`.</span>
-
-<span class="sd">        Args:</span>
-<span class="sd">            batch: the current batch.</span>
-<span class="sd">            output: the predict output from the model.</span>
-<span class="sd">            target: the validate target to compare output with.</span>
-<span class="sd">            copy_attn: the copy attention value.</span>
-<span class="sd">            align: the align info.</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">target</span> <span class="o">=</span> <span class="n">target</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-        <span class="n">align</span> <span class="o">=</span> <span class="n">align</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-        <span class="n">scores</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">generator</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_bottle</span><span class="p">(</span><span class="n">output</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_bottle</span><span class="p">(</span><span class="n">copy_attn</span><span class="p">),</span> <span class="n">batch</span><span class="o">.</span><span class="n">src_map</span><span class="p">)</span>
-        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">criterion</span><span class="p">(</span><span class="n">scores</span><span class="p">,</span> <span class="n">align</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">lambda_coverage</span> <span class="o">!=</span> <span class="mf">0.0</span><span class="p">:</span>
-            <span class="n">coverage_loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_compute_coverage_loss</span><span class="p">(</span><span class="n">std_attn</span><span class="p">,</span> <span class="n">coverage_attn</span><span class="p">)</span>
-            <span class="n">loss</span> <span class="o">+=</span> <span class="n">coverage_loss</span>
-
-        <span class="c1"># this block does not depend on the loss value computed above</span>
-        <span class="c1"># and is used only for stats</span>
-        <span class="n">scores_data</span> <span class="o">=</span> <span class="n">collapse_copy_scores</span><span class="p">(</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">_unbottle</span><span class="p">(</span><span class="n">scores</span><span class="o">.</span><span class="n">clone</span><span class="p">(),</span> <span class="n">batch</span><span class="o">.</span><span class="n">batch_size</span><span class="p">),</span> <span class="n">batch</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">tgt_vocab</span><span class="p">,</span> <span class="kc">None</span>
-        <span class="p">)</span>
-        <span class="n">scores_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_bottle</span><span class="p">(</span><span class="n">scores_data</span><span class="p">)</span>
-
-        <span class="c1"># this block does not depend on the loss value computed above</span>
-        <span class="c1"># and is used only for stats</span>
-        <span class="c1"># Correct target copy token instead of &lt;unk&gt;</span>
-        <span class="c1"># tgt[i] = align[i] + len(tgt_vocab)</span>
-        <span class="c1"># for i such that tgt[i] == 0 and align[i] != 0</span>
-        <span class="n">target_data</span> <span class="o">=</span> <span class="n">target</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
-        <span class="n">unk</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">criterion</span><span class="o">.</span><span class="n">unk_index</span>
-        <span class="n">correct_mask</span> <span class="o">=</span> <span class="p">(</span><span class="n">target_data</span> <span class="o">==</span> <span class="n">unk</span><span class="p">)</span> <span class="o">&amp;</span> <span class="p">(</span><span class="n">align</span> <span class="o">!=</span> <span class="n">unk</span><span class="p">)</span>
-        <span class="n">offset_align</span> <span class="o">=</span> <span class="n">align</span><span class="p">[</span><span class="n">correct_mask</span><span class="p">]</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tgt_vocab</span><span class="p">)</span>
-        <span class="n">target_data</span><span class="p">[</span><span class="n">correct_mask</span><span class="p">]</span> <span class="o">+=</span> <span class="n">offset_align</span>
-
-        <span class="c1"># Compute sum of perplexities for stats</span>
-        <span class="n">stats</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_stats</span><span class="p">(</span><span class="n">loss</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">clone</span><span class="p">(),</span> <span class="n">scores_data</span><span class="p">,</span> <span class="n">target_data</span><span class="p">)</span>
-
-        <span class="c1"># this part looks like it belongs in CopyGeneratorLoss</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalize_by_length</span><span class="p">:</span>
-            <span class="c1"># Compute Loss as NLL divided by seq length</span>
-            <span class="n">tgt_lens</span> <span class="o">=</span> <span class="n">batch</span><span class="o">.</span><span class="n">tgt</span><span class="p">[:,</span> <span class="p">:,</span> <span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">ne</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">padding_idx</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span>
-            <span class="c1"># Compute Total Loss per sequence in batch</span>
-            <span class="n">loss</span> <span class="o">=</span> <span class="n">loss</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">batch</span><span class="o">.</span><span class="n">batch_size</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
-            <span class="c1"># Divide by length of each sequence and sum</span>
-            <span class="n">loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">div</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="n">tgt_lens</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">loss</span> <span class="o">=</span> <span class="n">loss</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
-
-        <span class="k">return</span> <span class="n">loss</span><span class="p">,</span> <span class="n">stats</span>
-
-    <span class="k">def</span> <span class="nf">_make_shard_state</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">output</span><span class="p">,</span> <span class="n">range_</span><span class="p">,</span> <span class="n">attns</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;See base class for args description.&quot;&quot;&quot;</span>
-        <span class="n">shard_state</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">CommonCopyGeneratorLossCompute</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_make_shard_state</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">output</span><span class="p">,</span> <span class="n">range_</span><span class="p">,</span> <span class="n">attns</span><span class="p">)</span>
-
-        <span class="n">start_range</span> <span class="o">=</span> <span class="n">range_</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">tgt_shift_index</span>
-        <span class="n">end_range</span> <span class="o">=</span> <span class="n">range_</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
-        <span class="n">shard_state</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s2">&quot;copy_attn&quot;</span><span class="p">:</span> <span class="n">attns</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;copy&quot;</span><span class="p">),</span> <span class="s2">&quot;align&quot;</span><span class="p">:</span> <span class="n">batch</span><span class="o">.</span><span class="n">alignment</span><span class="p">[</span><span class="n">start_range</span><span class="p">:</span><span class="n">end_range</span><span class="p">]})</span>
-        <span class="k">return</span> <span class="n">shard_state</span>
-
-
-<span class="k">class</span> <span class="nc">CopyGeneratorLossCompute</span><span class="p">(</span><span class="n">CommonCopyGeneratorLossCompute</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Copy Generator Loss Computation.&quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">criterion</span><span class="p">,</span> <span class="n">generator</span><span class="p">,</span> <span class="n">tgt_vocab</span><span class="p">,</span> <span class="n">normalize_by_length</span><span class="p">,</span> <span class="n">lambda_coverage</span><span class="o">=</span><span class="mf">0.0</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">CopyGeneratorLossCompute</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-            <span class="n">criterion</span><span class="p">,</span> <span class="n">generator</span><span class="p">,</span> <span class="n">tgt_vocab</span><span class="p">,</span> <span class="n">normalize_by_length</span><span class="p">,</span> <span class="n">lambda_coverage</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tgt_shift_index</span><span class="o">=</span><span class="mi">1</span>
-        <span class="p">)</span>
-
-
-<span class="k">class</span> <span class="nc">CopyGeneratorLMLossCompute</span><span class="p">(</span><span class="n">CommonCopyGeneratorLossCompute</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Copy Generator LM Loss Computation.&quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">criterion</span><span class="p">,</span> <span class="n">generator</span><span class="p">,</span> <span class="n">tgt_vocab</span><span class="p">,</span> <span class="n">normalize_by_length</span><span class="p">,</span> <span class="n">lambda_coverage</span><span class="o">=</span><span class="mf">0.0</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">CopyGeneratorLMLossCompute</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-            <span class="n">criterion</span><span class="p">,</span> <span class="n">generator</span><span class="p">,</span> <span class="n">tgt_vocab</span><span class="p">,</span> <span class="n">normalize_by_length</span><span class="p">,</span> <span class="n">lambda_coverage</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">tgt_shift_index</span><span class="o">=</span><span class="mi">0</span>
-        <span class="p">)</span>
-</pre></div>
-
-           </div>
-           
-          </div>
-          <footer>
-  
-
-  <hr/>
-
-  <div role="contentinfo">
-    <p>
-        &copy; Copyright 2023, HelsinkiNLP
-
-    </p>
-  </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
-
-</footer>
-
-        </div>
-      </div>
-
-    </section>
-
-  </div>
-  
-
-
-  <script type="text/javascript">
-      jQuery(function () {
-          SphinxRtdTheme.Navigation.enable(true);
-      });
-  </script>
-
-  
-  
-    
-   
-
-</body>
-</html>
\ No newline at end of file
diff --git a/_modules/mammoth/modules/embeddings.html b/_modules/mammoth/modules/embeddings.html
index ba96f1f7..5d19914f 100644
--- a/_modules/mammoth/modules/embeddings.html
+++ b/_modules/mammoth/modules/embeddings.html
@@ -186,7 +186,6 @@ <h1>Source code for mammoth.modules.embeddings</h1><div class="highlight"><pre>
 <span class="c1"># from mammoth.utils.logging import logger</span>
 
 <span class="c1"># import bitsandbytes as bnb</span>
-<span class="c1"># from mammoth.modules.stable_embeddings import StableEmbedding</span>
 
 
 <span class="k">class</span> <span class="nc">SequenceTooLongError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
@@ -243,7 +242,7 @@ <h1>Source code for mammoth.modules.embeddings</h1><div class="highlight"><pre>
 <div class="viewcode-block" id="Embeddings"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.Embeddings">[docs]</a><span class="k">class</span> <span class="nc">Embeddings</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Words embeddings for encoder/decoder.</span>
 
-<span class="sd">    Additionally includes ability to add sparse input features</span>
+<span class="sd">    Additionally includes ability to add input features</span>
 <span class="sd">    based on &quot;Linguistic Input Features Improve Neural Machine Translation&quot;</span>
 <span class="sd">    :cite:`sennrich2016linguistic`.</span>
 
@@ -293,7 +292,6 @@ <h1>Source code for mammoth.modules.embeddings</h1><div class="highlight"><pre>
         <span class="n">feat_padding_idx</span><span class="o">=</span><span class="p">[],</span>
         <span class="n">feat_vocab_sizes</span><span class="o">=</span><span class="p">[],</span>
         <span class="n">dropout</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-        <span class="n">sparse</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
         <span class="n">freeze_word_vecs</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
     <span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">_validate_args</span><span class="p">(</span><span class="n">feat_merge</span><span class="p">,</span> <span class="n">feat_vocab_sizes</span><span class="p">,</span> <span class="n">feat_vec_exponent</span><span class="p">,</span> <span class="n">feat_vec_size</span><span class="p">,</span> <span class="n">feat_padding_idx</span><span class="p">)</span>
@@ -324,7 +322,7 @@ <h1>Source code for mammoth.modules.embeddings</h1><div class="highlight"><pre>
         <span class="c1"># The embedding matrix look-up tables. The first look-up table</span>
         <span class="c1"># is for words. Subsequent ones are for features, if any exist.</span>
         <span class="n">emb_params</span> <span class="o">=</span> <span class="nb">zip</span><span class="p">(</span><span class="n">vocab_sizes</span><span class="p">,</span> <span class="n">emb_dims</span><span class="p">,</span> <span class="n">pad_indices</span><span class="p">)</span>
-        <span class="n">embeddings</span> <span class="o">=</span> <span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">vocab</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="n">pad</span><span class="p">,</span> <span class="n">sparse</span><span class="o">=</span><span class="n">sparse</span><span class="p">)</span> <span class="k">for</span> <span class="n">vocab</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">pad</span> <span class="ow">in</span> <span class="n">emb_params</span><span class="p">]</span>
+        <span class="n">embeddings</span> <span class="o">=</span> <span class="p">[</span><span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="n">vocab</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="n">pad</span><span class="p">)</span> <span class="k">for</span> <span class="n">vocab</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">pad</span> <span class="ow">in</span> <span class="n">emb_params</span><span class="p">]</span>
         <span class="n">emb_luts</span> <span class="o">=</span> <span class="n">Elementwise</span><span class="p">(</span><span class="n">feat_merge</span><span class="p">,</span> <span class="n">embeddings</span><span class="p">)</span>
 
         <span class="c1"># The final output size of word + feature vectors. This can vary</span>
diff --git a/_modules/mammoth/modules/global_attention.html b/_modules/mammoth/modules/global_attention.html
deleted file mode 100644
index 09864e21..00000000
--- a/_modules/mammoth/modules/global_attention.html
+++ /dev/null
@@ -1,444 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
-  <meta charset="utf-8">
-  
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  
-  <title>mammoth.modules.global_attention &mdash; MAMMOTH  documentation</title>
-  
-
-  
-  
-  
-  
-
-  
-  <script type="text/javascript" src="../../../_static/js/modernizr.min.js"></script>
-  
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
-        <script src="../../../_static/jquery.js"></script>
-        <script src="../../../_static/underscore.js"></script>
-        <script src="../../../_static/doctools.js"></script>
-        <script src="../../../_static/language_data.js"></script>
-        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
-        <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["\\(", "\\)"]], "displayMath": [["\\[", "\\]"]]}})</script>
-        <script src="https://unpkg.com/mermaid@8.4.8/dist/mermaid.min.js"></script>
-    
-    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-    
-
-  
-  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/theme_overrides.css" type="text/css" />
-    <link rel="index" title="Index" href="../../../genindex.html" />
-    <link rel="search" title="Search" href="../../../search.html" /> 
-</head>
-
-<body class="wy-body-for-nav">
-
-   
-  <div class="wy-grid-for-nav">
-    
-    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
-      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search" >
-          
-
-          
-            <a href="../../../index.html" class="icon icon-home"> MAMMOTH
-          
-
-          
-          </a>
-
-          
-            
-            
-          
-
-          
-<div role="search">
-  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
-    <input type="text" name="q" placeholder="Search docs" />
-    <input type="hidden" name="check_keywords" value="yes" />
-    <input type="hidden" name="area" value="default" />
-  </form>
-</div>
-
-          
-        </div>
-
-        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-          
-            
-            
-              
-            
-            
-              <p class="caption"><span class="caption-text">Getting Started</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../main.html">Overview</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../quickstart.html">Quickstart</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../CONTRIBUTING.html">Contributors</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ref.html">References</a></li>
-</ul>
-<p class="caption"><span class="caption-text">MAMMOTH features</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../config_config.html">Config-config tool</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../attention_bridges.html">Attention Bridge</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Tutorials</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../prepare_data.html">Prepare Data</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Scripts</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/build_vocab.html">Build Vocab</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/train.html">Train</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/translate.html">Translate</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/server.html">Server</a></li>
-</ul>
-<p class="caption"><span class="caption-text">API</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.html">Framework</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.modules.html">Modules</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translation.html">Translation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translate.translation_server.html">Server</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.inputters.html">Data Loaders</a></li>
-</ul>
-
-            
-          
-        </div>
-      </div>
-    </nav>
-
-    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-      
-      <nav class="wy-nav-top" aria-label="top navigation">
-        
-          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-          <a href="../../../index.html">MAMMOTH</a>
-        
-      </nav>
-
-
-      <div class="wy-nav-content">
-        
-        <div class="rst-content">
-        
-          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
-  <ul class="wy-breadcrumbs">
-    
-      <li><a href="../../../index.html">Docs</a> &raquo;</li>
-        
-          <li><a href="../../index.html">Module code</a> &raquo;</li>
-        
-      <li>mammoth.modules.global_attention</li>
-    
-    
-      <li class="wy-breadcrumbs-aside">
-        
-      </li>
-    
-  </ul>
-
-  
-  <hr/>
-</div>
-          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
-           <div itemprop="articleBody">
-            
-  <h1>Source code for mammoth.modules.global_attention</h1><div class="highlight"><pre>
-<span></span><span class="sd">&quot;&quot;&quot;Global attention modules (Luong / Bahdanau)&quot;&quot;&quot;</span>
-<span class="kn">import</span> <span class="nn">torch</span>
-<span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
-<span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>
-
-<span class="kn">from</span> <span class="nn">mammoth.modules.sparse_activations</span> <span class="kn">import</span> <span class="n">sparsemax</span>
-<span class="kn">from</span> <span class="nn">mammoth.utils.misc</span> <span class="kn">import</span> <span class="n">aeq</span><span class="p">,</span> <span class="n">sequence_mask</span>
-
-<span class="c1"># This class is mainly used by decoder.py for RNNs but also</span>
-<span class="c1"># by the CNN / transformer decoder when copy attention is used</span>
-<span class="c1"># CNN has its own attention mechanism ConvMultiStepAttention</span>
-<span class="c1"># Transformer has its own MultiHeadedAttention</span>
-
-
-<div class="viewcode-block" id="GlobalAttention"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.GlobalAttention">[docs]</a><span class="k">class</span> <span class="nc">GlobalAttention</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
-<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Global attention takes a matrix and a query vector. It</span>
-<span class="sd">    then computes a parameterized convex combination of the matrix</span>
-<span class="sd">    based on the input query.</span>
-
-<span class="sd">    Constructs a unit mapping a query `q` of size `dim`</span>
-<span class="sd">    and a source matrix `H` of size `n x dim`, to an output</span>
-<span class="sd">    of size `dim`.</span>
-
-
-<span class="sd">    .. mermaid::</span>
-
-<span class="sd">       graph BT</span>
-<span class="sd">          A[Query]</span>
-<span class="sd">          subgraph RNN</span>
-<span class="sd">            C[H 1]</span>
-<span class="sd">            D[H 2]</span>
-<span class="sd">            E[H N]</span>
-<span class="sd">          end</span>
-<span class="sd">          F[Attn]</span>
-<span class="sd">          G[Output]</span>
-<span class="sd">          A --&gt; F</span>
-<span class="sd">          C --&gt; F</span>
-<span class="sd">          D --&gt; F</span>
-<span class="sd">          E --&gt; F</span>
-<span class="sd">          C -.-&gt; G</span>
-<span class="sd">          D -.-&gt; G</span>
-<span class="sd">          E -.-&gt; G</span>
-<span class="sd">          F --&gt; G</span>
-
-<span class="sd">    All models compute the output as</span>
-<span class="sd">    :math:`c = \sum_{j=1}^{\text{SeqLength}} a_j H_j` where</span>
-<span class="sd">    :math:`a_j` is the softmax of a score function.</span>
-<span class="sd">    Then then apply a projection layer to [q, c].</span>
-
-<span class="sd">    However they</span>
-<span class="sd">    differ on how they compute the attention score.</span>
-
-<span class="sd">    * Luong Attention (dot, general):</span>
-<span class="sd">       * dot: :math:`\text{score}(H_j,q) = H_j^T q`</span>
-<span class="sd">       * general: :math:`\text{score}(H_j, q) = H_j^T W_a q`</span>
-
-
-<span class="sd">    * Bahdanau Attention (mlp):</span>
-<span class="sd">       * :math:`\text{score}(H_j, q) = v_a^T \text{tanh}(W_a q + U_a h_j)`</span>
-
-
-<span class="sd">    Args:</span>
-<span class="sd">       dim (int): dimensionality of query and key</span>
-<span class="sd">       coverage (bool): use coverage term</span>
-<span class="sd">       attn_type (str): type of attention to use, options [dot,general,mlp]</span>
-<span class="sd">       attn_func (str): attention function to use, options [softmax,sparsemax]</span>
-
-<span class="sd">    &quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">coverage</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">attn_type</span><span class="o">=</span><span class="s2">&quot;dot&quot;</span><span class="p">,</span> <span class="n">attn_func</span><span class="o">=</span><span class="s2">&quot;softmax&quot;</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">GlobalAttention</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">dim</span> <span class="o">=</span> <span class="n">dim</span>
-        <span class="k">assert</span> <span class="n">attn_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;dot&quot;</span><span class="p">,</span> <span class="s2">&quot;general&quot;</span><span class="p">,</span> <span class="s2">&quot;mlp&quot;</span><span class="p">],</span> <span class="s2">&quot;Please select a valid attention type (got </span><span class="si">{:s}</span><span class="s2">).&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
-            <span class="n">attn_type</span>
-        <span class="p">)</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">attn_type</span> <span class="o">=</span> <span class="n">attn_type</span>
-        <span class="k">assert</span> <span class="n">attn_func</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;softmax&quot;</span><span class="p">,</span> <span class="s2">&quot;sparsemax&quot;</span><span class="p">],</span> <span class="s2">&quot;Please select a valid attention function.&quot;</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">attn_func</span> <span class="o">=</span> <span class="n">attn_func</span>
-
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_type</span> <span class="o">==</span> <span class="s2">&quot;general&quot;</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">linear_in</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
-        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_type</span> <span class="o">==</span> <span class="s2">&quot;mlp&quot;</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">linear_context</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">linear_query</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">v</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
-        <span class="c1"># mlp wants it with bias</span>
-        <span class="n">out_bias</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_type</span> <span class="o">==</span> <span class="s2">&quot;mlp&quot;</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">linear_out</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">dim</span> <span class="o">*</span> <span class="mi">2</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="n">out_bias</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="n">coverage</span><span class="p">:</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">linear_cover</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">dim</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
-
-<div class="viewcode-block" id="GlobalAttention.score"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.GlobalAttention.score">[docs]</a>    <span class="k">def</span> <span class="nf">score</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">h_t</span><span class="p">,</span> <span class="n">h_s</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">        Args:</span>
-<span class="sd">          h_t (FloatTensor): sequence of queries ``(batch, tgt_len, dim)``</span>
-<span class="sd">          h_s (FloatTensor): sequence of sources ``(batch, src_len, dim``</span>
-
-<span class="sd">        Returns:</span>
-<span class="sd">          FloatTensor: raw attention scores (unnormalized) for each src index</span>
-<span class="sd">            ``(batch, tgt_len, src_len)``</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-
-        <span class="c1"># Check input sizes</span>
-        <span class="n">src_batch</span><span class="p">,</span> <span class="n">src_len</span><span class="p">,</span> <span class="n">src_dim</span> <span class="o">=</span> <span class="n">h_s</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">tgt_batch</span><span class="p">,</span> <span class="n">tgt_len</span><span class="p">,</span> <span class="n">tgt_dim</span> <span class="o">=</span> <span class="n">h_t</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">src_batch</span><span class="p">,</span> <span class="n">tgt_batch</span><span class="p">)</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">src_dim</span><span class="p">,</span> <span class="n">tgt_dim</span><span class="p">)</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dim</span><span class="p">,</span> <span class="n">src_dim</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;general&quot;</span><span class="p">,</span> <span class="s2">&quot;dot&quot;</span><span class="p">]:</span>
-            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_type</span> <span class="o">==</span> <span class="s2">&quot;general&quot;</span><span class="p">:</span>
-                <span class="n">h_t_</span> <span class="o">=</span> <span class="n">h_t</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">tgt_batch</span> <span class="o">*</span> <span class="n">tgt_len</span><span class="p">,</span> <span class="n">tgt_dim</span><span class="p">)</span>
-                <span class="n">h_t_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear_in</span><span class="p">(</span><span class="n">h_t_</span><span class="p">)</span>
-                <span class="n">h_t</span> <span class="o">=</span> <span class="n">h_t_</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">tgt_batch</span><span class="p">,</span> <span class="n">tgt_len</span><span class="p">,</span> <span class="n">tgt_dim</span><span class="p">)</span>
-            <span class="n">h_s_</span> <span class="o">=</span> <span class="n">h_s</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
-            <span class="c1"># (batch, t_len, d) x (batch, d, s_len) --&gt; (batch, t_len, s_len)</span>
-            <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">bmm</span><span class="p">(</span><span class="n">h_t</span><span class="p">,</span> <span class="n">h_s_</span><span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dim</span>
-            <span class="n">wq</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear_query</span><span class="p">(</span><span class="n">h_t</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">dim</span><span class="p">))</span>
-            <span class="n">wq</span> <span class="o">=</span> <span class="n">wq</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">tgt_batch</span><span class="p">,</span> <span class="n">tgt_len</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
-            <span class="n">wq</span> <span class="o">=</span> <span class="n">wq</span><span class="o">.</span><span class="n">expand</span><span class="p">(</span><span class="n">tgt_batch</span><span class="p">,</span> <span class="n">tgt_len</span><span class="p">,</span> <span class="n">src_len</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
-
-            <span class="n">uh</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear_context</span><span class="p">(</span><span class="n">h_s</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">dim</span><span class="p">))</span>
-            <span class="n">uh</span> <span class="o">=</span> <span class="n">uh</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">src_batch</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">src_len</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
-            <span class="n">uh</span> <span class="o">=</span> <span class="n">uh</span><span class="o">.</span><span class="n">expand</span><span class="p">(</span><span class="n">src_batch</span><span class="p">,</span> <span class="n">tgt_len</span><span class="p">,</span> <span class="n">src_len</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
-
-            <span class="c1"># (batch, t_len, s_len, d)</span>
-            <span class="n">wquh</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="n">wq</span> <span class="o">+</span> <span class="n">uh</span><span class="p">)</span>
-
-            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">v</span><span class="p">(</span><span class="n">wquh</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">dim</span><span class="p">))</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">tgt_batch</span><span class="p">,</span> <span class="n">tgt_len</span><span class="p">,</span> <span class="n">src_len</span><span class="p">)</span></div>
-
-<div class="viewcode-block" id="GlobalAttention.forward"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.GlobalAttention.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source</span><span class="p">,</span> <span class="n">memory_bank</span><span class="p">,</span> <span class="n">memory_lengths</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">coverage</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
-<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
-
-<span class="sd">        Args:</span>
-<span class="sd">          source (FloatTensor): query vectors ``(batch, tgt_len, dim)``</span>
-<span class="sd">          memory_bank (FloatTensor): source vectors ``(batch, src_len, dim)``</span>
-<span class="sd">          memory_lengths (LongTensor): the source context lengths ``(batch,)``</span>
-<span class="sd">          coverage (FloatTensor): None (not supported yet)</span>
-
-<span class="sd">        Returns:</span>
-<span class="sd">          (FloatTensor, FloatTensor):</span>
-
-<span class="sd">          * Computed vector ``(tgt_len, batch, dim)``</span>
-<span class="sd">          * Attention distribtutions for each query</span>
-<span class="sd">            ``(tgt_len, batch, src_len)``</span>
-<span class="sd">        &quot;&quot;&quot;</span>
-
-        <span class="c1"># one step input</span>
-        <span class="k">if</span> <span class="n">source</span><span class="o">.</span><span class="n">dim</span><span class="p">()</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
-            <span class="n">one_step</span> <span class="o">=</span> <span class="kc">True</span>
-            <span class="n">source</span> <span class="o">=</span> <span class="n">source</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">one_step</span> <span class="o">=</span> <span class="kc">False</span>
-
-        <span class="n">batch</span><span class="p">,</span> <span class="n">source_l</span><span class="p">,</span> <span class="n">dim</span> <span class="o">=</span> <span class="n">memory_bank</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">batch_</span><span class="p">,</span> <span class="n">target_l</span><span class="p">,</span> <span class="n">dim_</span> <span class="o">=</span> <span class="n">source</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">batch_</span><span class="p">)</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="n">dim_</span><span class="p">)</span>
-        <span class="n">aeq</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dim</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
-        <span class="k">if</span> <span class="n">coverage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="n">batch_</span><span class="p">,</span> <span class="n">source_l_</span> <span class="o">=</span> <span class="n">coverage</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">batch_</span><span class="p">)</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">source_l</span><span class="p">,</span> <span class="n">source_l_</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="n">coverage</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="n">cover</span> <span class="o">=</span> <span class="n">coverage</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-            <span class="n">memory_bank</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear_cover</span><span class="p">(</span><span class="n">cover</span><span class="p">)</span><span class="o">.</span><span class="n">view_as</span><span class="p">(</span><span class="n">memory_bank</span><span class="p">)</span>
-            <span class="n">memory_bank</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="n">memory_bank</span><span class="p">)</span>
-
-        <span class="c1"># compute attention scores, as in Luong et al.</span>
-        <span class="n">align</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">source</span><span class="p">,</span> <span class="n">memory_bank</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="n">memory_lengths</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
-            <span class="n">mask</span> <span class="o">=</span> <span class="n">sequence_mask</span><span class="p">(</span><span class="n">memory_lengths</span><span class="p">,</span> <span class="n">max_len</span><span class="o">=</span><span class="n">align</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span>
-            <span class="n">mask</span> <span class="o">=</span> <span class="n">mask</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>  <span class="c1"># Make it broadcastable.</span>
-            <span class="n">align</span><span class="o">.</span><span class="n">masked_fill_</span><span class="p">(</span><span class="o">~</span><span class="n">mask</span><span class="p">,</span> <span class="o">-</span><span class="nb">float</span><span class="p">(</span><span class="s1">&#39;inf&#39;</span><span class="p">))</span>
-
-        <span class="c1"># Softmax or sparsemax to normalize attention weights</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_func</span> <span class="o">==</span> <span class="s2">&quot;softmax&quot;</span><span class="p">:</span>
-            <span class="n">align_vectors</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">align</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch</span> <span class="o">*</span> <span class="n">target_l</span><span class="p">,</span> <span class="n">source_l</span><span class="p">),</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">align_vectors</span> <span class="o">=</span> <span class="n">sparsemax</span><span class="p">(</span><span class="n">align</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch</span> <span class="o">*</span> <span class="n">target_l</span><span class="p">,</span> <span class="n">source_l</span><span class="p">),</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-        <span class="n">align_vectors</span> <span class="o">=</span> <span class="n">align_vectors</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">target_l</span><span class="p">,</span> <span class="n">source_l</span><span class="p">)</span>
-
-        <span class="c1"># each context vector c_t is the weighted average</span>
-        <span class="c1"># over all the source hidden states</span>
-        <span class="n">c</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">bmm</span><span class="p">(</span><span class="n">align_vectors</span><span class="p">,</span> <span class="n">memory_bank</span><span class="p">)</span>
-
-        <span class="c1"># concatenate</span>
-        <span class="n">concat_c</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">c</span><span class="p">,</span> <span class="n">source</span><span class="p">],</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch</span> <span class="o">*</span> <span class="n">target_l</span><span class="p">,</span> <span class="n">dim</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
-        <span class="n">attn_h</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">linear_out</span><span class="p">(</span><span class="n">concat_c</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">target_l</span><span class="p">,</span> <span class="n">dim</span><span class="p">)</span>
-        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">attn_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;general&quot;</span><span class="p">,</span> <span class="s2">&quot;dot&quot;</span><span class="p">]:</span>
-            <span class="n">attn_h</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="n">attn_h</span><span class="p">)</span>
-
-        <span class="k">if</span> <span class="n">one_step</span><span class="p">:</span>
-            <span class="n">attn_h</span> <span class="o">=</span> <span class="n">attn_h</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-            <span class="n">align_vectors</span> <span class="o">=</span> <span class="n">align_vectors</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-
-            <span class="c1"># Check output sizes</span>
-            <span class="n">batch_</span><span class="p">,</span> <span class="n">dim_</span> <span class="o">=</span> <span class="n">attn_h</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">batch_</span><span class="p">)</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="n">dim_</span><span class="p">)</span>
-            <span class="n">batch_</span><span class="p">,</span> <span class="n">source_l_</span> <span class="o">=</span> <span class="n">align_vectors</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">batch_</span><span class="p">)</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">source_l</span><span class="p">,</span> <span class="n">source_l_</span><span class="p">)</span>
-
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">attn_h</span> <span class="o">=</span> <span class="n">attn_h</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span>
-            <span class="n">align_vectors</span> <span class="o">=</span> <span class="n">align_vectors</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span>
-            <span class="c1"># Check output sizes</span>
-            <span class="n">target_l_</span><span class="p">,</span> <span class="n">batch_</span><span class="p">,</span> <span class="n">dim_</span> <span class="o">=</span> <span class="n">attn_h</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">target_l</span><span class="p">,</span> <span class="n">target_l_</span><span class="p">)</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">batch_</span><span class="p">)</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="n">dim_</span><span class="p">)</span>
-            <span class="n">target_l_</span><span class="p">,</span> <span class="n">batch_</span><span class="p">,</span> <span class="n">source_l_</span> <span class="o">=</span> <span class="n">align_vectors</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">target_l</span><span class="p">,</span> <span class="n">target_l_</span><span class="p">)</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">batch</span><span class="p">,</span> <span class="n">batch_</span><span class="p">)</span>
-            <span class="n">aeq</span><span class="p">(</span><span class="n">source_l</span><span class="p">,</span> <span class="n">source_l_</span><span class="p">)</span>
-
-        <span class="k">return</span> <span class="n">attn_h</span><span class="p">,</span> <span class="n">align_vectors</span></div></div>
-</pre></div>
-
-           </div>
-           
-          </div>
-          <footer>
-  
-
-  <hr/>
-
-  <div role="contentinfo">
-    <p>
-        &copy; Copyright 2023, HelsinkiNLP
-
-    </p>
-  </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
-
-</footer>
-
-        </div>
-      </div>
-
-    </section>
-
-  </div>
-  
-
-
-  <script type="text/javascript">
-      jQuery(function () {
-          SphinxRtdTheme.Navigation.enable(true);
-      });
-  </script>
-
-  
-  
-    
-   
-
-</body>
-</html>
\ No newline at end of file
diff --git a/_modules/mammoth/modules/structured_attention.html b/_modules/mammoth/modules/structured_attention.html
deleted file mode 100644
index c0d18ab1..00000000
--- a/_modules/mammoth/modules/structured_attention.html
+++ /dev/null
@@ -1,254 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
-  <meta charset="utf-8">
-  
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  
-  <title>mammoth.modules.structured_attention &mdash; MAMMOTH  documentation</title>
-  
-
-  
-  
-  
-  
-
-  
-  <script type="text/javascript" src="../../../_static/js/modernizr.min.js"></script>
-  
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
-        <script src="../../../_static/jquery.js"></script>
-        <script src="../../../_static/underscore.js"></script>
-        <script src="../../../_static/doctools.js"></script>
-        <script src="../../../_static/language_data.js"></script>
-        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
-        <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["\\(", "\\)"]], "displayMath": [["\\[", "\\]"]]}})</script>
-        <script src="https://unpkg.com/mermaid@8.4.8/dist/mermaid.min.js"></script>
-    
-    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-    
-
-  
-  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/theme_overrides.css" type="text/css" />
-    <link rel="index" title="Index" href="../../../genindex.html" />
-    <link rel="search" title="Search" href="../../../search.html" /> 
-</head>
-
-<body class="wy-body-for-nav">
-
-   
-  <div class="wy-grid-for-nav">
-    
-    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
-      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search" >
-          
-
-          
-            <a href="../../../index.html" class="icon icon-home"> MAMMOTH
-          
-
-          
-          </a>
-
-          
-            
-            
-          
-
-          
-<div role="search">
-  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
-    <input type="text" name="q" placeholder="Search docs" />
-    <input type="hidden" name="check_keywords" value="yes" />
-    <input type="hidden" name="area" value="default" />
-  </form>
-</div>
-
-          
-        </div>
-
-        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-          
-            
-            
-              
-            
-            
-              <p class="caption"><span class="caption-text">Getting Started</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../main.html">Overview</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../quickstart.html">Quickstart</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../CONTRIBUTING.html">Contributors</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ref.html">References</a></li>
-</ul>
-<p class="caption"><span class="caption-text">MAMMOTH features</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../config_config.html">Config-config tool</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../attention_bridges.html">Attention Bridge</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Tutorials</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../prepare_data.html">Prepare Data</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Scripts</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/build_vocab.html">Build Vocab</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/train.html">Train</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/translate.html">Translate</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/server.html">Server</a></li>
-</ul>
-<p class="caption"><span class="caption-text">API</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.html">Framework</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.modules.html">Modules</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translation.html">Translation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translate.translation_server.html">Server</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.inputters.html">Data Loaders</a></li>
-</ul>
-
-            
-          
-        </div>
-      </div>
-    </nav>
-
-    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-      
-      <nav class="wy-nav-top" aria-label="top navigation">
-        
-          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-          <a href="../../../index.html">MAMMOTH</a>
-        
-      </nav>
-
-
-      <div class="wy-nav-content">
-        
-        <div class="rst-content">
-        
-          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
-  <ul class="wy-breadcrumbs">
-    
-      <li><a href="../../../index.html">Docs</a> &raquo;</li>
-        
-          <li><a href="../../index.html">Module code</a> &raquo;</li>
-        
-      <li>mammoth.modules.structured_attention</li>
-    
-    
-      <li class="wy-breadcrumbs-aside">
-        
-      </li>
-    
-  </ul>
-
-  
-  <hr/>
-</div>
-          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
-           <div itemprop="articleBody">
-            
-  <h1>Source code for mammoth.modules.structured_attention</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
-<span class="kn">import</span> <span class="nn">torch</span>
-<span class="kn">import</span> <span class="nn">torch.cuda</span>
-
-
-<div class="viewcode-block" id="MatrixTree"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.structured_attention.MatrixTree">[docs]</a><span class="k">class</span> <span class="nc">MatrixTree</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Implementation of the matrix-tree theorem for computing marginals</span>
-<span class="sd">    of non-projective dependency parsing. This attention layer is used</span>
-<span class="sd">    in the paper &quot;Learning Structured Text Representations&quot;</span>
-<span class="sd">    :cite:`DBLP:journals/corr/LiuL17d`.</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">eps</span><span class="o">=</span><span class="mf">1e-5</span><span class="p">):</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">MatrixTree</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
-
-<div class="viewcode-block" id="MatrixTree.forward"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.structured_attention.MatrixTree.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">):</span>
-        <span class="n">laplacian</span> <span class="o">=</span> <span class="nb">input</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span>
-        <span class="n">output</span> <span class="o">=</span> <span class="nb">input</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
-        <span class="k">for</span> <span class="n">b</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)):</span>
-            <span class="n">lap</span> <span class="o">=</span> <span class="n">laplacian</span><span class="p">[</span><span class="n">b</span><span class="p">]</span><span class="o">.</span><span class="n">masked_fill</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">device</span><span class="o">=</span><span class="nb">input</span><span class="o">.</span><span class="n">device</span><span class="p">)</span><span class="o">.</span><span class="n">ne</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="mi">0</span><span class="p">)</span>
-            <span class="n">lap</span> <span class="o">=</span> <span class="o">-</span><span class="n">lap</span> <span class="o">+</span> <span class="n">torch</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">lap</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span>
-            <span class="c1"># store roots on diagonal</span>
-            <span class="n">lap</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="nb">input</span><span class="p">[</span><span class="n">b</span><span class="p">]</span><span class="o">.</span><span class="n">diag</span><span class="p">()</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span>
-            <span class="n">inv_laplacian</span> <span class="o">=</span> <span class="n">lap</span><span class="o">.</span><span class="n">inverse</span><span class="p">()</span>
-
-            <span class="n">factor</span> <span class="o">=</span> <span class="n">inv_laplacian</span><span class="o">.</span><span class="n">diag</span><span class="p">()</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="nb">input</span><span class="p">[</span><span class="n">b</span><span class="p">])</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-            <span class="n">term1</span> <span class="o">=</span> <span class="nb">input</span><span class="p">[</span><span class="n">b</span><span class="p">]</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span><span class="o">.</span><span class="n">mul</span><span class="p">(</span><span class="n">factor</span><span class="p">)</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
-            <span class="n">term2</span> <span class="o">=</span> <span class="nb">input</span><span class="p">[</span><span class="n">b</span><span class="p">]</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span><span class="o">.</span><span class="n">mul</span><span class="p">(</span><span class="n">inv_laplacian</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
-            <span class="n">term1</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
-            <span class="n">term2</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mi">0</span>
-            <span class="n">output</span><span class="p">[</span><span class="n">b</span><span class="p">]</span> <span class="o">=</span> <span class="n">term1</span> <span class="o">-</span> <span class="n">term2</span>
-            <span class="n">roots_output</span> <span class="o">=</span> <span class="nb">input</span><span class="p">[</span><span class="n">b</span><span class="p">]</span><span class="o">.</span><span class="n">diag</span><span class="p">()</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span><span class="o">.</span><span class="n">mul</span><span class="p">(</span><span class="n">inv_laplacian</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
-            <span class="n">output</span><span class="p">[</span><span class="n">b</span><span class="p">]</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="n">b</span><span class="p">]</span> <span class="o">+</span> <span class="n">torch</span><span class="o">.</span><span class="n">diag</span><span class="p">(</span><span class="n">roots_output</span><span class="p">)</span>
-        <span class="k">return</span> <span class="n">output</span></div></div>
-</pre></div>
-
-           </div>
-           
-          </div>
-          <footer>
-  
-
-  <hr/>
-
-  <div role="contentinfo">
-    <p>
-        &copy; Copyright 2023, HelsinkiNLP
-
-    </p>
-  </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
-
-</footer>
-
-        </div>
-      </div>
-
-    </section>
-
-  </div>
-  
-
-
-  <script type="text/javascript">
-      jQuery(function () {
-          SphinxRtdTheme.Navigation.enable(true);
-      });
-  </script>
-
-  
-  
-    
-   
-
-</body>
-</html>
\ No newline at end of file
diff --git a/_modules/mammoth/modules/weight_norm.html b/_modules/mammoth/modules/weight_norm.html
deleted file mode 100644
index a2e952a9..00000000
--- a/_modules/mammoth/modules/weight_norm.html
+++ /dev/null
@@ -1,443 +0,0 @@
-
-
-<!DOCTYPE html>
-<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
-<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
-<head>
-  <meta charset="utf-8">
-  
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  
-  <title>mammoth.modules.weight_norm &mdash; MAMMOTH  documentation</title>
-  
-
-  
-  
-  
-  
-
-  
-  <script type="text/javascript" src="../../../_static/js/modernizr.min.js"></script>
-  
-    
-      <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
-        <script src="../../../_static/jquery.js"></script>
-        <script src="../../../_static/underscore.js"></script>
-        <script src="../../../_static/doctools.js"></script>
-        <script src="../../../_static/language_data.js"></script>
-        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
-        <script type="text/x-mathjax-config">MathJax.Hub.Config({"tex2jax": {"inlineMath": [["\\(", "\\)"]], "displayMath": [["\\[", "\\]"]]}})</script>
-        <script src="https://unpkg.com/mermaid@8.4.8/dist/mermaid.min.js"></script>
-    
-    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-    
-
-  
-  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
-  <link rel="stylesheet" href="../../../_static/theme_overrides.css" type="text/css" />
-    <link rel="index" title="Index" href="../../../genindex.html" />
-    <link rel="search" title="Search" href="../../../search.html" /> 
-</head>
-
-<body class="wy-body-for-nav">
-
-   
-  <div class="wy-grid-for-nav">
-    
-    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
-      <div class="wy-side-scroll">
-        <div class="wy-side-nav-search" >
-          
-
-          
-            <a href="../../../index.html" class="icon icon-home"> MAMMOTH
-          
-
-          
-          </a>
-
-          
-            
-            
-          
-
-          
-<div role="search">
-  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
-    <input type="text" name="q" placeholder="Search docs" />
-    <input type="hidden" name="check_keywords" value="yes" />
-    <input type="hidden" name="area" value="default" />
-  </form>
-</div>
-
-          
-        </div>
-
-        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
-          
-            
-            
-              
-            
-            
-              <p class="caption"><span class="caption-text">Getting Started</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../main.html">Overview</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../quickstart.html">Quickstart</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../CONTRIBUTING.html">Contributors</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../ref.html">References</a></li>
-</ul>
-<p class="caption"><span class="caption-text">MAMMOTH features</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../config_config.html">Config-config tool</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../attention_bridges.html">Attention Bridge</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Tutorials</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../prepare_data.html">Prepare Data</a></li>
-</ul>
-<p class="caption"><span class="caption-text">Scripts</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/build_vocab.html">Build Vocab</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/train.html">Train</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/translate.html">Translate</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../options/server.html">Server</a></li>
-</ul>
-<p class="caption"><span class="caption-text">API</span></p>
-<ul>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.html">Framework</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.modules.html">Modules</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translation.html">Translation</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.translate.translation_server.html">Server</a></li>
-<li class="toctree-l1"><a class="reference internal" href="../../../mammoth.inputters.html">Data Loaders</a></li>
-</ul>
-
-            
-          
-        </div>
-      </div>
-    </nav>
-
-    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
-
-      
-      <nav class="wy-nav-top" aria-label="top navigation">
-        
-          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
-          <a href="../../../index.html">MAMMOTH</a>
-        
-      </nav>
-
-
-      <div class="wy-nav-content">
-        
-        <div class="rst-content">
-        
-          
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-<div role="navigation" aria-label="breadcrumbs navigation">
-
-  <ul class="wy-breadcrumbs">
-    
-      <li><a href="../../../index.html">Docs</a> &raquo;</li>
-        
-          <li><a href="../../index.html">Module code</a> &raquo;</li>
-        
-      <li>mammoth.modules.weight_norm</li>
-    
-    
-      <li class="wy-breadcrumbs-aside">
-        
-      </li>
-    
-  </ul>
-
-  
-  <hr/>
-</div>
-          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
-           <div itemprop="articleBody">
-            
-  <h1>Source code for mammoth.modules.weight_norm</h1><div class="highlight"><pre>
-<span></span><span class="sd">&quot;&quot;&quot;  Weights normalization modules  &quot;&quot;&quot;</span>
-<span class="kn">import</span> <span class="nn">torch</span>
-<span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
-<span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>
-<span class="kn">from</span> <span class="nn">torch.nn</span> <span class="kn">import</span> <span class="n">Parameter</span>
-
-
-<span class="k">def</span> <span class="nf">get_var_maybe_avg</span><span class="p">(</span><span class="n">namespace</span><span class="p">,</span> <span class="n">var_name</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">polyak_decay</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;utility for retrieving polyak averaged params</span>
-<span class="sd">    Update average</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="n">v</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">namespace</span><span class="p">,</span> <span class="n">var_name</span><span class="p">)</span>
-    <span class="n">v_avg</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">namespace</span><span class="p">,</span> <span class="n">var_name</span> <span class="o">+</span> <span class="s1">&#39;_avg&#39;</span><span class="p">)</span>
-    <span class="n">v_avg</span> <span class="o">-=</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">polyak_decay</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">v_avg</span> <span class="o">-</span> <span class="n">v</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-
-    <span class="k">if</span> <span class="n">training</span><span class="p">:</span>
-        <span class="k">return</span> <span class="n">v</span>
-    <span class="k">else</span><span class="p">:</span>
-        <span class="k">return</span> <span class="n">v_avg</span>
-
-
-<span class="k">def</span> <span class="nf">get_vars_maybe_avg</span><span class="p">(</span><span class="n">namespace</span><span class="p">,</span> <span class="n">var_names</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">polyak_decay</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;utility for retrieving polyak averaged params&quot;&quot;&quot;</span>
-    <span class="nb">vars</span> <span class="o">=</span> <span class="p">[]</span>
-    <span class="k">for</span> <span class="n">vn</span> <span class="ow">in</span> <span class="n">var_names</span><span class="p">:</span>
-        <span class="nb">vars</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">get_var_maybe_avg</span><span class="p">(</span><span class="n">namespace</span><span class="p">,</span> <span class="n">vn</span><span class="p">,</span> <span class="n">training</span><span class="p">,</span> <span class="n">polyak_decay</span><span class="p">))</span>
-    <span class="k">return</span> <span class="nb">vars</span>
-
-
-<span class="k">class</span> <span class="nc">WeightNormLinear</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
-<span class="sd">    Implementation of &quot;Weight Normalization: A Simple Reparameterization</span>
-<span class="sd">    to Accelerate Training of Deep Neural Networks&quot;</span>
-<span class="sd">    :cite:`DBLP:journals/corr/SalimansK16`</span>
-
-<span class="sd">    As a reparameterization method, weight normalization is same</span>
-<span class="sd">    as BatchNormalization, but it doesn&#39;t depend on minibatch.</span>
-
-<span class="sd">    NOTE: This is used nowhere in the code at this stage</span>
-<span class="sd">          Vincent Nguyen 05/18/2018</span>
-<span class="sd">    &quot;&quot;&quot;</span>
-
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">in_features</span><span class="p">,</span> <span class="n">out_features</span><span class="p">,</span> <span class="n">init_scale</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">polyak_decay</span><span class="o">=</span><span class="mf">0.9995</span><span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">WeightNormLinear</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">in_features</span><span class="p">,</span> <span class="n">out_features</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">V</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">g</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">out_features</span><span class="p">))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">b</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;V_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">out_features</span><span class="p">,</span> <span class="n">in_features</span><span class="p">))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;g_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">out_features</span><span class="p">))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;b_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">out_features</span><span class="p">))</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">init_scale</span> <span class="o">=</span> <span class="n">init_scale</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">polyak_decay</span> <span class="o">=</span> <span class="n">polyak_decay</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">reset_parameters</span><span class="p">()</span>
-
-    <span class="k">def</span> <span class="nf">reset_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span>
-
-    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
-        <span class="k">if</span> <span class="n">init</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
-            <span class="c1"># out_features * in_features</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span><span class="o">.</span><span class="n">type_as</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> <span class="o">*</span> <span class="mf">0.05</span><span class="p">)</span>
-            <span class="c1"># norm is out_features * 1</span>
-            <span class="n">v_norm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="c1"># batch_size * out_features</span>
-            <span class="n">x_init</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">linear</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">v_norm</span><span class="p">)</span><span class="o">.</span><span class="n">data</span>
-            <span class="c1"># out_features</span>
-            <span class="n">m_init</span><span class="p">,</span> <span class="n">v_init</span> <span class="o">=</span> <span class="n">x_init</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="n">x_init</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
-            <span class="c1"># out_features</span>
-            <span class="n">scale_init</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_scale</span> <span class="o">/</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">v_init</span> <span class="o">+</span> <span class="mf">1e-10</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">g</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="n">scale_init</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">b</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="o">-</span><span class="n">m_init</span> <span class="o">*</span> <span class="n">scale_init</span><span class="p">)</span>
-            <span class="n">x_init</span> <span class="o">=</span> <span class="n">scale_init</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x_init</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">x_init</span> <span class="o">-</span> <span class="n">m_init</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x_init</span><span class="p">))</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">V_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">g_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">g</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">b_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">b</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="k">return</span> <span class="n">x_init</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">v</span><span class="p">,</span> <span class="n">g</span><span class="p">,</span> <span class="n">b</span> <span class="o">=</span> <span class="n">get_vars_maybe_avg</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="s1">&#39;V&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">,</span> <span class="n">polyak_decay</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">polyak_decay</span><span class="p">)</span>
-            <span class="c1"># batch_size * out_features</span>
-            <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">linear</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">v</span><span class="p">)</span>
-            <span class="n">scalar</span> <span class="o">=</span> <span class="n">g</span> <span class="o">/</span> <span class="n">torch</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-            <span class="n">x</span> <span class="o">=</span> <span class="n">scalar</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">*</span> <span class="n">x</span> <span class="o">+</span> <span class="n">b</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
-            <span class="k">return</span> <span class="n">x</span>
-
-
-<div class="viewcode-block" id="WeightNormConv2d"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.WeightNormConv2d">[docs]</a><span class="k">class</span> <span class="nc">WeightNormConv2d</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">):</span>
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-        <span class="bp">self</span><span class="p">,</span>
-        <span class="n">in_channels</span><span class="p">,</span>
-        <span class="n">out_channels</span><span class="p">,</span>
-        <span class="n">kernel_size</span><span class="p">,</span>
-        <span class="n">stride</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-        <span class="n">padding</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-        <span class="n">dilation</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-        <span class="n">groups</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-        <span class="n">init_scale</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
-        <span class="n">polyak_decay</span><span class="o">=</span><span class="mf">0.9995</span><span class="p">,</span>
-    <span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">WeightNormConv2d</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-            <span class="n">in_channels</span><span class="p">,</span> <span class="n">out_channels</span><span class="p">,</span> <span class="n">kernel_size</span><span class="p">,</span> <span class="n">stride</span><span class="p">,</span> <span class="n">padding</span><span class="p">,</span> <span class="n">dilation</span><span class="p">,</span> <span class="n">groups</span>
-        <span class="p">)</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">V</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">g</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">out_channels</span><span class="p">))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">b</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;V_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">size</span><span class="p">()))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;g_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">out_channels</span><span class="p">))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;b_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">out_channels</span><span class="p">))</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">init_scale</span> <span class="o">=</span> <span class="n">init_scale</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">polyak_decay</span> <span class="o">=</span> <span class="n">polyak_decay</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">reset_parameters</span><span class="p">()</span>
-
-    <span class="k">def</span> <span class="nf">reset_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span>
-
-<div class="viewcode-block" id="WeightNormConv2d.forward"><a class="viewcode-back" href="../../../mammoth.modules.html#mammoth.modules.WeightNormConv2d.forward">[docs]</a>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
-        <span class="k">if</span> <span class="n">init</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
-            <span class="c1"># out_channels, in_channels // groups, * kernel_size</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span><span class="o">.</span><span class="n">type_as</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> <span class="o">*</span> <span class="mf">0.05</span><span class="p">)</span>
-            <span class="n">v_norm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kernel_size</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
-            <span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="n">x_init</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">conv2d</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">v_norm</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">stride</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">padding</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dilation</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="p">)</span><span class="o">.</span><span class="n">data</span>
-            <span class="n">t_x_init</span> <span class="o">=</span> <span class="n">x_init</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-            <span class="n">m_init</span><span class="p">,</span> <span class="n">v_init</span> <span class="o">=</span> <span class="n">t_x_init</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">t_x_init</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-            <span class="c1"># out_features</span>
-            <span class="n">scale_init</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_scale</span> <span class="o">/</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">v_init</span> <span class="o">+</span> <span class="mf">1e-10</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">g</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="n">scale_init</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">b</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="o">-</span><span class="n">m_init</span> <span class="o">*</span> <span class="n">scale_init</span><span class="p">)</span>
-            <span class="n">scale_init_shape</span> <span class="o">=</span> <span class="n">scale_init</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">x_init</span><span class="o">.</span><span class="n">size</span><span class="p">())</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)))</span>
-            <span class="n">m_init_shape</span> <span class="o">=</span> <span class="n">m_init</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">x_init</span><span class="o">.</span><span class="n">size</span><span class="p">())</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)))</span>
-            <span class="n">x_init</span> <span class="o">=</span> <span class="n">scale_init_shape</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x_init</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">x_init</span> <span class="o">-</span> <span class="n">m_init_shape</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x_init</span><span class="p">))</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">V_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">g_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">g</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">b_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">b</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="k">return</span> <span class="n">x_init</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">v</span><span class="p">,</span> <span class="n">g</span><span class="p">,</span> <span class="n">b</span> <span class="o">=</span> <span class="n">get_vars_maybe_avg</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="s1">&#39;V&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">,</span> <span class="n">polyak_decay</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">polyak_decay</span><span class="p">)</span>
-
-            <span class="n">scalar</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">),</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
-            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">scalar</span><span class="o">.</span><span class="n">size</span><span class="p">())</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
-                <span class="n">scalar</span> <span class="o">=</span> <span class="n">g</span> <span class="o">/</span> <span class="n">scalar</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="n">scalar</span> <span class="o">=</span> <span class="n">g</span> <span class="o">/</span> <span class="n">scalar</span>
-
-            <span class="n">w</span> <span class="o">=</span> <span class="n">scalar</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">size</span><span class="p">())</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)))</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="o">*</span> <span class="n">v</span>
-
-            <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">conv2d</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">stride</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">padding</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">dilation</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="p">)</span>
-            <span class="k">return</span> <span class="n">x</span></div></div>
-
-
-<span class="c1"># This is used nowhere in the code at the moment (Vincent Nguyen 05/18/2018)</span>
-
-
-<span class="k">class</span> <span class="nc">WeightNormConvTranspose2d</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">ConvTranspose2d</span><span class="p">):</span>
-    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
-        <span class="bp">self</span><span class="p">,</span>
-        <span class="n">in_channels</span><span class="p">,</span>
-        <span class="n">out_channels</span><span class="p">,</span>
-        <span class="n">kernel_size</span><span class="p">,</span>
-        <span class="n">stride</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-        <span class="n">padding</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-        <span class="n">output_padding</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-        <span class="n">groups</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
-        <span class="n">init_scale</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
-        <span class="n">polyak_decay</span><span class="o">=</span><span class="mf">0.9995</span><span class="p">,</span>
-    <span class="p">):</span>
-        <span class="nb">super</span><span class="p">(</span><span class="n">WeightNormConvTranspose2d</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
-            <span class="n">in_channels</span><span class="p">,</span> <span class="n">out_channels</span><span class="p">,</span> <span class="n">kernel_size</span><span class="p">,</span> <span class="n">stride</span><span class="p">,</span> <span class="n">padding</span><span class="p">,</span> <span class="n">output_padding</span><span class="p">,</span> <span class="n">groups</span>
-        <span class="p">)</span>
-        <span class="c1"># in_channels, out_channels, *kernel_size</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">V</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">weight</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">g</span> <span class="o">=</span> <span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">(</span><span class="n">out_channels</span><span class="p">))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">b</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bias</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;V_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">size</span><span class="p">()))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;g_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">out_channels</span><span class="p">))</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;b_avg&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">out_channels</span><span class="p">))</span>
-
-        <span class="bp">self</span><span class="o">.</span><span class="n">init_scale</span> <span class="o">=</span> <span class="n">init_scale</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">polyak_decay</span> <span class="o">=</span> <span class="n">polyak_decay</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">reset_parameters</span><span class="p">()</span>
-
-    <span class="k">def</span> <span class="nf">reset_parameters</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
-        <span class="k">return</span>
-
-    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
-        <span class="k">if</span> <span class="n">init</span> <span class="ow">is</span> <span class="kc">True</span><span class="p">:</span>
-            <span class="c1"># in_channels, out_channels, *kernel_size</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span><span class="o">.</span><span class="n">type_as</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> <span class="o">*</span> <span class="mf">0.05</span><span class="p">)</span>
-            <span class="n">v_norm</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">in_channels</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">kernel_size</span><span class="p">))</span>
-            <span class="p">)</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="n">x_init</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">conv_transpose2d</span><span class="p">(</span>
-                <span class="n">x</span><span class="p">,</span> <span class="n">v_norm</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">stride</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">padding</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">output_padding</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">groups</span>
-            <span class="p">)</span><span class="o">.</span><span class="n">data</span>
-            <span class="c1"># self.out_channels, 1</span>
-            <span class="n">t_x_init</span> <span class="o">=</span> <span class="n">x_init</span><span class="o">.</span><span class="n">tranpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span>
-            <span class="c1"># out_features</span>
-            <span class="n">m_init</span><span class="p">,</span> <span class="n">v_init</span> <span class="o">=</span> <span class="n">t_x_init</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">t_x_init</span><span class="o">.</span><span class="n">var</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-            <span class="c1"># out_features</span>
-            <span class="n">scale_init</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">init_scale</span> <span class="o">/</span> <span class="n">torch</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">v_init</span> <span class="o">+</span> <span class="mf">1e-10</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">g</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="n">scale_init</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">b</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="o">-</span><span class="n">m_init</span> <span class="o">*</span> <span class="n">scale_init</span><span class="p">)</span>
-            <span class="n">scale_init_shape</span> <span class="o">=</span> <span class="n">scale_init</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">x_init</span><span class="o">.</span><span class="n">size</span><span class="p">())</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)))</span>
-            <span class="n">m_init_shape</span> <span class="o">=</span> <span class="n">m_init</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">x_init</span><span class="o">.</span><span class="n">size</span><span class="p">())</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)))</span>
-
-            <span class="n">x_init</span> <span class="o">=</span> <span class="n">scale_init_shape</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x_init</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="n">x_init</span> <span class="o">-</span> <span class="n">m_init_shape</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">x_init</span><span class="p">))</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">V_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">V</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">g_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">g</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="bp">self</span><span class="o">.</span><span class="n">b_avg</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">b</span><span class="o">.</span><span class="n">data</span><span class="p">)</span>
-            <span class="k">return</span> <span class="n">x_init</span>
-        <span class="k">else</span><span class="p">:</span>
-            <span class="n">v</span><span class="p">,</span> <span class="n">g</span><span class="p">,</span> <span class="n">b</span> <span class="o">=</span> <span class="n">get_vars_maybe_avg</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="p">[</span><span class="s1">&#39;V&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">training</span><span class="p">,</span> <span class="n">polyak_decay</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">polyak_decay</span><span class="p">)</span>
-            <span class="n">scalar</span> <span class="o">=</span> <span class="n">g</span> <span class="o">/</span> <span class="n">torch</span><span class="o">.</span><span class="n">norm</span><span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">),</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
-            <span class="n">w</span> <span class="o">=</span> <span class="n">scalar</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">in_channels</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">out_channels</span><span class="p">,</span> <span class="o">*</span><span class="p">([</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">size</span><span class="p">())</span> <span class="o">-</span> <span class="mi">2</span><span class="p">)))</span><span class="o">.</span><span class="n">expand_as</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="o">*</span> <span class="n">v</span>
-
-            <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">conv_transpose2d</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">w</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">stride</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">padding</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">output_padding</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">groups</span><span class="p">)</span>
-            <span class="k">return</span> <span class="n">x</span>
-</pre></div>
-
-           </div>
-           
-          </div>
-          <footer>
-  
-
-  <hr/>
-
-  <div role="contentinfo">
-    <p>
-        &copy; Copyright 2023, HelsinkiNLP
-
-    </p>
-  </div>
-  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
-
-</footer>
-
-        </div>
-      </div>
-
-    </section>
-
-  </div>
-  
-
-
-  <script type="text/javascript">
-      jQuery(function () {
-          SphinxRtdTheme.Navigation.enable(true);
-      });
-  </script>
-
-  
-  
-    
-   
-
-</body>
-</html>
\ No newline at end of file
diff --git a/_modules/mammoth/translate/translator.html b/_modules/mammoth/translate/translator.html
index 1a320fb6..a47098e2 100644
--- a/_modules/mammoth/translate/translator.html
+++ b/_modules/mammoth/translate/translator.html
@@ -192,7 +192,6 @@ <h1>Source code for mammoth.translate.translator</h1><div class="highlight"><pre
 <span class="kn">from</span> <span class="nn">mammoth.translate.greedy_search</span> <span class="kn">import</span> <span class="n">GreedySearch</span><span class="p">,</span> <span class="n">GreedySearchLM</span>
 <span class="kn">from</span> <span class="nn">mammoth.utils.misc</span> <span class="kn">import</span> <span class="n">tile</span><span class="p">,</span> <span class="n">set_random_seed</span><span class="p">,</span> <span class="n">report_matrix</span>
 <span class="kn">from</span> <span class="nn">mammoth.utils.alignment</span> <span class="kn">import</span> <span class="n">extract_alignment</span><span class="p">,</span> <span class="n">build_align_pharaoh</span>
-<span class="kn">from</span> <span class="nn">mammoth.modules.copy_generator</span> <span class="kn">import</span> <span class="n">collapse_copy_scores</span>
 <span class="kn">from</span> <span class="nn">mammoth.constants</span> <span class="kn">import</span> <span class="n">ModelTask</span><span class="p">,</span> <span class="n">DefaultTokens</span>
 <span class="kn">from</span> <span class="nn">mammoth.inputters.dataset</span> <span class="kn">import</span> <span class="n">ParallelCorpus</span>
 <span class="kn">from</span> <span class="nn">mammoth.inputters.dataloader</span> <span class="kn">import</span> <span class="n">build_dataloader</span>
@@ -847,39 +846,14 @@ <h1>Source code for mammoth.translate.translator</h1><div class="highlight"><pre
         <span class="p">)</span>
 
         <span class="c1"># Generator forward.</span>
-        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">copy_attn</span><span class="p">:</span>
-            <span class="k">if</span> <span class="s2">&quot;std&quot;</span> <span class="ow">in</span> <span class="n">dec_attn</span><span class="p">:</span>
-                <span class="n">attn</span> <span class="o">=</span> <span class="n">dec_attn</span><span class="p">[</span><span class="s2">&quot;std&quot;</span><span class="p">]</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="n">attn</span> <span class="o">=</span> <span class="kc">None</span>
-            <span class="n">log_probs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">generator</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;generator_</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">tgt_lang</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">](</span><span class="n">dec_out</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span>
-            <span class="c1"># returns [(batch_size x beam_size) , vocab ] when 1 step</span>
-            <span class="c1"># or [ tgt_len, batch_size, vocab ] when full sentence</span>
+        <span class="k">if</span> <span class="s2">&quot;std&quot;</span> <span class="ow">in</span> <span class="n">dec_attn</span><span class="p">:</span>
+            <span class="n">attn</span> <span class="o">=</span> <span class="n">dec_attn</span><span class="p">[</span><span class="s2">&quot;std&quot;</span><span class="p">]</span>
         <span class="k">else</span><span class="p">:</span>
-            <span class="n">attn</span> <span class="o">=</span> <span class="n">dec_attn</span><span class="p">[</span><span class="s2">&quot;copy&quot;</span><span class="p">]</span>
-            <span class="n">scores</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">generator</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;generator_</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">tgt_lang</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">](</span>
-                <span class="n">dec_out</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">dec_out</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">2</span><span class="p">)),</span>
-                <span class="n">attn</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">attn</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">2</span><span class="p">)),</span>
-                <span class="n">src_map</span><span class="p">,</span>
-            <span class="p">)</span>
-            <span class="c1"># here we have scores [tgt_lenxbatch, vocab] or [beamxbatch, vocab]</span>
-            <span class="k">if</span> <span class="n">batch_offset</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
-                <span class="n">scores</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">batch</span><span class="o">.</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">scores</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span>
-                <span class="n">scores</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">contiguous</span><span class="p">()</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="n">scores</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">beam_size</span><span class="p">,</span> <span class="n">scores</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span>
-            <span class="n">scores</span> <span class="o">=</span> <span class="n">collapse_copy_scores</span><span class="p">(</span>
-                <span class="n">scores</span><span class="p">,</span>
-                <span class="n">batch</span><span class="p">,</span>
-                <span class="bp">self</span><span class="o">.</span><span class="n">_tgt_vocab</span><span class="p">,</span>
-                <span class="n">src_vocabs</span><span class="p">,</span>
-                <span class="n">batch_dim</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
-                <span class="n">batch_offset</span><span class="o">=</span><span class="n">batch_offset</span><span class="p">,</span>
-            <span class="p">)</span>
-            <span class="n">scores</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">decoder_in</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="n">scores</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span>
-            <span class="n">log_probs</span> <span class="o">=</span> <span class="n">scores</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="n">log</span><span class="p">()</span>
-            <span class="c1"># returns [(batch_size x beam_size) , vocab ] when 1 step</span>
-            <span class="c1"># or [ tgt_len, batch_size, vocab ] when full sentence</span>
+            <span class="n">attn</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">log_probs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">generator</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;generator_</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">tgt_lang</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">](</span><span class="n">dec_out</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">0</span><span class="p">))</span>
+        <span class="c1"># returns [(batch_size x beam_size) , vocab ] when 1 step</span>
+        <span class="c1"># or [ tgt_len, batch_size, vocab ] when full sentence</span>
+
         <span class="k">return</span> <span class="n">log_probs</span><span class="p">,</span> <span class="n">attn</span>
 
     <span class="k">def</span> <span class="nf">translate_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">src_vocabs</span><span class="p">,</span> <span class="n">attn_debug</span><span class="p">):</span>
diff --git a/_modules/mammoth/utils/loss.html b/_modules/mammoth/utils/loss.html
index ac753673..7cf7db3e 100644
--- a/_modules/mammoth/utils/loss.html
+++ b/_modules/mammoth/utils/loss.html
@@ -184,8 +184,6 @@ <h1>Source code for mammoth.utils.loss</h1><div class="highlight"><pre>
 <span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>
 
 <span class="kn">import</span> <span class="nn">mammoth</span>
-<span class="kn">from</span> <span class="nn">mammoth.modules.sparse_losses</span> <span class="kn">import</span> <span class="n">SparsemaxLoss</span>
-<span class="kn">from</span> <span class="nn">mammoth.modules.sparse_activations</span> <span class="kn">import</span> <span class="n">LogSparsemax</span>
 <span class="kn">from</span> <span class="nn">mammoth.constants</span> <span class="kn">import</span> <span class="n">ModelTask</span><span class="p">,</span> <span class="n">DefaultTokens</span>
 
 
@@ -212,16 +210,14 @@ <h1>Source code for mammoth.utils.loss</h1><div class="highlight"><pre>
         <span class="p">)</span>
     <span class="k">elif</span> <span class="n">opt</span><span class="o">.</span><span class="n">label_smoothing</span> <span class="o">&gt;</span> <span class="mi">0</span> <span class="ow">and</span> <span class="n">train</span><span class="p">:</span>
         <span class="n">criterion</span> <span class="o">=</span> <span class="n">LabelSmoothingLoss</span><span class="p">(</span><span class="n">opt</span><span class="o">.</span><span class="n">label_smoothing</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">tgt_vocab</span><span class="p">),</span> <span class="n">ignore_index</span><span class="o">=</span><span class="n">padding_idx</span><span class="p">)</span>
-    <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">generator</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="n">LogSparsemax</span><span class="p">):</span>  <span class="c1"># elif isinstance(model.generator[-1], LogSparsemax):</span>
-        <span class="n">criterion</span> <span class="o">=</span> <span class="n">SparsemaxLoss</span><span class="p">(</span><span class="n">ignore_index</span><span class="o">=</span><span class="n">padding_idx</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;sum&#39;</span><span class="p">)</span>
     <span class="k">else</span><span class="p">:</span>
         <span class="n">criterion</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">NLLLoss</span><span class="p">(</span><span class="n">ignore_index</span><span class="o">=</span><span class="n">padding_idx</span><span class="p">,</span> <span class="n">reduction</span><span class="o">=</span><span class="s1">&#39;sum&#39;</span><span class="p">)</span>
 
     <span class="c1"># if the loss function operates on vectors of raw logits instead of</span>
     <span class="c1"># probabilities, only the first part of the generator needs to be</span>
-    <span class="c1"># passed to the NMTLossCompute. At the moment, the only supported</span>
-    <span class="c1"># loss function of this kind is the sparsemax loss.</span>
-    <span class="n">use_raw_logits</span> <span class="o">=</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">criterion</span><span class="p">,</span> <span class="n">SparsemaxLoss</span><span class="p">)</span>
+    <span class="c1"># passed to the NMTLossCompute. At the moment, there is no supported</span>
+    <span class="c1"># loss function of this kind.</span>
+    <span class="n">use_raw_logits</span> <span class="o">=</span> <span class="kc">False</span>
     <span class="n">loss_gen</span> <span class="o">=</span> <span class="p">(</span>
         <span class="n">generator</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="n">use_raw_logits</span> <span class="k">else</span> <span class="n">generator</span>
     <span class="p">)</span>  <span class="c1"># loss_gen = model.generator[0] if use_raw_logits else model.generator</span>
diff --git a/_modules/mammoth/utils/optimizers.html b/_modules/mammoth/utils/optimizers.html
index 2508ccc2..eee78357 100644
--- a/_modules/mammoth/utils/optimizers.html
+++ b/_modules/mammoth/utils/optimizers.html
@@ -288,21 +288,6 @@ <h1>Source code for mammoth.utils.optimizers</h1><div class="highlight"><pre>
                 <span class="n">params</span><span class="p">,</span> <span class="n">lr</span><span class="o">=</span><span class="n">opt</span><span class="o">.</span><span class="n">learning_rate</span><span class="p">,</span> <span class="n">betas</span><span class="o">=</span><span class="n">betas</span><span class="p">,</span> <span class="n">eps</span><span class="o">=</span><span class="mf">1e-9</span><span class="p">,</span> <span class="n">weight_decay</span><span class="o">=</span><span class="n">opt</span><span class="o">.</span><span class="n">weight_decay</span>
             <span class="p">)</span>
         <span class="p">)</span>
-    <span class="k">elif</span> <span class="n">opt</span><span class="o">.</span><span class="n">optim</span> <span class="o">==</span> <span class="s1">&#39;sparseadam&#39;</span><span class="p">:</span>
-        <span class="n">encs</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="n">decs</span> <span class="o">=</span> <span class="p">[]</span>
-        <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
-            <span class="k">if</span> <span class="ow">not</span> <span class="n">param</span><span class="o">.</span><span class="n">requires_grad</span><span class="p">:</span>
-                <span class="k">continue</span>
-            <span class="c1"># TODO: Find a better way to check for sparse gradients.</span>
-            <span class="k">if</span> <span class="s1">&#39;decoder&#39;</span> <span class="ow">in</span> <span class="n">name</span><span class="p">:</span>
-                <span class="c1"># print(name)</span>
-                <span class="n">decs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">param</span><span class="p">)</span>
-            <span class="k">else</span><span class="p">:</span>
-                <span class="n">encs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">param</span><span class="p">)</span>
-        <span class="n">optimizer</span> <span class="o">=</span> <span class="n">MultipleOptimizer</span><span class="p">(</span>
-            <span class="p">[</span><span class="n">optim</span><span class="o">.</span><span class="n">Adam</span><span class="p">(</span><span class="n">encs</span><span class="p">,</span> <span class="n">lr</span><span class="o">=</span><span class="n">opt</span><span class="o">.</span><span class="n">learning_rate</span><span class="p">,</span> <span class="n">betas</span><span class="o">=</span><span class="n">betas</span><span class="p">,</span> <span class="n">eps</span><span class="o">=</span><span class="mf">1e-9</span><span class="p">),</span> <span class="n">AdaFactorFairSeq</span><span class="p">(</span><span class="n">decs</span><span class="p">,</span> <span class="n">warmup_init</span><span class="o">=</span><span class="kc">True</span><span class="p">)]</span>
-        <span class="p">)</span>
     <span class="k">elif</span> <span class="n">opt</span><span class="o">.</span><span class="n">optim</span> <span class="o">==</span> <span class="s1">&#39;fusedadam&#39;</span><span class="p">:</span>
         <span class="c1"># we use here a FusedAdam() copy of an old Apex repo</span>
         <span class="n">optimizer</span> <span class="o">=</span> <span class="n">FusedAdam</span><span class="p">(</span><span class="n">params</span><span class="p">,</span> <span class="n">lr</span><span class="o">=</span><span class="n">opt</span><span class="o">.</span><span class="n">learning_rate</span><span class="p">,</span> <span class="n">betas</span><span class="o">=</span><span class="n">betas</span><span class="p">)</span>
@@ -389,7 +374,7 @@ <h1>Source code for mammoth.utils.optimizers</h1><div class="highlight"><pre>
 
 
 <span class="k">class</span> <span class="nc">MultipleOptimizer</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Implement multiple optimizers needed for sparse adam&quot;&quot;&quot;</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Implement multiple optimizers&quot;&quot;&quot;</span>
 
     <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">op</span><span class="p">,</span> <span class="n">multiOptims_Langs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">optimizers</span> <span class="o">=</span> <span class="n">op</span>
@@ -911,11 +896,7 @@ <h1>Source code for mammoth.utils.optimizers</h1><div class="highlight"><pre>
                 <span class="k">if</span> <span class="n">grad</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                     <span class="n">grad</span> <span class="o">=</span> <span class="n">p</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">data</span>
                 <span class="k">if</span> <span class="n">grad</span><span class="o">.</span><span class="n">is_sparse</span><span class="p">:</span>
-                    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
-                        <span class="s1">&#39;FusedAdam does not support sparse </span><span class="se">\</span>
-<span class="s1">                                       gradients, please consider </span><span class="se">\</span>
-<span class="s1">                                       SparseAdam instead&#39;</span>
-                    <span class="p">)</span>
+                    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;sparse gradient not supported&#39;</span><span class="p">)</span>
 
                 <span class="n">state</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span><span class="p">[</span><span class="n">p</span><span class="p">]</span>
 
diff --git a/genindex.html b/genindex.html
index bbf3522f..f6479b04 100644
--- a/genindex.html
+++ b/genindex.html
@@ -213,15 +213,13 @@ <h2 id="A">A</h2>
         <li><a href="mammoth.translation.html#mammoth.translate.GreedySearch.advance">(mammoth.translate.GreedySearch method)</a>
 </li>
       </ul></li>
-      <li><a href="mammoth.html#mammoth.utils.Statistics.all_gather_stats">all_gather_stats() (mammoth.utils.Statistics static method)</a>
-</li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
+      <li><a href="mammoth.html#mammoth.utils.Statistics.all_gather_stats">all_gather_stats() (mammoth.utils.Statistics static method)</a>
+</li>
       <li><a href="mammoth.html#mammoth.utils.Statistics.all_gather_stats_list">all_gather_stats_list() (mammoth.utils.Statistics static method)</a>
 </li>
       <li><a href="mammoth.html#mammoth.utils.Optimizer.amp">amp() (mammoth.utils.Optimizer property)</a>
-</li>
-      <li><a href="mammoth.modules.html#mammoth.modules.ConvMultiStepAttention.apply_mask">apply_mask() (mammoth.modules.ConvMultiStepAttention method)</a>
 </li>
       <li><a href="mammoth.modules.html#mammoth.modules.AverageAttention">AverageAttention (class in mammoth.modules)</a>
 </li>
@@ -248,17 +246,13 @@ <h2 id="C">C</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="mammoth.translate.translation_server.html#mammoth.translate.translation_server.TranslationServer.clone_model">clone_model() (mammoth.translate.translation_server.TranslationServer method)</a>
-</li>
-      <li><a href="mammoth.modules.html#mammoth.modules.ConvMultiStepAttention">ConvMultiStepAttention (class in mammoth.modules)</a>
-</li>
-      <li><a href="mammoth.modules.html#mammoth.modules.CopyGenerator">CopyGenerator (class in mammoth.modules)</a>
 </li>
       <li><a href="mammoth.html#mammoth.models.NMTModel.count_parameters">count_parameters() (mammoth.models.NMTModel method)</a>
 </li>
-  </ul></td>
-  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="mammoth.translation.html#mammoth.translate.penalties.PenaltyBuilder.coverage_none">coverage_none() (mammoth.translate.penalties.PenaltyBuilder method)</a>
 </li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="mammoth.translation.html#mammoth.translate.penalties.PenaltyBuilder.coverage_summary">coverage_summary() (mammoth.translate.penalties.PenaltyBuilder method)</a>
 </li>
       <li><a href="mammoth.translation.html#mammoth.translate.penalties.PenaltyBuilder.coverage_wu">coverage_wu() (mammoth.translate.penalties.PenaltyBuilder method)</a>
@@ -305,24 +299,14 @@ <h2 id="F">F</h2>
 
       <ul>
         <li><a href="mammoth.modules.html#mammoth.modules.AverageAttention.forward">(mammoth.modules.AverageAttention method)</a>
-</li>
-        <li><a href="mammoth.modules.html#mammoth.modules.ConvMultiStepAttention.forward">(mammoth.modules.ConvMultiStepAttention method)</a>
-</li>
-        <li><a href="mammoth.modules.html#mammoth.modules.CopyGenerator.forward">(mammoth.modules.CopyGenerator method)</a>
 </li>
         <li><a href="mammoth.modules.html#mammoth.modules.Embeddings.forward">(mammoth.modules.Embeddings method)</a>
-</li>
-        <li><a href="mammoth.modules.html#mammoth.modules.GlobalAttention.forward">(mammoth.modules.GlobalAttention method)</a>
 </li>
         <li><a href="mammoth.modules.html#mammoth.modules.MultiHeadedAttention.forward">(mammoth.modules.MultiHeadedAttention method)</a>
 </li>
         <li><a href="mammoth.modules.html#mammoth.modules.position_ffn.PositionwiseFeedForward.forward">(mammoth.modules.position_ffn.PositionwiseFeedForward method)</a>
 </li>
         <li><a href="mammoth.modules.html#mammoth.modules.PositionalEncoding.forward">(mammoth.modules.PositionalEncoding method)</a>
-</li>
-        <li><a href="mammoth.modules.html#mammoth.modules.structured_attention.MatrixTree.forward">(mammoth.modules.structured_attention.MatrixTree method)</a>
-</li>
-        <li><a href="mammoth.modules.html#mammoth.modules.WeightNormConv2d.forward">(mammoth.modules.WeightNormConv2d method)</a>
 </li>
       </ul></li>
   </ul></td>
@@ -335,12 +319,10 @@ <h2 id="F">F</h2>
 <h2 id="G">G</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="mammoth.modules.html#mammoth.modules.GlobalAttention">GlobalAttention (class in mammoth.modules)</a>
+      <li><a href="mammoth.translation.html#mammoth.translate.GNMTGlobalScorer">GNMTGlobalScorer (class in mammoth.translate)</a>
 </li>
   </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="mammoth.translation.html#mammoth.translate.GNMTGlobalScorer">GNMTGlobalScorer (class in mammoth.translate)</a>
-</li>
       <li><a href="mammoth.translation.html#mammoth.translate.GreedySearch">GreedySearch (class in mammoth.translate)</a>
 </li>
   </ul></td>
@@ -391,8 +373,6 @@ <h2 id="L">L</h2>
 <h2 id="M">M</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="mammoth.modules.html#mammoth.modules.structured_attention.MatrixTree">MatrixTree (class in mammoth.modules.structured_attention)</a>
-</li>
       <li><a href="mammoth.translate.translation_server.html#mammoth.translate.translation_server.ServerModel.maybe_convert_align">maybe_convert_align() (mammoth.translate.translation_server.ServerModel method)</a>
 </li>
       <li><a href="mammoth.translate.translation_server.html#mammoth.translate.translation_server.ServerModel.maybe_detokenize">maybe_detokenize() (mammoth.translate.translation_server.ServerModel method)</a>
@@ -476,15 +456,13 @@ <h2 id="S">S</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="mammoth.translation.html#mammoth.translate.greedy_search.sample_with_temperature">sample_with_temperature() (in module mammoth.translate.greedy_search)</a>
-</li>
-      <li><a href="mammoth.modules.html#mammoth.modules.GlobalAttention.score">score() (mammoth.modules.GlobalAttention method)</a>
 </li>
       <li><a href="mammoth.translate.translation_server.html#mammoth.translate.translation_server.ServerModel">ServerModel (class in mammoth.translate.translation_server)</a>
 </li>
-  </ul></td>
-  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="mammoth.translate.translation_server.html#mammoth.translate.translation_server.ServerModelError">ServerModelError</a>
 </li>
+  </ul></td>
+  <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="mammoth.translate.translation_server.html#mammoth.translate.translation_server.TranslationServer.start">start() (mammoth.translate.translation_server.TranslationServer method)</a>
 </li>
       <li><a href="mammoth.html#mammoth.utils.Statistics">Statistics (class in mammoth.utils)</a>
@@ -560,10 +538,6 @@ <h2 id="V">V</h2>
 
 <h2 id="W">W</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
-  <td style="width: 33%; vertical-align: top;"><ul>
-      <li><a href="mammoth.modules.html#mammoth.modules.WeightNormConv2d">WeightNormConv2d (class in mammoth.modules)</a>
-</li>
-  </ul></td>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="mammoth.modules.html#mammoth.modules.Embeddings.word_lut">word_lut() (mammoth.modules.Embeddings property)</a>
 </li>
diff --git a/mammoth.modules.html b/mammoth.modules.html
index 38bea3b3..14f95676 100644
--- a/mammoth.modules.html
+++ b/mammoth.modules.html
@@ -195,10 +195,10 @@ <h1>Modules<a class="headerlink" href="#modules" title="Permalink to this headli
 <h2>Core Modules<a class="headerlink" href="#core-modules" title="Permalink to this headline">¶</a></h2>
 <dl class="class">
 <dt id="mammoth.modules.Embeddings">
-<em class="property">class </em><code class="sig-prename descclassname">mammoth.modules.</code><code class="sig-name descname">Embeddings</code><span class="sig-paren">(</span><em class="sig-param">word_vec_size</em>, <em class="sig-param">word_vocab_size</em>, <em class="sig-param">word_padding_idx</em>, <em class="sig-param">position_encoding=False</em>, <em class="sig-param">feat_merge='concat'</em>, <em class="sig-param">feat_vec_exponent=0.7</em>, <em class="sig-param">feat_vec_size=-1</em>, <em class="sig-param">feat_padding_idx=[]</em>, <em class="sig-param">feat_vocab_sizes=[]</em>, <em class="sig-param">dropout=0</em>, <em class="sig-param">sparse=False</em>, <em class="sig-param">freeze_word_vecs=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/embeddings.html#Embeddings"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.Embeddings" title="Permalink to this definition">¶</a></dt>
+<em class="property">class </em><code class="sig-prename descclassname">mammoth.modules.</code><code class="sig-name descname">Embeddings</code><span class="sig-paren">(</span><em class="sig-param">word_vec_size</em>, <em class="sig-param">word_vocab_size</em>, <em class="sig-param">word_padding_idx</em>, <em class="sig-param">position_encoding=False</em>, <em class="sig-param">feat_merge='concat'</em>, <em class="sig-param">feat_vec_exponent=0.7</em>, <em class="sig-param">feat_vec_size=-1</em>, <em class="sig-param">feat_padding_idx=[]</em>, <em class="sig-param">feat_vocab_sizes=[]</em>, <em class="sig-param">dropout=0</em>, <em class="sig-param">freeze_word_vecs=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/embeddings.html#Embeddings"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.Embeddings" title="Permalink to this definition">¶</a></dt>
 <dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.modules.module.Module</span></code></p>
 <p>Words embeddings for encoder/decoder.</p>
-<p>Additionally includes ability to add sparse input features
+<p>Additionally includes ability to add input features
 based on “Linguistic Input Features Improve Neural Machine Translation”
 <a class="bibtex reference internal" href="ref.html#sennrich2016linguistic" id="id1">[SH16]</a>.</p>
 <script>mermaid.initialize({startOnLoad:true});</script><div class="mermaid">
@@ -377,118 +377,6 @@ <h2>Attention<a class="headerlink" href="#attention" title="Permalink to this he
 
 </dd></dl>
 
-<dl class="class">
-<dt id="mammoth.modules.GlobalAttention">
-<em class="property">class </em><code class="sig-prename descclassname">mammoth.modules.</code><code class="sig-name descname">GlobalAttention</code><span class="sig-paren">(</span><em class="sig-param">dim</em>, <em class="sig-param">coverage=False</em>, <em class="sig-param">attn_type='dot'</em>, <em class="sig-param">attn_func='softmax'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/global_attention.html#GlobalAttention"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.GlobalAttention" title="Permalink to this definition">¶</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.modules.module.Module</span></code></p>
-<p>Global attention takes a matrix and a query vector. It
-then computes a parameterized convex combination of the matrix
-based on the input query.</p>
-<p>Constructs a unit mapping a query <cite>q</cite> of size <cite>dim</cite>
-and a source matrix <cite>H</cite> of size <cite>n x dim</cite>, to an output
-of size <cite>dim</cite>.</p>
-<div class="mermaid">
-            graph BT
-   A[Query]
-   subgraph RNN
-     C[H 1]
-     D[H 2]
-     E[H N]
-   end
-   F[Attn]
-   G[Output]
-   A --&gt; F
-   C --&gt; F
-   D --&gt; F
-   E --&gt; F
-   C -.-&gt; G
-   D -.-&gt; G
-   E -.-&gt; G
-   F --&gt; G
-        </div><p>All models compute the output as
-<span class="math notranslate nohighlight">\(c = \sum_{j=1}^{\text{SeqLength}} a_j H_j\)</span> where
-<span class="math notranslate nohighlight">\(a_j\)</span> is the softmax of a score function.
-Then then apply a projection layer to [q, c].</p>
-<p>However they
-differ on how they compute the attention score.</p>
-<ul class="simple">
-<li><dl class="simple">
-<dt>Luong Attention (dot, general):</dt><dd><ul>
-<li><p>dot: <span class="math notranslate nohighlight">\(\text{score}(H_j,q) = H_j^T q\)</span></p></li>
-<li><p>general: <span class="math notranslate nohighlight">\(\text{score}(H_j, q) = H_j^T W_a q\)</span></p></li>
-</ul>
-</dd>
-</dl>
-</li>
-<li><dl class="simple">
-<dt>Bahdanau Attention (mlp):</dt><dd><ul>
-<li><p><span class="math notranslate nohighlight">\(\text{score}(H_j, q) = v_a^T \text{tanh}(W_a q + U_a h_j)\)</span></p></li>
-</ul>
-</dd>
-</dl>
-</li>
-</ul>
-<dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>dim</strong> (<em>int</em>) – dimensionality of query and key</p></li>
-<li><p><strong>coverage</strong> (<em>bool</em>) – use coverage term</p></li>
-<li><p><strong>attn_type</strong> (<em>str</em>) – type of attention to use, options [dot,general,mlp]</p></li>
-<li><p><strong>attn_func</strong> (<em>str</em>) – attention function to use, options [softmax,sparsemax]</p></li>
-</ul>
-</dd>
-</dl>
-<dl class="method">
-<dt id="mammoth.modules.GlobalAttention.forward">
-<code class="sig-name descname">forward</code><span class="sig-paren">(</span><em class="sig-param">source</em>, <em class="sig-param">memory_bank</em>, <em class="sig-param">memory_lengths=None</em>, <em class="sig-param">coverage=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/global_attention.html#GlobalAttention.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.GlobalAttention.forward" title="Permalink to this definition">¶</a></dt>
-<dd><dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>source</strong> (<em>FloatTensor</em>) – query vectors <code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">tgt_len,</span> <span class="pre">dim)</span></code></p></li>
-<li><p><strong>memory_bank</strong> (<em>FloatTensor</em>) – source vectors <code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">src_len,</span> <span class="pre">dim)</span></code></p></li>
-<li><p><strong>memory_lengths</strong> (<em>LongTensor</em>) – the source context lengths <code class="docutils literal notranslate"><span class="pre">(batch,)</span></code></p></li>
-<li><p><strong>coverage</strong> (<em>FloatTensor</em>) – None (not supported yet)</p></li>
-</ul>
-</dd>
-<dt class="field-even">Returns</dt>
-<dd class="field-even"><p><ul class="simple">
-<li><p>Computed vector <code class="docutils literal notranslate"><span class="pre">(tgt_len,</span> <span class="pre">batch,</span> <span class="pre">dim)</span></code></p></li>
-<li><p>Attention distribtutions for each query
-<code class="docutils literal notranslate"><span class="pre">(tgt_len,</span> <span class="pre">batch,</span> <span class="pre">src_len)</span></code></p></li>
-</ul>
-</p>
-</dd>
-<dt class="field-odd">Return type</dt>
-<dd class="field-odd"><p>(FloatTensor, FloatTensor)</p>
-</dd>
-</dl>
-</dd></dl>
-
-<dl class="method">
-<dt id="mammoth.modules.GlobalAttention.score">
-<code class="sig-name descname">score</code><span class="sig-paren">(</span><em class="sig-param">h_t</em>, <em class="sig-param">h_s</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/global_attention.html#GlobalAttention.score"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.GlobalAttention.score" title="Permalink to this definition">¶</a></dt>
-<dd><dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>h_t</strong> (<em>FloatTensor</em>) – sequence of queries <code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">tgt_len,</span> <span class="pre">dim)</span></code></p></li>
-<li><p><strong>h_s</strong> (<em>FloatTensor</em>) – sequence of sources <code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">src_len,</span> <span class="pre">dim</span></code></p></li>
-</ul>
-</dd>
-<dt class="field-even">Returns</dt>
-<dd class="field-even"><p><dl class="simple">
-<dt>raw attention scores (unnormalized) for each src index</dt><dd><p><code class="docutils literal notranslate"><span class="pre">(batch,</span> <span class="pre">tgt_len,</span> <span class="pre">src_len)</span></code></p>
-</dd>
-</dl>
-</p>
-</dd>
-<dt class="field-odd">Return type</dt>
-<dd class="field-odd"><p>FloatTensor</p>
-</dd>
-</dl>
-</dd></dl>
-
-</dd></dl>
-
 </div>
 <div class="section" id="architecture-transformer">
 <h2>Architecture: Transformer<a class="headerlink" href="#architecture-transformer" title="Permalink to this headline">¶</a></h2>
@@ -648,158 +536,15 @@ <h2>Architecture: Transformer<a class="headerlink" href="#architecture-transform
 <h2>Architecture: Conv2Conv<a class="headerlink" href="#architecture-conv2conv" title="Permalink to this headline">¶</a></h2>
 <p>(These methods are from a user contribution
 and have not been thoroughly tested.)</p>
-<dl class="class">
-<dt id="mammoth.modules.ConvMultiStepAttention">
-<em class="property">class </em><code class="sig-prename descclassname">mammoth.modules.</code><code class="sig-name descname">ConvMultiStepAttention</code><span class="sig-paren">(</span><em class="sig-param">input_size</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/conv_multi_step_attention.html#ConvMultiStepAttention"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.ConvMultiStepAttention" title="Permalink to this definition">¶</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.modules.module.Module</span></code></p>
-<p>Conv attention takes a key matrix, a value matrix and a query vector.
-Attention weight is calculated by key matrix with the query vector
-and sum on the value matrix. And the same operation is applied
-in each decode conv layer.</p>
-<dl class="method">
-<dt id="mammoth.modules.ConvMultiStepAttention.apply_mask">
-<code class="sig-name descname">apply_mask</code><span class="sig-paren">(</span><em class="sig-param">mask</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/conv_multi_step_attention.html#ConvMultiStepAttention.apply_mask"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.ConvMultiStepAttention.apply_mask" title="Permalink to this definition">¶</a></dt>
-<dd><p>Apply mask</p>
-</dd></dl>
-
-<dl class="method">
-<dt id="mammoth.modules.ConvMultiStepAttention.forward">
-<code class="sig-name descname">forward</code><span class="sig-paren">(</span><em class="sig-param">base_target_emb</em>, <em class="sig-param">input_from_dec</em>, <em class="sig-param">encoder_out_top</em>, <em class="sig-param">encoder_out_combine</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/conv_multi_step_attention.html#ConvMultiStepAttention.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.ConvMultiStepAttention.forward" title="Permalink to this definition">¶</a></dt>
-<dd><dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>base_target_emb</strong> – target emb tensor</p></li>
-<li><p><strong>input_from_dec</strong> – output of decode conv</p></li>
-<li><p><strong>encoder_out_top</strong> – the key matrix for calculation of attetion weight,
-which is the top output of encode conv</p></li>
-<li><p><strong>encoder_out_combine</strong> – the value matrix for the attention-weighted sum,
-which is the combination of base emb and top output of encode</p></li>
-</ul>
-</dd>
-</dl>
-</dd></dl>
-
-</dd></dl>
-
-<dl class="class">
-<dt id="mammoth.modules.WeightNormConv2d">
-<em class="property">class </em><code class="sig-prename descclassname">mammoth.modules.</code><code class="sig-name descname">WeightNormConv2d</code><span class="sig-paren">(</span><em class="sig-param">in_channels</em>, <em class="sig-param">out_channels</em>, <em class="sig-param">kernel_size</em>, <em class="sig-param">stride=1</em>, <em class="sig-param">padding=0</em>, <em class="sig-param">dilation=1</em>, <em class="sig-param">groups=1</em>, <em class="sig-param">init_scale=1.0</em>, <em class="sig-param">polyak_decay=0.9995</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/weight_norm.html#WeightNormConv2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.WeightNormConv2d" title="Permalink to this definition">¶</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.modules.conv.Conv2d</span></code></p>
-<dl class="method">
-<dt id="mammoth.modules.WeightNormConv2d.forward">
-<code class="sig-name descname">forward</code><span class="sig-paren">(</span><em class="sig-param">x</em>, <em class="sig-param">init=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/weight_norm.html#WeightNormConv2d.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.WeightNormConv2d.forward" title="Permalink to this definition">¶</a></dt>
-<dd><p>Defines the computation performed at every call.</p>
-<p>Should be overridden by all subclasses.</p>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Although the recipe for forward pass needs to be defined within
-this function, one should call the <code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code> instance afterwards
-instead of this since the former takes care of running the
-registered hooks while the latter silently ignores them.</p>
-</div>
-</dd></dl>
-
-</dd></dl>
-
 </div>
 <div class="section" id="architecture-sru">
 <h2>Architecture: SRU<a class="headerlink" href="#architecture-sru" title="Permalink to this headline">¶</a></h2>
 </div>
 <div class="section" id="copy-attention">
 <h2>Copy Attention<a class="headerlink" href="#copy-attention" title="Permalink to this headline">¶</a></h2>
-<dl class="class">
-<dt id="mammoth.modules.CopyGenerator">
-<em class="property">class </em><code class="sig-prename descclassname">mammoth.modules.</code><code class="sig-name descname">CopyGenerator</code><span class="sig-paren">(</span><em class="sig-param">input_size</em>, <em class="sig-param">output_size</em>, <em class="sig-param">pad_idx</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/copy_generator.html#CopyGenerator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.CopyGenerator" title="Permalink to this definition">¶</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.modules.module.Module</span></code></p>
-<p>An implementation of pointer-generator networks
-<a class="bibtex reference internal" href="ref.html#dblp-journals-corr-seelm17" id="id7">[SLM17]</a>.</p>
-<p>These networks consider copying words
-directly from the source sequence.</p>
-<p>The copy generator is an extended version of the standard
-generator that computes three values.</p>
-<ul class="simple">
-<li><p><span class="math notranslate nohighlight">\(p_{softmax}\)</span> the standard softmax over <cite>tgt_dict</cite></p></li>
-<li><p><span class="math notranslate nohighlight">\(p(z)\)</span> the probability of copying a word from
-the source</p></li>
-<li><p><span class="math notranslate nohighlight">\(p_{copy}\)</span> the probility of copying a particular word.
-taken from the attention distribution directly.</p></li>
-</ul>
-<p>The model returns a distribution over the extend dictionary,
-computed as</p>
-<p><span class="math notranslate nohighlight">\(p(w) = p(z=1)  p_{copy}(w)  +  p(z=0)  p_{softmax}(w)\)</span></p>
-<div class="mermaid">
-            graph BT
-   A[input]
-   S[src_map]
-   B[softmax]
-   BB[switch]
-   C[attn]
-   D[copy]
-   O[output]
-   A --&gt; B
-   A --&gt; BB
-   S --&gt; D
-   C --&gt; D
-   D --&gt; O
-   B --&gt; O
-   BB --&gt; O
-        </div><dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>input_size</strong> (<em>int</em>) – size of input representation</p></li>
-<li><p><strong>output_size</strong> (<em>int</em>) – size of output vocabulary</p></li>
-<li><p><strong>pad_idx</strong> (<em>int</em>) – </p></li>
-</ul>
-</dd>
-</dl>
-<dl class="method">
-<dt id="mammoth.modules.CopyGenerator.forward">
-<code class="sig-name descname">forward</code><span class="sig-paren">(</span><em class="sig-param">hidden</em>, <em class="sig-param">attn</em>, <em class="sig-param">src_map</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/copy_generator.html#CopyGenerator.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.CopyGenerator.forward" title="Permalink to this definition">¶</a></dt>
-<dd><p>Compute a distribution over the target dictionary
-extended by the dynamic dictionary implied by copying
-source words.</p>
-<dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>hidden</strong> (<em>FloatTensor</em>) – hidden outputs <code class="docutils literal notranslate"><span class="pre">(batch</span> <span class="pre">x</span> <span class="pre">tlen,</span> <span class="pre">input_size)</span></code></p></li>
-<li><p><strong>attn</strong> (<em>FloatTensor</em>) – attn for each <code class="docutils literal notranslate"><span class="pre">(batch</span> <span class="pre">x</span> <span class="pre">tlen,</span> <span class="pre">slen)</span></code></p></li>
-<li><p><strong>src_map</strong> (<em>FloatTensor</em>) – A sparse indicator matrix mapping each source word to
-its index in the “extended” vocab containing.
-<code class="docutils literal notranslate"><span class="pre">(src_len,</span> <span class="pre">batch,</span> <span class="pre">extra_words)</span></code></p></li>
-</ul>
-</dd>
-</dl>
-</dd></dl>
-
-</dd></dl>
-
 </div>
 <div class="section" id="structured-attention">
 <h2>Structured Attention<a class="headerlink" href="#structured-attention" title="Permalink to this headline">¶</a></h2>
-<dl class="class">
-<dt id="mammoth.modules.structured_attention.MatrixTree">
-<em class="property">class </em><code class="sig-prename descclassname">mammoth.modules.structured_attention.</code><code class="sig-name descname">MatrixTree</code><span class="sig-paren">(</span><em class="sig-param">eps=1e-05</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/structured_attention.html#MatrixTree"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.structured_attention.MatrixTree" title="Permalink to this definition">¶</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.modules.module.Module</span></code></p>
-<p>Implementation of the matrix-tree theorem for computing marginals
-of non-projective dependency parsing. This attention layer is used
-in the paper “Learning Structured Text Representations”
-<a class="bibtex reference internal" href="ref.html#dblp-journals-corr-liul17d" id="id8">[LL17]</a>.</p>
-<dl class="method">
-<dt id="mammoth.modules.structured_attention.MatrixTree.forward">
-<code class="sig-name descname">forward</code><span class="sig-paren">(</span><em class="sig-param">input</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mammoth/modules/structured_attention.html#MatrixTree.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mammoth.modules.structured_attention.MatrixTree.forward" title="Permalink to this definition">¶</a></dt>
-<dd><p>Defines the computation performed at every call.</p>
-<p>Should be overridden by all subclasses.</p>
-<div class="admonition note">
-<p class="admonition-title">Note</p>
-<p>Although the recipe for forward pass needs to be defined within
-this function, one should call the <code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code> instance afterwards
-instead of this since the former takes care of running the
-registered hooks while the latter silently ignores them.</p>
-</div>
-</dd></dl>
-
-</dd></dl>
-
 </div>
 </div>
 
diff --git a/objects.inv b/objects.inv
index 5f67f66112eb021707aeb74de2e99feb3a6cb8db..d05e677914bf70f648a50dcf10307417b6c31aa9 100644
GIT binary patch
delta 1460
zcmV;l1xxyo4Ves(cYoh=+Bgt?=dbY0-78M$O@B<^nl|ZmrfGXk?p`&rwHOsy@<}p;
z`|EFIVX%oI#y0SfV5HselXh3Dl|@R^RM}bhs417AgKYIa7Sxzv5~S;{nN7uSzX#({
zT9*y2`*a#9*qLH&R2EclMlH0{N)-cP!f*GF_wK3H;C6JvSbycxl8kDKe4tGy=5VU?
zl4{mkITv8U4-3#Vfe+S#v|P!i@t3l#LBcpsb3rX%0HMCGaa2P9la!kIK<I}o_#)bE
z8UbTmCZ>}Ox9TCJ5fQmHeaA7>d_tipBbkO`F##hBh^^B7$G23PxtVqYy3d8>e*$H|
z`h{;3P(Rx>B7YXv!+ASMYOUr`j!C93YM!}VMVMBZ>T1a!%5Vop#jT7LTaWQ*J3fc$
zXT@xdN2XJ`0bg<hcL3~ei+uDz+W>bPI{H`|Oi&Qs08Jl&>Uh>P#8S?AuoXy%p<_Aj
zO#d~f(w2aq038Dl*kB=9^%z+lD5PvbWjx@}!bqriPJd*Adm_=00o{`iyP==Lnqf*;
z5s>wQ#PEqYV<3rDbCBj#JeKMNtjo?*{xl#-4R@Up27)dvgibIsV11B3WA*b{sECS(
zSn1~4uq~9v#lPykWUDYkEQ=c6FkxGR9MBn$Td^Jnte{}wkqk7To16>QD?UmvH4MnM
zkwnz`kbmns&!liDG2WwwrkMb;<aS0Zq!|{uHrGYOOmi6*?m3zw1Kidvce{M?wCZor
zeUNw@T9@`x$zFl}GK0y}l<L)KNtXFZ@CC>OU3}<(H4w$g{IJIFXP}B0E&+|^2p@`%
zy$AAxx}BNA$j!=c_n4ve!8r{~&7h<`E2|=wr+={T+;JjSU;e~OVv8lX8z%Dn^p`RC
zF1Jte7cv2!qHLyB48~l59M1%o;=Y9EGpY>`m7R5^UC6_i@vR1*zYQcLqwyT~)U)zi
zMUpH(FXQ>+c(Xeath?rC-6U_d_O&}a#gOeOg9o5|#4S;uxa_E$Z2Sz*%@@W3i$ao?
zC4bf_efXSQaAQ5Uxd6F)8-AgQV`{u4z0T$NNsXo2o}V_CgIYSbT$u1X=AE=W9K3i>
zxT6!|!M@$?-mYSV`&s+D!CmdCR_Wf-<3()&RJ&^rqFdjo?m`#~9VcGixsdJ<_=h{Q
zrbij?!D2KR%VTqdqqj;SjpJO?c-4(Fq<=z?gxVQyv$%S!83=~ByxS2A1kDVv;=tGu
zLa2mTkVff99U06O6g;hFdDmHFndqqL-?Mg9egb*ah=npH{EGiRsuU*+H`KtvK>Yaq
z+vCqKH-A5V`Sv+5mc3^x4#tz9+FnQpAC>f1mr9!vEw3R-q&ag1b4B-Arw7>H)qma;
z%_BU?PkBQ8M-`yfv0=yO@mkg@B}Oa~+52BA*OCfz6)UZ;Xk^Bdch|4S*Vk|Um=utL
zMwrGCRKwIy+v$x<y|L39@92#WV80t*zkNM<bu@OV*NDBKW=SPCS^g`TXOp+>(VuCK
zNAOg<2R2vL*YXwXHuZZWR~kkMR)6b~!Bc%GWviF};vPp^u{A+O!UYXF-JlRA5+Ukh
zMN<h?eDT~OS2B5ZJ$W7NF@z6%UYnNt?y<h(KD^+tEZ0U|YP$e}JNLVrrPA}wD&i7X
zP4P>d2Axf(*ZxE;4ZdQjfGxW>%}wgnyW`;Ae`!^tG=aTXg9N^s88y69J%5DP7z?Sp
z(|;BV*=k&NROhsg3RN7m%#}Y|FH}sUDsMx#2PxhV2miQ+4aRdb)O%_ZF1!2>z#4Rl
zj?E;wtZI0h?$KHsyk;I4xtA7H17R*4DvH&1uk*vV6oE7PH|O!(SGX0WZL3%LYx8}C
z!C#du*Fx|Url3K_7)<^is4ecL_1)0gLPh@%ipkf2l$ysgTxM?s7D{=s;~ZFT%iM`3
ON+kjgi2V;#%f1w}a@dIg

delta 1581
zcmV+|2GaSN43Q0xcYob;+Bgt@=dWPq?iDBWrXSO{CT(+_Y0_T9-92h#YcVRa<db9w
z{p)XKVX%oI#y0Sf0MhPnKeVe|Sx#t@C_4)#nsNzxn6BPOf*La%50mxO%qHTu--n|?
zQl$;7`*a)$*qLHYSQb=pMlCesN)!WOf`|Lbz5CWkxgDM$Qh%AWB&C|7JkX|<<Y21w
zl4{nNITK)l4-3#Vh7Z<)v|PzL{R`U0AVHKRnV^<0fKWeIt5qMsB%x;B6Z#<!zKC|K
z<l;UPmOolZyZN&oDxhg9Rvql!1KZK+bae-qHq<KJ(%2G0H)TF0Auz^eY+7luWe*?;
ziO8(!sz+b;1b;$NMp6y=eg=#zAVPO_*KesbbC$LPx)Um-qFb=<V081?E|`gwZn^j5
zner!42CSQbbwJ&$tVPTt1J~3+QfoC2Gpu1<QM1&QLc+94Rof`}t_*izRD6<=Vrw0b
z_TzJyZeIKscs#QYN+)h(e8pNRwF}ZsrF3l~nNH=#-hU-G@EL%8-Xb5ZW)t9UsvE1b
z$YvV%Cc259dx~-RG|>R9w0;4zox+$W(qIn(!41&l5vY!4b=nnlPG%#J0MpNMY#x8r
z?>G$sKLQ3fJmP?bc-29(Do{Y#g372z)q+r{Xij8|Z7b1`0^L(D+o7Mrn?XWXA&~W^
zhT#)(#(zK(tL7lhspMFyW3aAzPx(`iLMpiHU9~6Z!b7+SW(KSa@*N&fd|7^U#aMTU
z=sQe$w(6X+WtcwFqJq~?*j6BWT#U(<tb+qfDA<_BJt=gPaltypM**h#0oe|ckXj#d
zZS9Qn1trF>sG&(JfGoM45erG`4nLW1RHvDY@_*$yT0{!i8d>f-rTp8fyF>Ru29a8y
zzNM0#0{vwPCQA~kSEm)S%ua$YK*qSlhn|7-B;sW2u*UD^pb{CTfJS!&kNL;mXJ!X^
zTT2Q;XO%DaxI^oMHuWr;K_T|6QWdd0fqiGU6G`>$PlP8T(3W5|jOF>oU&i6Pvi&B1
zA%7R(vCpPjMPSSYgyWguQnD|x@+s8@h)U0zX%|}I%dD*izkeHuNlK$R*3`4gw~EAR
zc3z3+kK;{sB-nS&&srouY3+M=c#0t1(+2k-a)KpMpSbNPoou{?=av^l0-HjT6fM>%
z*YLS;!HxB@%>{(J+mZ_zj$7j;#p_I-Uw^2vRNM25&1A2h&MX%u_=fvVQXCFmyvpC<
z3GiUw>;it5F~oY-{Q7>Ed#Y8kxAl0DTL9JW>WJvZuRZKS7z-^8FYc{KdkFl)%B<;8
z&U>^N4aV}w9I?<_rhrCKrfIZlCo`ZzkeJ#TmRZamYkGnqZr^Q*1%jpqSbkt^34b9}
zOe{#Fbf}ID<`N2?R@1Dl6=^D3a{A}2Et#J{9;LBR#spvS->pi}Sh%4E4hG`ww};7(
zFE@WrzC7Fwjb-ndiiV@{usohq4{w$9*S|`e!RF$OgqkzgFqe3rO?rUsefLd1JiwFu
zl*hz>lmRLe8|nCcw5GL6i2(~m`hWhX%Cw}yTt!OjD;k>7_}%sE(e?G4KgKzvpdoJK
z2&!S~&31gFf^X#bMmv0?1K96I*Kc2sUmcBI@YP}qo{8~W)*Q58s6{gXA{oDJj#e<#
zqAjRdQpru4{fy`7F|f}#M+11upHZ8u@|8hp<?H5s0F(v>F`gtOgQu!0#eb-$|Kh$5
zwq$DoB?=eN7?1{uaFY|FN|p~5Q27rpTVzVcudc_h!#$4hNy#f_x!)e^8}<kb4$C5M
zt5n<M9Da7M?Jbp_Z!(-q%;@|@z8ap*r04!fEggQvHU>v_FTm^A%lF#Dd;g<R|5OL|
zVt+LBoz|e{dG8^-+E_^3JAWI`Vj*3PiixsK>!?uqA<$g;bN@m`G%U-lnf4(0)9c|s
zZeWA)+zr*4iowN{e}A-wC!g3D$z_?t`*aW2+Tj)Vz`#9Fp&AHt;ZR~Mk9(f)ek2c^
z!G9T#=DzzbF>OaZ%U_${0}TE$WswWRA20<CGQwc;cU09?THjl(JtmaTf2UX&3`nVY
fG{ZD|Be03f@@jY>-<Fvpjg^W893cB2W<g8uh~*Ts

diff --git a/options/train.html b/options/train.html
index 92c22bdc..377d425f 100644
--- a/options/train.html
+++ b/options/train.html
@@ -272,13 +272,12 @@ <h1>Train<a class="headerlink" href="#train" title="Permalink to this headline">
                 <span class="p">[</span><span class="o">--</span><span class="n">enc_layers</span> <span class="n">ENC_LAYERS</span> <span class="p">[</span><span class="n">ENC_LAYERS</span> <span class="o">...</span><span class="p">]]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">dec_layers</span> <span class="n">DEC_LAYERS</span> <span class="p">[</span><span class="n">DEC_LAYERS</span> <span class="o">...</span><span class="p">]]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">rnn_size</span> <span class="n">RNN_SIZE</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">pos_ffn_activation_fn</span> <span class="p">{</span><span class="n">relu</span><span class="p">,</span><span class="n">gelu</span><span class="p">}]</span>
-                <span class="p">[</span><span class="o">--</span><span class="n">bridge</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">context_gate</span> <span class="p">{</span><span class="n">source</span><span class="p">,</span><span class="n">target</span><span class="p">,</span><span class="n">both</span><span class="p">}]</span>
-                <span class="p">[</span><span class="o">--</span><span class="n">bridge_extra_node</span> <span class="n">BRIDGE_EXTRA_NODE</span><span class="p">]</span>
+                <span class="p">[</span><span class="o">--</span><span class="n">bridge</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">bridge_extra_node</span> <span class="n">BRIDGE_EXTRA_NODE</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">bidir_edges</span> <span class="n">BIDIR_EDGES</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">state_dim</span> <span class="n">STATE_DIM</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">n_edge_types</span> <span class="n">N_EDGE_TYPES</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">n_node</span> <span class="n">N_NODE</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">n_steps</span> <span class="n">N_STEPS</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">src_ggnn_size</span> <span class="n">SRC_GGNN_SIZE</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">global_attention</span> <span class="p">{</span><span class="n">dot</span><span class="p">,</span><span class="n">general</span><span class="p">,</span><span class="n">mlp</span><span class="p">,</span><span class="n">none</span><span class="p">}]</span>
-                <span class="p">[</span><span class="o">--</span><span class="n">global_attention_function</span> <span class="p">{</span><span class="n">softmax</span><span class="p">,</span><span class="n">sparsemax</span><span class="p">}]</span>
+                <span class="p">[</span><span class="o">--</span><span class="n">global_attention_function</span> <span class="p">{</span><span class="n">softmax</span><span class="p">}]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">self_attn_type</span> <span class="n">SELF_ATTN_TYPE</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">max_relative_positions</span> <span class="n">MAX_RELATIVE_POSITIONS</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">heads</span> <span class="n">HEADS</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">transformer_ff</span> <span class="n">TRANSFORMER_FF</span><span class="p">]</span>
@@ -286,7 +285,7 @@ <h1>Train<a class="headerlink" href="#train" title="Permalink to this headline">
                 <span class="p">[</span><span class="o">--</span><span class="n">alignment_layer</span> <span class="n">ALIGNMENT_LAYER</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">alignment_heads</span> <span class="n">ALIGNMENT_HEADS</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">full_context_alignment</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">copy_attn</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">copy_attn_type</span> <span class="p">{</span><span class="n">dot</span><span class="p">,</span><span class="n">general</span><span class="p">,</span><span class="n">mlp</span><span class="p">,</span><span class="n">none</span><span class="p">}]</span>
-                <span class="p">[</span><span class="o">--</span><span class="n">generator_function</span> <span class="p">{</span><span class="n">softmax</span><span class="p">,</span><span class="n">sparsemax</span><span class="p">}]</span> <span class="p">[</span><span class="o">--</span><span class="n">copy_attn_force</span><span class="p">]</span>
+                <span class="p">[</span><span class="o">--</span><span class="n">generator_function</span> <span class="p">{</span><span class="n">softmax</span><span class="p">}]</span> <span class="p">[</span><span class="o">--</span><span class="n">copy_attn_force</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">reuse_copy_attn</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">copy_loss_by_seqlength</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">coverage_attn</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">lambda_coverage</span> <span class="n">LAMBDA_COVERAGE</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">loss_scale</span> <span class="n">LOSS_SCALE</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">apex_opt_level</span> <span class="p">{</span><span class="n">O0</span><span class="p">,</span><span class="n">O1</span><span class="p">,</span><span class="n">O2</span><span class="p">,</span><span class="n">O3</span><span class="p">}]</span>
@@ -323,7 +322,7 @@ <h1>Train<a class="headerlink" href="#train" title="Permalink to this headline">
                 <span class="p">[</span><span class="o">--</span><span class="n">train_steps</span> <span class="n">TRAIN_STEPS</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">single_pass</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">epochs</span> <span class="n">EPOCHS</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">early_stopping</span> <span class="n">EARLY_STOPPING</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">early_stopping_criteria</span> <span class="p">[</span><span class="n">EARLY_STOPPING_CRITERIA</span> <span class="p">[</span><span class="n">EARLY_STOPPING_CRITERIA</span> <span class="o">...</span><span class="p">]]]</span>
-                <span class="p">[</span><span class="o">--</span><span class="n">optim</span> <span class="p">{</span><span class="n">sgd</span><span class="p">,</span><span class="n">adagrad</span><span class="p">,</span><span class="n">adadelta</span><span class="p">,</span><span class="n">adam</span><span class="p">,</span><span class="n">adamw</span><span class="p">,</span><span class="n">sparseadam</span><span class="p">,</span><span class="n">adafactor</span><span class="p">,</span><span class="n">fusedadam</span><span class="p">}]</span>
+                <span class="p">[</span><span class="o">--</span><span class="n">optim</span> <span class="p">{</span><span class="n">sgd</span><span class="p">,</span><span class="n">adagrad</span><span class="p">,</span><span class="n">adadelta</span><span class="p">,</span><span class="n">adam</span><span class="p">,</span><span class="n">adamw</span><span class="p">,</span><span class="n">adafactor</span><span class="p">,</span><span class="n">fusedadam</span><span class="p">}]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">adagrad_accumulator_init</span> <span class="n">ADAGRAD_ACCUMULATOR_INIT</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">max_grad_norm</span> <span class="n">MAX_GRAD_NORM</span><span class="p">]</span> <span class="p">[</span><span class="o">--</span><span class="n">weight_decay</span> <span class="n">WEIGHT_DECAY</span><span class="p">]</span>
                 <span class="p">[</span><span class="o">--</span><span class="n">dropout</span> <span class="n">DROPOUT</span> <span class="p">[</span><span class="n">DROPOUT</span> <span class="o">...</span><span class="p">]]</span>
@@ -737,10 +736,6 @@ <h2>Model- Encoder-Decoder<a class="headerlink" href="#Model- Encoder-Decoder" t
 <dd><p>Have an additional layer between the last encoder state and the first decoder state</p>
 <p>Default: False</p>
 </dd>
-<dt><kbd>--context_gate, -context_gate</kbd></dt>
-<dd><p>Possible choices: source, target, both</p>
-<p>Type of context gate to use. Do not select for no context gate.</p>
-</dd>
 <dt><kbd>--bridge_extra_node, -bridge_extra_node</kbd></dt>
 <dd><p>Graph encoder bridges only extra node to decoder as input</p>
 <p>Default: True</p>
@@ -780,7 +775,7 @@ <h2>Model- Attention<a class="headerlink" href="#Model- Attention" title="Permal
 <p>Default: “general”</p>
 </dd>
 <dt><kbd>--global_attention_function, -global_attention_function</kbd></dt>
-<dd><p>Possible choices: softmax, sparsemax</p>
+<dd><p>Possible choices: softmax</p>
 <p>Default: “softmax”</p>
 </dd>
 <dt><kbd>--self_attn_type, -self_attn_type</kbd></dt>
@@ -840,8 +835,8 @@ <h2>Generator<a class="headerlink" href="#Generator" title="Permalink to this he
 <p>The copy attention type to use. Leave as None to use the same as -global_attention.</p>
 </dd>
 <dt><kbd>--generator_function, -generator_function</kbd></dt>
-<dd><p>Possible choices: softmax, sparsemax</p>
-<p>Which function to use for generating probabilities over the target vocabulary (choices: softmax, sparsemax)</p>
+<dd><p>Possible choices: softmax</p>
+<p>Which function to use for generating probabilities over the target vocabulary (choices: softmax)</p>
 <p>Default: “softmax”</p>
 </dd>
 <dt><kbd>--copy_attn_force, -copy_attn_force</kbd></dt>
@@ -1080,7 +1075,7 @@ <h2>Optimization- Type<a class="headerlink" href="#Optimization- Type" title="Pe
 <dd><p>Criteria to use for early stopping.</p>
 </dd>
 <dt><kbd>--optim, -optim</kbd></dt>
-<dd><p>Possible choices: sgd, adagrad, adadelta, adam, adamw, sparseadam, adafactor, fusedadam</p>
+<dd><p>Possible choices: sgd, adagrad, adadelta, adam, adamw, adafactor, fusedadam</p>
 <p>Optimization method.</p>
 <p>Default: “sgd”</p>
 </dd>
diff --git a/ref.html b/ref.html
index 5c2babf4..7f5457a4 100644
--- a/ref.html
+++ b/ref.html
@@ -182,15 +182,9 @@
 <h1>References<a class="headerlink" href="#references" title="Permalink to this headline">¶</a></h1>
 <p>References</p>
 <p id="bibtex-bibliography-ref-0"><dl class="citation">
-<dt class="bibtex label" id="dblp-journals-corr-liul17d"><span class="brackets">LL17</span></dt>
-<dd><p>Yang Liu and Mirella Lapata. Learning structured text representations. <em>CoRR</em>, 2017. URL: <a class="reference external" href="http://arxiv.org/abs/1705.09207">http://arxiv.org/abs/1705.09207</a>, <a class="reference external" href="https://arxiv.org/abs/1705.09207">arXiv:1705.09207</a>.</p>
-</dd>
 <dt class="bibtex label" id="luong2015b"><span class="brackets">LSL+15</span></dt>
 <dd><p>Minh-Thang Luong, Ilya Sutskever, Quoc Le, Oriol Vinyals, and Wojciech Zaremba. Addressing the Rare Word Problem in Neural Machine Translation. In <em>Proc of ACL</em>. 2015.</p>
 </dd>
-<dt class="bibtex label" id="dblp-journals-corr-seelm17"><span class="brackets">SLM17</span></dt>
-<dd><p>Abigail See, Peter J. Liu, and Christopher D. Manning. Get to the point: summarization with pointer-generator networks. <em>CoRR</em>, 2017. URL: <a class="reference external" href="http://arxiv.org/abs/1704.04368">http://arxiv.org/abs/1704.04368</a>, <a class="reference external" href="https://arxiv.org/abs/1704.04368">arXiv:1704.04368</a>.</p>
-</dd>
 <dt class="bibtex label" id="sennrich2016linguistic"><span class="brackets">SH16</span></dt>
 <dd><p>Rico Sennrich and Barry Haddow. Linguistic input features improve neural machine translation. <em>arXiv preprint arXiv:1606.02892</em>, 2016.</p>
 </dd>
diff --git a/searchindex.js b/searchindex.js
index 78ff8c13..87806067 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["CONTRIBUTING","attention_bridges","config_config","examples/Translation","index","install","main","mammoth","mammoth.inputters","mammoth.modules","mammoth.translate.translation_server","mammoth.translation","options/build_vocab","options/server","options/train","options/translate","prepare_data","quickstart","ref"],envversion:{"sphinx.domains.c":1,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":1,"sphinx.domains.index":1,"sphinx.domains.javascript":1,"sphinx.domains.math":2,"sphinx.domains.python":1,"sphinx.domains.rst":1,"sphinx.domains.std":1,"sphinx.ext.viewcode":1,sphinx:56},filenames:["CONTRIBUTING.md","attention_bridges.md","config_config.md","examples/Translation.md","index.rst","install.md","main.md","mammoth.rst","mammoth.inputters.rst","mammoth.modules.rst","mammoth.translate.translation_server.rst","mammoth.translation.rst","options/build_vocab.rst","options/server.rst","options/train.rst","options/translate.rst","prepare_data.md","quickstart.md","ref.rst"],objects:{"mammoth.Trainer":{train:[7,1,1,""],validate:[7,1,1,""]},"mammoth.models":{NMTModel:[7,0,1,""]},"mammoth.models.NMTModel":{count_parameters:[7,1,1,""],forward:[7,1,1,""]},"mammoth.modules":{AverageAttention:[9,0,1,""],ConvMultiStepAttention:[9,0,1,""],CopyGenerator:[9,0,1,""],Embeddings:[9,0,1,""],GlobalAttention:[9,0,1,""],MultiHeadedAttention:[9,0,1,""],PositionalEncoding:[9,0,1,""],WeightNormConv2d:[9,0,1,""]},"mammoth.modules.AverageAttention":{cumulative_average:[9,1,1,""],cumulative_average_mask:[9,1,1,""],forward:[9,1,1,""]},"mammoth.modules.ConvMultiStepAttention":{apply_mask:[9,1,1,""],forward:[9,1,1,""]},"mammoth.modules.CopyGenerator":{forward:[9,1,1,""]},"mammoth.modules.Embeddings":{emb_luts:[9,1,1,""],forward:[9,1,1,""],load_pretrained_vectors:[9,1,1,""],word_lut:[9,1,1,""]},"mammoth.modules.GlobalAttention":{forward:[9,1,1,""],score:[9,1,1,""]},"mammoth.modules.MultiHeadedAttention":{forward:[9,1,1,""],training:[9,2,1,""],update_dropout:[9,1,1,""]},"mammoth.modules.PositionalEncoding":{forward:[9,1,1,""]},"mammoth.modules.WeightNormConv2d":{forward:[9,1,1,""]},"mammoth.modules.position_ffn":{PositionwiseFeedForward:[9,0,1,""]},"mammoth.modules.position_ffn.PositionwiseFeedForward":{forward:[9,1,1,""]},"mammoth.modules.structured_attention":{MatrixTree:[9,0,1,""]},"mammoth.modules.structured_attention.MatrixTree":{forward:[9,1,1,""]},"mammoth.translate":{BeamSearch:[11,0,1,""],DecodeStrategy:[11,0,1,""],GNMTGlobalScorer:[11,0,1,""],GreedySearch:[11,0,1,""],Translation:[11,0,1,""],TranslationBuilder:[11,0,1,""],Translator:[11,0,1,""]},"mammoth.translate.BeamSearch":{initialize:[11,1,1,""]},"mammoth.translate.DecodeStrategy":{advance:[11,1,1,""],block_ngram_repeats:[11,1,1,""],initialize:[11,1,1,""],maybe_update_forbidden_tokens:[11,1,1,""],maybe_update_target_prefix:[11,1,1,""],target_prefixing:[11,1,1,""],update_finished:[11,1,1,""]},"mammoth.translate.GreedySearch":{advance:[11,1,1,""],initialize:[11,1,1,""],update_finished:[11,1,1,""]},"mammoth.translate.Translation":{log:[11,1,1,""]},"mammoth.translate.Translator":{translate_batch:[11,1,1,""]},"mammoth.translate.greedy_search":{sample_with_temperature:[11,3,1,""]},"mammoth.translate.penalties":{PenaltyBuilder:[11,0,1,""]},"mammoth.translate.penalties.PenaltyBuilder":{coverage_none:[11,1,1,""],coverage_summary:[11,1,1,""],coverage_wu:[11,1,1,""],length_average:[11,1,1,""],length_none:[11,1,1,""],length_wu:[11,1,1,""]},"mammoth.translate.translation_server":{ServerModel:[10,0,1,""],ServerModelError:[10,4,1,""],Timer:[10,0,1,""],TranslationServer:[10,0,1,""]},"mammoth.translate.translation_server.ServerModel":{build_tokenizer:[10,1,1,""],detokenize:[10,1,1,""],do_timeout:[10,1,1,""],maybe_convert_align:[10,1,1,""],maybe_detokenize:[10,1,1,""],maybe_detokenize_with_align:[10,1,1,""],maybe_postprocess:[10,1,1,""],maybe_preprocess:[10,1,1,""],maybe_tokenize:[10,1,1,""],parse_opt:[10,1,1,""],postprocess:[10,1,1,""],preprocess:[10,1,1,""],rebuild_seg_packages:[10,1,1,""],to_gpu:[10,1,1,""],tokenize:[10,1,1,""],tokenizer_marker:[10,1,1,""]},"mammoth.translate.translation_server.TranslationServer":{clone_model:[10,1,1,""],list_models:[10,1,1,""],load_model:[10,1,1,""],preload_model:[10,1,1,""],run:[10,1,1,""],start:[10,1,1,""],unload_model:[10,1,1,""]},"mammoth.utils":{Optimizer:[7,0,1,""],Statistics:[7,0,1,""]},"mammoth.utils.Optimizer":{amp:[7,1,1,""],backward:[7,1,1,""],from_opt:[7,1,1,""],learning_rate:[7,1,1,""],step:[7,1,1,""],training_step:[7,1,1,""],zero_grad:[7,1,1,""]},"mammoth.utils.Statistics":{accuracy:[7,1,1,""],all_gather_stats:[7,1,1,""],all_gather_stats_list:[7,1,1,""],elapsed_time:[7,1,1,""],log_tensorboard:[7,1,1,""],output:[7,1,1,""],ppl:[7,1,1,""],update:[7,1,1,""],xent:[7,1,1,""]},"mammoth.utils.loss":{LossComputeBase:[7,0,1,""]},mammoth:{Trainer:[7,0,1,""]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","function","Python function"],"4":["py","exception","Python exception"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:function","4":"py:exception"},terms:{"25g":5,"boolean":[7,11],"break":16,"class":[0,4,7,9,10],"default":[10,12,13,14,15,16],"export":5,"final":[1,3,11],"float":[2,9,11],"function":[0,1,2,7,9,10,11,14],"import":0,"int":[7,9,10,11],"long":0,"new":[0,1,3],"public":5,"return":[0,7,9,10,11],"static":[7,14],"true":[2,3,7,11,14,15,16],"try":[0,5],"while":[2,9],And:[0,9],EOS:11,For:[0,2,11,14,17],IDs:11,IFS:16,LPs:2,Not:0,One:2,The:[1,3,7,9,10,11,14,15],Then:[0,3,9],There:[1,2],These:[1,2,9,11],Use:[2,14,15],Used:11,Will:2,__init__:10,_compute_loss:7,a_j:9,aan:14,aan_useffn:[9,14],ab_fixed_length:14,ab_lay:14,ab_layer_norm:14,abbrevi:0,abigail:18,abil:9,about:0,abov:[0,11],abs:[1,14,15,18],acceler:[9,18],accept:[0,2,11],access:[1,2,5],accord:2,account:[2,5],accross:7,accum:7,accum_count:[3,7,14],accum_step:[3,7,14],accumul:[7,14],accuraci:[7,11],achiev:2,achin:18,acl:[6,18],aclweb:14,action:[9,11,14],activ:[1,5,9,14],activation_fn:9,activationfunct:9,actual:11,adadelta:14,adafactor:14,adagrad:14,adagrad_accumulator_init:14,adam:[3,14],adam_beta1:14,adam_beta2:[3,14],adamoptim:14,adamw:14,adapt:[4,6],adapter_nam:2,add:[0,3,9],added:2,adding:0,addit:[0,9,12,14,15],addition:9,address:11,adjust:2,adopt:14,advanc:[11,14],advic:0,after:[0,1,11,14],afterward:9,again:0,aidan:18,alexand:6,algorithm:18,align:[4,7,10,11,15],align_debug:15,alignment_head:14,alignment_lay:14,aliv:11,alive_attn:11,alive_seq:11,all:[0,2,7,9,11,12,14,15,18],all_gather_stat:7,all_gather_stats_list:7,all_preprocess:10,allennlp:0,alloc:2,allow:[0,1,2,14],almost:[11,14],alon:0,along:1,alpha:[1,11,15],alphabet:2,alreadi:[12,14,15],also:[0,2,5,7,9,14],although:9,alwai:[0,2],amp:[7,14],ani:[0,2,11,12,14],anoth:[0,1,7],antholog:14,apex:14,apex_opt_level:14,api:[0,4],api_doc:14,appear:2,append:[5,16],appli:[1,2,9,11,12,14,15],applic:15,apply_mask:9,appropri:11,approxim:14,architectur:[1,4],arg:[0,10],argmax:15,argpars:10,argument:[0,4],arxiv:[0,1,14,15,18],ashish:18,assig:2,assign:[2,15],assing:2,assum:[9,11],att_typ:1,attend:1,attent:[0,4,7,11,15,18],attention_bridg:7,attention_dropout:[3,14],attentionbridgenorm:1,attet:9,attn:[9,11,15],attn_debug:[11,15],attn_func:9,attn_typ:9,attr:10,attribut:11,augment:18,author:6,autodoc:0,autogener:14,avail:[7,10,14,15],available_model:13,averag:[9,14,15,18],average_decai:[3,7,14],average_everi:[7,14],average_output:9,averageattent:9,avg:15,avg_raw_prob:15,avoid:[0,2],aws:5,axi:11,back:7,backend:14,backward:7,bahdanau:[9,14],ban_unk_token:[11,15],barri:18,bart:[12,14,15],base:[0,1,2,3,5,6,7,9,10,11,12,14,15],base_target_emb:9,baselin:14,basemodel:7,basenam:[3,16],bash:5,batch:[1,3,7,9,11,14,15],batch_siz:[3,9,11,14,15],batch_size_multipl:[3,14],batch_typ:[3,14,15],beam:[4,11],beam_search:11,beam_siz:[3,11,15],beamsearch:11,beamsearchbas:11,becaus:[2,15],becom:2,been:[9,11,12,14,15],befor:[0,3,10,11,14,15],begin:[7,11],below:0,ben:2,bengali:2,best:[11,15],beta1:14,beta2:14,beta:[11,15],better:[0,12,14,15],between:[1,12,14,15,18],beyond:7,biao:18,bib:0,bibtex:0,bibtext:0,bidir_edg:14,bidirect:14,bin:[5,14],binari:[3,9],bit:15,blank:0,bleu:3,block:[11,15],block_ngram_repeat:[11,15],booktitl:6,bool:[7,9,10,11],bos:11,both:[2,11,14],both_embed:14,boundari:[12,14,15],bpe:[12,14,15],bptt:[7,14],bridg:[4,18],bridge_extra_nod:14,browser:0,bucket_s:[3,14],buffer:7,build:[0,4,7,9,10,11,15,16],build_token:10,build_vocab:12,built:7,bytetensor:11,cach:9,calcul:[1,7,9,11],call:[9,11],callabl:11,callback:7,can:[1,2,3,5,7,10,11,12,14,15],cancel:10,candid:[2,12,14,15],cao:18,capit:0,captur:1,care:9,cat:16,categor:11,categori:11,challeng:4,chang:[0,2,7,14],channel:1,charact:[0,15],character_coverag:16,check:[0,6,17],checklist:0,checkpoint:[3,7,14],chen:18,chmod:[3,5],choic:[0,9,12,14,15],choos:[0,12,14,15],chosen:11,christoph:18,citat:[0,4],cite:[0,6],classmethod:7,clear:0,clone:[6,10,17],clone_model:10,close:0,cls:7,cluster:[2,6,17],clutter:0,code:[0,2,5,15],code_dir:5,codebas:5,column:2,com:[6,17],combin:[9,15],comma:2,command:[3,4],comment:0,commentari:3,common:[0,4],commoncrawl:3,commun:0,complet:11,complex:[2,11],compon:[1,2],composit:14,comput:[1,2,3,7,9,14,15],concat:[9,14],condit:[11,14,15],conf:[13,15],config:[3,4,10,12,13,14,15],config_fil:10,configur:[2,3,4],connect:1,consid:[2,9,16],consider:14,consist:0,constant:2,construct:9,constructor:0,consum:14,contain:[2,9,10,11],content:[0,15],context:[1,9,14],context_g:14,continu:0,contribut:[0,1,9],contributor:4,control:[2,7],conv2conv:4,conv2d:9,conv:9,conveni:2,convent:0,convers:11,convert:10,convex:9,convmultistepattent:9,copi:[0,2,4,5,14,15],copy_attn:[11,14],copy_attn_forc:14,copy_attn_typ:14,copy_loss_by_seqlength:14,copygener:9,core:[1,4,7],corpora:3,corpu:[2,3,12,14,16],corr:[0,18],correct:2,correspand:10,correspond:[1,15],could:11,count:[2,7,11,12,14,15],count_paramet:7,cov:11,cov_pen:11,coverag:[9,11,14,15],coverage_attn:14,coverage_non:11,coverage_penalti:[11,15],coverage_summari:11,coverage_wu:11,cpu:[10,14,15],crai:5,crayon:14,creat:[2,5,7],creation:2,criteria:14,criterion:7,critic:[14,15],cross:[7,14],csc:16,csv:2,ct2_model:10,ct2_translate_batch_arg:10,ct2_translator_arg:10,ctrl:0,cumbersom:2,cumul:[9,11,15],cumulative_averag:9,cumulative_average_mask:9,cur_dir:16,cur_len:11,current:[2,7,9,11,14],curricula:2,curriculum:2,custom:[10,14],custom_opt:10,cut:[0,16],cutoff:11,d_ff:9,d_model:9,dai:18,data:[1,2,4,7,11,18],data_path:16,data_typ:[7,11,14,15],dataset:[3,4,12,14,15,16],datastructur:10,dblp:0,ddress:18,deal:2,debug:[13,14,15],dec:2,dec_lay:[3,14],decai:14,decay_method:[3,14],decay_step:14,decod:[1,2,4,7],decode_strategi:11,decoder_typ:[3,14],decoderbas:7,decodestrategi:11,def:0,defin:[2,3,9,12,14,15],definit:9,delai:2,delet:[12,14,15],delimit:15,deng:6,denois:[2,4],denoising_object:[12,14,15],denot:1,depend:[0,2,5,7,9,10],deprec:[14,15],describ:[1,9,10,14],descript:0,desir:[2,3],detail:[6,12,14],determin:2,detoken:[3,10],dev:[5,16],develop:0,devic:[2,9,11,15],device_context:7,deyi:18,diagon:2,dict:[2,7,10,11,12,14,15],dict_kei:14,dictionari:[7,9,11,14],differ:[0,1,2,9,10,15],dilat:9,dim:9,dimens:[1,9,11,14],dimension:[1,9],dir:16,direct:[0,2,11],directli:[0,9,15],directori:[2,5,10,14],disabl:14,discard:14,discourag:14,disk:14,displai:7,dist:7,distanc:14,distribtut:9,distribut:[2,7,9,11,12,14,15],divers:[1,12,14,15],divid:[1,2,14,15],divis:9,do_timeout:10,doc:0,document:[0,6],doe:[2,15],doesn:16,doi:6,doing:[2,15],don:0,done:[3,11,16],dot:[1,9,14],dotprod:14,down:[11,12],download:5,dropout:[3,7,9,12,14,15],dropout_step:[3,7,14],due:14,dump:[12,14,15],dump_beam:[11,15],dump_sampl:12,dump_transform:14,dure:[10,14,15],dynam:[4,9,15],each:[1,2,9,11,12,14,15],earli:14,earlier:[1,12,14,15],early_stop:14,early_stopping_criteria:14,earlystopp:7,eas:2,easi:0,easili:2,echo:[3,16],edg:14,effect:[1,10,12],effici:[4,7,18],either:[11,14],elaps:7,elapsed_tim:7,element:[1,2],els:16,emb:9,emb_fil:9,emb_lut:9,embed:[1,4,9,12],embedding_s:9,embeddings_typ:14,emerg:1,emploi:[1,7],empti:[3,11,12,14],enabl:15,enc:2,enc_lay:[3,14],encapsul:1,encod:[1,2,4,7,11],encoder_out_combin:9,encoder_out_top:9,encoder_typ:[3,14],encoderbas:7,encordec:[12,14],encount:[12,14],encout:[12,14],end:11,eng:2,english:[2,3,16],enhanc:1,ensembl:15,ensur:1,entir:16,entri:0,entropi:7,env_dir:5,environ:5,eos:11,epoch:14,eps:9,epsilon:14,equal:[11,14],equat:9,equival:14,error:[0,12,14,15],especi:2,essenti:11,establish:1,eural:18,europarl:3,evalu:7,even:2,event:11,everi:[7,9,14,15],exactli:0,exampl:[0,2,3,12,14,17],exce:14,except:[0,10,12,14,15],exclusion_token:11,execut:[3,12,14],exist:[12,14,15,16],exp:14,exp_host:14,expect:[2,11],experi:[12,14,15],experiment:14,exponenti:14,extend:[0,9],extern:0,extra:[5,14],extra_word:9,extract:16,facilit:1,fail:11,fairseq:0,fals:[7,9,10,11,12,13,14,15],familiar:6,faster:14,feat_0:15,feat_1:15,feat_dim_expon:9,feat_merg:[9,14],feat_merge_s:14,feat_padding_idx:9,feat_vec_expon:[9,14],feat_vec_s:[9,14],feat_vocab_s:9,feats0:15,feats1:15,featur:[1,4,7,9,12,15,18],fed:1,feed:[2,9,14],feedforward:[1,14],feedforwardattentionbridgelay:4,feel:0,few:0,ffn:[9,14],figur:9,file:[0,2,10,12,14,15,16],filenam:14,filter:[3,4,16],filterfeat:[12,14,15],filtertoolong:[2,3,12,14,15],find:0,firefox:0,first:[0,2,9,11,14],five:1,fix:[0,11,14],flag:7,flake8:0,floattensor:[7,9,11],flow:1,fly:3,fnn:9,focu:[0,1],folder:0,follow:[0,1,2,3,15,17],foo:0,forbidden:11,forbidden_token:11,forc:[11,15],format:[0,10,12,14,15,16],former:9,forward:[2,7,9,14],fotran:2,found:16,foundat:1,fp16:[14,15],fp32:[3,7,14,15],frac:1,fraction:[12,14,15],framework:[4,14],free:[0,10],freez:[9,14],freeze_word_vec:9,freeze_word_vecs_dec:14,freeze_word_vecs_enc:14,frequenc:[12,14,15],from:[1,2,7,9,11,14,15,16],from_opt:7,frozenset:11,full:[0,2,10,12,14,15,16],full_context_align:14,fulli:2,further:[12,14],fusedadam:14,gao:18,gap:18,garg:14,gate:14,gather:7,gating_output:9,gelu:14,gener:[0,1,2,3,4,7,9,11,15,18],generator_funct:14,german:3,get:[4,5,18],git:[6,17],github:[6,14,17],give:[2,14,15],given:[1,2,10],global:9,global_attent:14,global_attention_funct:14,global_scor:11,globalattent:9,glove:14,gnmt:11,gnmtglobalscor:11,going:11,gold:11,gold_scor:11,gold_sent:11,gomez:18,gone:14,good:[0,14],googl:[0,11,15,18],gpu:[2,3,5,10,11,14,15],gpu_backend:14,gpu_rank:[3,14],gpu_verbose_level:[7,14],gpuid:14,grad:7,gradient:[7,14],graham:18,gram:11,graph:14,gre:5,greater:11,greedy_search:11,greedysearch:11,group:[9,14,15],groupwis:2,grow:11,gtx1080:15,guid:[6,17],guidelin:4,guillaum:6,h_j:9,h_s:9,h_t:9,had:15,haddow:18,hand:2,handl:[0,7],happen:11,has:[1,2,11,12,14,15],has_cov_pen:11,has_len_pen:11,has_tgt:11,have:[0,2,3,9,11,14,15],head:[1,3,9,14],head_count:9,help:[0,1,15],helsinki:[6,17],here:[1,11,16],hidden:[7,9,14],hidden_ab_s:14,hidden_dim:1,hieu:18,high:2,higher:[11,14,15],highest:15,hold:11,hook:9,hop:1,host:5,how:[0,9],howev:[0,7,9],html:[0,14],http:[1,5,6,14,15,16,17,18],huge:14,human:[2,18],hyp_:3,hyperbol:1,hyphen:2,hypothesi:3,identifi:15,idl:2,ids:2,ignor:[3,9,12,14,15],ignore_when_block:[11,15],illia:18,ilya:18,imag:7,impact:14,implement:[1,7,9,14],impli:[1,9],improv:[9,11,14,18],in_channel:9,in_config:2,includ:[0,2,9,12,14,15],incompat:[12,14,15],incorpor:14,increas:2,index:[5,9,14],indic:[1,7,9,11,12,14,15],individu:2,inf:11,infer:11,inferfeat:4,info:[14,15],inform:[1,2,14,15],ingredi:11,init:[9,14],init_scal:9,init_st:7,initi:[4,7,10,11],initial_accumulator_valu:14,inp:11,inp_seq_len:11,inproceed:6,input:[1,4,7,9,10,11,12,14,15,16,18],input_format:3,input_from_dec:9,input_len:9,input_s:9,input_sentence_s:16,inputs_len:9,inputt:11,insert:[12,14,15],insert_ratio:[12,14,15],instal:[0,3,4],instanc:[7,9,11],instanti:7,instead:[0,2,5,9,12,14,15],instruct:14,int8:15,integ:11,integr:0,interact:5,interfac:7,intermedi:1,intermediate_output:1,intern:10,interv:14,introduc:[1,2],introduct:2,invalid:[12,14,15],involv:1,is_finish:11,isn:11,item:9,iter:7,its:[0,2,9],itself:2,jakob:18,jean:6,jinsong:18,job:5,joiner:[12,14,15],jone:18,journal:0,json:13,kaiser:18,keep:[10,11,14],keep_checkpoint:[3,14],keep_stat:14,keep_topk:11,keep_topp:11,kei:9,kera:14,kernel_s:9,key_len:9,kim:6,klau:18,klein:6,krikun:18,label:14,label_smooth:[3,14],lambda:[12,14,15],lambda_align:14,lambda_coverag:14,lang:2,lang_a:2,lang_b:2,lang_pair:[2,15],languag:[1,4,12,14,16],language_pair:16,lapata:18,last:[2,14,15],latter:9,layer:[1,9,14,15],layer_cach:9,layer_type_to_cl:1,layernorm:14,layerstack:2,lead:11,learn:[1,7,9,14,18],learning_r:[3,7,14],learning_rate_decai:14,learning_rate_decay_fn:7,least:0,leav:[2,14],left:1,len:[7,9,11],length:[2,7,9,11,12,14,15,16],length_averag:11,length_non:11,length_pen:11,length_penalti:[11,15],length_wu:11,less:2,let:[2,3],level:[12,14],lib:5,librari:14,like:[0,11,15],limit:15,lin:[1,14],linattentionbridgelay:4,line:[0,3,12,14,15],linear:1,linear_warmup:14,linguist:[9,18],link:[0,1,5],list:[0,2,7,9,10,11,12,14,15],list_model:10,literatur:14,liu:18,ll17:[9,18],llion:18,load:[5,7,9,10,14],load_model:10,load_pretrained_vector:9,loader:4,local:[0,2],localhost:14,log:[4,7,11],log_fil:[14,15],log_file_level:[14,15],log_prob:11,log_tensorboard:7,logger:11,login:5,logit:[11,15],logsumexp:11,longer:15,longest:11,longtensor:[7,9,11],look:[0,6,9,15],loop:7,loss:[4,14],loss_scal:14,losscomputebas:7,love:0,lower:[2,14],lsl:[11,18],lstm:14,lua:10,lukasz:18,luong:[9,14,18],lustrep1:5,lustrep2:5,macherei:18,machin:[6,9,11,18],made:2,magic:11,mai:[2,7,10,11,12,14],main:[0,6,7,12,14,15],maintain:11,make:[0,5,7,12,14,15],make_shard_st:7,mammoth:[0,4,5,6,7,9,10,11,14],man:18,manag:7,mani:[7,11,14],manipul:7,manual:[10,11],map:[2,7,9],margin:9,marian:14,mark:14,marker:10,mask:[9,12,14,15],mask_length:[12,14,15],mask_or_step:9,mask_ratio:[12,14,15],mass:[12,14,15],massiv:[2,6],master:14,master_ip:14,master_port:14,match:10,mathbb:1,mathbf:1,mathemat:1,matric:1,matrix:[1,9,14],matrixtre:9,max:[7,11,16],max_generator_batch:[3,14],max_grad_norm:[3,7,14],max_len:9,max_length:[11,15],max_relative_posit:[9,14],max_sent_length:15,max_sentence_length:16,max_siz:7,maxim:18,maximum:[12,14,15],maybe_convert_align:10,maybe_detoken:10,maybe_detokenize_with_align:10,maybe_postprocess:10,maybe_preprocess:10,maybe_token:10,maybe_update_forbidden_token:11,maybe_update_target_prefix:11,mean:[2,10,14,15],mechan:[1,2],mem:5,memori:[10,14],memory_bank:[9,11],memory_length:9,merg:[9,14],meta:2,metadata:7,method:[7,9,14],metric:15,mi250:5,mike:18,min_length:[11,15],minh:18,minimum:15,mirella:18,mirror:14,mix:7,mkdir:[5,16],mlp:[9,14],mode:[2,12,14,15],model:[1,2,4,9,11,12],model_dim:9,model_dtyp:[3,7,14],model_id:10,model_kwarg:10,model_prefix:16,model_root:10,model_sav:7,model_step:3,model_task:14,model_typ:14,modelsaverbas:7,modif:7,modifi:[0,11],modul:[0,1,4,5,7,14,15],modular:6,mohammad:18,monolingu:2,more:[0,2,11,12,14,15],most:[11,15],mostli:7,move:[10,14],moving_averag:[7,14],much:14,multi:[0,1,9],multiheadedattent:[1,9],multilingu:[2,6],multipl:[0,1,2,7,9,14,15],multipli:1,multplic:0,must:[2,9,10,14],mymodul:5,n_batch:7,n_best:[10,11,15],n_bucket:14,n_correct:7,n_edge_typ:14,n_node:14,n_sampl:[3,12,14],n_seg:10,n_src_word:7,n_step:14,n_word:7,name:[0,2,4,11,12,14,16],namespac:10,napoleon:0,nccl:14,necessari:[0,3,5,7,11,14,15],necessit:2,need:[0,2,3,7,9,14,18],neg:[10,14],network:[9,18],neubig:18,neural:[6,9,11,18],never:11,news_commentari:3,next:[2,7,11,15],nfeat:9,ngram:[11,15],nightmar:2,niki:18,nlp:[6,17],nmt:[7,11,14,15],nmtmodel:7,noam:[3,14,18],noamwd:14,node:[2,5,7,14],node_rank:14,nois:2,non:[9,11,14],none:[7,9,10,11,12,14,15],nonetyp:[9,11],norm:[9,14],norm_method:7,normal:[1,3,7,14],normalz:7,norouzi:18,note:[0,2,3,5,11],noth:[0,7],notset:[14,15],ntask:5,nucleu:15,num_step:7,num_thread:12,number:[1,2,7,9,11,12,14,15],nvidia:14,obj:[0,7],object:[0,7,10,11,12,14,15,16],oder:2,off:14,ofi:5,often:[12,14,15],on_timemout:10,on_timeout:10,onc:[11,14],one:[0,1,2,7,9,12,14,15],onli:[2,7,11,12,14,15],onmt:16,onmt_build_vocab:3,onmt_token:[12,14,15],onmt_transl:3,onmttok:4,open:6,opennmt:[0,2,5,6,7,13],oper:[1,9],operatornam:1,opt:[3,7,10,14,15],opt_level:14,optim:[3,4],option:[0,2,3,5,7,9,10,11,12,14,15,16],opu:4,opus100:[16,17],ord:18,order:[2,14],org:[1,5,6,14,15,18],origin:[1,14,16],oriol:18,other:[1,5,7,11,12,14,15,16,18],other_lang:16,otherwis:[2,9,14,15],our:[5,11],our_stat:7,out:[1,2,6,7,17],out_channel:9,out_config:2,out_fil:11,outcom:1,output:[1,2,3,7,9,10,11,12,14,15],output_model:15,output_s:9,over:[0,2,3,7,9,11,14,15,16],overal:1,overrid:[11,12,14],overridden:9,overview:4,overwrit:[5,12,14],own:[7,15],ownership:7,p17:6,p18:14,packag:[5,10],pad:[7,9,11],pad_idx:9,pair:[2,7,10,14,15,16],paper:[0,1,9,14],parallel:[9,11,12,14],parallel_path:11,parallelcorpu:11,param:7,param_init:[3,14],param_init_glorot:[3,14],paramet:[3,7,9,10,11,12,14,15],parameter:9,parenthes:0,parmar:18,pars:[9,10],parse_opt:10,part:[1,11],particular:[0,2,9],partit:5,pass:[1,2,7,9,10,14],past:[0,14],path:[2,5,9,10,11,12,14,15],path_src:3,path_tgt:3,patienc:7,pattern:2,pdf:14,pen:11,penalti:[4,11,14],penaltybuild:11,peopl:5,per:[0,2,12,14,15],perceiv:[1,14],perceiverattentionbridgelay:4,percentag:[12,14,15],perfom:14,perform:[1,9,14],permut:[12,14,15],permute_sent_ratio:[12,14,15],perplex:7,peter:18,pfs:5,pham:18,phrase_t:[11,15],piec:3,pip3:[5,6,17],pip:[0,5],pipelin:[12,14,15],pleas:[0,6],plu:14,point:18,pointer:[9,18],poisson:[12,14,15],poisson_lambda:[12,14,15],polosukhin:18,polyak_decai:9,pool:14,port:[13,14],portal:6,pos_ffn_activation_fn:[9,14],posit:[9,14],position_encod:[9,14],position_ffn:9,positionalencod:9,positionwisefeedforward:[9,14],possibl:[2,7,10,11,12,14,15],postprocess:10,postprocess_opt:10,potenti:11,pouta:16,ppl:7,pre:[7,10,11],pre_word_vecs_dec:14,pre_word_vecs_enc:14,preced:2,precis:7,pred:15,pred_scor:11,pred_sent:11,predict:[7,11,15],prefer:0,prefix:[2,7,12,14,15],prefix_seq_len:11,preliminari:3,preload:10,preload_model:10,prepar:[4,11],prepare_wmt_data:3,preprint:18,preprocess:10,preprocess_opt:10,presenc:2,presum:11,pretrain:[9,14],prevent:[11,15],previou:[1,2,9,11],previous:1,primari:2,prime:1,print:[7,14,15],prior:3,prior_token:[12,14,15],prob:11,proba:15,probabl:[9,11,12,14,15],probil:9,problem:11,proc:[6,18],procedur:2,process:[1,7,10,12,14],processu:10,produc:[1,11,12,14,15],product:1,projappl:5,project:[0,1,5,6,9],project_2005099:5,project_462000125:5,propag:7,proper:10,properli:5,properti:[7,9],proport:[2,12,14,15],provid:[6,15],prune:4,pty:5,pull_request_chk:0,punctuat:0,put:11,pwd:16,pyonmttok:[12,14,15],python3:[2,5],python:[0,2,5,14],pythonpath:5,pythonuserbas:5,pytorch:[0,5],qin:18,quantiz:15,queri:9,query_len:9,queue:[12,14],queue_siz:[3,14],quickstart:[4,6],quoc:18,quot:0,rais:[12,14],random:[4,12,14],random_ratio:[12,14,15],random_sampling_temp:[11,15],random_sampling_topk:[11,15],random_sampling_topp:[11,15],randomli:11,rang:15,rank:[11,14],ranslat:18,rare:11,rate:[4,7],rather:0,ratio:[11,15],raw:[9,11,15],rccl:5,reach:11,read:[0,2,10,16],readabl:[0,2],reader:4,readm:14,rebuild:10,rebuild_seg_packag:10,receiv:2,recent:14,recip:9,recommend:14,recommonmark:0,rectifi:1,recurr:9,redund:2,ref:0,refer:[0,1,4],regardless:2,regist:9,regular:[12,14,15],rel:14,relat:[3,12,14,15],relationship:1,relev:[9,11],relu:[1,9,14],rememb:0,remov:2,renorm:14,reorder:11,repeat:[11,15],repetit:15,replac:[11,12,14,15],replace_length:[12,14,15],replace_unk:[11,15],report:[6,7,14,15],report_align:[11,15],report_everi:[3,14],report_manag:7,report_scor:11,report_stats_from_paramet:[7,14],report_tim:[11,15],reportmgrbas:7,repres:[1,7],represent:[1,9,14,18],reproduc:4,requir:[0,7,14],research:6,reset:7,reset_optim:14,resett:14,residu:9,resourc:2,respect:[1,2],respons:7,rest:13,restrict:[12,14,15],result:[1,10,14],return_attent:11,reus:14,reuse_copy_attn:14,revers:[12,14,15],reversible_token:[12,14,15],rico:18,right:[0,1],rmsnorm:14,rnn:[7,14],rnn_size:[3,14],roblem:18,rocm5:5,rocm:5,root:[1,2],rotat:[12,14,15],rotate_ratio:[12,14,15],roundrobin:14,row:2,rsqrt:14,rst:0,run:[0,2,3,7,9,10,14,15],rush:6,sacrebleu:[3,5,6,17],sai:2,samantao:5,same:[0,2,3,9,10,14],sampl:[4,11,12,14,16],sample_with_temperatur:11,sampling_temp:11,saniti:15,save:[7,12,14,15,16],save_all_gpu:14,save_checkpoint_step:[3,7,14],save_config:[12,14,15],save_data:[3,12,14],save_model:[3,14],saver:7,scale:[11,14],schedul:[7,14],schuster:18,score:[4,9,10,15],scorer:11,scratch:5,script:[0,3,4,5],search:[0,2,4,11],second:[1,2,9,10],secur:[12,14],see:[2,9,10,11,12,14,18],seed:[3,11,12,14,15],seemingli:14,seen:1,segment:[2,10,15],select:[9,11,14],select_index:11,self:[1,9,10,11,14],self_attn_typ:14,send:[0,14],senellart:6,sennrich:18,sensibl:0,sent:[7,14,15],sent_numb:11,sentenc:[11,12,14,15,16],sentencepiec:[2,3,5,6,12,14,15,17],separ:2,seper:10,seq2seq:[11,14],seq:11,seq_len:[1,9,11],seqlength:9,sequenc:[1,2,7,9,10,11,12,14,15],serial:9,serv:1,server:[4,14,16],servermodel:10,servermodelerror:10,session:5,set:[1,2,3,5,7,9,10,11,12,14,15],setup:3,sever:[2,9,11],sgd:14,sh16:[9,18],shape:[0,9,11],shard:[7,14,15],shard_siz:[7,15],share:[5,12,14,15],share_decoder_embed:[3,14],share_embed:[3,14],share_vocab:[12,14],shazeer:18,shortest:11,shot:2,should:[2,3,9,11,14],shuf:16,shuffle_input_sent:16,side:[2,7,10,12,14,15],side_a:2,side_b:2,silent:[3,9,12,14],similar:[1,2,9,14],simpl:[1,7,14],simpleattentionbridgelay:4,simulatan:9,sin:14,sinc:9,singl:[0,10,14],single_pass:14,sinusoid:9,site:5,size:[2,7,9,11,12,14,15,16],skip:[2,12,14],skip_empty_level:[3,12,14],slen:9,slm17:[9,18],slow:[12,15],slurm:[2,5],smaller:[12,14,15],smooth:[12,14,15],softmax:[1,9,14,15],some:[0,2,7,15],someth:0,sometim:0,sort:[10,16],sorted_pair:2,sourc:[0,2,4,5,6,7,9,10,11,12,14],sp_path:16,space:[0,1,14],spacer:[12,14,15],span:[12,14,15],spars:9,sparseadam:14,sparsemax:[9,14],specif:[1,2,6,11,12,14,17],specifi:[1,12,14,15],sphinx:0,sphinx_rtd_them:0,sphinxcontrib:0,spill:0,spm_decod:3,spm_encod:[3,16],spm_train:16,sqrt:1,squar:[1,2],src:[2,3,7,9,10,11,12,14,15,16],src_embed:14,src_feat:15,src_feats_vocab:[12,14],src_file_path:11,src_ggnn_size:14,src_group:2,src_lang:[2,15],src_languag:2,src_len:[7,9],src_length:11,src_map:[9,11],src_onmttok_kwarg:[12,14,15],src_raw:11,src_seq_length:[3,12,14,15],src_seq_length_trunc:14,src_subword_alpha:[3,12,14,15],src_subword_model:[3,12,14,15],src_subword_nbest:[3,12,14,15],src_subword_typ:[12,14,15],src_subword_vocab:[12,14,15],src_vocab:[3,11,12,14],src_vocab_s:14,src_vocab_threshold:[12,14,15],src_word_vec_s:14,src_words_min_frequ:14,sru:4,srun:5,stabl:1,stack:[14,15],stage:1,stand:0,standard:[9,14,15],start:[2,4,5,7,10,14,16],start_decay_step:14,stat:[7,14],stat_list:7,state:[7,11,14],state_dict:14,state_dim:14,statist:[7,14],stdout:7,step:[1,2,4,7,9,11,14,15],stepwis:9,stepwise_penalti:[11,15],still:0,stop:[12,14,15],store:14,str:[0,7,9,10,11],strategi:[4,7,14],stride:9,string:[7,9,12,14,15],structur:[1,4,18],structured_attent:9,style:[0,12,14,15],styleguid:0,subclass:[7,9,11],subcompon:2,subdirectori:5,subsequ:1,subset:16,substitut:2,subword:[2,4],suggest:14,sum:[7,9,11,14],sum_:9,sume:7,summar:18,summari:[0,11,15],superclass:0,supervis:[2,14],support:[0,2,9,14],suppos:16,sure:[5,11],sutskev:18,switchout:[4,18],switchout_temperatur:[12,14,15],symmetr:2,system:[11,14,18],tab:[12,14],tabl:[9,15],take:[1,2,6,9,12,14,15],taken:9,tangent:1,tanh:[1,9],tar:16,target:[2,4,7,9,10,11,12,14],target_prefix:11,task:[2,3,4,7,11],task_distribution_strategi:14,task_queue_manag:7,tatoeba:[2,4],tau:[12,14,15],technic:6,temperatur:[2,11,12,14,15],templat:2,tensor:[0,7,9,11],tensorboard:[7,14],tensorboard_log_dir:14,tensorflow:14,term:[1,9],test:[0,3,5,9],testset:3,text:[7,9,11,14,15,18],tgt:[2,3,7,10,12,14,15],tgt_dict:9,tgt_embed:14,tgt_file_path:11,tgt_group:2,tgt_lang:[2,15],tgt_languag:2,tgt_len:[7,9],tgt_onmttok_kwarg:[12,14,15],tgt_prefix:[11,15],tgt_sent:11,tgt_seq_length:[3,12,14,15],tgt_seq_length_trunc:14,tgt_subword_alpha:[3,12,14,15],tgt_subword_model:[3,12,14,15],tgt_subword_nbest:[3,12,14,15],tgt_subword_typ:[12,14,15],tgt_subword_vocab:[12,14,15],tgt_vocab:[3,7,12,14],tgt_vocab_s:14,tgt_vocab_threshold:[12,14,15],tgt_word_vec_s:14,tgt_words_min_frequ:14,than:[0,11,14,16],thang:18,thant:11,thei:[1,9,11],them:[2,9],theorem:9,thi:[0,1,2,3,5,6,7,9,11,12,14,15],thin:7,thing:[0,2],thoroughli:9,thread:12,three:[1,9],through:[1,2,7],thu:7,tic:0,tick:0,time:[1,2,5,7,11,14,15],timeout:10,timer:10,titl:6,tlen:9,to_cpu:10,to_gpu:10,todo:[5,16],tok:10,token:[3,7,10,11,12,14,15],token_drop:4,token_mask:4,tokendrop:[12,14,15],tokendrop_temperatur:[12,14,15],tokenizer_mark:10,tokenizer_opt:10,tokenmask:[12,14,15],tokenmask_temperatur:[12,14,15],too:11,tool:4,toolkit:6,top:[1,9,11,15],topk_id:11,topk_scor:11,torch:[0,5,7,9,14],torchtext:7,total:[2,7,14],trail:0,train:[2,4,5,6,7,9],train_extremely_large_corpu:16,train_from:14,train_it:7,train_loss:7,train_loss_md:7,train_step:[3,7,14],trainabl:7,trainer:4,training_step:7,transform:[1,3,4,7,18],transformer_ff:[3,14],transformerattentionbridgelay:4,transformerencoderlay:1,translat:[2,4,6,7,9,10,13,18],translate_batch:11,translation_serv:10,translationbuild:11,translationserv:10,travi:0,tree:9,trg:2,triang:2,trick:[4,9],trunc_siz:7,truncat:[7,14],truncated_decod:14,trust:16,turn:14,tutori:[4,17],two:[1,2,9],txt:[0,15,16],type:[0,1,2,4,7,9,10,11,12,15],typic:[7,14],u_a:9,under:[2,14,15],undergo:1,undergon:1,underli:11,uniform:14,unigram:[12,14,15],union:0,unit:[1,9],unittest:0,unk:[11,15],unknown:11,unless:2,unload:10,unload_model:10,unmodifi:11,unnecessari:[0,2],unnorm:9,unset:2,until:[11,15],unwieldli:2,updat:[5,7,10,11,14],update_dropout:9,update_finish:11,update_learning_r:14,update_n_src_word:7,update_vocab:14,upgrad:5,upper:2,url:[5,6,18],url_root:13,usag:[4,12,13,14,15],use:[0,1,2,3,5,7,9,10,11,12,14,15,16],used:[1,2,3,7,9,10,11,12,14,15],useful:7,user:[5,7,9,10],uses:[0,2,9,11,14],using:[0,1,2,6,9,10,11,12,14,15],uszkoreit:18,util:[1,7],v11:3,v_a:9,valid:[3,7,12,14,15],valid_batch_s:[3,14],valid_it:7,valid_loss:7,valid_loss_md:7,valid_step:[3,7,14],valu:[1,2,7,9,10,11,12,14,15],variabl:[2,5,11],variat:0,vaswani:18,vaswanispujgkp17:0,vector:[9,14],venv:5,verbos:[11,14,15],veri:[0,15],version:[9,10,11],via:[9,18],vinyal:18,virtual:5,visit:0,visual:14,vocab:[3,4,7,9,11],vocab_path:[12,14],vocab_s:[11,14,16],vocab_sample_queue_s:12,vocab_size_multipl:14,vocabulari:[2,7,9,12,14,15,16],vsp:[9,18],w_a:9,wai:[2,11],wait:2,wang:18,want:[2,15],warmup:14,warmup_step:[3,14],warn:[12,14,15],weight:[1,2,3,9,14,15],weight_decai:14,weighted_sampl:14,weightnormconv2d:9,well:[0,14],wget:16,what:[2,7,10],when:[0,2,6,9,11,12,14,15,16],where:[1,3,5,9,11,12,14,15],wherea:[11,14],whether:[7,10,11,12,14,15],which:[2,9,11,14],whl:5,whole:[3,11],whose:15,why:1,wiki:14,wikipedia:14,window:[12,14,15],wise:1,with_align:7,within:[1,9,10],without:[0,14],wmt14_en_d:3,wmt:3,wmtend:3,wojciech:18,wolfgang:18,word2vec:14,word:[1,9,11,12,14,15],word_align:11,word_lut:9,word_padding_idx:9,word_vec_s:[3,9,14],word_vocab_s:9,work:[0,2,11,14],workflow:6,world_siz:[3,14],would:[2,11,14],wpdn18:[12,14,15,18],wrap:10,wrapper:7,writabl:2,write:[2,7],writer:7,written:3,wsc:[11,18],www:14,xavier_uniform:14,xent:7,xinyi:18,xiong:18,xzvf:16,yaml:[3,12,14,15],yang:18,year:6,yet:[9,11],yml:0,yonghui:18,yoon:6,you:[0,2,3,5,9,14,15,18],your:[0,2,5,15,16],your_venv_nam:5,your_vevn_nam:5,yourself:6,yuan:18,yuntian:6,zaremba:18,zero:[2,7,9,11,14,15],zero_grad:7,zhang:18,zhifeng:18,zihang:18,zxs18:[9,18]},titles:["Contributors","Attention Bridge","Config-config tool","Translation","Contents","Installation","Overview","Framework","Data Loaders","Modules","Server","Translation","Build Vocab","Server","Train","Translate","Prepare Data","Quickstart","References"],titleterms:{"class":11,The:2,actual:2,adapt:[2,14],adapter_config:2,ae_path:2,ae_transform:2,align:14,allocate_devic:2,altern:2,architectur:9,argument:13,attent:[1,9,14],autoencod:2,beam:15,bridg:[1,14],build:[3,12],challeng:16,citat:6,cluster_languag:2,command:2,common:[12,14,15],complete_language_pair:2,config:2,config_al:2,config_config:2,configur:[12,14,15],content:4,contributor:0,conv2conv:9,copi:9,core:[9,10],corpora:2,corpora_schedul:2,data:[3,8,12,14,15,16,17],dataset:8,dec_sharing_group:2,decod:[9,11,14,15],denois:[12,14,15],direct:16,distanc:2,distance_matrix:2,docstr:0,download:[3,16],dynam:14,effici:15,embed:14,enc_sharing_group:2,encod:[9,14],evalu:3,featur:14,feedforwardattentionbridgelay:1,filter:[12,14,15],framework:7,gener:14,get:16,group:2,guidelin:0,inferfeat:[12,14,15],initi:14,input:2,instal:[5,6,17],kei:2,languag:[2,15],level:2,linattentionbridgelay:1,line:2,loader:8,log:[14,15],loss:7,lumi:5,mahti:5,mammoth:17,manual:2,matrix:2,model:[3,7,10,14,15,16],modul:9,n_gpus_per_nod:2,n_group:2,n_node:2,name:13,onmttok:[12,14,15],optim:[7,14],opu:16,other:2,overrid:2,overview:6,paramet:2,pars:16,path:16,penalti:15,perceiverattentionbridgelay:1,prepar:[3,16,17],prune:14,puhti:5,quickstart:17,random:15,rate:14,reader:8,refer:18,relev:16,remove_temporary_kei:2,reproduc:[12,14,15],run:5,sampl:15,score:11,search:15,sentencepiec:16,server:[10,13],set:16,set_transform:2,share:2,sharing_group:2,shot:16,simpleattentionbridgelay:1,sourc:15,specifi:2,src_path:2,sru:9,stage:2,step:[3,16,17],strategi:11,structur:9,subword:[3,12,14,15],supervis:16,switchout:[12,14,15],target:15,task:14,tatoeba:16,test:16,tgt_path:2,than:2,token_drop:[12,14,15],token_mask:[12,14,15],tool:2,top:2,train:[3,14,16],trainer:7,transform:[2,9,12,14,15],transformerattentionbridgelay:1,translat:[3,11,15,16],translation_config:2,translation_config_dir:2,trick:15,type:14,usag:2,use_introduce_at_training_step:2,use_weight:2,valid:16,variabl:16,vocab:[12,14,16],vocabulari:3,yaml:2,zero:16,zero_shot:2}})
\ No newline at end of file
+Search.setIndex({docnames:["CONTRIBUTING","attention_bridges","config_config","examples/Translation","index","install","main","mammoth","mammoth.inputters","mammoth.modules","mammoth.translate.translation_server","mammoth.translation","options/build_vocab","options/server","options/train","options/translate","prepare_data","quickstart","ref"],envversion:{"sphinx.domains.c":1,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":1,"sphinx.domains.index":1,"sphinx.domains.javascript":1,"sphinx.domains.math":2,"sphinx.domains.python":1,"sphinx.domains.rst":1,"sphinx.domains.std":1,"sphinx.ext.viewcode":1,sphinx:56},filenames:["CONTRIBUTING.md","attention_bridges.md","config_config.md","examples/Translation.md","index.rst","install.md","main.md","mammoth.rst","mammoth.inputters.rst","mammoth.modules.rst","mammoth.translate.translation_server.rst","mammoth.translation.rst","options/build_vocab.rst","options/server.rst","options/train.rst","options/translate.rst","prepare_data.md","quickstart.md","ref.rst"],objects:{"mammoth.Trainer":{train:[7,1,1,""],validate:[7,1,1,""]},"mammoth.models":{NMTModel:[7,0,1,""]},"mammoth.models.NMTModel":{count_parameters:[7,1,1,""],forward:[7,1,1,""]},"mammoth.modules":{AverageAttention:[9,0,1,""],Embeddings:[9,0,1,""],MultiHeadedAttention:[9,0,1,""],PositionalEncoding:[9,0,1,""]},"mammoth.modules.AverageAttention":{cumulative_average:[9,1,1,""],cumulative_average_mask:[9,1,1,""],forward:[9,1,1,""]},"mammoth.modules.Embeddings":{emb_luts:[9,1,1,""],forward:[9,1,1,""],load_pretrained_vectors:[9,1,1,""],word_lut:[9,1,1,""]},"mammoth.modules.MultiHeadedAttention":{forward:[9,1,1,""],training:[9,2,1,""],update_dropout:[9,1,1,""]},"mammoth.modules.PositionalEncoding":{forward:[9,1,1,""]},"mammoth.modules.position_ffn":{PositionwiseFeedForward:[9,0,1,""]},"mammoth.modules.position_ffn.PositionwiseFeedForward":{forward:[9,1,1,""]},"mammoth.translate":{BeamSearch:[11,0,1,""],DecodeStrategy:[11,0,1,""],GNMTGlobalScorer:[11,0,1,""],GreedySearch:[11,0,1,""],Translation:[11,0,1,""],TranslationBuilder:[11,0,1,""],Translator:[11,0,1,""]},"mammoth.translate.BeamSearch":{initialize:[11,1,1,""]},"mammoth.translate.DecodeStrategy":{advance:[11,1,1,""],block_ngram_repeats:[11,1,1,""],initialize:[11,1,1,""],maybe_update_forbidden_tokens:[11,1,1,""],maybe_update_target_prefix:[11,1,1,""],target_prefixing:[11,1,1,""],update_finished:[11,1,1,""]},"mammoth.translate.GreedySearch":{advance:[11,1,1,""],initialize:[11,1,1,""],update_finished:[11,1,1,""]},"mammoth.translate.Translation":{log:[11,1,1,""]},"mammoth.translate.Translator":{translate_batch:[11,1,1,""]},"mammoth.translate.greedy_search":{sample_with_temperature:[11,3,1,""]},"mammoth.translate.penalties":{PenaltyBuilder:[11,0,1,""]},"mammoth.translate.penalties.PenaltyBuilder":{coverage_none:[11,1,1,""],coverage_summary:[11,1,1,""],coverage_wu:[11,1,1,""],length_average:[11,1,1,""],length_none:[11,1,1,""],length_wu:[11,1,1,""]},"mammoth.translate.translation_server":{ServerModel:[10,0,1,""],ServerModelError:[10,4,1,""],Timer:[10,0,1,""],TranslationServer:[10,0,1,""]},"mammoth.translate.translation_server.ServerModel":{build_tokenizer:[10,1,1,""],detokenize:[10,1,1,""],do_timeout:[10,1,1,""],maybe_convert_align:[10,1,1,""],maybe_detokenize:[10,1,1,""],maybe_detokenize_with_align:[10,1,1,""],maybe_postprocess:[10,1,1,""],maybe_preprocess:[10,1,1,""],maybe_tokenize:[10,1,1,""],parse_opt:[10,1,1,""],postprocess:[10,1,1,""],preprocess:[10,1,1,""],rebuild_seg_packages:[10,1,1,""],to_gpu:[10,1,1,""],tokenize:[10,1,1,""],tokenizer_marker:[10,1,1,""]},"mammoth.translate.translation_server.TranslationServer":{clone_model:[10,1,1,""],list_models:[10,1,1,""],load_model:[10,1,1,""],preload_model:[10,1,1,""],run:[10,1,1,""],start:[10,1,1,""],unload_model:[10,1,1,""]},"mammoth.utils":{Optimizer:[7,0,1,""],Statistics:[7,0,1,""]},"mammoth.utils.Optimizer":{amp:[7,1,1,""],backward:[7,1,1,""],from_opt:[7,1,1,""],learning_rate:[7,1,1,""],step:[7,1,1,""],training_step:[7,1,1,""],zero_grad:[7,1,1,""]},"mammoth.utils.Statistics":{accuracy:[7,1,1,""],all_gather_stats:[7,1,1,""],all_gather_stats_list:[7,1,1,""],elapsed_time:[7,1,1,""],log_tensorboard:[7,1,1,""],output:[7,1,1,""],ppl:[7,1,1,""],update:[7,1,1,""],xent:[7,1,1,""]},"mammoth.utils.loss":{LossComputeBase:[7,0,1,""]},mammoth:{Trainer:[7,0,1,""]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","function","Python function"],"4":["py","exception","Python exception"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:function","4":"py:exception"},terms:{"25g":5,"boolean":[7,11],"break":16,"class":[0,4,7,9,10],"default":[10,12,13,14,15,16],"export":5,"final":[1,3,11],"float":[2,9,11],"function":[0,1,2,7,9,10,11,14],"import":0,"int":[7,9,10,11],"long":0,"new":[0,1,3],"public":5,"return":[0,7,9,10,11],"static":[7,14],"true":[2,3,7,11,14,15,16],"try":[0,5],"while":2,And:0,EOS:11,For:[0,2,11,14,17],IDs:11,IFS:16,LPs:2,Not:0,One:2,The:[1,3,7,10,11,14,15],Then:[0,3],There:[1,2],These:[1,2,9,11],Use:[2,14,15],Used:11,Will:2,__init__:10,_compute_loss:7,aan:14,aan_useffn:[9,14],ab_fixed_length:14,ab_lay:14,ab_layer_norm:14,abbrevi:0,abil:9,about:0,abov:[0,11],abs:[1,14,15,18],acceler:[9,18],accept:[0,2,11],access:[1,2,5],accord:2,account:[2,5],accross:7,accum:7,accum_count:[3,7,14],accum_step:[3,7,14],accumul:[7,14],accuraci:[7,11],achiev:2,achin:18,acl:[6,18],aclweb:14,action:[9,11,14],activ:[1,5,9,14],activation_fn:9,activationfunct:9,actual:11,adadelta:14,adafactor:14,adagrad:14,adagrad_accumulator_init:14,adam:[3,14],adam_beta1:14,adam_beta2:[3,14],adamoptim:14,adamw:14,adapt:[4,6],adapter_nam:2,add:[0,3,9],added:2,adding:0,addit:[0,9,12,14,15],addition:9,address:11,adjust:2,adopt:14,advanc:[11,14],advic:0,after:[0,1,11,14],again:0,aidan:18,alexand:6,algorithm:18,align:[4,7,10,11,15],align_debug:15,alignment_head:14,alignment_lay:14,aliv:11,alive_attn:11,alive_seq:11,all:[0,2,7,9,11,12,14,15,18],all_gather_stat:7,all_gather_stats_list:7,all_preprocess:10,allennlp:0,alloc:2,allow:[0,1,2,14],almost:[11,14],alon:0,along:1,alpha:[1,11,15],alphabet:2,alreadi:[12,14,15],also:[0,2,5,7,9,14],alwai:[0,2],amp:[7,14],ani:[0,2,11,12,14],anoth:[0,1,7],antholog:14,apex:14,apex_opt_level:14,api:[0,4],api_doc:14,appear:2,append:[5,16],appli:[1,2,11,12,14,15],applic:15,appropri:11,approxim:14,architectur:[1,4],arg:[0,10],argmax:15,argpars:10,argument:[0,4],arxiv:[0,1,14,15,18],ashish:18,assig:2,assign:[2,15],assing:2,assum:[9,11],att_typ:1,attend:1,attent:[0,4,7,11,15,18],attention_bridg:7,attention_dropout:[3,14],attentionbridgenorm:1,attn:[11,15],attn_debug:[11,15],attn_typ:9,attr:10,attribut:11,augment:18,author:6,autodoc:0,autogener:14,avail:[7,10,14,15],available_model:13,averag:[9,14,15,18],average_decai:[3,7,14],average_everi:[7,14],average_output:9,averageattent:9,avg:15,avg_raw_prob:15,avoid:[0,2],aws:5,axi:11,back:7,backend:14,backward:7,bahdanau:14,ban_unk_token:[11,15],barri:18,bart:[12,14,15],base:[0,1,2,3,5,6,7,9,10,11,12,14,15],baselin:14,basemodel:7,basenam:[3,16],bash:5,batch:[1,3,7,9,11,14,15],batch_siz:[3,9,11,14,15],batch_size_multipl:[3,14],batch_typ:[3,14,15],beam:[4,11],beam_search:11,beam_siz:[3,11,15],beamsearch:11,beamsearchbas:11,becaus:[2,15],becom:2,been:[9,11,12,14,15],befor:[0,3,10,11,14,15],begin:[7,11],below:0,ben:2,bengali:2,best:[11,15],beta1:14,beta2:14,beta:[11,15],better:[0,12,14,15],between:[1,12,14,15,18],beyond:7,biao:18,bib:0,bibtex:0,bibtext:0,bidir_edg:14,bidirect:14,bin:[5,14],binari:[3,9],bit:15,blank:0,bleu:3,block:[11,15],block_ngram_repeat:[11,15],booktitl:6,bool:[7,9,10,11],bos:11,both:[2,11,14],both_embed:14,boundari:[12,14,15],bpe:[12,14,15],bptt:[7,14],bridg:[4,18],bridge_extra_nod:14,browser:0,bucket_s:[3,14],buffer:7,build:[0,4,7,9,10,11,15,16],build_token:10,build_vocab:12,built:7,bytetensor:11,cach:9,calcul:[1,7,11],call:11,callabl:11,callback:7,can:[1,2,3,5,7,10,11,12,14,15],cancel:10,candid:[2,12,14,15],cao:18,capit:0,captur:1,cat:16,categor:11,categori:11,challeng:4,chang:[0,2,7,14],channel:1,charact:[0,15],character_coverag:16,check:[0,6,17],checklist:0,checkpoint:[3,7,14],chen:18,chmod:[3,5],choic:[0,9,12,14,15],choos:[0,12,14,15],chosen:11,citat:[0,4],cite:[0,6],classmethod:7,clear:0,clone:[6,10,17],clone_model:10,close:0,cls:7,cluster:[2,6,17],clutter:0,code:[0,2,5,15],code_dir:5,codebas:5,column:2,com:[6,17],combin:15,comma:2,command:[3,4],comment:0,commentari:3,common:[0,4],commoncrawl:3,commun:0,complet:11,complex:[2,11],compon:[1,2],composit:14,comput:[1,2,3,7,9,14,15],concat:[9,14],condit:[11,14,15],conf:[13,15],config:[3,4,10,12,13,14,15],config_fil:10,configur:[2,3,4],connect:1,consid:[2,16],consider:14,consist:0,constant:2,constructor:0,consum:14,contain:[2,9,10,11],content:[0,15],context:[1,9,14],continu:0,contribut:[0,1,9],contributor:4,control:[2,7],conv2conv:4,conveni:2,convent:0,convers:11,convert:10,copi:[0,2,4,5,14,15],copy_attn:[11,14],copy_attn_forc:14,copy_attn_typ:14,copy_loss_by_seqlength:14,core:[1,4,7],corpora:3,corpu:[2,3,12,14,16],corr:[0,18],correct:2,correspand:10,correspond:[1,15],could:11,count:[2,7,11,12,14,15],count_paramet:7,cov:11,cov_pen:11,coverag:[11,14,15],coverage_attn:14,coverage_non:11,coverage_penalti:[11,15],coverage_summari:11,coverage_wu:11,cpu:[10,14,15],crai:5,crayon:14,creat:[2,5,7],creation:2,criteria:14,criterion:7,critic:[14,15],cross:[7,14],csc:16,csv:2,ct2_model:10,ct2_translate_batch_arg:10,ct2_translator_arg:10,ctrl:0,cumbersom:2,cumul:[9,11,15],cumulative_averag:9,cumulative_average_mask:9,cur_dir:16,cur_len:11,current:[2,7,9,11,14],curricula:2,curriculum:2,custom:[10,14],custom_opt:10,cut:[0,16],cutoff:11,d_ff:9,d_model:9,dai:18,data:[1,2,4,7,11,18],data_path:16,data_typ:[7,11,14,15],dataset:[3,4,12,14,15,16],datastructur:10,dblp:0,ddress:18,deal:2,debug:[13,14,15],dec:2,dec_lay:[3,14],decai:14,decay_method:[3,14],decay_step:14,decod:[1,2,4,7],decode_strategi:11,decoder_typ:[3,14],decoderbas:7,decodestrategi:11,def:0,defin:[2,3,12,14,15],definit:9,delai:2,delet:[12,14,15],delimit:15,deng:6,denois:[2,4],denoising_object:[12,14,15],denot:1,depend:[0,2,5,7,10],deprec:[14,15],describ:[1,9,10,14],descript:0,desir:[2,3],detail:[6,12,14],determin:2,detoken:[3,10],dev:[5,16],develop:0,devic:[2,9,11,15],device_context:7,deyi:18,diagon:2,dict:[2,7,10,11,12,14,15],dict_kei:14,dictionari:[7,9,11,14],differ:[0,1,2,10,15],dim:9,dimens:[1,9,11,14],dimension:1,dir:16,direct:[0,2,11],directli:[0,15],directori:[2,5,10,14],disabl:14,discard:14,discourag:14,disk:14,displai:7,dist:7,distanc:14,distribut:[2,7,9,11,12,14,15],divers:[1,12,14,15],divid:[1,2,14,15],divis:9,do_timeout:10,doc:0,document:[0,6],doe:[2,15],doesn:16,doi:6,doing:[2,15],don:0,done:[3,11,16],dot:[1,9,14],dotprod:14,down:[11,12],download:5,dropout:[3,7,9,12,14,15],dropout_step:[3,7,14],due:14,dump:[12,14,15],dump_beam:[11,15],dump_sampl:12,dump_transform:14,dure:[10,14,15],dynam:[4,9,15],each:[1,2,9,11,12,14,15],earli:14,earlier:[1,12,14,15],early_stop:14,early_stopping_criteria:14,earlystopp:7,eas:2,easi:0,easili:2,echo:[3,16],edg:14,effect:[1,10,12],effici:[4,7,18],either:[11,14],elaps:7,elapsed_tim:7,element:[1,2],els:16,emb:9,emb_fil:9,emb_lut:9,embed:[1,4,9,12],embedding_s:9,embeddings_typ:14,emerg:1,emploi:[1,7],empti:[3,11,12,14],enabl:15,enc:2,enc_lay:[3,14],encapsul:1,encod:[1,2,4,7,11],encoder_typ:[3,14],encoderbas:7,encordec:[12,14],encount:[12,14],encout:[12,14],end:11,eng:2,english:[2,3,16],enhanc:1,ensembl:15,ensur:1,entir:16,entri:0,entropi:7,env_dir:5,environ:5,eos:11,epoch:14,epsilon:14,equal:[11,14],equat:9,equival:14,error:[0,12,14,15],especi:2,essenti:11,establish:1,eural:18,europarl:3,evalu:7,even:2,event:11,everi:[7,14,15],exactli:0,exampl:[0,2,3,12,14,17],exce:14,except:[0,10,12,14,15],exclusion_token:11,execut:[3,12,14],exist:[12,14,15,16],exp:14,exp_host:14,expect:[2,11],experi:[12,14,15],experiment:14,exponenti:14,extend:0,extern:0,extra:[5,14],extract:16,facilit:1,fail:11,fairseq:0,fals:[7,9,10,11,12,13,14,15],familiar:6,faster:14,feat_0:15,feat_1:15,feat_dim_expon:9,feat_merg:[9,14],feat_merge_s:14,feat_padding_idx:9,feat_vec_expon:[9,14],feat_vec_s:[9,14],feat_vocab_s:9,feats0:15,feats1:15,featur:[1,4,7,9,12,15,18],fed:1,feed:[2,9,14],feedforward:[1,14],feedforwardattentionbridgelay:4,feel:0,few:0,ffn:[9,14],figur:9,file:[0,2,10,12,14,15,16],filenam:14,filter:[3,4,16],filterfeat:[12,14,15],filtertoolong:[2,3,12,14,15],find:0,firefox:0,first:[0,2,9,11,14],five:1,fix:[0,11,14],flag:7,flake8:0,floattensor:[7,9,11],flow:1,fly:3,fnn:9,focu:[0,1],folder:0,follow:[0,1,2,3,15,17],foo:0,forbidden:11,forbidden_token:11,forc:[11,15],format:[0,10,12,14,15,16],forward:[2,7,9,14],fotran:2,found:16,foundat:1,fp16:[14,15],fp32:[3,7,14,15],frac:1,fraction:[12,14,15],framework:[4,14],free:[0,10],freez:[9,14],freeze_word_vec:9,freeze_word_vecs_dec:14,freeze_word_vecs_enc:14,frequenc:[12,14,15],from:[1,2,7,9,11,14,15,16],from_opt:7,frozenset:11,full:[0,2,10,12,14,15,16],full_context_align:14,fulli:2,further:[12,14],fusedadam:14,gao:18,gap:18,garg:14,gather:7,gating_output:9,gelu:14,gener:[0,1,2,3,4,7,11,15],generator_funct:14,german:3,get:[4,5],git:[6,17],github:[6,14,17],give:[2,14,15],given:[1,2,10],global_attent:14,global_attention_funct:14,global_scor:11,glove:14,gnmt:11,gnmtglobalscor:11,going:11,gold:11,gold_scor:11,gold_sent:11,gomez:18,gone:14,good:[0,14],googl:[0,11,15,18],gpu:[2,3,5,10,11,14,15],gpu_backend:14,gpu_rank:[3,14],gpu_verbose_level:[7,14],gpuid:14,grad:7,gradient:[7,14],graham:18,gram:11,graph:14,gre:5,greater:11,greedy_search:11,greedysearch:11,group:[14,15],groupwis:2,grow:11,gtx1080:15,guid:[6,17],guidelin:4,guillaum:6,had:15,haddow:18,hand:2,handl:[0,7],happen:11,has:[1,2,11,12,14,15],has_cov_pen:11,has_len_pen:11,has_tgt:11,have:[0,2,3,9,11,14,15],head:[1,3,9,14],head_count:9,help:[0,1,15],helsinki:[6,17],here:[1,11,16],hidden:[7,9,14],hidden_ab_s:14,hidden_dim:1,hieu:18,high:2,higher:[11,14,15],highest:15,hold:11,hop:1,host:5,how:0,howev:[0,7],html:[0,14],http:[1,5,6,14,15,16,17,18],huge:14,human:[2,18],hyp_:3,hyperbol:1,hyphen:2,hypothesi:3,identifi:15,idl:2,ids:2,ignor:[3,12,14,15],ignore_when_block:[11,15],illia:18,ilya:18,imag:7,impact:14,implement:[1,7,9,14],impli:1,improv:[9,11,14,18],in_config:2,includ:[0,2,9,12,14,15],incompat:[12,14,15],incorpor:14,increas:2,index:[5,9,14],indic:[1,7,9,11,12,14,15],individu:2,inf:11,infer:11,inferfeat:4,info:[14,15],inform:[1,2,14,15],ingredi:11,init:14,init_st:7,initi:[4,7,10,11],initial_accumulator_valu:14,inp:11,inp_seq_len:11,inproceed:6,input:[1,4,7,9,10,11,12,14,15,16,18],input_format:3,input_len:9,input_sentence_s:16,inputs_len:9,inputt:11,insert:[12,14,15],insert_ratio:[12,14,15],instal:[0,3,4],instanc:[7,11],instanti:7,instead:[0,2,5,12,14,15],instruct:14,int8:15,integ:11,integr:0,interact:5,interfac:7,intermedi:1,intermediate_output:1,intern:10,interv:14,introduc:[1,2],introduct:2,invalid:[12,14,15],involv:1,is_finish:11,isn:11,item:9,iter:7,its:[0,2],itself:2,jakob:18,jean:6,jinsong:18,job:5,joiner:[12,14,15],jone:18,journal:0,json:13,kaiser:18,keep:[10,11,14],keep_checkpoint:[3,14],keep_stat:14,keep_topk:11,keep_topp:11,kei:9,kera:14,key_len:9,kim:6,klau:18,klein:6,krikun:18,label:14,label_smooth:[3,14],lambda:[12,14,15],lambda_align:14,lambda_coverag:14,lang:2,lang_a:2,lang_b:2,lang_pair:[2,15],languag:[1,4,12,14,16],language_pair:16,last:[2,14,15],layer:[1,9,14,15],layer_cach:9,layer_type_to_cl:1,layernorm:14,layerstack:2,lead:11,learn:[1,7,14],learning_r:[3,7,14],learning_rate_decai:14,learning_rate_decay_fn:7,least:0,leav:[2,14],left:1,len:[7,9,11],length:[2,7,9,11,12,14,15,16],length_averag:11,length_non:11,length_pen:11,length_penalti:[11,15],length_wu:11,less:2,let:[2,3],level:[12,14],lib:5,librari:14,like:[0,11,15],limit:15,lin:[1,14],linattentionbridgelay:4,line:[0,3,12,14,15],linear:1,linear_warmup:14,linguist:[9,18],link:[0,1,5],list:[0,2,7,9,10,11,12,14,15],list_model:10,literatur:14,llion:18,load:[5,7,9,10,14],load_model:10,load_pretrained_vector:9,loader:4,local:[0,2],localhost:14,log:[4,7,11],log_fil:[14,15],log_file_level:[14,15],log_prob:11,log_tensorboard:7,logger:11,login:5,logit:[11,15],logsumexp:11,longer:15,longest:11,longtensor:[7,9,11],look:[0,6,9,15],loop:7,loss:[4,14],loss_scal:14,losscomputebas:7,love:0,lower:[2,14],lsl:[11,18],lstm:14,lua:10,lukasz:18,luong:[14,18],lustrep1:5,lustrep2:5,macherei:18,machin:[6,9,11,18],made:2,magic:11,mai:[2,7,10,11,12,14],main:[0,6,7,12,14,15],maintain:11,make:[0,5,7,12,14,15],make_shard_st:7,mammoth:[0,4,5,6,7,9,10,11,14],manag:7,mani:[7,11,14],manipul:7,manual:[10,11],map:[2,7],marian:14,mark:14,marker:10,mask:[9,12,14,15],mask_length:[12,14,15],mask_or_step:9,mask_ratio:[12,14,15],mass:[12,14,15],massiv:[2,6],master:14,master_ip:14,master_port:14,match:10,mathbb:1,mathbf:1,mathemat:1,matric:1,matrix:[1,9,14],max:[7,11,16],max_generator_batch:[3,14],max_grad_norm:[3,7,14],max_len:9,max_length:[11,15],max_relative_posit:[9,14],max_sent_length:15,max_sentence_length:16,max_siz:7,maxim:18,maximum:[12,14,15],maybe_convert_align:10,maybe_detoken:10,maybe_detokenize_with_align:10,maybe_postprocess:10,maybe_preprocess:10,maybe_token:10,maybe_update_forbidden_token:11,maybe_update_target_prefix:11,mean:[2,10,14,15],mechan:[1,2],mem:5,memori:[10,14],memory_bank:11,merg:[9,14],meta:2,metadata:7,method:[7,9,14],metric:15,mi250:5,mike:18,min_length:[11,15],minh:18,minimum:15,mirror:14,mix:7,mkdir:[5,16],mlp:[9,14],mode:[2,12,14,15],model:[1,2,4,11,12],model_dim:9,model_dtyp:[3,7,14],model_id:10,model_kwarg:10,model_prefix:16,model_root:10,model_sav:7,model_step:3,model_task:14,model_typ:14,modelsaverbas:7,modif:7,modifi:[0,11],modul:[0,1,4,5,7,14,15],modular:6,mohammad:18,monolingu:2,more:[0,2,11,12,14,15],most:[11,15],mostli:7,move:[10,14],moving_averag:[7,14],much:14,multi:[0,1,9],multiheadedattent:[1,9],multilingu:[2,6],multipl:[0,1,2,7,9,14,15],multipli:1,multplic:0,must:[2,9,10,14],mymodul:5,n_batch:7,n_best:[10,11,15],n_bucket:14,n_correct:7,n_edge_typ:14,n_node:14,n_sampl:[3,12,14],n_seg:10,n_src_word:7,n_step:14,n_word:7,name:[0,2,4,11,12,14,16],namespac:10,napoleon:0,nccl:14,necessari:[0,3,5,7,11,14,15],necessit:2,need:[0,2,3,7,9,14,18],neg:[10,14],network:[9,18],neubig:18,neural:[6,9,11,18],never:11,news_commentari:3,next:[2,7,11,15],nfeat:9,ngram:[11,15],nightmar:2,niki:18,nlp:[6,17],nmt:[7,11,14,15],nmtmodel:7,noam:[3,14,18],noamwd:14,node:[2,5,7,14],node_rank:14,nois:2,non:[9,11,14],none:[7,9,10,11,12,14,15],nonetyp:[9,11],norm:[9,14],norm_method:7,normal:[1,3,7,14],normalz:7,norouzi:18,note:[0,2,3,5,11],noth:[0,7],notset:[14,15],ntask:5,nucleu:15,num_step:7,num_thread:12,number:[1,2,7,9,11,12,14,15],nvidia:14,obj:[0,7],object:[0,7,10,11,12,14,15,16],oder:2,off:14,ofi:5,often:[12,14,15],on_timemout:10,on_timeout:10,onc:[11,14],one:[0,1,2,7,12,14,15],onli:[2,7,11,12,14,15],onmt:16,onmt_build_vocab:3,onmt_token:[12,14,15],onmt_transl:3,onmttok:4,open:6,opennmt:[0,2,5,6,7,13],oper:1,operatornam:1,opt:[3,7,10,14,15],opt_level:14,optim:[3,4],option:[0,2,3,5,7,9,10,11,12,14,15,16],opu:4,opus100:[16,17],ord:18,order:[2,14],org:[1,5,6,14,15,18],origin:[1,14,16],oriol:18,other:[1,5,7,11,12,14,15,16,18],other_lang:16,otherwis:[2,9,14,15],our:[5,11],our_stat:7,out:[1,2,6,7,17],out_config:2,out_fil:11,outcom:1,output:[1,2,3,7,9,10,11,12,14,15],output_model:15,over:[0,2,3,7,11,14,15,16],overal:1,overrid:[11,12,14],overview:4,overwrit:[5,12,14],own:[7,15],ownership:7,p17:6,p18:14,packag:[5,10],pad:[7,9,11],pair:[2,7,10,14,15,16],paper:[0,1,14],parallel:[9,11,12,14],parallel_path:11,parallelcorpu:11,param:7,param_init:[3,14],param_init_glorot:[3,14],paramet:[3,7,9,10,11,12,14,15],parenthes:0,parmar:18,pars:10,parse_opt:10,part:[1,11],particular:[0,2],partit:5,pass:[1,2,7,10,14],past:[0,14],path:[2,5,9,10,11,12,14,15],path_src:3,path_tgt:3,patienc:7,pattern:2,pdf:14,pen:11,penalti:[4,11,14],penaltybuild:11,peopl:5,per:[0,2,12,14,15],perceiv:[1,14],perceiverattentionbridgelay:4,percentag:[12,14,15],perfom:14,perform:[1,14],permut:[12,14,15],permute_sent_ratio:[12,14,15],perplex:7,pfs:5,pham:18,phrase_t:[11,15],piec:3,pip3:[5,6,17],pip:[0,5],pipelin:[12,14,15],pleas:[0,6],plu:14,poisson:[12,14,15],poisson_lambda:[12,14,15],polosukhin:18,pool:14,port:[13,14],portal:6,pos_ffn_activation_fn:[9,14],posit:[9,14],position_encod:[9,14],position_ffn:9,positionalencod:9,positionwisefeedforward:[9,14],possibl:[2,7,10,11,12,14,15],postprocess:10,postprocess_opt:10,potenti:11,pouta:16,ppl:7,pre:[7,10,11],pre_word_vecs_dec:14,pre_word_vecs_enc:14,preced:2,precis:7,pred:15,pred_scor:11,pred_sent:11,predict:[7,11,15],prefer:0,prefix:[2,7,12,14,15],prefix_seq_len:11,preliminari:3,preload:10,preload_model:10,prepar:[4,11],prepare_wmt_data:3,preprint:18,preprocess:10,preprocess_opt:10,presenc:2,presum:11,pretrain:[9,14],prevent:[11,15],previou:[1,2,9,11],previous:1,primari:2,prime:1,print:[7,14,15],prior:3,prior_token:[12,14,15],prob:11,proba:15,probabl:[9,11,12,14,15],problem:11,proc:[6,18],procedur:2,process:[1,7,10,12,14],processu:10,produc:[1,11,12,14,15],product:1,projappl:5,project:[0,1,5,6],project_2005099:5,project_462000125:5,propag:7,proper:10,properli:5,properti:[7,9],proport:[2,12,14,15],provid:[6,15],prune:4,pty:5,pull_request_chk:0,punctuat:0,put:11,pwd:16,pyonmttok:[12,14,15],python3:[2,5],python:[0,2,5,14],pythonpath:5,pythonuserbas:5,pytorch:[0,5],qin:18,quantiz:15,queri:9,query_len:9,queue:[12,14],queue_siz:[3,14],quickstart:[4,6],quoc:18,quot:0,rais:[12,14],random:[4,12,14],random_ratio:[12,14,15],random_sampling_temp:[11,15],random_sampling_topk:[11,15],random_sampling_topp:[11,15],randomli:11,rang:15,rank:[11,14],ranslat:18,rare:11,rate:[4,7],rather:0,ratio:[11,15],raw:[11,15],rccl:5,reach:11,read:[0,2,10,16],readabl:[0,2],reader:4,readm:14,rebuild:10,rebuild_seg_packag:10,receiv:2,recent:14,recommend:14,recommonmark:0,rectifi:1,recurr:9,redund:2,ref:0,refer:[0,1,4],regardless:2,regular:[12,14,15],rel:14,relat:[3,12,14,15],relationship:1,relev:[9,11],relu:[1,9,14],rememb:0,remov:2,renorm:14,reorder:11,repeat:[11,15],repetit:15,replac:[11,12,14,15],replace_length:[12,14,15],replace_unk:[11,15],report:[6,7,14,15],report_align:[11,15],report_everi:[3,14],report_manag:7,report_scor:11,report_stats_from_paramet:[7,14],report_tim:[11,15],reportmgrbas:7,repres:[1,7],represent:[1,14],reproduc:4,requir:[0,7,14],research:6,reset:7,reset_optim:14,resett:14,residu:9,resourc:2,respect:[1,2],respons:7,rest:13,restrict:[12,14,15],result:[1,10,14],return_attent:11,reus:14,reuse_copy_attn:14,revers:[12,14,15],reversible_token:[12,14,15],rico:18,right:[0,1],rmsnorm:14,rnn:[7,14],rnn_size:[3,14],roblem:18,rocm5:5,rocm:5,root:[1,2],rotat:[12,14,15],rotate_ratio:[12,14,15],roundrobin:14,row:2,rsqrt:14,rst:0,run:[0,2,3,7,10,14,15],rush:6,sacrebleu:[3,5,6,17],sai:2,samantao:5,same:[0,2,3,9,10,14],sampl:[4,11,12,14,16],sample_with_temperatur:11,sampling_temp:11,saniti:15,save:[7,12,14,15,16],save_all_gpu:14,save_checkpoint_step:[3,7,14],save_config:[12,14,15],save_data:[3,12,14],save_model:[3,14],saver:7,scale:[11,14],schedul:[7,14],schuster:18,score:[4,10,15],scorer:11,scratch:5,script:[0,3,4,5],search:[0,2,4,11],second:[1,2,9,10],secur:[12,14],see:[2,9,10,11,12,14],seed:[3,11,12,14,15],seemingli:14,seen:1,segment:[2,10,15],select:[9,11],select_index:11,self:[1,9,10,11,14],self_attn_typ:14,send:[0,14],senellart:6,sennrich:18,sensibl:0,sent:[7,14,15],sent_numb:11,sentenc:[11,12,14,15,16],sentencepiec:[2,3,5,6,12,14,15,17],separ:2,seper:10,seq2seq:[11,14],seq:11,seq_len:[1,9,11],sequenc:[1,2,7,9,10,11,12,14,15],serial:9,serv:1,server:[4,14,16],servermodel:10,servermodelerror:10,session:5,set:[1,2,3,5,7,9,10,11,12,14,15],setup:3,sever:[2,9,11],sgd:14,sh16:[9,18],shape:[0,9,11],shard:[7,14,15],shard_siz:[7,15],share:[5,12,14,15],share_decoder_embed:[3,14],share_embed:[3,14],share_vocab:[12,14],shazeer:18,shortest:11,shot:2,should:[2,3,11,14],shuf:16,shuffle_input_sent:16,side:[2,7,10,12,14,15],side_a:2,side_b:2,silent:[3,12,14],similar:[1,2,9,14],simpl:[1,7,14],simpleattentionbridgelay:4,simulatan:9,sin:14,singl:[0,10,14],single_pass:14,sinusoid:9,site:5,size:[2,7,9,11,12,14,15,16],skip:[2,12,14],skip_empty_level:[3,12,14],slow:[12,15],slurm:[2,5],smaller:[12,14,15],smooth:[12,14,15],softmax:[1,14,15],some:[0,2,7,15],someth:0,sometim:0,sort:[10,16],sorted_pair:2,sourc:[0,2,4,5,6,7,9,10,11,12,14],sp_path:16,space:[0,1,14],spacer:[12,14,15],span:[12,14,15],specif:[1,2,6,11,12,14,17],specifi:[1,12,14,15],sphinx:0,sphinx_rtd_them:0,sphinxcontrib:0,spill:0,spm_decod:3,spm_encod:[3,16],spm_train:16,sqrt:1,squar:[1,2],src:[2,3,7,10,11,12,14,15,16],src_embed:14,src_feat:15,src_feats_vocab:[12,14],src_file_path:11,src_ggnn_size:14,src_group:2,src_lang:[2,15],src_languag:2,src_len:7,src_length:11,src_map:11,src_onmttok_kwarg:[12,14,15],src_raw:11,src_seq_length:[3,12,14,15],src_seq_length_trunc:14,src_subword_alpha:[3,12,14,15],src_subword_model:[3,12,14,15],src_subword_nbest:[3,12,14,15],src_subword_typ:[12,14,15],src_subword_vocab:[12,14,15],src_vocab:[3,11,12,14],src_vocab_s:14,src_vocab_threshold:[12,14,15],src_word_vec_s:14,src_words_min_frequ:14,sru:4,srun:5,stabl:1,stack:[14,15],stage:1,stand:0,standard:[9,14,15],start:[2,4,5,7,10,14,16],start_decay_step:14,stat:[7,14],stat_list:7,state:[7,11,14],state_dict:14,state_dim:14,statist:[7,14],stdout:7,step:[1,2,4,7,9,11,14,15],stepwis:9,stepwise_penalti:[11,15],still:0,stop:[12,14,15],store:14,str:[0,7,9,10,11],strategi:[4,7,14],string:[7,9,12,14,15],structur:[1,4],style:[0,12,14,15],styleguid:0,subclass:[7,11],subcompon:2,subdirectori:5,subsequ:1,subset:16,substitut:2,subword:[2,4],suggest:14,sum:[7,9,11,14],sume:7,summari:[0,11,15],superclass:0,supervis:[2,14],support:[0,2,14],suppos:16,sure:[5,11],sutskev:18,switchout:[4,18],switchout_temperatur:[12,14,15],symmetr:2,system:[11,14,18],tab:[12,14],tabl:[9,15],take:[1,2,6,9,12,14,15],tangent:1,tanh:1,tar:16,target:[2,4,7,10,11,12,14],target_prefix:11,task:[2,3,4,7,11],task_distribution_strategi:14,task_queue_manag:7,tatoeba:[2,4],tau:[12,14,15],technic:6,temperatur:[2,11,12,14,15],templat:2,tensor:[0,7,9,11],tensorboard:[7,14],tensorboard_log_dir:14,tensorflow:14,term:1,test:[0,3,5,9],testset:3,text:[7,11,14,15],tgt:[2,3,7,10,12,14,15],tgt_embed:14,tgt_file_path:11,tgt_group:2,tgt_lang:[2,15],tgt_languag:2,tgt_len:7,tgt_onmttok_kwarg:[12,14,15],tgt_prefix:[11,15],tgt_sent:11,tgt_seq_length:[3,12,14,15],tgt_seq_length_trunc:14,tgt_subword_alpha:[3,12,14,15],tgt_subword_model:[3,12,14,15],tgt_subword_nbest:[3,12,14,15],tgt_subword_typ:[12,14,15],tgt_subword_vocab:[12,14,15],tgt_vocab:[3,7,12,14],tgt_vocab_s:14,tgt_vocab_threshold:[12,14,15],tgt_word_vec_s:14,tgt_words_min_frequ:14,than:[0,11,14,16],thang:18,thant:11,thei:[1,11],them:2,thi:[0,1,2,3,5,6,7,9,11,12,14,15],thin:7,thing:[0,2],thoroughli:9,thread:12,three:1,through:[1,2,7],thu:7,tic:0,tick:0,time:[1,2,5,7,11,14,15],timeout:10,timer:10,titl:6,to_cpu:10,to_gpu:10,todo:[5,16],tok:10,token:[3,7,10,11,12,14,15],token_drop:4,token_mask:4,tokendrop:[12,14,15],tokendrop_temperatur:[12,14,15],tokenizer_mark:10,tokenizer_opt:10,tokenmask:[12,14,15],tokenmask_temperatur:[12,14,15],too:11,tool:4,toolkit:6,top:[1,11,15],topk_id:11,topk_scor:11,torch:[0,5,7,9,14],torchtext:7,total:[2,7,14],trail:0,train:[2,4,5,6,7,9],train_extremely_large_corpu:16,train_from:14,train_it:7,train_loss:7,train_loss_md:7,train_step:[3,7,14],trainabl:7,trainer:4,training_step:7,transform:[1,3,4,7,18],transformer_ff:[3,14],transformerattentionbridgelay:4,transformerencoderlay:1,translat:[2,4,6,7,9,10,13,18],translate_batch:11,translation_serv:10,translationbuild:11,translationserv:10,travi:0,trg:2,triang:2,trick:[4,9],trunc_siz:7,truncat:[7,14],truncated_decod:14,trust:16,turn:14,tutori:[4,17],two:[1,2,9],txt:[0,15,16],type:[0,1,2,4,7,9,10,11,12,15],typic:[7,14],under:[2,14,15],undergo:1,undergon:1,underli:11,uniform:14,unigram:[12,14,15],union:0,unit:1,unittest:0,unk:[11,15],unknown:11,unless:2,unload:10,unload_model:10,unmodifi:11,unnecessari:[0,2],unset:2,until:[11,15],unwieldli:2,updat:[5,7,10,11,14],update_dropout:9,update_finish:11,update_learning_r:14,update_n_src_word:7,update_vocab:14,upgrad:5,upper:2,url:[5,6,18],url_root:13,usag:[4,12,13,14,15],use:[0,1,2,3,5,7,9,10,11,12,14,15,16],used:[1,2,3,7,9,10,11,12,14,15],useful:7,user:[5,7,9,10],uses:[0,2,9,11,14],using:[0,1,2,6,9,10,11,12,14,15],uszkoreit:18,util:[1,7],v11:3,valid:[3,7,12,14,15],valid_batch_s:[3,14],valid_it:7,valid_loss:7,valid_loss_md:7,valid_step:[3,7,14],valu:[1,2,7,9,10,11,12,14,15],variabl:[2,5,11],variat:0,vaswani:18,vaswanispujgkp17:0,vector:[9,14],venv:5,verbos:[11,14,15],veri:[0,15],version:[10,11],via:[9,18],vinyal:18,virtual:5,visit:0,visual:14,vocab:[3,4,7,11],vocab_path:[12,14],vocab_s:[11,14,16],vocab_sample_queue_s:12,vocab_size_multipl:14,vocabulari:[2,7,12,14,15,16],vsp:[9,18],wai:[2,11],wait:2,wang:18,want:[2,15],warmup:14,warmup_step:[3,14],warn:[12,14,15],weight:[1,2,3,9,14,15],weight_decai:14,weighted_sampl:14,well:[0,14],wget:16,what:[2,7,10],when:[0,2,6,9,11,12,14,15,16],where:[1,3,5,9,11,12,14,15],wherea:[11,14],whether:[7,10,11,12,14,15],which:[2,9,11,14],whl:5,whole:[3,11],whose:15,why:1,wiki:14,wikipedia:14,window:[12,14,15],wise:1,with_align:7,within:[1,10],without:[0,14],wmt14_en_d:3,wmt:3,wmtend:3,wojciech:18,wolfgang:18,word2vec:14,word:[1,9,11,12,14,15],word_align:11,word_lut:9,word_padding_idx:9,word_vec_s:[3,9,14],word_vocab_s:9,work:[0,2,11,14],workflow:6,world_siz:[3,14],would:[2,11,14],wpdn18:[12,14,15,18],wrap:10,wrapper:7,writabl:2,write:[2,7],writer:7,written:3,wsc:[11,18],www:14,xavier_uniform:14,xent:7,xinyi:18,xiong:18,xzvf:16,yaml:[3,12,14,15],year:6,yet:11,yml:0,yonghui:18,yoon:6,you:[0,2,3,5,9,14,15,18],your:[0,2,5,15,16],your_venv_nam:5,your_vevn_nam:5,yourself:6,yuan:18,yuntian:6,zaremba:18,zero:[2,7,9,11,14,15],zero_grad:7,zhang:18,zhifeng:18,zihang:18,zxs18:[9,18]},titles:["Contributors","Attention Bridge","Config-config tool","Translation","Contents","Installation","Overview","Framework","Data Loaders","Modules","Server","Translation","Build Vocab","Server","Train","Translate","Prepare Data","Quickstart","References"],titleterms:{"class":11,The:2,actual:2,adapt:[2,14],adapter_config:2,ae_path:2,ae_transform:2,align:14,allocate_devic:2,altern:2,architectur:9,argument:13,attent:[1,9,14],autoencod:2,beam:15,bridg:[1,14],build:[3,12],challeng:16,citat:6,cluster_languag:2,command:2,common:[12,14,15],complete_language_pair:2,config:2,config_al:2,config_config:2,configur:[12,14,15],content:4,contributor:0,conv2conv:9,copi:9,core:[9,10],corpora:2,corpora_schedul:2,data:[3,8,12,14,15,16,17],dataset:8,dec_sharing_group:2,decod:[9,11,14,15],denois:[12,14,15],direct:16,distanc:2,distance_matrix:2,docstr:0,download:[3,16],dynam:14,effici:15,embed:14,enc_sharing_group:2,encod:[9,14],evalu:3,featur:14,feedforwardattentionbridgelay:1,filter:[12,14,15],framework:7,gener:14,get:16,group:2,guidelin:0,inferfeat:[12,14,15],initi:14,input:2,instal:[5,6,17],kei:2,languag:[2,15],level:2,linattentionbridgelay:1,line:2,loader:8,log:[14,15],loss:7,lumi:5,mahti:5,mammoth:17,manual:2,matrix:2,model:[3,7,10,14,15,16],modul:9,n_gpus_per_nod:2,n_group:2,n_node:2,name:13,onmttok:[12,14,15],optim:[7,14],opu:16,other:2,overrid:2,overview:6,paramet:2,pars:16,path:16,penalti:15,perceiverattentionbridgelay:1,prepar:[3,16,17],prune:14,puhti:5,quickstart:17,random:15,rate:14,reader:8,refer:18,relev:16,remove_temporary_kei:2,reproduc:[12,14,15],run:5,sampl:15,score:11,search:15,sentencepiec:16,server:[10,13],set:16,set_transform:2,share:2,sharing_group:2,shot:16,simpleattentionbridgelay:1,sourc:15,specifi:2,src_path:2,sru:9,stage:2,step:[3,16,17],strategi:11,structur:9,subword:[3,12,14,15],supervis:16,switchout:[12,14,15],target:15,task:14,tatoeba:16,test:16,tgt_path:2,than:2,token_drop:[12,14,15],token_mask:[12,14,15],tool:2,top:2,train:[3,14,16],trainer:7,transform:[2,9,12,14,15],transformerattentionbridgelay:1,translat:[3,11,15,16],translation_config:2,translation_config_dir:2,trick:15,type:14,usag:2,use_introduce_at_training_step:2,use_weight:2,valid:16,variabl:16,vocab:[12,14,16],vocabulari:3,yaml:2,zero:16,zero_shot:2}})
\ No newline at end of file
