# from mammoth.utils.logging import logger
# import bitsandbytes as bnb
-# from mammoth.modules.stable_embeddings import StableEmbedding
class SequenceTooLongError ( Exception ):
@@ -243,7 +242,7 @@ Source code for mammoth.modules.embeddings
[docs] class Embeddings ( nn . Module ):
"""Words embeddings for encoder/decoder.
-
Additionally includes ability to add sparse input features
+
Additionally includes ability to add input features
based on "Linguistic Input Features Improve Neural Machine Translation"
:cite:`sennrich2016linguistic`.
@@ -293,7 +292,6 @@
Source code for mammoth.modules.embeddings
feat_padding_idx = [],
feat_vocab_sizes = [],
dropout = 0 ,
- sparse = False ,
freeze_word_vecs = False ,
):
self . _validate_args ( feat_merge , feat_vocab_sizes , feat_vec_exponent , feat_vec_size , feat_padding_idx )
@@ -324,7 +322,7 @@ Source code for mammoth.modules.embeddings
# The embedding matrix look-up tables. The first look-up table
# is for words. Subsequent ones are for features, if any exist.
emb_params = zip ( vocab_sizes , emb_dims , pad_indices )
- embeddings = [ nn . Embedding ( vocab , dim , padding_idx = pad , sparse = sparse ) for vocab , dim , pad in emb_params ]
+ embeddings = [ nn . Embedding ( vocab , dim , padding_idx = pad ) for vocab , dim , pad in emb_params ]
emb_luts = Elementwise ( feat_merge , embeddings )
# The final output size of word + feature vectors. This can vary
diff --git a/_modules/mammoth/modules/global_attention.html b/_modules/mammoth/modules/global_attention.html
deleted file mode 100644
index 09864e21..00000000
--- a/_modules/mammoth/modules/global_attention.html
+++ /dev/null
@@ -1,444 +0,0 @@
-
-
-
-
-
-
- mammoth.modules.global_attention — MAMMOTH documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- MAMMOTH
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Docs »Module code »mammoth.modules.global_attention
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Source code for mammoth.modules.global_attention
-"""Global attention modules (Luong / Bahdanau)"""
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-
-from mammoth.modules.sparse_activations import sparsemax
-from mammoth.utils.misc import aeq , sequence_mask
-
-# This class is mainly used by decoder.py for RNNs but also
-# by the CNN / transformer decoder when copy attention is used
-# CNN has its own attention mechanism ConvMultiStepAttention
-# Transformer has its own MultiHeadedAttention
-
-
-[docs] class GlobalAttention ( nn . Module ):
-
r """
-
Global attention takes a matrix and a query vector. It
-
then computes a parameterized convex combination of the matrix
-
based on the input query.
-
-
Constructs a unit mapping a query `q` of size `dim`
-
and a source matrix `H` of size `n x dim`, to an output
-
of size `dim`.
-
-
-
.. mermaid::
-
-
graph BT
-
A[Query]
-
subgraph RNN
-
C[H 1]
-
D[H 2]
-
E[H N]
-
end
-
F[Attn]
-
G[Output]
-
A --> F
-
C --> F
-
D --> F
-
E --> F
-
C -.-> G
-
D -.-> G
-
E -.-> G
-
F --> G
-
-
All models compute the output as
-
:math:`c = \sum_{j=1}^{\text{SeqLength}} a_j H_j` where
-
:math:`a_j` is the softmax of a score function.
-
Then then apply a projection layer to [q, c].
-
-
However they
-
differ on how they compute the attention score.
-
-
* Luong Attention (dot, general):
-
* dot: :math:`\text{score}(H_j,q) = H_j^T q`
-
* general: :math:`\text{score}(H_j, q) = H_j^T W_a q`
-
-
-
* Bahdanau Attention (mlp):
-
* :math:`\text{score}(H_j, q) = v_a^T \text{tanh}(W_a q + U_a h_j)`
-
-
-
Args:
-
dim (int): dimensionality of query and key
-
coverage (bool): use coverage term
-
attn_type (str): type of attention to use, options [dot,general,mlp]
-
attn_func (str): attention function to use, options [softmax,sparsemax]
-
-
"""
-
-
def __init__ ( self , dim , coverage = False , attn_type = "dot" , attn_func = "softmax" ):
-
super ( GlobalAttention , self ) . __init__ ()
-
-
self . dim = dim
-
assert attn_type in [ "dot" , "general" , "mlp" ], "Please select a valid attention type (got {:s} )." . format (
-
attn_type
-
)
-
self . attn_type = attn_type
-
assert attn_func in [ "softmax" , "sparsemax" ], "Please select a valid attention function."
-
self . attn_func = attn_func
-
-
if self . attn_type == "general" :
-
self . linear_in = nn . Linear ( dim , dim , bias = False )
-
elif self . attn_type == "mlp" :
-
self . linear_context = nn . Linear ( dim , dim , bias = False )
-
self . linear_query = nn . Linear ( dim , dim , bias = True )
-
self . v = nn . Linear ( dim , 1 , bias = False )
-
# mlp wants it with bias
-
out_bias = self . attn_type == "mlp"
-
self . linear_out = nn . Linear ( dim * 2 , dim , bias = out_bias )
-
-
if coverage :
-
self . linear_cover = nn . Linear ( 1 , dim , bias = False )
-
-
[docs] def score ( self , h_t , h_s ):
-
"""
-
Args:
-
h_t (FloatTensor): sequence of queries ``(batch, tgt_len, dim)``
-
h_s (FloatTensor): sequence of sources ``(batch, src_len, dim``
-
-
Returns:
-
FloatTensor: raw attention scores (unnormalized) for each src index
-
``(batch, tgt_len, src_len)``
-
"""
-
-
# Check input sizes
-
src_batch , src_len , src_dim = h_s . size ()
-
tgt_batch , tgt_len , tgt_dim = h_t . size ()
-
aeq ( src_batch , tgt_batch )
-
aeq ( src_dim , tgt_dim )
-
aeq ( self . dim , src_dim )
-
-
if self . attn_type in [ "general" , "dot" ]:
-
if self . attn_type == "general" :
-
h_t_ = h_t . view ( tgt_batch * tgt_len , tgt_dim )
-
h_t_ = self . linear_in ( h_t_ )
-
h_t = h_t_ . view ( tgt_batch , tgt_len , tgt_dim )
-
h_s_ = h_s . transpose ( 1 , 2 )
-
# (batch, t_len, d) x (batch, d, s_len) --> (batch, t_len, s_len)
-
return torch . bmm ( h_t , h_s_ )
-
else :
-
dim = self . dim
-
wq = self . linear_query ( h_t . view ( - 1 , dim ))
-
wq = wq . view ( tgt_batch , tgt_len , 1 , dim )
-
wq = wq . expand ( tgt_batch , tgt_len , src_len , dim )
-
-
uh = self . linear_context ( h_s . contiguous () . view ( - 1 , dim ))
-
uh = uh . view ( src_batch , 1 , src_len , dim )
-
uh = uh . expand ( src_batch , tgt_len , src_len , dim )
-
-
# (batch, t_len, s_len, d)
-
wquh = torch . tanh ( wq + uh )
-
-
return self . v ( wquh . view ( - 1 , dim )) . view ( tgt_batch , tgt_len , src_len )
-
-
[docs] def forward ( self , source , memory_bank , memory_lengths = None , coverage = None ):
-
"""
-
-
Args:
-
source (FloatTensor): query vectors ``(batch, tgt_len, dim)``
-
memory_bank (FloatTensor): source vectors ``(batch, src_len, dim)``
-
memory_lengths (LongTensor): the source context lengths ``(batch,)``
-
coverage (FloatTensor): None (not supported yet)
-
-
Returns:
-
(FloatTensor, FloatTensor):
-
-
* Computed vector ``(tgt_len, batch, dim)``
-
* Attention distribtutions for each query
-
``(tgt_len, batch, src_len)``
-
"""
-
-
# one step input
-
if source . dim () == 2 :
-
one_step = True
-
source = source . unsqueeze ( 1 )
-
else :
-
one_step = False
-
-
batch , source_l , dim = memory_bank . size ()
-
batch_ , target_l , dim_ = source . size ()
-
aeq ( batch , batch_ )
-
aeq ( dim , dim_ )
-
aeq ( self . dim , dim )
-
if coverage is not None :
-
batch_ , source_l_ = coverage . size ()
-
aeq ( batch , batch_ )
-
aeq ( source_l , source_l_ )
-
-
if coverage is not None :
-
cover = coverage . view ( - 1 ) . unsqueeze ( 1 )
-
memory_bank += self . linear_cover ( cover ) . view_as ( memory_bank )
-
memory_bank = torch . tanh ( memory_bank )
-
-
# compute attention scores, as in Luong et al.
-
align = self . score ( source , memory_bank )
-
-
if memory_lengths is not None :
-
mask = sequence_mask ( memory_lengths , max_len = align . size ( - 1 ))
-
mask = mask . unsqueeze ( 1 ) # Make it broadcastable.
-
align . masked_fill_ ( ~ mask , - float ( 'inf' ))
-
-
# Softmax or sparsemax to normalize attention weights
-
if self . attn_func == "softmax" :
-
align_vectors = F . softmax ( align . view ( batch * target_l , source_l ), - 1 )
-
else :
-
align_vectors = sparsemax ( align . view ( batch * target_l , source_l ), - 1 )
-
align_vectors = align_vectors . view ( batch , target_l , source_l )
-
-
# each context vector c_t is the weighted average
-
# over all the source hidden states
-
c = torch . bmm ( align_vectors , memory_bank )
-
-
# concatenate
-
concat_c = torch . cat ([ c , source ], 2 ) . view ( batch * target_l , dim * 2 )
-
attn_h = self . linear_out ( concat_c ) . view ( batch , target_l , dim )
-
if self . attn_type in [ "general" , "dot" ]:
-
attn_h = torch . tanh ( attn_h )
-
-
if one_step :
-
attn_h = attn_h . squeeze ( 1 )
-
align_vectors = align_vectors . squeeze ( 1 )
-
-
# Check output sizes
-
batch_ , dim_ = attn_h . size ()
-
aeq ( batch , batch_ )
-
aeq ( dim , dim_ )
-
batch_ , source_l_ = align_vectors . size ()
-
aeq ( batch , batch_ )
-
aeq ( source_l , source_l_ )
-
-
else :
-
attn_h = attn_h . transpose ( 0 , 1 ) . contiguous ()
-
align_vectors = align_vectors . transpose ( 0 , 1 ) . contiguous ()
-
# Check output sizes
-
target_l_ , batch_ , dim_ = attn_h . size ()
-
aeq ( target_l , target_l_ )
-
aeq ( batch , batch_ )
-
aeq ( dim , dim_ )
-
target_l_ , batch_ , source_l_ = align_vectors . size ()
-
aeq ( target_l , target_l_ )
-
aeq ( batch , batch_ )
-
aeq ( source_l , source_l_ )
-
-
return attn_h , align_vectors
-
-
-
-
-
-
-
-
-
-
mammoth.modules.structured_attention — MAMMOTH documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- MAMMOTH
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Docs »Module code »mammoth.modules.structured_attention
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Source code for mammoth.modules.structured_attention
-import torch.nn as nn
-import torch
-import torch.cuda
-
-
-[docs] class MatrixTree ( nn . Module ):
-
"""Implementation of the matrix-tree theorem for computing marginals
-
of non-projective dependency parsing. This attention layer is used
-
in the paper "Learning Structured Text Representations"
-
:cite:`DBLP:journals/corr/LiuL17d`.
-
"""
-
-
def __init__ ( self , eps = 1e-5 ):
-
self . eps = eps
-
super ( MatrixTree , self ) . __init__ ()
-
-
[docs] def forward ( self , input ):
-
laplacian = input . exp () + self . eps
-
output = input . clone ()
-
for b in range ( input . size ( 0 )):
-
lap = laplacian [ b ] . masked_fill ( torch . eye ( input . size ( 1 ), device = input . device ) . ne ( 0 ), 0 )
-
lap = - lap + torch . diag ( lap . sum ( 0 ))
-
# store roots on diagonal
-
lap [ 0 ] = input [ b ] . diag () . exp ()
-
inv_laplacian = lap . inverse ()
-
-
factor = inv_laplacian . diag () . unsqueeze ( 1 ) . expand_as ( input [ b ]) . transpose ( 0 , 1 )
-
term1 = input [ b ] . exp () . mul ( factor ) . clone ()
-
term2 = input [ b ] . exp () . mul ( inv_laplacian . transpose ( 0 , 1 )) . clone ()
-
term1 [:, 0 ] = 0
-
term2 [ 0 ] = 0
-
output [ b ] = term1 - term2
-
roots_output = input [ b ] . diag () . exp () . mul ( inv_laplacian . transpose ( 0 , 1 )[ 0 ])
-
output [ b ] = output [ b ] + torch . diag ( roots_output )
-
return output
-
-
-
-
-
-
-
-
-
-
mammoth.modules.weight_norm — MAMMOTH documentation
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- MAMMOTH
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Source code for mammoth.modules.weight_norm
-""" Weights normalization modules """
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torch.nn import Parameter
-
-
-def get_var_maybe_avg ( namespace , var_name , training , polyak_decay ):
- """utility for retrieving polyak averaged params
- Update average
- """
- v = getattr ( namespace , var_name )
- v_avg = getattr ( namespace , var_name + '_avg' )
- v_avg -= ( 1 - polyak_decay ) * ( v_avg - v . data )
-
- if training :
- return v
- else :
- return v_avg
-
-
-def get_vars_maybe_avg ( namespace , var_names , training , polyak_decay ):
- """utility for retrieving polyak averaged params"""
- vars = []
- for vn in var_names :
- vars . append ( get_var_maybe_avg ( namespace , vn , training , polyak_decay ))
- return vars
-
-
-class WeightNormLinear ( nn . Linear ):
- """
- Implementation of "Weight Normalization: A Simple Reparameterization
- to Accelerate Training of Deep Neural Networks"
- :cite:`DBLP:journals/corr/SalimansK16`
-
- As a reparameterization method, weight normalization is same
- as BatchNormalization, but it doesn't depend on minibatch.
-
- NOTE: This is used nowhere in the code at this stage
- Vincent Nguyen 05/18/2018
- """
-
- def __init__ ( self , in_features , out_features , init_scale = 1.0 , polyak_decay = 0.9995 ):
- super ( WeightNormLinear , self ) . __init__ ( in_features , out_features , bias = True )
-
- self . V = self . weight
- self . g = Parameter ( torch . Tensor ( out_features ))
- self . b = self . bias
-
- self . register_buffer ( 'V_avg' , torch . zeros ( out_features , in_features ))
- self . register_buffer ( 'g_avg' , torch . zeros ( out_features ))
- self . register_buffer ( 'b_avg' , torch . zeros ( out_features ))
-
- self . init_scale = init_scale
- self . polyak_decay = polyak_decay
- self . reset_parameters ()
-
- def reset_parameters ( self ):
- return
-
- def forward ( self , x , init = False ):
- if init is True :
- # out_features * in_features
- self . V . data . copy_ ( torch . randn ( self . V . data . size ()) . type_as ( self . V . data ) * 0.05 )
- # norm is out_features * 1
- v_norm = self . V . data / self . V . data . norm ( 2 , 1 ) . expand_as ( self . V . data )
- # batch_size * out_features
- x_init = F . linear ( x , v_norm ) . data
- # out_features
- m_init , v_init = x_init . mean ( 0 ) . squeeze ( 0 ), x_init . var ( 0 ) . squeeze ( 0 )
- # out_features
- scale_init = self . init_scale / torch . sqrt ( v_init + 1e-10 )
- self . g . data . copy_ ( scale_init )
- self . b . data . copy_ ( - m_init * scale_init )
- x_init = scale_init . view ( 1 , - 1 ) . expand_as ( x_init ) * ( x_init - m_init . view ( 1 , - 1 ) . expand_as ( x_init ))
- self . V_avg . copy_ ( self . V . data )
- self . g_avg . copy_ ( self . g . data )
- self . b_avg . copy_ ( self . b . data )
- return x_init
- else :
- v , g , b = get_vars_maybe_avg ( self , [ 'V' , 'g' , 'b' ], self . training , polyak_decay = self . polyak_decay )
- # batch_size * out_features
- x = F . linear ( x , v )
- scalar = g / torch . norm ( v , 2 , 1 ) . squeeze ( 1 )
- x = scalar . view ( 1 , - 1 ) . expand_as ( x ) * x + b . view ( 1 , - 1 ) . expand_as ( x )
- return x
-
-
-[docs] class WeightNormConv2d ( nn . Conv2d ):
-
def __init__ (
-
self ,
-
in_channels ,
-
out_channels ,
-
kernel_size ,
-
stride = 1 ,
-
padding = 0 ,
-
dilation = 1 ,
-
groups = 1 ,
-
init_scale = 1.0 ,
-
polyak_decay = 0.9995 ,
-
):
-
super ( WeightNormConv2d , self ) . __init__ (
-
in_channels , out_channels , kernel_size , stride , padding , dilation , groups
-
)
-
-
self . V = self . weight
-
self . g = Parameter ( torch . Tensor ( out_channels ))
-
self . b = self . bias
-
-
self . register_buffer ( 'V_avg' , torch . zeros ( self . V . size ()))
-
self . register_buffer ( 'g_avg' , torch . zeros ( out_channels ))
-
self . register_buffer ( 'b_avg' , torch . zeros ( out_channels ))
-
-
self . init_scale = init_scale
-
self . polyak_decay = polyak_decay
-
self . reset_parameters ()
-
-
def reset_parameters ( self ):
-
return
-
-
[docs] def forward ( self , x , init = False ):
-
if init is True :
-
# out_channels, in_channels // groups, * kernel_size
-
self . V . data . copy_ ( torch . randn ( self . V . data . size ()) . type_as ( self . V . data ) * 0.05 )
-
v_norm = self . V . data / self . V . data . view ( self . out_channels , - 1 ) . norm ( 2 , 1 ) . view (
-
self . out_channels , * ([ 1 ] * ( len ( self . kernel_size ) + 1 ))
-
) . expand_as ( self . V . data )
-
x_init = F . conv2d ( x , v_norm , None , self . stride , self . padding , self . dilation , self . groups ) . data
-
t_x_init = x_init . transpose ( 0 , 1 ) . contiguous () . view ( self . out_channels , - 1 )
-
m_init , v_init = t_x_init . mean ( 1 ) . squeeze ( 1 ), t_x_init . var ( 1 ) . squeeze ( 1 )
-
# out_features
-
scale_init = self . init_scale / torch . sqrt ( v_init + 1e-10 )
-
self . g . data . copy_ ( scale_init )
-
self . b . data . copy_ ( - m_init * scale_init )
-
scale_init_shape = scale_init . view ( 1 , self . out_channels , * ([ 1 ] * ( len ( x_init . size ()) - 2 )))
-
m_init_shape = m_init . view ( 1 , self . out_channels , * ([ 1 ] * ( len ( x_init . size ()) - 2 )))
-
x_init = scale_init_shape . expand_as ( x_init ) * ( x_init - m_init_shape . expand_as ( x_init ))
-
self . V_avg . copy_ ( self . V . data )
-
self . g_avg . copy_ ( self . g . data )
-
self . b_avg . copy_ ( self . b . data )
-
return x_init
-
else :
-
v , g , b = get_vars_maybe_avg ( self , [ 'V' , 'g' , 'b' ], self . training , polyak_decay = self . polyak_decay )
-
-
scalar = torch . norm ( v . view ( self . out_channels , - 1 ), 2 , 1 )
-
if len ( scalar . size ()) == 2 :
-
scalar = g / scalar . squeeze ( 1 )
-
else :
-
scalar = g / scalar
-
-
w = scalar . view ( self . out_channels , * ([ 1 ] * ( len ( v . size ()) - 1 ))) . expand_as ( v ) * v
-
-
x = F . conv2d ( x , w , b , self . stride , self . padding , self . dilation , self . groups )
-
return x # This is used nowhere in the code at the moment (Vincent Nguyen 05/18/2018)
-
-
-class WeightNormConvTranspose2d ( nn . ConvTranspose2d ):
- def __init__ (
- self ,
- in_channels ,
- out_channels ,
- kernel_size ,
- stride = 1 ,
- padding = 0 ,
- output_padding = 0 ,
- groups = 1 ,
- init_scale = 1.0 ,
- polyak_decay = 0.9995 ,
- ):
- super ( WeightNormConvTranspose2d , self ) . __init__ (
- in_channels , out_channels , kernel_size , stride , padding , output_padding , groups
- )
- # in_channels, out_channels, *kernel_size
- self . V = self . weight
- self . g = Parameter ( torch . Tensor ( out_channels ))
- self . b = self . bias
-
- self . register_buffer ( 'V_avg' , torch . zeros ( self . V . size ()))
- self . register_buffer ( 'g_avg' , torch . zeros ( out_channels ))
- self . register_buffer ( 'b_avg' , torch . zeros ( out_channels ))
-
- self . init_scale = init_scale
- self . polyak_decay = polyak_decay
- self . reset_parameters ()
-
- def reset_parameters ( self ):
- return
-
- def forward ( self , x , init = False ):
- if init is True :
- # in_channels, out_channels, *kernel_size
- self . V . data . copy_ ( torch . randn ( self . V . data . size ()) . type_as ( self . V . data ) * 0.05 )
- v_norm = self . V . data / self . V . data . transpose ( 0 , 1 ) . contiguous () . view ( self . out_channels , - 1 ) . norm ( 2 , 1 ) . view (
- self . in_channels , self . out_channels , * ([ 1 ] * len ( self . kernel_size ))
- ) . expand_as ( self . V . data )
- x_init = F . conv_transpose2d (
- x , v_norm , None , self . stride , self . padding , self . output_padding , self . groups
- ) . data
- # self.out_channels, 1
- t_x_init = x_init . tranpose ( 0 , 1 ) . contiguous () . view ( self . out_channels , - 1 )
- # out_features
- m_init , v_init = t_x_init . mean ( 1 ) . squeeze ( 1 ), t_x_init . var ( 1 ) . squeeze ( 1 )
- # out_features
- scale_init = self . init_scale / torch . sqrt ( v_init + 1e-10 )
- self . g . data . copy_ ( scale_init )
- self . b . data . copy_ ( - m_init * scale_init )
- scale_init_shape = scale_init . view ( 1 , self . out_channels , * ([ 1 ] * ( len ( x_init . size ()) - 2 )))
- m_init_shape = m_init . view ( 1 , self . out_channels , * ([ 1 ] * ( len ( x_init . size ()) - 2 )))
-
- x_init = scale_init_shape . expand_as ( x_init ) * ( x_init - m_init_shape . expand_as ( x_init ))
- self . V_avg . copy_ ( self . V . data )
- self . g_avg . copy_ ( self . g . data )
- self . b_avg . copy_ ( self . b . data )
- return x_init
- else :
- v , g , b = get_vars_maybe_avg ( self , [ 'V' , 'g' , 'b' ], self . training , polyak_decay = self . polyak_decay )
- scalar = g / torch . norm ( v . transpose ( 0 , 1 ) . contiguous () . view ( self . out_channels , - 1 ), 2 , 1 ) . squeeze ( 1 )
- w = scalar . view ( self . in_channels , self . out_channels , * ([ 1 ] * ( len ( v . size ()) - 2 ))) . expand_as ( v ) * v
-
- x = F . conv_transpose2d ( x , w , b , self . stride , self . padding , self . output_padding , self . groups )
- return x
-
-
-
-
-
-
-
-
-
-
-
Source code for mammoth.translate.translator from mammoth.translate.greedy_search import GreedySearch , GreedySearchLM
from mammoth.utils.misc import tile , set_random_seed , report_matrix
from mammoth.utils.alignment import extract_alignment , build_align_pharaoh
-from mammoth.modules.copy_generator import collapse_copy_scores
from mammoth.constants import ModelTask , DefaultTokens
from mammoth.inputters.dataset import ParallelCorpus
from mammoth.inputters.dataloader import build_dataloader
@@ -847,39 +846,14 @@ Source code for mammoth.translate.translator )
# Generator forward.
- if not self . copy_attn :
- if "std" in dec_attn :
- attn = dec_attn [ "std" ]
- else :
- attn = None
- log_probs = self . model . generator [ f "generator_ { self . task . tgt_lang } " ]( dec_out . squeeze ( 0 ))
- # returns [(batch_size x beam_size) , vocab ] when 1 step
- # or [ tgt_len, batch_size, vocab ] when full sentence
+ if "std" in dec_attn :
+ attn = dec_attn [ "std" ]
else :
- attn = dec_attn [ "copy" ]
- scores = self . model . generator [ f "generator_ { self . task . tgt_lang } " ](
- dec_out . view ( - 1 , dec_out . size ( 2 )),
- attn . view ( - 1 , attn . size ( 2 )),
- src_map ,
- )
- # here we have scores [tgt_lenxbatch, vocab] or [beamxbatch, vocab]
- if batch_offset is None :
- scores = scores . view ( - 1 , batch . batch_size , scores . size ( - 1 ))
- scores = scores . transpose ( 0 , 1 ) . contiguous ()
- else :
- scores = scores . view ( - 1 , self . beam_size , scores . size ( - 1 ))
- scores = collapse_copy_scores (
- scores ,
- batch ,
- self . _tgt_vocab ,
- src_vocabs ,
- batch_dim = 0 ,
- batch_offset = batch_offset ,
- )
- scores = scores . view ( decoder_in . size ( 0 ), - 1 , scores . size ( - 1 ))
- log_probs = scores . squeeze ( 0 ) . log ()
- # returns [(batch_size x beam_size) , vocab ] when 1 step
- # or [ tgt_len, batch_size, vocab ] when full sentence
+ attn = None
+ log_probs = self . model . generator [ f "generator_ { self . task . tgt_lang } " ]( dec_out . squeeze ( 0 ))
+ # returns [(batch_size x beam_size) , vocab ] when 1 step
+ # or [ tgt_len, batch_size, vocab ] when full sentence
+
return log_probs , attn
def translate_batch ( self , batch , src_vocabs , attn_debug ):
diff --git a/_modules/mammoth/utils/loss.html b/_modules/mammoth/utils/loss.html
index ac753673..7cf7db3e 100644
--- a/_modules/mammoth/utils/loss.html
+++ b/_modules/mammoth/utils/loss.html
@@ -184,8 +184,6 @@ Source code for mammoth.utils.loss
import torch.nn.functional as F
import mammoth
-from mammoth.modules.sparse_losses import SparsemaxLoss
-from mammoth.modules.sparse_activations import LogSparsemax
from mammoth.constants import ModelTask , DefaultTokens
@@ -212,16 +210,14 @@ Source code for mammoth.utils.loss
)
elif opt . label_smoothing > 0 and train :
criterion = LabelSmoothingLoss ( opt . label_smoothing , len ( tgt_vocab ), ignore_index = padding_idx )
- elif isinstance ( generator [ - 1 ], LogSparsemax ): # elif isinstance(model.generator[-1], LogSparsemax):
- criterion = SparsemaxLoss ( ignore_index = padding_idx , reduction = 'sum' )
else :
criterion = nn . NLLLoss ( ignore_index = padding_idx , reduction = 'sum' )
# if the loss function operates on vectors of raw logits instead of
# probabilities, only the first part of the generator needs to be
- # passed to the NMTLossCompute. At the moment, the only supported
- # loss function of this kind is the sparsemax loss.
- use_raw_logits = isinstance ( criterion , SparsemaxLoss )
+ # passed to the NMTLossCompute. At the moment, there is no supported
+ # loss function of this kind.
+ use_raw_logits = False
loss_gen = (
generator [ 0 ] if use_raw_logits else generator
) # loss_gen = model.generator[0] if use_raw_logits else model.generator
diff --git a/_modules/mammoth/utils/optimizers.html b/_modules/mammoth/utils/optimizers.html
index 2508ccc2..eee78357 100644
--- a/_modules/mammoth/utils/optimizers.html
+++ b/_modules/mammoth/utils/optimizers.html
@@ -288,21 +288,6 @@ Source code for mammoth.utils.optimizers
params , lr = opt . learning_rate , betas = betas , eps = 1e-9 , weight_decay = opt . weight_decay
)
)
- elif opt . optim == 'sparseadam' :
- encs = []
- decs = []
- for name , param in model . named_parameters ():
- if not param . requires_grad :
- continue
- # TODO: Find a better way to check for sparse gradients.
- if 'decoder' in name :
- # print(name)
- decs . append ( param )
- else :
- encs . append ( param )
- optimizer = MultipleOptimizer (
- [ optim . Adam ( encs , lr = opt . learning_rate , betas = betas , eps = 1e-9 ), AdaFactorFairSeq ( decs , warmup_init = True )]
- )
elif opt . optim == 'fusedadam' :
# we use here a FusedAdam() copy of an old Apex repo
optimizer = FusedAdam ( params , lr = opt . learning_rate , betas = betas )
@@ -389,7 +374,7 @@ Source code for mammoth.utils.optimizers
class MultipleOptimizer ( object ):
- """Implement multiple optimizers needed for sparse adam"""
+ """Implement multiple optimizers"""
def __init__ ( self , op , multiOptims_Langs = None ):
self . optimizers = op
@@ -911,11 +896,7 @@ Source code for mammoth.utils.optimizers
if grad is None :
grad = p . grad . data
if grad . is_sparse :
- raise RuntimeError (
- 'FusedAdam does not support sparse \
- gradients, please consider \
- SparseAdam instead'
- )
+ raise RuntimeError ( 'sparse gradient not supported' )
state = self . state [ p ]
diff --git a/genindex.html b/genindex.html
index bbf3522f..f6479b04 100644
--- a/genindex.html
+++ b/genindex.html
@@ -213,15 +213,13 @@ A
(mammoth.translate.GreedySearch method)
-
all_gather_stats() (mammoth.utils.Statistics static method)
-
+ all_gather_stats() (mammoth.utils.Statistics static method)
+all_gather_stats_list() (mammoth.utils.Statistics static method)
amp() (mammoth.utils.Optimizer property)
-apply_mask() (mammoth.modules.ConvMultiStepAttention method)
AverageAttention (class in mammoth.modules)
C
F
G
L
M
- MatrixTree (class in mammoth.modules.structured_attention)
-maybe_convert_align() (mammoth.translate.translation_server.ServerModel method)
maybe_detokenize() (mammoth.translate.translation_server.ServerModel method)
@@ -476,15 +456,13 @@ S
start() (mammoth.translate.translation_server.TranslationServer method)
Statistics (class in mammoth.utils)
@@ -560,10 +538,6 @@ V
W
-
word_lut() (mammoth.modules.Embeddings property)
Modules
-class mammoth.modules.Embeddings( word_vec_size , word_vocab_size , word_padding_idx , position_encoding=False , feat_merge='concat' , feat_vec_exponent=0.7 , feat_vec_size=-1 , feat_padding_idx=[] , feat_vocab_sizes=[] , dropout=0 , sparse=False , freeze_word_vecs=False ) [source]
+class mammoth.modules.Embeddings( word_vec_size , word_vocab_size , word_padding_idx , position_encoding=False , feat_merge='concat' , feat_vec_exponent=0.7 , feat_vec_size=-1 , feat_padding_idx=[] , feat_vocab_sizes=[] , dropout=0 , freeze_word_vecs=False ) [source] Bases: torch.nn.modules.module.Module
Words embeddings for encoder/decoder.
-Additionally includes ability to add sparse input features
+
Additionally includes ability to add input features
based on “Linguistic Input Features Improve Neural Machine Translation”
[SH16] .
@@ -377,118 +377,6 @@
Attention
Architecture: SRU
Copy Attention
-
-
-class mammoth.modules.CopyGenerator( input_size , output_size , pad_idx ) [source]
-Bases: torch.nn.modules.module.Module
-An implementation of pointer-generator networks
-[SLM17] .
-These networks consider copying words
-directly from the source sequence.
-The copy generator is an extended version of the standard
-generator that computes three values.
-
-\(p_{softmax}\) the standard softmax over tgt_dict
\(p(z)\) the probability of copying a word from
-the source
\(p_{copy}\) the probility of copying a particular word.
-taken from the attention distribution directly.
-The model returns a distribution over the extend dictionary,
-computed as
-\(p(w) = p(z=1) p_{copy}(w) + p(z=0) p_{softmax}(w)\)
-
- graph BT
- A[input]
- S[src_map]
- B[softmax]
- BB[switch]
- C[attn]
- D[copy]
- O[output]
- A --> B
- A --> BB
- S --> D
- C --> D
- D --> O
- B --> O
- BB --> O
-
-Parameters
-
-
-
-
-
-forward( hidden , attn , src_map ) [source]
-Compute a distribution over the target dictionary
-extended by the dynamic dictionary implied by copying
-source words.
-
-Parameters
-
-hidden (FloatTensor ) – hidden outputs (batch x tlen, input_size)
attn (FloatTensor ) – attn for each (batch x tlen, slen)
src_map (FloatTensor ) – A sparse indicator matrix mapping each source word to
-its index in the “extended” vocab containing.
-(src_len, batch, extra_words)
-
-
-
-
-
Structured Attention
-
-
-class mammoth.modules.structured_attention.MatrixTree( eps=1e-05 ) [source]
-Bases: torch.nn.modules.module.Module
-Implementation of the matrix-tree theorem for computing marginals
-of non-projective dependency parsing. This attention layer is used
-in the paper “Learning Structured Text Representations”
-[LL17] .
-
-
-forward( input ) [source]
-Defines the computation performed at every call.
-Should be overridden by all subclasses.
-
-
Note
-
Although the recipe for forward pass needs to be defined within
-this function, one should call the Module
-
-
-
-
diff --git a/objects.inv b/objects.inv
index 5f67f661..d05e6779 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/options/train.html b/options/train.html
index 92c22bdc..377d425f 100644
--- a/options/train.html
+++ b/options/train.html
@@ -272,13 +272,12 @@ Train --context_gate, -context_gate Possible choices: source, target, both
-Type of context gate to use. Do not select for no context gate.
---bridge_extra_node, -bridge_extra_node Graph encoder bridges only extra node to decoder as input
Default: True
@@ -780,7 +775,7 @@ Model- Attention
References
-LL17 Yang Liu and Mirella Lapata. Learning structured text representations. CoRR , 2017. URL: http://arxiv.org/abs/1705.09207 , arXiv:1705.09207 .
-LSL+15 Minh-Thang Luong, Ilya Sutskever, Quoc Le, Oriol Vinyals, and Wojciech Zaremba. Addressing the Rare Word Problem in Neural Machine Translation. In Proc of ACL . 2015.
SLM17 Abigail See, Peter J. Liu, and Christopher D. Manning. Get to the point: summarization with pointer-generator networks. CoRR , 2017. URL: http://arxiv.org/abs/1704.04368 , arXiv:1704.04368 .
-SH16 Rico Sennrich and Barry Haddow. Linguistic input features improve neural machine translation. arXiv preprint arXiv:1606.02892 , 2016.