Added "random" sentence embeddings

examples/models/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+from .esn import ESN
+from .lstm import RandLSTM
+from .borep import BOREP

examples/models/borep.py

@@ -0,0 +1,77 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+import torch
+import torch.nn as nn
+
+from . import model_utils
+
+class BOREP(nn.Module):
+
+    def __init__(self, params):
+        super(BOREP, self).__init__()
+        self.params = params
+        self.max_seq_len = params.max_seq_len
+
+        self.projection = params.projection
+        self.proj = self.get_projection()
+
+        self.position_enc = None
+        if params.pos_enc:
+            self.position_enc = torch.nn.Embedding(self.max_seq_len + 1, params.word_emb_dim, padding_idx=0)
+            self.position_enc.weight.data = model_utils.position_encoding_init(self.max_seq_len + 1,
+                                                                               params.word_emb_dim)
+
+        if params.gpu:
+            self.cuda()
+
+    def get_projection(self):
+        proj = nn.Linear(self.params.input_dim, self.params.output_dim)
+        if self.params.init == "orthogonal":
+            nn.init.orthogonal_(proj.weight)
+        elif self.params.init == "sparse":
+            nn.init.sparse_(proj.weight, sparsity=0.1)
+        elif self.params.init == "normal":
+            nn.init.normal_(proj.weight, std=0.1)
+        elif self.params.init == "uniform":
+            nn.init.uniform_(proj.weight, a=-0.1, b=0.1)
+        elif self.params.init == "kaiming":
+            nn.init.kaiming_uniform_(proj.weight)
+        elif self.params.init == "xavier":
+            nn.init.xavier_uniform_(proj.weight)
+
+        nn.init.constant_(proj.bias, 0)
+
+        if self.params.gpu:
+            proj = proj.cuda()
+        return proj
+
+    def borep(self, x):
+        batch_sz, seq_len = x.size(1), x.size(0)
+        out = torch.FloatTensor(seq_len, batch_sz, self.params.output_dim).zero_()
+        for i in range(seq_len):
+            if self.projection:
+                emb = self.proj(x[i])
+            else:
+                emb = x[i]
+            out[i] = emb
+        return out
+
+    def forward(self, batch, se_params):
+        lengths, out, word_pos = model_utils.embed(batch, self.params, se_params,
+                                                   position_enc=self.position_enc, to_reverse=0)
+
+        out = self.borep(out)
+        out = model_utils.pool(out, lengths, self.params)
+
+        if self.params.activation is not None:
+            out = self.params.activation(out)
+
+        return out
+
+    def encode(self, batch, params):
+        return self.forward(batch, params).cpu().detach().numpy()

examples/models/esn.py

@@ -0,0 +1,131 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+import math
+
+import torch.nn as nn
+import torch
+
+import numpy as np
+
+from . import model_utils
+
+class ESN(nn.Module):
+
+    def __init__(self, params):
+        super(ESN, self).__init__()
+        self.params = params
+
+        self.bidirectional = params.bidirectional
+        self.max_seq_len = params.max_seq_len
+
+        self.n_inputs = params.input_dim
+        self.n_reservoir = params.output_dim
+        self.spectral_radius = params.spectral_radius
+        self.leaky = params.leaky
+        self.concat_inp = params.concat_inp
+        self.stdv = params.stdv
+        self.sparsity = params.sparsity
+
+        if self.concat_inp:
+            self.n_reservoir = self.n_reservoir - self.n_inputs
+
+        self.W = nn.Parameter(torch.Tensor(self.n_reservoir , self.n_reservoir))
+        self.Win = nn.Parameter(torch.Tensor(self.n_reservoir , self.n_inputs))
+        self.W.data.uniform_(-0.5, 0.5)
+        self.Win.data.uniform_(-self.stdv, self.stdv)
+
+        if self.bidirectional:
+            self.W_rev = nn.Parameter(torch.Tensor(self.n_reservoir, self.n_reservoir))
+            self.Win_rev = nn.Parameter(torch.Tensor(self.n_reservoir, self.n_inputs))
+            self.W_rev.data.uniform_(-0.5, 0.5)
+            self.Win_rev.data.uniform_(-self.stdv, self.stdv)
+
+        if self.spectral_radius > 0:
+            radius = np.max(np.abs(np.linalg.eigvals(self.W.data)))
+            self.W.data = self.W.data * (self.spectral_radius / radius)
+            if self.bidirectional:
+                radius = np.max(np.abs(np.linalg.eigvals(self.W_rev.data)))
+                self.W_rev.data = self.W_rev.data * (self.spectral_radius / radius)
+
+        self.input_layer = nn.Linear(self.n_inputs, self.n_reservoir, bias=False)
+        self.input_layer.weight = self.Win
+        self.recurrent_layer = nn.Linear(self.n_reservoir, self.n_reservoir, bias=False)
+        self.recurrent_layer.weight = self.W
+
+        if self.bidirectional:
+            self.input_layer_rev = nn.Linear(self.n_inputs, self.n_reservoir, bias=True)
+            self.input_layer_rev.weight = self.Win_rev
+            self.recurrent_layer_rev = nn.Linear(self.n_reservoir, self.n_reservoir, bias=True)
+            self.recurrent_layer_rev.weight = self.W_rev
+
+        if self.sparsity > 0:
+            self.sparse(self.recurrent_layer.weight.data, self.sparsity)
+
+            if self.bidirectional:
+                self.sparse(self.recurrent_layer_rev.weight.data, self.sparsity)
+
+        if params.gpu:
+            self.cuda()
+
+    def sparse(self, tensor, sparsity):
+        rows, cols = tensor.shape
+        num_zeros = int(math.ceil(sparsity * rows))
+
+        with torch.no_grad():
+            for col_idx in range(cols):
+                row_indices = torch.randperm(rows)
+                zero_indices = row_indices[:num_zeros]
+                tensor[zero_indices, col_idx] = 0
+
+    def esn(self, out, torev):
+        hidden_states = torch.zeros(out.size()[1], out.size()[0], self.n_reservoir) #SxBxD
+        curr_hid = torch.zeros(1, 1, self.n_reservoir)
+        if self.params.gpu:
+            curr_hid = curr_hid.cuda()
+            hidden_states = hidden_states.cuda()
+
+        curr_hid.expand(1, out.size()[0], self.n_reservoir).contiguous()
+        for i in range(out.size()[1]):
+            curr_embs = out[:,i,:]
+            if not torev:
+                hid_i = self.input_layer(curr_embs) + self.recurrent_layer(curr_hid)
+            else:
+                hid_i = self.input_layer_rev(curr_embs) + self.recurrent_layer_rev(curr_hid)
+            if self.params.activation is not None:
+                hid_i = self.params.activation(hid_i)
+            hidden_states[i] = hid_i
+            if i > 1 and self.leaky > 0:
+                hidden_states[i] = (1 - self.leaky) * hidden_states[i] + (self.leaky) * hidden_states[i - 1]
+            curr_hid = hidden_states[i]
+
+        if self.concat_inp:
+            out = torch.cat([hidden_states, out.transpose(1,0)], dim=2)
+        else:
+            out = hidden_states
+
+        return out
+
+    def forward(self, batch, se_params):
+        lengths, emb_fwd, _ = model_utils.embed(batch, self.params, se_params)
+        _, emb_rev, _ = model_utils.embed(batch, self.params, se_params, to_reverse=1)
+
+        emb_fwd = emb_fwd.transpose(1, 0)
+        emb_rev = emb_rev.transpose(1, 0)
+
+        if self.bidirectional:
+            out_fwd = self.esn(emb_fwd, False)
+            out_rev = self.esn(emb_rev, True)
+            out = torch.cat([out_fwd, out_rev], dim=2)
+        else:
+            out = self.esn(emb_fwd, False)
+
+        out = model_utils.pool(out, lengths, self.params)
+        return out
+
+    def encode(self, batch, params):
+        return self.forward(batch, params).cpu().detach().numpy()

examples/models/lstm.py

@@ -0,0 +1,71 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+import torch
+import torch.nn as nn
+from torch.nn.utils.rnn import pad_packed_sequence as unpack
+from torch.nn.utils.rnn import pack_padded_sequence as pack
+
+from . import model_utils
+
+class RandLSTM(nn.Module):
+
+    def __init__(self, params):
+        super(RandLSTM, self).__init__()
+        self.params = params
+
+        self.bidirectional = params.bidirectional
+        self.max_seq_len = params.max_seq_len
+
+        self.e_hid_init = torch.zeros(1, 1, params.output_dim)
+        self.e_cell_init = torch.zeros(1, 1, params.output_dim)
+
+        self.output_dim = params.output_dim
+        self.num_layers = params.num_layers
+        self.lm = nn.LSTM(params.input_dim, params.output_dim, num_layers=self.num_layers,
+                          bidirectional=bool(params.bidirectional), batch_first=True)
+
+        self.bidirectional += 1
+
+        if params.init != "none":
+            model_utils.param_init(self, params)
+
+        if params.gpu:
+            self.e_hid_init = self.e_hid_init.cuda()
+            self.e_cell_init = self.e_cell_init.cuda()
+            self.cuda()
+
+    def lstm(self, inputs, lengths):
+        bsz, max_len, _ = inputs.size()
+        in_embs = inputs
+        lens, indices = torch.sort(lengths, 0, True)
+
+        e_hid_init = self.e_hid_init.expand(1*self.num_layers*self.bidirectional, bsz, self.output_dim).contiguous()
+        e_cell_init = self.e_cell_init.expand(1*self.num_layers*self.bidirectional, bsz, self.output_dim).contiguous()
+        all_hids, (enc_last_hid, _) = self.lm(pack(in_embs[indices],
+                                                        lens.tolist(), batch_first=True), (e_hid_init, e_cell_init))
+        _, _indices = torch.sort(indices, 0)
+        all_hids = unpack(all_hids, batch_first=True)[0][_indices]
+
+        return all_hids
+
+    def forward(self, batch, se_params):
+        lengths, out, _ = model_utils.embed(batch, self.params, se_params)
+        out = out.transpose(1, 0)
+
+        out = self.lstm(out, lengths)
+        out = out.transpose(1,0)
+
+        out = model_utils.pool(out, lengths, self.params)
+
+        if self.params.activation is not None:
+            out = self.params.activation(out)
+
+        return out
+
+    def encode(self, batch, params):
+        return self.forward(batch, params).cpu().detach().numpy()