From b0068c3f74bb2e8d4f44256a2c6c18d1781e1025 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 18:08:22 +0800 Subject: [PATCH 01/31] no message --- .github/workflows/ci.yml | 54 +++ examples/run_dssm_negsampling.py | 2 +- examples/run_youtubednn_sampledsoftmax.py | 2 +- tests/__init__.py | 0 tests/models/DSSM_test.py | 25 ++ tests/models/FM_test.py | 25 ++ tests/models/MIND_test.py | 26 ++ tests/models/YoutubeDNN_test.py | 24 ++ tests/models/__init__.py | 0 tests/utils.py | 392 ++++++++++++++++++++++ 10 files changed, 548 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/__init__.py create mode 100644 tests/models/DSSM_test.py create mode 100644 tests/models/FM_test.py create mode 100644 tests/models/MIND_test.py create mode 100644 tests/models/YoutubeDNN_test.py create mode 100644 tests/models/__init__.py create mode 100644 tests/utils.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8a44485 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +on: + push: + path: + - 'deepmatch/*' + - 'tests/*' + pull_request: + path: + - 'deepmatch/*' + - 'tests/*' + +jobs: + build: + + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + matrix: + python-version: [3.5,3.6,3.7] + tf-version: [1.4.0,1.15.0,2.0.0] + + exclude: + - python-version: 3.7 + tf-version: 1.4.0 + + steps: + + - uses: actions/checkout@v1 + + - name: Setup python environment + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip3 install -q tensorflow==${{ matrix.tf-version }} + pip install -q requests + pip install -e . + - name: Test with pytest + timeout-minutes: 120 + run: | + pip install -q pytest + pip install -q pytest-cov + pip install -q python-coveralls + pytest --cov=deepctr --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1.0.2 + with: + token: ${{secrets.CODECOV_TOKEN}} + file: ./coverage.xml + flags: pytest + name: py${{ matrix.python-version }}-tf${{ matrix.tf-version }} \ No newline at end of file diff --git a/examples/run_dssm_negsampling.py b/examples/run_dssm_negsampling.py index 65568d3..52c372f 100644 --- a/examples/run_dssm_negsampling.py +++ b/examples/run_dssm_negsampling.py @@ -62,7 +62,7 @@ # 4. Generate user features for testing and full item features for retrieval test_user_model_input = test_model_input - all_item_model_input = {"movie_id": item_profile['movie_id'].values, "movie_idx": item_profile['movie_id'].values} + all_item_model_input = {"movie_id": item_profile['movie_id'].values} user_embedding_model = Model(inputs=model.user_input, outputs=model.user_embedding) item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding) diff --git a/examples/run_youtubednn_sampledsoftmax.py b/examples/run_youtubednn_sampledsoftmax.py index 3e18afe..23c432c 100644 --- a/examples/run_youtubednn_sampledsoftmax.py +++ b/examples/run_youtubednn_sampledsoftmax.py @@ -67,7 +67,7 @@ # 4. Generate user features for testing and full item features for retrieval test_user_model_input = test_model_input - all_item_model_input = {"movie_id": item_profile['movie_id'].values, "movie_idx": item_profile['movie_id'].values} + all_item_model_input = {"movie_id": item_profile['movie_id'].values} user_embedding_model = Model(inputs=model.user_input, outputs=model.user_embedding) item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/models/DSSM_test.py b/tests/models/DSSM_test.py new file mode 100644 index 0000000..2ae2670 --- /dev/null +++ b/tests/models/DSSM_test.py @@ -0,0 +1,25 @@ +import numpy as np + +from deepmatch.models import DSSM +from deepmatch.utils import sampledsoftmaxloss +from deepctr.inputs import SparseFeat,VarLenSparseFeat,DenseFeat,get_feature_names +from tensorflow.python.keras import backend as K +from ..utils import check_model,get_xy_fd + + +#@pytest.mark.xfail(reason="There is a bug when save model use Dice") +#@pytest.mark.skip(reason="misunderstood the API") + + +def test_DSSM(): + model_name = "DSSM" + + x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) + model = DSSM(user_feature_columns, item_feature_columns, ) + + model.compile('adam', "binary_crossentropy") + check_model(model,model_name,x,y,check_model_io=False) + + +if __name__ == "__main__": + pass diff --git a/tests/models/FM_test.py b/tests/models/FM_test.py new file mode 100644 index 0000000..dee90e0 --- /dev/null +++ b/tests/models/FM_test.py @@ -0,0 +1,25 @@ +import numpy as np + +from deepmatch.models import FM +from deepmatch.utils import sampledsoftmaxloss +from deepctr.inputs import SparseFeat,VarLenSparseFeat,DenseFeat,get_feature_names +from tensorflow.python.keras import backend as K +from ..utils import check_model,get_xy_fd + + +#@pytest.mark.xfail(reason="There is a bug when save model use Dice") +#@pytest.mark.skip(reason="misunderstood the API") + + +def test_FM(): + model_name = "FM" + + x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) + model = FM(user_feature_columns, item_feature_columns, ) + + model.compile('adam', "binary_crossentropy") + check_model(model,model_name,x,y,check_model_io=False) + + +if __name__ == "__main__": + pass diff --git a/tests/models/MIND_test.py b/tests/models/MIND_test.py new file mode 100644 index 0000000..05bff74 --- /dev/null +++ b/tests/models/MIND_test.py @@ -0,0 +1,26 @@ +import numpy as np + +from deepmatch.models import MIND +from deepmatch.utils import sampledsoftmaxloss +from deepctr.inputs import SparseFeat,VarLenSparseFeat,DenseFeat,get_feature_names +from tensorflow.python.keras import backend as K +from ..utils import check_model,get_xy_fd + + +#@pytest.mark.xfail(reason="There is a bug when save model use Dice") +#@pytest.mark.skip(reason="misunderstood the API") + + +def test_MIND(): + model_name = "MIND" + + x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) + K.set_learning_phase(True) + model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) + + model.compile('adam', sampledsoftmaxloss) + check_model(model,model_name,x,y,check_model_io=False) + + +if __name__ == "__main__": + pass diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py new file mode 100644 index 0000000..309969c --- /dev/null +++ b/tests/models/YoutubeDNN_test.py @@ -0,0 +1,24 @@ +from deepmatch.models import YoutubeDNN +from deepmatch.utils import sampledsoftmaxloss +from tensorflow.python.keras import backend as K + +from ..utils import check_model,get_xy_fd + + +#@pytest.mark.xfail(reason="There is a bug when save model use Dice") +#@pytest.mark.skip(reason="misunderstood the API") + + +def test_YoutubeDNN(): + model_name = "YoutubeDNN" + + x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) + K.set_learning_phase(True) + model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) + model.compile('adam', sampledsoftmaxloss) + + check_model(model,model_name,x,y,check_model_io=False) + + +if __name__ == "__main__": + pass diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..c77a2cb --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,392 @@ +from __future__ import absolute_import, division, print_function + +import inspect +import os +import sys + +import numpy as np +import tensorflow as tf +from numpy.testing import assert_allclose +from tensorflow.python.keras import backend as K +from tensorflow.python.keras.layers import Input, Masking +from tensorflow.python.keras.models import Model, load_model, save_model + +from deepctr.inputs import SparseFeat, DenseFeat, VarLenSparseFeat,DEFAULT_GROUP_NAME +from deepctr.layers import custom_objects + +SAMPLE_SIZE = 8 +VOCABULARY_SIZE = 4 + + +def gen_sequence(dim, max_len, sample_size): + return np.array([np.random.randint(0, dim, max_len) for _ in range(sample_size)]), np.random.randint(1, max_len + 1, + sample_size) + + +def get_test_data(sample_size=1000, embedding_size=4, sparse_feature_num=1, dense_feature_num=1, + sequence_feature=['sum', 'mean', 'max', 'weight'], classification=True, include_length=False, + hash_flag=False, prefix='', use_group=False): + feature_columns = [] + model_input = {} + + if 'weight' in sequence_feature: + feature_columns.append( + VarLenSparseFeat(SparseFeat(prefix + "weighted_seq", vocabulary_size=2, embedding_dim=embedding_size), + maxlen=3, length_name=prefix + "weighted_seq" + "_seq_length", + weight_name=prefix + "weight")) + s_input, s_len_input = gen_sequence( + 2, 3, sample_size) + + model_input[prefix + "weighted_seq"] = s_input + model_input[prefix + 'weight'] = np.random.randn(sample_size, 3, 1) + model_input[prefix + "weighted_seq" + "_seq_length"] = s_len_input + sequence_feature.pop(sequence_feature.index('weight')) + + for i in range(sparse_feature_num): + if use_group: + group_name = str(i%3) + else: + group_name = DEFAULT_GROUP_NAME + dim = np.random.randint(1, 10) + feature_columns.append( + SparseFeat(prefix + 'sparse_feature_' + str(i), dim, embedding_size, use_hash=hash_flag, dtype=tf.int32,group_name=group_name)) + + for i in range(dense_feature_num): + feature_columns.append(DenseFeat(prefix + 'dense_feature_' + str(i), 1, dtype=tf.float32)) + for i, mode in enumerate(sequence_feature): + dim = np.random.randint(1, 10) + maxlen = np.random.randint(1, 10) + feature_columns.append( + VarLenSparseFeat(SparseFeat(prefix + 'sequence_' + mode, vocabulary_size=dim, embedding_dim=embedding_size), + maxlen=maxlen, combiner=mode)) + + for fc in feature_columns: + if isinstance(fc, SparseFeat): + model_input[fc.name] = np.random.randint(0, fc.vocabulary_size, sample_size) + elif isinstance(fc, DenseFeat): + model_input[fc.name] = np.random.random(sample_size) + else: + s_input, s_len_input = gen_sequence( + fc.vocabulary_size, fc.maxlen, sample_size) + model_input[fc.name] = s_input + if include_length: + fc.length_name = prefix + "sequence_" + str(i) + '_seq_length' + model_input[prefix + "sequence_" + str(i) + '_seq_length'] = s_len_input + + if classification: + y = np.random.randint(0, 2, sample_size) + else: + y = np.random.random(sample_size) + + return model_input, y, feature_columns + + +def layer_test(layer_cls, kwargs={}, input_shape=None, input_dtype=None, + + input_data=None, expected_output=None, + + expected_output_dtype=None, fixed_batch_size=False, supports_masking=False): + # generate input data + + if input_data is None: + + if not input_shape: + raise AssertionError() + + if not input_dtype: + input_dtype = K.floatx() + + input_data_shape = list(input_shape) + + for i, e in enumerate(input_data_shape): + + if e is None: + input_data_shape[i] = np.random.randint(1, 4) + input_mask = [] + if all(isinstance(e, tuple) for e in input_data_shape): + input_data = [] + + for e in input_data_shape: + input_data.append( + (10 * np.random.random(e)).astype(input_dtype)) + if supports_masking: + a = np.full(e[:2], False) + a[:, :e[1] // 2] = True + input_mask.append(a) + + else: + + input_data = (10 * np.random.random(input_data_shape)) + + input_data = input_data.astype(input_dtype) + if supports_masking: + a = np.full(input_data_shape[:2], False) + a[:, :input_data_shape[1] // 2] = True + + print(a) + print(a.shape) + input_mask.append(a) + + else: + + if input_shape is None: + input_shape = input_data.shape + + if input_dtype is None: + input_dtype = input_data.dtype + + if expected_output_dtype is None: + expected_output_dtype = input_dtype + + # instantiation + + layer = layer_cls(**kwargs) + + # test get_weights , set_weights at layer level + + weights = layer.get_weights() + + layer.set_weights(weights) + + try: + expected_output_shape = layer.compute_output_shape(input_shape) + except Exception: + expected_output_shape = layer._compute_output_shape(input_shape) + + # test in functional API + if isinstance(input_shape, list): + if fixed_batch_size: + + x = [Input(batch_shape=e, dtype=input_dtype) for e in input_shape] + if supports_masking: + mask = [Input(batch_shape=e[0:2], dtype=bool) + for e in input_shape] + + else: + + x = [Input(shape=e[1:], dtype=input_dtype) for e in input_shape] + if supports_masking: + mask = [Input(shape=(e[1],), dtype=bool) for e in input_shape] + + else: + if fixed_batch_size: + + x = Input(batch_shape=input_shape, dtype=input_dtype) + if supports_masking: + mask = Input(batch_shape=input_shape[0:2], dtype=bool) + + else: + + x = Input(shape=input_shape[1:], dtype=input_dtype) + if supports_masking: + mask = Input(shape=(input_shape[1],), dtype=bool) + + if supports_masking: + + y = layer(Masking()(x), mask=mask) + else: + y = layer(x) + + if not (K.dtype(y) == expected_output_dtype): + raise AssertionError() + + # check with the functional API + if supports_masking: + model = Model([x, mask], y) + + actual_output = model.predict([input_data, input_mask[0]]) + else: + model = Model(x, y) + + actual_output = model.predict(input_data) + + actual_output_shape = actual_output.shape + for expected_dim, actual_dim in zip(expected_output_shape, + + actual_output_shape): + + if expected_dim is not None: + + if not (expected_dim == actual_dim): + raise AssertionError("expected_shape", expected_output_shape, "actual_shape", actual_output_shape) + + if expected_output is not None: + assert_allclose(actual_output, expected_output, rtol=1e-3) + + # test serialization, weight setting at model level + + model_config = model.get_config() + + recovered_model = model.__class__.from_config(model_config) + + if model.weights: + weights = model.get_weights() + + recovered_model.set_weights(weights) + + _output = recovered_model.predict(input_data) + + assert_allclose(_output, actual_output, rtol=1e-3) + + # test training mode (e.g. useful when the layer has a + + # different behavior at training and testing time). + + if has_arg(layer.call, 'training'): + model.compile('rmsprop', 'mse') + + model.train_on_batch(input_data, actual_output) + + # test instantiation from layer config + + layer_config = layer.get_config() + + layer_config['batch_input_shape'] = input_shape + + layer = layer.__class__.from_config(layer_config) + + # for further checks in the caller function + + return actual_output + + +def has_arg(fn, name, accept_all=False): + """Checks if a callable accepts a given keyword argument. + + + + For Python 2, checks if there is an argument with the given name. + + + + For Python 3, checks if there is an argument with the given name, and + + also whether this argument can be called with a keyword (i.e. if it is + + not a positional-only argument). + + + + # Arguments + + fn: Callable to inspect. + + name: Check if `fn` can be called with `name` as a keyword argument. + + accept_all: What to return if there is no parameter called `name` + + but the function accepts a `**kwargs` argument. + + + + # Returns + + bool, whether `fn` accepts a `name` keyword argument. + + """ + + if sys.version_info < (3,): + + arg_spec = inspect.getargspec(fn) + + if accept_all and arg_spec.keywords is not None: + return True + + return (name in arg_spec.args) + + elif sys.version_info < (3, 3): + + arg_spec = inspect.getfullargspec(fn) + + if accept_all and arg_spec.varkw is not None: + return True + + return (name in arg_spec.args or + + name in arg_spec.kwonlyargs) + + else: + + signature = inspect.signature(fn) + + parameter = signature.parameters.get(name) + + if parameter is None: + + if accept_all: + + for param in signature.parameters.values(): + + if param.kind == inspect.Parameter.VAR_KEYWORD: + return True + + return False + + return (parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, + + inspect.Parameter.KEYWORD_ONLY)) + + +def check_model(model, model_name, x, y, check_model_io=True): + """ + compile model,train and evaluate it,then save/load weight and model file. + :param model: + :param model_name: + :param x: + :param y: + :param check_model_io: test save/load model file or not + :return: + """ + + model.fit(x, y, batch_size=10, epochs=1, validation_split=0.5) + + + + print(model_name + " test train valid pass!") + + + user_embedding_model = Model(inputs=model.user_input, outputs=model.user_embedding) + item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding) + + user_embs = user_embedding_model.predict(x, batch_size=2 ** 12) + # user_embs = user_embs[:, i, :] i in [0,k_max) if MIND + print(model_name + " user_emb pass!") + item_embs = item_embedding_model.predict(x, batch_size=2 ** 12) + + print(model_name + " item_emb pass!") + + model.save_weights(model_name + '_weights.h5') + model.load_weights(model_name + '_weights.h5') + os.remove(model_name + '_weights.h5') + print(model_name + " test save load weight pass!") + if check_model_io: + save_model(model, model_name + '.h5') + model = load_model(model_name + '.h5', custom_objects) + os.remove(model_name + '.h5') + print(model_name + " test save load model pass!") + + print(model_name + " test pass!") + + +def get_xy_fd(hash_flag=False): + + + user_feature_columns = [SparseFeat('user',3),SparseFeat( + 'gender', 2),VarLenSparseFeat(SparseFeat('hist_item', vocabulary_size=3 + 1,embedding_dim=4,embedding_name='item'), maxlen=4,length_name="hist_len") ] + item_feature_columns = [SparseFeat('item', 3 + 1,embedding_dim=4,)] + + + uid = np.array([0, 1, 2]) + ugender = np.array([0, 1, 0]) + iid = np.array([1, 2, 3]) # 0 is mask value + + hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]]) + hist_len = np.array([3,3,2]) + + feature_dict = {'user': uid, 'gender': ugender, 'item': iid, + 'hist_item': hist_iid, "hist_len":hist_len} + + #feature_names = get_feature_names(feature_columns) + x = feature_dict + y = [1, 1, 1] + return x, y, user_feature_columns,item_feature_columns \ No newline at end of file From aed9d1232b4bd39f702e1e2b1d2e81c37b8a0d36 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 19:26:18 +0800 Subject: [PATCH 02/31] no message --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c26e2af..2036b2a 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ [![Documentation Status](https://readthedocs.org/projects/deepmatch/badge/?version=latest)](https://deepmatch.readthedocs.io/) +![CI status](https://github.com/shenweichen/deepmatch/workflows/CI/badge.svg) +[![codecov](https://codecov.io/gh/shenweichen/DeepMatch/branch/master/graph/badge.svg)](https://codecov.io/gh/shenweichen/DeepMatch) [![Disscussion](https://img.shields.io/badge/chat-wechat-brightgreen?style=flat)](./README.md#disscussiongroup) [![License](https://img.shields.io/github/license/shenweichen/deepmatch.svg)](https://github.com/shenweichen/deepmatch/blob/master/LICENSE) From 7fb46cb80449ba4a272329bfc8045c7678affa3a Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 20:14:54 +0800 Subject: [PATCH 03/31] no message --- deepmatch/layers/__init__.py | 17 ++++++++++++ deepmatch/layers/core.py | 46 +++++++++++++++++++++++++++++++++ deepmatch/models/youtubednn.py | 28 ++++++++++++++------ deepmatch/utils.py | 4 +++ tests/models/DSSM_test.py | 13 +++------- tests/models/FM_test.py | 13 +++------- tests/models/MIND_test.py | 20 +++++++------- tests/models/YoutubeDNN_test.py | 2 +- tests/utils.py | 2 +- 9 files changed, 107 insertions(+), 38 deletions(-) diff --git a/deepmatch/layers/__init__.py b/deepmatch/layers/__init__.py index e69de29..df87605 100644 --- a/deepmatch/layers/__init__.py +++ b/deepmatch/layers/__init__.py @@ -0,0 +1,17 @@ +from deepctr.layers import custom_objects +from deepctr.layers.utils import reduce_sum + +from .core import PoolingLayer, SampledSoftmaxLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayerv2 +from ..utils import sampledsoftmaxloss + +_custom_objects = {'PoolingLayer': PoolingLayer, + 'SampledSoftmaxLayer': SampledSoftmaxLayer, + 'Similarity': Similarity, + 'LabelAwareAttention': LabelAwareAttention, + 'CapsuleLayer': CapsuleLayer, + 'reduce_sum':reduce_sum, + 'SampledSoftmaxLayerv2':SampledSoftmaxLayerv2, + 'sampledsoftmaxloss':sampledsoftmaxloss + } + +custom_objects = dict(custom_objects, **_custom_objects) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index bf5f1c4..3e09785 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -81,6 +81,52 @@ def get_config(self, ): return dict(list(base_config.items()) + list(config.items())) +class SampledSoftmaxLayerv2(Layer): + def __init__(self, num_sampled=5, **kwargs): + self.num_sampled = num_sampled + # self.target_song_size = item_embedding.input_dim + # self.item_embedding = item_embedding + super(SampledSoftmaxLayerv2, self).__init__(**kwargs) + + def build(self, input_shape): + self.size = input_shape[0][0] + self.zero_bias = self.add_weight(shape=[self.size], + initializer=Zeros, + dtype=tf.float32, + trainable=False, + name="bias") + # if not self.item_embedding.built: + # self.item_embedding.build([]) + # self.trainable_weights.append(self.item_embedding.embeddings) + super(SampledSoftmaxLayerv2, self).build(input_shape) + + def call(self, inputs_with_label_idx, training=None, **kwargs): + """ + The first input should be the model as it were, and the second the + target (i.e., a repeat of the training data) to compute the labels + argument + """ + embeddings,inputs, label_idx = inputs_with_label_idx + + loss = tf.nn.sampled_softmax_loss(weights=embeddings,#self.item_embedding. + biases=self.zero_bias, + labels=label_idx, + inputs=inputs, + num_sampled=self.num_sampled, + num_classes=self.size,#self.target_song_size + ) + return tf.expand_dims(loss, axis=1) + + def compute_output_shape(self, input_shape): + return (None, 1) + + def get_config(self, ): + config = { 'num_sampled': self.num_sampled} + base_config = super(SampledSoftmaxLayerv2, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + + class LabelAwareAttention(Layer): def __init__(self, k_max, pow_p=1, **kwargs): self.k_max = k_max diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 2c1fe29..93f9e45 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -4,14 +4,16 @@ Reference: Deep Neural Networks for YouTube Recommendations """ - +import tensorflow as tf from deepctr.inputs import input_from_feature_columns, build_input_features, combined_dnn_input, create_embedding_matrix from deepctr.layers.core import DNN +from deepctr.layers.utils import NoMask from tensorflow.python.keras.models import Model +from tensorflow.python.keras.layers import Input -from deepmatch.utils import get_item_embedding +from deepmatch.utils import get_item_embedding,get_item_embeddingv2 from ..inputs import input_from_feature_columns -from ..layers.core import SampledSoftmaxLayer +from ..layers.core import SampledSoftmaxLayer,SampledSoftmaxLayerv2 def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, @@ -38,6 +40,8 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, if len(item_feature_columns) > 1: raise ValueError("Now YoutubeNN only support 1 item feature like item_id") item_feature_name = item_feature_columns[0].name + item_vocabulary_size = item_feature_columns[0].vocabulary_size + item_idx = Input(tensor=tf.range(0,item_vocabulary_size),name="item_idx") embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, seed, prefix="") @@ -55,16 +59,24 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_out = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed, )(user_dnn_input) - item_embedding = embedding_matrix_dict[item_feature_name] + item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) + # if not item_embedding.built: + # item_embedding.build([]) - output = SampledSoftmaxLayer(item_embedding, num_sampled=num_sampled)( - inputs=(user_dnn_out, item_features[item_feature_name])) - model = Model(inputs=user_inputs_list + item_inputs_list, outputs=output) + output = SampledSoftmaxLayerv2( num_sampled=num_sampled)( + inputs=(item_embedding,user_dnn_out, item_features[item_feature_name])) + model = Model(inputs=user_inputs_list + item_inputs_list+[item_idx], outputs=output) model.__setattr__("user_input", user_inputs_list) model.__setattr__("user_embedding", user_dnn_out) model.__setattr__("item_input", item_inputs_list) - model.__setattr__("item_embedding", get_item_embedding(item_embedding, item_features[item_feature_name])) + model.__setattr__("item_embedding", get_item_embeddingv2(item_embedding, item_features[item_feature_name])) return model + +from tensorflow.python.keras import backend as K + + +def loss(y_true, y_pred): + return K.mean(K.log(K.sigmoid(y_true*y_pred))) \ No newline at end of file diff --git a/deepmatch/utils.py b/deepmatch/utils.py index d8166c9..1b28881 100644 --- a/deepmatch/utils.py +++ b/deepmatch/utils.py @@ -34,6 +34,10 @@ def get_item_embedding(item_embedding_layer, item_input_layer): return Lambda(lambda x: tf.squeeze(tf.gather(item_embedding_layer.embeddings, x), axis=1))( item_input_layer) +def get_item_embeddingv2(item_embedding_layer, item_input_layer): + return Lambda(lambda x: tf.squeeze(tf.gather(item_embedding_layer, x), axis=1))( + item_input_layer) + def check_version(version): diff --git a/tests/models/DSSM_test.py b/tests/models/DSSM_test.py index 2ae2670..90c5e24 100644 --- a/tests/models/DSSM_test.py +++ b/tests/models/DSSM_test.py @@ -1,14 +1,9 @@ -import numpy as np - from deepmatch.models import DSSM -from deepmatch.utils import sampledsoftmaxloss -from deepctr.inputs import SparseFeat,VarLenSparseFeat,DenseFeat,get_feature_names -from tensorflow.python.keras import backend as K -from ..utils import check_model,get_xy_fd +from ..utils import check_model, get_xy_fd -#@pytest.mark.xfail(reason="There is a bug when save model use Dice") -#@pytest.mark.skip(reason="misunderstood the API") +# @pytest.mark.xfail(reason="There is a bug when save model use Dice") +# @pytest.mark.skip(reason="misunderstood the API") def test_DSSM(): @@ -18,7 +13,7 @@ def test_DSSM(): model = DSSM(user_feature_columns, item_feature_columns, ) model.compile('adam', "binary_crossentropy") - check_model(model,model_name,x,y,check_model_io=False) + check_model(model, model_name, x, y) if __name__ == "__main__": diff --git a/tests/models/FM_test.py b/tests/models/FM_test.py index dee90e0..0ae1222 100644 --- a/tests/models/FM_test.py +++ b/tests/models/FM_test.py @@ -1,14 +1,9 @@ -import numpy as np - from deepmatch.models import FM -from deepmatch.utils import sampledsoftmaxloss -from deepctr.inputs import SparseFeat,VarLenSparseFeat,DenseFeat,get_feature_names -from tensorflow.python.keras import backend as K -from ..utils import check_model,get_xy_fd +from ..utils import check_model, get_xy_fd -#@pytest.mark.xfail(reason="There is a bug when save model use Dice") -#@pytest.mark.skip(reason="misunderstood the API") +# @pytest.mark.xfail(reason="There is a bug when save model use Dice") +# @pytest.mark.skip(reason="misunderstood the API") def test_FM(): @@ -18,7 +13,7 @@ def test_FM(): model = FM(user_feature_columns, item_feature_columns, ) model.compile('adam', "binary_crossentropy") - check_model(model,model_name,x,y,check_model_io=False) + check_model(model, model_name, x, y,) if __name__ == "__main__": diff --git a/tests/models/MIND_test.py b/tests/models/MIND_test.py index 05bff74..0ff3098 100644 --- a/tests/models/MIND_test.py +++ b/tests/models/MIND_test.py @@ -11,16 +11,16 @@ #@pytest.mark.skip(reason="misunderstood the API") -def test_MIND(): - model_name = "MIND" - - x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) - K.set_learning_phase(True) - model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) - - model.compile('adam', sampledsoftmaxloss) - check_model(model,model_name,x,y,check_model_io=False) - +# def test_MIND(): +# model_name = "MIND" +# +# x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) +# K.set_learning_phase(True) +# model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) +# +# model.compile('adam', sampledsoftmaxloss) +# check_model(model,model_name,x,y,check_model_io=True) +# if __name__ == "__main__": pass diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index 309969c..997886d 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -17,7 +17,7 @@ def test_YoutubeDNN(): model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss) - check_model(model,model_name,x,y,check_model_io=False) + check_model(model,model_name,x,y,check_model_io=True) if __name__ == "__main__": diff --git a/tests/utils.py b/tests/utils.py index c77a2cb..7281f89 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,7 +12,7 @@ from tensorflow.python.keras.models import Model, load_model, save_model from deepctr.inputs import SparseFeat, DenseFeat, VarLenSparseFeat,DEFAULT_GROUP_NAME -from deepctr.layers import custom_objects +from deepmatch.layers import custom_objects SAMPLE_SIZE = 8 VOCABULARY_SIZE = 4 From d5284750157777190996cdcf6a30b6be83ed3548 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 20:27:54 +0800 Subject: [PATCH 04/31] no message --- deepmatch/models/youtubednn.py | 5 ----- tests/utils.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 93f9e45..38a1e35 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -75,8 +75,3 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, return model -from tensorflow.python.keras import backend as K - - -def loss(y_true, y_pred): - return K.mean(K.log(K.sigmoid(y_true*y_pred))) \ No newline at end of file diff --git a/tests/utils.py b/tests/utils.py index 7281f89..ae48b7c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -388,5 +388,5 @@ def get_xy_fd(hash_flag=False): #feature_names = get_feature_names(feature_columns) x = feature_dict - y = [1, 1, 1] + y = np.array([1, 1, 1]) return x, y, user_feature_columns,item_feature_columns \ No newline at end of file From 015c9b329f606e239640d3125b40ce448e594855 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 20:33:40 +0800 Subject: [PATCH 05/31] no message --- deepmatch/layers/core.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index 3e09785..8983908 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -106,14 +106,14 @@ def call(self, inputs_with_label_idx, training=None, **kwargs): target (i.e., a repeat of the training data) to compute the labels argument """ - embeddings,inputs, label_idx = inputs_with_label_idx + embeddings, inputs, label_idx = inputs_with_label_idx - loss = tf.nn.sampled_softmax_loss(weights=embeddings,#self.item_embedding. + loss = tf.nn.sampled_softmax_loss(weights=embeddings, # self.item_embedding. biases=self.zero_bias, labels=label_idx, inputs=inputs, num_sampled=self.num_sampled, - num_classes=self.size,#self.target_song_size + num_classes=self.size, # self.target_song_size ) return tf.expand_dims(loss, axis=1) @@ -121,12 +121,11 @@ def compute_output_shape(self, input_shape): return (None, 1) def get_config(self, ): - config = { 'num_sampled': self.num_sampled} + config = {'num_sampled': self.num_sampled} base_config = super(SampledSoftmaxLayerv2, self).get_config() return dict(list(base_config.items()) + list(config.items())) - class LabelAwareAttention(Layer): def __init__(self, k_max, pow_p=1, **kwargs): self.k_max = k_max @@ -197,6 +196,11 @@ def call(self, inputs, **kwargs): def compute_output_shape(self, input_shape): return (None, 1) + def get_config(self, ): + config = {'gamma': self.gamma, 'axis': self.axis, 'type': self.type} + base_config = super(Similarity, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + class CapsuleLayer(Layer): def __init__(self, input_units, out_units, max_len, k_max, iteration_times=3, @@ -241,6 +245,12 @@ def call(self, inputs, **kwargs): def compute_output_shape(self, input_shape): return (None, self.k_max, self.out_units) + def get_config(self, ): + config = {'input_units': self.input_units, 'out_units': self.out_units, 'max_len': self.max_len, + 'k_max': self.k_max, 'iteration_times': self.iteration_times, "initializer": self.initializer} + base_config = super(CapsuleLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + def squash(inputs): vec_squared_norm = tf.reduce_sum(tf.square(inputs), axis=-1, keep_dims=True) From 4c80b8ad828a030871c36b14b41d9af403e0ca6d Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 20:44:33 +0800 Subject: [PATCH 06/31] no message --- tests/models/YoutubeDNN_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index 997886d..3ca4c02 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -1,6 +1,7 @@ from deepmatch.models import YoutubeDNN from deepmatch.utils import sampledsoftmaxloss from tensorflow.python.keras import backend as K +import tensorflow as tf from ..utils import check_model,get_xy_fd @@ -17,6 +18,9 @@ def test_YoutubeDNN(): model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss) + if tf.__version__ >= '2.0.0': + tf.compat.v1.disable_eager_execution() + check_model(model,model_name,x,y,check_model_io=True) From 85c1f4e22ad0c6ac32de056497ca57cdb243851f Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 21:30:59 +0800 Subject: [PATCH 07/31] no message --- deepmatch/models/youtubednn.py | 2 ++ deepmatch/utils.py | 4 ++-- tests/models/YoutubeDNN_test.py | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 38a1e35..f234159 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -60,6 +60,8 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, dnn_use_bn, seed, )(user_dnn_input) item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) + #item_embedding = get_item_embeddingv2() + # if not item_embedding.built: # item_embedding.build([]) diff --git a/deepmatch/utils.py b/deepmatch/utils.py index 1b28881..552c51d 100644 --- a/deepmatch/utils.py +++ b/deepmatch/utils.py @@ -34,8 +34,8 @@ def get_item_embedding(item_embedding_layer, item_input_layer): return Lambda(lambda x: tf.squeeze(tf.gather(item_embedding_layer.embeddings, x), axis=1))( item_input_layer) -def get_item_embeddingv2(item_embedding_layer, item_input_layer): - return Lambda(lambda x: tf.squeeze(tf.gather(item_embedding_layer, x), axis=1))( +def get_item_embeddingv2(item_embedding, item_input_layer): + return Lambda(lambda x: tf.squeeze(tf.gather(item_embedding, x), axis=1))( item_input_layer) diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index 3ca4c02..b0f9a34 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -15,12 +15,13 @@ def test_YoutubeDNN(): x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) K.set_learning_phase(True) - model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) - model.compile('adam', sampledsoftmaxloss) if tf.__version__ >= '2.0.0': tf.compat.v1.disable_eager_execution() + model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) + model.compile('adam', sampledsoftmaxloss) + check_model(model,model_name,x,y,check_model_io=True) From 2694cb79916a288d9479b3061494f94faa40f350 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Wed, 8 Apr 2020 21:38:56 +0800 Subject: [PATCH 08/31] no message --- deepmatch/models/youtubednn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index f234159..70b5f8b 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -41,7 +41,7 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, raise ValueError("Now YoutubeNN only support 1 item feature like item_id") item_feature_name = item_feature_columns[0].name item_vocabulary_size = item_feature_columns[0].vocabulary_size - item_idx = Input(tensor=tf.range(0,item_vocabulary_size),name="item_idx") + item_idx = Input(shape=[None,],tensor=tf.range(0,item_vocabulary_size),name="item_idx") embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, seed, prefix="") From ef2583f3b8004803837df27b054b16cced9e76f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E6=A2=A6?= Date: Thu, 9 Apr 2020 13:19:14 +0800 Subject: [PATCH 09/31] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a44485..a469b74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: python-version: [3.5,3.6,3.7] - tf-version: [1.4.0,1.15.0,2.0.0] + tf-version: [1.4.0,1.15.0,1.14.0] exclude: - python-version: 3.7 @@ -51,4 +51,4 @@ jobs: token: ${{secrets.CODECOV_TOKEN}} file: ./coverage.xml flags: pytest - name: py${{ matrix.python-version }}-tf${{ matrix.tf-version }} \ No newline at end of file + name: py${{ matrix.python-version }}-tf${{ matrix.tf-version }} From f150d1470746168a93584b4edc3b5be316f35fa4 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Thu, 9 Apr 2020 17:06:22 +0800 Subject: [PATCH 10/31] no message --- deepmatch/models/youtubednn.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 70b5f8b..4793c1d 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -9,7 +9,7 @@ from deepctr.layers.core import DNN from deepctr.layers.utils import NoMask from tensorflow.python.keras.models import Model -from tensorflow.python.keras.layers import Input +from tensorflow.python.keras.layers import Input,Lambda from deepmatch.utils import get_item_embedding,get_item_embeddingv2 from ..inputs import input_from_feature_columns @@ -41,7 +41,7 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, raise ValueError("Now YoutubeNN only support 1 item feature like item_id") item_feature_name = item_feature_columns[0].name item_vocabulary_size = item_feature_columns[0].vocabulary_size - item_idx = Input(shape=[None,],tensor=tf.range(0,item_vocabulary_size),name="item_idx") + #item_idx = Input(shape=[None,],tensor=tf.range(0,item_vocabulary_size),name="item_idx") embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, seed, prefix="") @@ -59,7 +59,11 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_out = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed, )(user_dnn_input) - item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) + #item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) + item_embedding = embedding_matrix_dict[item_feature_name]#Lambda( lambda x:embedding_matrix_dict[item_feature_name])(None) + # if not item_embedding.built: + # item_embedding.build() + item_embedding = Input(tensor=item_embedding.embeddings,name=item_feature_name+"_embeddings") #item_embedding = get_item_embeddingv2() # if not item_embedding.built: @@ -67,7 +71,7 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, output = SampledSoftmaxLayerv2( num_sampled=num_sampled)( inputs=(item_embedding,user_dnn_out, item_features[item_feature_name])) - model = Model(inputs=user_inputs_list + item_inputs_list+[item_idx], outputs=output) + model = Model(inputs=user_inputs_list + item_inputs_list+[item_embedding], outputs=output)#+[item_idx] model.__setattr__("user_input", user_inputs_list) model.__setattr__("user_embedding", user_dnn_out) From 1094ead6c4e72ff781383f41658bda792543e2bc Mon Sep 17 00:00:00 2001 From: shenweichen Date: Thu, 9 Apr 2020 17:26:49 +0800 Subject: [PATCH 11/31] no message --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ceb8d48..05996c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,10 +18,10 @@ jobs: strategy: matrix: python-version: [3.5,3.6,3.7] - tf-version: [1.4.0,1.15.0,1.14.0] + tf-version: [1.4.0,1.14.0,2.1.0] - exclude: - - python-version: 3.7 +# exclude: +# - python-version: 3.7 #tf-version: 1.4.0 steps: From d6847266f5743500e27c237f29518590d70131b6 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Thu, 9 Apr 2020 17:30:54 +0800 Subject: [PATCH 12/31] no message --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05996c5..7195104 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,9 @@ jobs: python-version: [3.5,3.6,3.7] tf-version: [1.4.0,1.14.0,2.1.0] -# exclude: -# - python-version: 3.7 - #tf-version: 1.4.0 + exclude: + - python-version: 3.7 + - tf-version: 1.4.0 steps: From 4cc38fba5ee5b7e90b1259896f1c1ca897d9fac9 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Thu, 9 Apr 2020 20:36:17 +0800 Subject: [PATCH 13/31] no message --- deepmatch/layers/core.py | 20 ++++++++-------- deepmatch/models/mind.py | 35 +++++++++++++++++----------- deepmatch/models/ncf.py | 2 +- deepmatch/models/youtubednn.py | 41 ++++++++++++++++++++++----------- tests/models/MIND_test.py | 20 ++++++++-------- tests/models/YoutubeDNN_test.py | 4 ++-- tests/utils.py | 12 +++++----- 7 files changed, 78 insertions(+), 56 deletions(-) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index 8983908..99a1390 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -26,6 +26,7 @@ def call(self, seq_value_len_list, mask=None, **kwargs): seq_value_len_list = [seq_value_len_list] if len(seq_value_len_list) == 1: return seq_value_len_list[0] + # seq_value_len_list[1] = tf.squeeze(seq_value_len_list[1],axis=0) expand_seq_value_len_list = list(map(lambda x: tf.expand_dims(x, axis=-1), seq_value_len_list)) a = concat_func(expand_seq_value_len_list) if self.mode == "mean": @@ -204,20 +205,21 @@ def get_config(self, ): class CapsuleLayer(Layer): def __init__(self, input_units, out_units, max_len, k_max, iteration_times=3, - initializer=RandomNormal(stddev=1.0), **kwargs): + init_std=1.0, **kwargs): self.input_units = input_units self.out_units = out_units self.max_len = max_len self.k_max = k_max self.iteration_times = iteration_times - self.initializer = initializer + self.init_std = init_std super(CapsuleLayer, self).__init__(**kwargs) def build(self, input_shape): - self.routing_logits = self.add_weight(shape=[1, self.k_max, self.max_len], initializer=self.initializer, + self.routing_logits = self.add_weight(shape=[1, self.k_max, self.max_len], + initializer=RandomNormal(stddev=self.init_std), trainable=False, name="B", dtype=tf.float32) self.bilinear_mapping_matrix = self.add_weight(shape=[self.input_units, self.out_units], - initializer=self.initializer, + initializer=RandomNormal(stddev=self.init_std), name="S", dtype=tf.float32) super(CapsuleLayer, self).build(input_shape) @@ -233,21 +235,21 @@ def call(self, inputs, **kwargs): weight = tf.nn.softmax(routing_logits_with_padding) behavior_embdding_mapping = tf.tensordot(behavior_embddings, self.bilinear_mapping_matrix, axes=1) Z = tf.matmul(weight, behavior_embdding_mapping) - interet_capsules = squash(Z) + interest_capsules = squash(Z) delta_routing_logits = tf.reduce_sum( - tf.matmul(interet_capsules, tf.transpose(behavior_embdding_mapping, perm=[0, 2, 1])), + tf.matmul(interest_capsules, tf.transpose(behavior_embdding_mapping, perm=[0, 2, 1])), axis=0, keep_dims=True ) self.routing_logits.assign_add(delta_routing_logits) - interet_capsules = tf.reshape(interet_capsules, [-1, self.k_max, self.out_units]) - return interet_capsules + interest_capsules = tf.reshape(interest_capsules, [-1, self.k_max, self.out_units]) + return interest_capsules def compute_output_shape(self, input_shape): return (None, self.k_max, self.out_units) def get_config(self, ): config = {'input_units': self.input_units, 'out_units': self.out_units, 'max_len': self.max_len, - 'k_max': self.k_max, 'iteration_times': self.iteration_times, "initializer": self.initializer} + 'k_max': self.k_max, 'iteration_times': self.iteration_times, "init_std": self.init_std} base_config = super(CapsuleLayer, self).get_config() return dict(list(base_config.items()) + list(config.items())) diff --git a/deepmatch/models/mind.py b/deepmatch/models/mind.py index 8bfb9e0..8e95871 100755 --- a/deepmatch/models/mind.py +++ b/deepmatch/models/mind.py @@ -12,12 +12,12 @@ combined_dnn_input from deepctr.layers.core import DNN from deepctr.layers.utils import NoMask -from tensorflow.python.keras.layers import Concatenate +from tensorflow.python.keras.layers import Concatenate,Input,Lambda from tensorflow.python.keras.models import Model -from deepmatch.utils import get_item_embedding +from deepmatch.utils import get_item_embedding,get_item_embeddingv2 from ..inputs import create_embedding_matrix -from ..layers.core import CapsuleLayer, SampledSoftmaxLayer, PoolingLayer, LabelAwareAttention +from ..layers.core import CapsuleLayer, SampledSoftmaxLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayerv2 def shape_target(target_emb_tmp, target_emb_size): @@ -57,6 +57,9 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 raise ValueError("Now MIND only support 1 item feature like item_id") item_feature_column = item_feature_columns[0] item_feature_name = item_feature_column.name + item_vocabulary_size = item_feature_columns[0].vocabulary_size + item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))])) + history_feature_list = [item_feature_name] features = build_input_features(user_feature_columns) @@ -78,21 +81,21 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 inputs_list = list(features.values()) - embedding_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, + embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, seed, prefix="") item_features = build_input_features(item_feature_columns) - query_emb_list = embedding_lookup(embedding_dict, item_features, item_feature_columns, + query_emb_list = embedding_lookup(embedding_matrix_dict, item_features, item_feature_columns, history_feature_list, history_feature_list, to_list=True) - keys_emb_list = embedding_lookup(embedding_dict, features, history_feature_columns, history_fc_names, + keys_emb_list = embedding_lookup(embedding_matrix_dict, features, history_feature_columns, history_fc_names, history_fc_names, to_list=True) - dnn_input_emb_list = embedding_lookup(embedding_dict, features, sparse_feature_columns, + dnn_input_emb_list = embedding_lookup(embedding_matrix_dict, features, sparse_feature_columns, mask_feat_list=history_feature_list, to_list=True) dense_value_list = get_dense_input(features, dense_feature_columns) - sequence_embed_dict = varlen_embedding_lookup(embedding_dict, features, sparse_varlen_feature_columns) + sequence_embed_dict = varlen_embedding_lookup(embedding_matrix_dict, features, sparse_varlen_feature_columns) sequence_embed_list = get_varlen_pooling_list(sequence_embed_dict, features, sparse_varlen_feature_columns, to_list=True) @@ -121,27 +124,31 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 else: user_deep_input = high_capsule - # user_deep_input._uses_learning_phase = True # attention_score._uses_learning_phase user_embeddings = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed, name="user_embedding")(user_deep_input) item_inputs_list = list(item_features.values()) - item_embedding = embedding_dict[item_feature_name] + item_embedding_matrix = embedding_matrix_dict[item_feature_name] + + item_embedding_weight = item_embedding_matrix(item_index) + item_embedding_weight = Lambda(lambda x: tf.squeeze(x, axis=0))(NoMask()(item_embedding_weight)) + + pooling_item_embedding_weight = PoolingLayer()([item_embedding_weight]) if dynamic_k: user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p, )((user_embeddings, target_emb, hist_len)) else: user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p, )((user_embeddings, target_emb)) - output = SampledSoftmaxLayer(item_embedding, num_sampled=num_sampled)( - inputs=(user_embedding_final, item_features[item_feature_name])) - model = Model(inputs=inputs_list + item_inputs_list, outputs=output) + output = SampledSoftmaxLayerv2( num_sampled=num_sampled)( + inputs=(pooling_item_embedding_weight,user_embedding_final, item_features[item_feature_name])) + model = Model(inputs=inputs_list + item_inputs_list+[item_index], outputs=output) model.__setattr__("user_input", inputs_list) model.__setattr__("user_embedding", user_embeddings) model.__setattr__("item_input", item_inputs_list) - model.__setattr__("item_embedding", get_item_embedding(item_embedding, item_features[item_feature_name])) + model.__setattr__("item_embedding", get_item_embeddingv2(item_embedding_weight, item_features[item_feature_name])) return model diff --git a/deepmatch/models/ncf.py b/deepmatch/models/ncf.py index 91b2d7b..3484278 100644 --- a/deepmatch/models/ncf.py +++ b/deepmatch/models/ncf.py @@ -3,7 +3,7 @@ Jieyu Yang , yangjieyu@zju.edu.cn Reference: -Neural Collaborative Filtering +He X, Liao L, Zhang H, et al. Neural collaborative filtering[C]//Proceedings of the 26th international conference on world wide web. 2017: 173-182. """ import math diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 4793c1d..a3fe157 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -2,18 +2,20 @@ Author: Weichen Shen, wcshen1994@163.com Reference: -Deep Neural Networks for YouTube Recommendations +Covington P, Adams J, Sargin E. Deep neural networks for youtube recommendations[C]//Proceedings of the 10th ACM conference on recommender systems. 2016: 191-198. """ import tensorflow as tf from deepctr.inputs import input_from_feature_columns, build_input_features, combined_dnn_input, create_embedding_matrix from deepctr.layers.core import DNN from deepctr.layers.utils import NoMask +import numpy as np from tensorflow.python.keras.models import Model -from tensorflow.python.keras.layers import Input,Lambda +from tensorflow.python.keras.layers import Input, Lambda -from deepmatch.utils import get_item_embedding,get_item_embeddingv2 +from deepmatch.utils import get_item_embedding, get_item_embeddingv2 +from deepmatch.layers import PoolingLayer from ..inputs import input_from_feature_columns -from ..layers.core import SampledSoftmaxLayer,SampledSoftmaxLayerv2 +from ..layers.core import SampledSoftmaxLayer, SampledSoftmaxLayerv2 def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, @@ -41,7 +43,8 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, raise ValueError("Now YoutubeNN only support 1 item feature like item_id") item_feature_name = item_feature_columns[0].name item_vocabulary_size = item_feature_columns[0].vocabulary_size - #item_idx = Input(shape=[None,],tensor=tf.range(0,item_vocabulary_size),name="item_idx") + item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))])) + # item_idskey = Input( tensor=tf.constant([[0]*209]), name="item_idx") embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, seed, prefix="") @@ -59,25 +62,35 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_out = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed, )(user_dnn_input) - #item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) - item_embedding = embedding_matrix_dict[item_feature_name]#Lambda( lambda x:embedding_matrix_dict[item_feature_name])(None) + # item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) + item_embedding_matrix = embedding_matrix_dict[ + item_feature_name] # Lambda( lambda x:embedding_matrix_dict[item_feature_name])(None) # if not item_embedding.built: # item_embedding.build() - item_embedding = Input(tensor=item_embedding.embeddings,name=item_feature_name+"_embeddings") - #item_embedding = get_item_embeddingv2() + item_embedding_weight = item_embedding_matrix(item_index) + # item_embedding_skey = item_embedding(item_idskey) + + # item_embedding_ip = Input(tensor=item_embedding.embeddings,name=item_feature_name+"_embeddings") + item_embedding_weight = Lambda(lambda x: tf.squeeze(x, axis=0))(NoMask()(item_embedding_weight)) + # item_embedding_skey = Lambda(lambda x: tf.squeeze(x, axis=0))(NoMask()(item_embedding_skey)) + + pooling_item_embedding_weight = PoolingLayer()([item_embedding_weight]) + + # print(item_idx,"itemidxitem_idxitem_idxitem_idxitem_idx") + # print(item_embedding_ip, "item_embedding_ipitem_embedding_ipitem_embedding_ipitem_embedding_ip") + # item_embedding = get_item_embeddingv2() # if not item_embedding.built: # item_embedding.build([]) - output = SampledSoftmaxLayerv2( num_sampled=num_sampled)( - inputs=(item_embedding,user_dnn_out, item_features[item_feature_name])) - model = Model(inputs=user_inputs_list + item_inputs_list+[item_embedding], outputs=output)#+[item_idx] + output = SampledSoftmaxLayerv2(num_sampled=num_sampled)( + inputs=(pooling_item_embedding_weight, user_dnn_out, item_features[item_feature_name])) + model = Model(inputs=user_inputs_list + item_inputs_list + [item_index], outputs=output) # +[item_idx] model.__setattr__("user_input", user_inputs_list) model.__setattr__("user_embedding", user_dnn_out) model.__setattr__("item_input", item_inputs_list) - model.__setattr__("item_embedding", get_item_embeddingv2(item_embedding, item_features[item_feature_name])) + model.__setattr__("item_embedding", get_item_embeddingv2(item_embedding_weight, item_features[item_feature_name])) return model - diff --git a/tests/models/MIND_test.py b/tests/models/MIND_test.py index 0ff3098..c6310b7 100644 --- a/tests/models/MIND_test.py +++ b/tests/models/MIND_test.py @@ -11,16 +11,16 @@ #@pytest.mark.skip(reason="misunderstood the API") -# def test_MIND(): -# model_name = "MIND" -# -# x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) -# K.set_learning_phase(True) -# model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) -# -# model.compile('adam', sampledsoftmaxloss) -# check_model(model,model_name,x,y,check_model_io=True) -# +def test_MIND(): + model_name = "MIND" + + x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) + K.set_learning_phase(True) + model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) + + model.compile('adam', sampledsoftmaxloss) + check_model(model,model_name,x,y) + if __name__ == "__main__": pass diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index b0f9a34..e75f768 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -16,8 +16,8 @@ def test_YoutubeDNN(): x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) K.set_learning_phase(True) - if tf.__version__ >= '2.0.0': - tf.compat.v1.disable_eager_execution() + #if tf.__version__ >= '2.0.0': + # tf.compat.v1.disable_eager_execution() model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss) diff --git a/tests/utils.py b/tests/utils.py index ae48b7c..7ba5bf7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -376,17 +376,17 @@ def get_xy_fd(hash_flag=False): item_feature_columns = [SparseFeat('item', 3 + 1,embedding_dim=4,)] - uid = np.array([0, 1, 2]) - ugender = np.array([0, 1, 0]) - iid = np.array([1, 2, 3]) # 0 is mask value + uid = np.array([0, 1, 2,1]) + ugender = np.array([0, 1, 0,1]) + iid = np.array([1, 2, 3,1]) # 0 is mask value - hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0]]) - hist_len = np.array([3,3,2]) + hist_iid = np.array([[1, 2, 3, 0], [1, 2, 3, 0], [1, 2, 0, 0],[3, 0, 0, 0]]) + hist_len = np.array([3,3,2,1]) feature_dict = {'user': uid, 'gender': ugender, 'item': iid, 'hist_item': hist_iid, "hist_len":hist_len} #feature_names = get_feature_names(feature_columns) x = feature_dict - y = np.array([1, 1, 1]) + y = np.array([1, 1, 1,1]) return x, y, user_feature_columns,item_feature_columns \ No newline at end of file From 9510d44e2671e2027ac12c529a487f5ad72381ea Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 11:18:55 +0800 Subject: [PATCH 14/31] no message --- .github/workflows/ci.yml | 6 +++--- deepmatch/layers/core.py | 5 +++++ deepmatch/models/youtubednn.py | 26 +++++++++++++++++++++-- examples/run_youtubednn_sampledsoftmax.py | 3 +++ tests/models/MIND_test.py | 25 +++++++++++++--------- tests/models/YoutubeDNN_test.py | 4 ++-- 6 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7195104..a50ba13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,11 +18,11 @@ jobs: strategy: matrix: python-version: [3.5,3.6,3.7] - tf-version: [1.4.0,1.14.0,2.1.0] + tf-version: [1.4.0,1.13.0,1.14.0,2.1.0] exclude: - python-version: 3.7 - - tf-version: 1.4.0 + - tf-version: 1.4.0 steps: @@ -44,7 +44,7 @@ jobs: pip install -q pytest pip install -q pytest-cov pip install -q python-coveralls - pytest --cov=deepctr --cov-report=xml + pytest --cov=deepmatch --cov-report=xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v1.0.2 with: diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index 99a1390..cf8cfbe 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -37,6 +37,11 @@ def call(self, seq_value_len_list, mask=None, **kwargs): hist = reduce_max(a, axis=-1, ) return hist + def get_config(self, ): + config = {'mode': self.mode, 'supports_masking': self.supports_masking} + base_config = super(PoolingLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + class SampledSoftmaxLayer(Layer): def __init__(self, item_embedding, num_sampled=5, **kwargs): diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index a3fe157..991ff82 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -43,7 +43,10 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, raise ValueError("Now YoutubeNN only support 1 item feature like item_id") item_feature_name = item_feature_columns[0].name item_vocabulary_size = item_feature_columns[0].vocabulary_size - item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))])) + #item_indexx = Input(tensor=tf.constant([list(range(item_vocabulary_size))]),batch_size=10,shape=[item_vocabulary_size]) + + #item_indexx = tf.Variable(tf.constant([list(range(item_vocabulary_size))])) + #item_indexx = tf.constant(list(range(item_vocabulary_size))) # item_idskey = Input( tensor=tf.constant([[0]*209]), name="item_idx") embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, @@ -56,6 +59,7 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, l2_reg_embedding, init_std, seed, embedding_matrix_dict=embedding_matrix_dict) user_dnn_input = combined_dnn_input(user_sparse_embedding_list, user_dense_value_list) + item_index = Lambda(input_idx,arguments={'item_vocabulary_size': item_vocabulary_size})(user_dnn_input) item_features = build_input_features(item_feature_columns) item_inputs_list = list(item_features.values()) @@ -85,7 +89,7 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, output = SampledSoftmaxLayerv2(num_sampled=num_sampled)( inputs=(pooling_item_embedding_weight, user_dnn_out, item_features[item_feature_name])) - model = Model(inputs=user_inputs_list + item_inputs_list + [item_index], outputs=output) # +[item_idx] + model = Model(inputs=user_inputs_list + item_inputs_list , outputs=output) # +[item_idx] model.__setattr__("user_input", user_inputs_list) model.__setattr__("user_embedding", user_dnn_out) @@ -94,3 +98,21 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, model.__setattr__("item_embedding", get_item_embeddingv2(item_embedding_weight, item_features[item_feature_name])) return model + + +# def softmax_fine_loss(labels, logits, transposed_W=None, b=None): +# res = tf.map_fn(lambda (__labels, __logits): tf.nn.sampled_softmax_loss(transposed_W, b, __labels, __logits, +# num_sampled=1000, +# num_classes=OUTPUT_COUNT + 1), +# (labels, logits), dtype=tf.float32) +# return res +# +# +# loss = lambda labels, logits: softmax_fine_loss(labels, logits, transposed_W=transposed_W, b=b) +# +# model_truncated.compile(optimizer=optimizer, loss=loss, sample_weight_mode='temporal') + +def input_idx(x,item_vocabulary_size): + #print(x) + #item_vocabulary_size = x[1] + return tf.constant([list(range(item_vocabulary_size))]) \ No newline at end of file diff --git a/examples/run_youtubednn_sampledsoftmax.py b/examples/run_youtubednn_sampledsoftmax.py index 23c432c..6775f34 100644 --- a/examples/run_youtubednn_sampledsoftmax.py +++ b/examples/run_youtubednn_sampledsoftmax.py @@ -56,6 +56,9 @@ # 3.Define Model and train K.set_learning_phase(True) + import tensorflow as tf + if tf.__version__ >= '2.0.0': + tf.compat.v1.disable_eager_execution() model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_hidden_units=(64, 16)) # model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,p=1,k_max=2,num_sampled=5,user_dnn_hidden_units=(64,16),init_std=0.001) diff --git a/tests/models/MIND_test.py b/tests/models/MIND_test.py index c6310b7..165e67c 100644 --- a/tests/models/MIND_test.py +++ b/tests/models/MIND_test.py @@ -5,22 +5,27 @@ from deepctr.inputs import SparseFeat,VarLenSparseFeat,DenseFeat,get_feature_names from tensorflow.python.keras import backend as K from ..utils import check_model,get_xy_fd +import tensorflow as tf #@pytest.mark.xfail(reason="There is a bug when save model use Dice") #@pytest.mark.skip(reason="misunderstood the API") -def test_MIND(): - model_name = "MIND" - - x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) - K.set_learning_phase(True) - model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) - - model.compile('adam', sampledsoftmaxloss) - check_model(model,model_name,x,y) - +# def test_MIND(): +# model_name = "MIND" +# +# x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) +# K.set_learning_phase(True) +# +# if tf.__version__ >= '2.0.0': +# tf.compat.v1.disable_eager_execution() +# +# model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) +# +# model.compile('adam', sampledsoftmaxloss) +# check_model(model,model_name,x,y) +# if __name__ == "__main__": pass diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index e75f768..ea7560c 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -16,8 +16,8 @@ def test_YoutubeDNN(): x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) K.set_learning_phase(True) - #if tf.__version__ >= '2.0.0': - # tf.compat.v1.disable_eager_execution() + if tf.__version__ >= '2.0.0': + tf.compat.v1.disable_eager_execution() model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss) From 8118f43a6b4ebd878ded06f63859e7301052f4d0 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 11:21:11 +0800 Subject: [PATCH 15/31] no message --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a50ba13..5a40c0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: python-version: [3.5,3.6,3.7] - tf-version: [1.4.0,1.13.0,1.14.0,2.1.0] + tf-version: [1.4.0,1.14.0,1.15.0,2.1.0] exclude: - python-version: 3.7 From a50dfb65813977d422973fc8c733a7dbba446907 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 11:31:34 +0800 Subject: [PATCH 16/31] no message --- deepmatch/layers/__init__.py | 5 +++-- deepmatch/layers/core.py | 22 ++++++++++++++++++++++ deepmatch/models/mind.py | 8 +++++--- deepmatch/models/youtubednn.py | 6 ++++-- 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/deepmatch/layers/__init__.py b/deepmatch/layers/__init__.py index df87605..822cb8f 100644 --- a/deepmatch/layers/__init__.py +++ b/deepmatch/layers/__init__.py @@ -1,7 +1,7 @@ from deepctr.layers import custom_objects from deepctr.layers.utils import reduce_sum -from .core import PoolingLayer, SampledSoftmaxLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayerv2 +from .core import PoolingLayer, SampledSoftmaxLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayerv2,EmbeddingIdx from ..utils import sampledsoftmaxloss _custom_objects = {'PoolingLayer': PoolingLayer, @@ -11,7 +11,8 @@ 'CapsuleLayer': CapsuleLayer, 'reduce_sum':reduce_sum, 'SampledSoftmaxLayerv2':SampledSoftmaxLayerv2, - 'sampledsoftmaxloss':sampledsoftmaxloss + 'sampledsoftmaxloss':sampledsoftmaxloss, + 'EmbeddingIdx':EmbeddingIdx } custom_objects = dict(custom_objects, **_custom_objects) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index cf8cfbe..8213670 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -264,3 +264,25 @@ def squash(inputs): scalar_factor = vec_squared_norm / (1 + vec_squared_norm) / tf.sqrt(vec_squared_norm + 1e-8) vec_squashed = scalar_factor * inputs return vec_squashed + + + + +class EmbeddingIdx(Layer): + + def __init__(self, vocabulary_size,**kwargs): + + self.vocabulary_size =vocabulary_size + super(EmbeddingIdx, self).__init__(**kwargs) + + def build(self, input_shape): + + super(EmbeddingIdx, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, x, **kwargs): + return tf.constant([list(range(self.vocabulary_size))]) + def get_config(self, ): + config = {'vocabulary_size': self.vocabulary_size,} + base_config = super(EmbeddingIdx, self).get_config() + return dict(list(base_config.items()) + list(config.items())) \ No newline at end of file diff --git a/deepmatch/models/mind.py b/deepmatch/models/mind.py index 8e95871..b95bf4a 100755 --- a/deepmatch/models/mind.py +++ b/deepmatch/models/mind.py @@ -17,7 +17,7 @@ from deepmatch.utils import get_item_embedding,get_item_embeddingv2 from ..inputs import create_embedding_matrix -from ..layers.core import CapsuleLayer, SampledSoftmaxLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayerv2 +from ..layers.core import CapsuleLayer, SampledSoftmaxLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayerv2,EmbeddingIdx def shape_target(target_emb_tmp, target_emb_size): @@ -58,7 +58,7 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 item_feature_column = item_feature_columns[0] item_feature_name = item_feature_column.name item_vocabulary_size = item_feature_columns[0].vocabulary_size - item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))])) + #item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))])) history_feature_list = [item_feature_name] @@ -131,6 +131,8 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 item_embedding_matrix = embedding_matrix_dict[item_feature_name] + item_index = EmbeddingIdx(item_vocabulary_size)(item_features[item_feature_name]) + item_embedding_weight = item_embedding_matrix(item_index) item_embedding_weight = Lambda(lambda x: tf.squeeze(x, axis=0))(NoMask()(item_embedding_weight)) @@ -143,7 +145,7 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 output = SampledSoftmaxLayerv2( num_sampled=num_sampled)( inputs=(pooling_item_embedding_weight,user_embedding_final, item_features[item_feature_name])) - model = Model(inputs=inputs_list + item_inputs_list+[item_index], outputs=output) + model = Model(inputs=inputs_list + item_inputs_list, outputs=output) model.__setattr__("user_input", inputs_list) model.__setattr__("user_embedding", user_embeddings) diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 991ff82..967d5b2 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -15,7 +15,7 @@ from deepmatch.utils import get_item_embedding, get_item_embeddingv2 from deepmatch.layers import PoolingLayer from ..inputs import input_from_feature_columns -from ..layers.core import SampledSoftmaxLayer, SampledSoftmaxLayerv2 +from ..layers.core import SampledSoftmaxLayer, SampledSoftmaxLayerv2,EmbeddingIdx def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, @@ -59,13 +59,15 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, l2_reg_embedding, init_std, seed, embedding_matrix_dict=embedding_matrix_dict) user_dnn_input = combined_dnn_input(user_sparse_embedding_list, user_dense_value_list) - item_index = Lambda(input_idx,arguments={'item_vocabulary_size': item_vocabulary_size})(user_dnn_input) item_features = build_input_features(item_feature_columns) item_inputs_list = list(item_features.values()) user_dnn_out = DNN(user_dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed, )(user_dnn_input) + + item_index = EmbeddingIdx(item_vocabulary_size)(item_features[item_feature_name]) + # item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) item_embedding_matrix = embedding_matrix_dict[ item_feature_name] # Lambda( lambda x:embedding_matrix_dict[item_feature_name])(None) From 6186fd83bf8c12e05aeaaf1f01fab75972b08108 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 12:31:48 +0800 Subject: [PATCH 17/31] no message --- deepmatch/layers/__init__.py | 4 ++-- deepmatch/layers/core.py | 18 +++++++--------- deepmatch/models/mind.py | 9 ++++---- deepmatch/models/youtubednn.py | 39 ++++++---------------------------- 4 files changed, 21 insertions(+), 49 deletions(-) diff --git a/deepmatch/layers/__init__.py b/deepmatch/layers/__init__.py index 822cb8f..c36b462 100644 --- a/deepmatch/layers/__init__.py +++ b/deepmatch/layers/__init__.py @@ -1,7 +1,7 @@ from deepctr.layers import custom_objects from deepctr.layers.utils import reduce_sum -from .core import PoolingLayer, SampledSoftmaxLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayerv2,EmbeddingIdx +from .core import PoolingLayer, SampledSoftmaxLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayerv2,EmbeddingIndex from ..utils import sampledsoftmaxloss _custom_objects = {'PoolingLayer': PoolingLayer, @@ -12,7 +12,7 @@ 'reduce_sum':reduce_sum, 'SampledSoftmaxLayerv2':SampledSoftmaxLayerv2, 'sampledsoftmaxloss':sampledsoftmaxloss, - 'EmbeddingIdx':EmbeddingIdx + 'EmbeddingIndex':EmbeddingIndex } custom_objects = dict(custom_objects, **_custom_objects) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index 8213670..a08f11a 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -268,21 +268,19 @@ def squash(inputs): -class EmbeddingIdx(Layer): +class EmbeddingIndex(Layer): - def __init__(self, vocabulary_size,**kwargs): - - self.vocabulary_size =vocabulary_size - super(EmbeddingIdx, self).__init__(**kwargs) + def __init__(self, index,**kwargs): + self.index =index + super(EmbeddingIndex, self).__init__(**kwargs) def build(self, input_shape): - super(EmbeddingIdx, self).build( + super(EmbeddingIndex, self).build( input_shape) # Be sure to call this somewhere! - def call(self, x, **kwargs): - return tf.constant([list(range(self.vocabulary_size))]) + return tf.constant(self.index) def get_config(self, ): - config = {'vocabulary_size': self.vocabulary_size,} - base_config = super(EmbeddingIdx, self).get_config() + config = {'index': self.index, } + base_config = super(EmbeddingIndex, self).get_config() return dict(list(base_config.items()) + list(config.items())) \ No newline at end of file diff --git a/deepmatch/models/mind.py b/deepmatch/models/mind.py index b95bf4a..5d61a55 100755 --- a/deepmatch/models/mind.py +++ b/deepmatch/models/mind.py @@ -17,7 +17,7 @@ from deepmatch.utils import get_item_embedding,get_item_embeddingv2 from ..inputs import create_embedding_matrix -from ..layers.core import CapsuleLayer, SampledSoftmaxLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayerv2,EmbeddingIdx +from ..layers.core import CapsuleLayer, SampledSoftmaxLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayerv2,EmbeddingIndex def shape_target(target_emb_tmp, target_emb_size): @@ -131,10 +131,9 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 item_embedding_matrix = embedding_matrix_dict[item_feature_name] - item_index = EmbeddingIdx(item_vocabulary_size)(item_features[item_feature_name]) + item_index = EmbeddingIndex(list(range(item_vocabulary_size)))(item_features[item_feature_name]) - item_embedding_weight = item_embedding_matrix(item_index) - item_embedding_weight = Lambda(lambda x: tf.squeeze(x, axis=0))(NoMask()(item_embedding_weight)) + item_embedding_weight = NoMask()(item_embedding_matrix(item_index)) pooling_item_embedding_weight = PoolingLayer()([item_embedding_weight]) @@ -151,6 +150,6 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 model.__setattr__("user_embedding", user_embeddings) model.__setattr__("item_input", item_inputs_list) - model.__setattr__("item_embedding", get_item_embeddingv2(item_embedding_weight, item_features[item_feature_name])) + model.__setattr__("item_embedding", get_item_embeddingv2(pooling_item_embedding_weight, item_features[item_feature_name])) return model diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 967d5b2..c4978a4 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -15,7 +15,7 @@ from deepmatch.utils import get_item_embedding, get_item_embeddingv2 from deepmatch.layers import PoolingLayer from ..inputs import input_from_feature_columns -from ..layers.core import SampledSoftmaxLayer, SampledSoftmaxLayerv2,EmbeddingIdx +from ..layers.core import SampledSoftmaxLayer, SampledSoftmaxLayerv2,EmbeddingIndex def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, @@ -43,11 +43,6 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, raise ValueError("Now YoutubeNN only support 1 item feature like item_id") item_feature_name = item_feature_columns[0].name item_vocabulary_size = item_feature_columns[0].vocabulary_size - #item_indexx = Input(tensor=tf.constant([list(range(item_vocabulary_size))]),batch_size=10,shape=[item_vocabulary_size]) - - #item_indexx = tf.Variable(tf.constant([list(range(item_vocabulary_size))])) - #item_indexx = tf.constant(list(range(item_vocabulary_size))) - # item_idskey = Input( tensor=tf.constant([[0]*209]), name="item_idx") embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, seed, prefix="") @@ -66,38 +61,23 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, dnn_use_bn, seed, )(user_dnn_input) - item_index = EmbeddingIdx(item_vocabulary_size)(item_features[item_feature_name]) + item_index = EmbeddingIndex(list(range(item_vocabulary_size)))(item_features[item_feature_name]) - # item_embedding = NoMask()(embedding_matrix_dict[item_feature_name](item_idx)) item_embedding_matrix = embedding_matrix_dict[ - item_feature_name] # Lambda( lambda x:embedding_matrix_dict[item_feature_name])(None) - # if not item_embedding.built: - # item_embedding.build() - item_embedding_weight = item_embedding_matrix(item_index) - # item_embedding_skey = item_embedding(item_idskey) - - # item_embedding_ip = Input(tensor=item_embedding.embeddings,name=item_feature_name+"_embeddings") - item_embedding_weight = Lambda(lambda x: tf.squeeze(x, axis=0))(NoMask()(item_embedding_weight)) - # item_embedding_skey = Lambda(lambda x: tf.squeeze(x, axis=0))(NoMask()(item_embedding_skey)) + item_feature_name] + item_embedding_weight = NoMask()(item_embedding_matrix(item_index)) pooling_item_embedding_weight = PoolingLayer()([item_embedding_weight]) - # print(item_idx,"itemidxitem_idxitem_idxitem_idxitem_idx") - # print(item_embedding_ip, "item_embedding_ipitem_embedding_ipitem_embedding_ipitem_embedding_ip") - # item_embedding = get_item_embeddingv2() - - # if not item_embedding.built: - # item_embedding.build([]) - output = SampledSoftmaxLayerv2(num_sampled=num_sampled)( inputs=(pooling_item_embedding_weight, user_dnn_out, item_features[item_feature_name])) - model = Model(inputs=user_inputs_list + item_inputs_list , outputs=output) # +[item_idx] + model = Model(inputs=user_inputs_list + item_inputs_list , outputs=output) model.__setattr__("user_input", user_inputs_list) model.__setattr__("user_embedding", user_dnn_out) model.__setattr__("item_input", item_inputs_list) - model.__setattr__("item_embedding", get_item_embeddingv2(item_embedding_weight, item_features[item_feature_name])) + model.__setattr__("item_embedding", get_item_embeddingv2(pooling_item_embedding_weight, item_features[item_feature_name])) return model @@ -112,9 +92,4 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, # # loss = lambda labels, logits: softmax_fine_loss(labels, logits, transposed_W=transposed_W, b=b) # -# model_truncated.compile(optimizer=optimizer, loss=loss, sample_weight_mode='temporal') - -def input_idx(x,item_vocabulary_size): - #print(x) - #item_vocabulary_size = x[1] - return tf.constant([list(range(item_vocabulary_size))]) \ No newline at end of file +# model_truncated.compile(optimizer=optimizer, loss=loss, sample_weight_mode='temporal') \ No newline at end of file From 1e0d4bb9395b20293b4ba514d480617f83b60416 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 12:38:37 +0800 Subject: [PATCH 18/31] no message --- .github/ISSUE_TEMPLATE/question.md | 2 +- .github/workflows/ci.yml | 2 +- deepmatch/models/youtubednn.py | 13 ------------- docs/source/FAQ.md | 24 +++++++++++++++++++++++- tests/models/MIND_test.py | 28 ++++++++++++++-------------- tests/models/YoutubeDNN_test.py | 4 ++-- 6 files changed, 41 insertions(+), 32 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 2e490d0..53e8451 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -6,7 +6,7 @@ labels: question assignees: '' --- -Please refer to the [FAQ](https://deepmatch.readthedocs.io/en/latest/FAQ.html) in doc and search for the [related issues](https://github.com/shenweichen/DeepCTR/issues) before you ask the question. +Please refer to the [FAQ](https://deepmatch.readthedocs.io/en/latest/FAQ.html) in doc and search for the [related issues](https://github.com/shenweichen/DeepMatch/issues) before you ask the question. **Describe the question(问题描述)** A clear and concise description of what the question is. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a40c0e..6abfd62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: exclude: - python-version: 3.7 - - tf-version: 1.4.0 + tf-version: 1.4.0 steps: diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index c4978a4..844e743 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -80,16 +80,3 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, model.__setattr__("item_embedding", get_item_embeddingv2(pooling_item_embedding_weight, item_features[item_feature_name])) return model - - -# def softmax_fine_loss(labels, logits, transposed_W=None, b=None): -# res = tf.map_fn(lambda (__labels, __logits): tf.nn.sampled_softmax_loss(transposed_W, b, __labels, __logits, -# num_sampled=1000, -# num_classes=OUTPUT_COUNT + 1), -# (labels, logits), dtype=tf.float32) -# return res -# -# -# loss = lambda labels, logits: softmax_fine_loss(labels, logits, transposed_W=transposed_W, b=b) -# -# model_truncated.compile(optimizer=optimizer, loss=loss, sample_weight_mode='temporal') \ No newline at end of file diff --git a/docs/source/FAQ.md b/docs/source/FAQ.md index 49f8a8e..cf5cef9 100644 --- a/docs/source/FAQ.md +++ b/docs/source/FAQ.md @@ -1,6 +1,28 @@ # FAQ -## 1. Set learning rate and use earlystopping + +## 1. Save or load weights/models +---------------------------------------- +To save/load weights,you can write codes just like any other keras models. + +```python +model = YoutubeDNN() +model.save_weights('YoutubeDNN_w.h5') +model.load_weights('YoutubeDNN_w.h5') +``` + +To save/load models,just a little different. + +```python +from tensorflow.python.keras.models import save_model,load_model +model = DeepFM() +save_model(model, 'YoutubeDNN.h5')# save_model, same as before + +from deepmatch.layers import custom_objects +model = load_model('YoutubeDNN.h5',custom_objects)# load_model,just add a parameter +``` + +## 2. Set learning rate and use earlystopping --------------------------------------------------- You can use any models in DeepCTR like a keras model object. Here is a example of how to set learning rate and earlystopping: diff --git a/tests/models/MIND_test.py b/tests/models/MIND_test.py index 165e67c..4bfec8a 100644 --- a/tests/models/MIND_test.py +++ b/tests/models/MIND_test.py @@ -12,20 +12,20 @@ #@pytest.mark.skip(reason="misunderstood the API") -# def test_MIND(): -# model_name = "MIND" -# -# x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) -# K.set_learning_phase(True) -# -# if tf.__version__ >= '2.0.0': -# tf.compat.v1.disable_eager_execution() -# -# model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) -# -# model.compile('adam', sampledsoftmaxloss) -# check_model(model,model_name,x,y) -# +def test_MIND(): + model_name = "MIND" + + x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) + K.set_learning_phase(True) + + # if tf.__version__ >= '2.0.0': + # tf.compat.v1.disable_eager_execution() + + model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) + + model.compile('adam', sampledsoftmaxloss) + check_model(model,model_name,x,y) + if __name__ == "__main__": pass diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index ea7560c..241b2d6 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -16,8 +16,8 @@ def test_YoutubeDNN(): x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) K.set_learning_phase(True) - if tf.__version__ >= '2.0.0': - tf.compat.v1.disable_eager_execution() + # if tf.__version__ >= '2.0.0': + # tf.compat.v1.disable_eager_execution() model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss) From e2ffd4fccf34e4180ce7e5c94d31e56c6e902cde Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 12:50:34 +0800 Subject: [PATCH 19/31] no message --- .github/workflows/ci.yml | 2 +- deepmatch/models/mind.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6abfd62..00096cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: python-version: [3.5,3.6,3.7] - tf-version: [1.4.0,1.14.0,1.15.0,2.1.0] + tf-version: [1.4.0,1.14.0,2.1.0] exclude: - python-version: 3.7 diff --git a/deepmatch/models/mind.py b/deepmatch/models/mind.py index 5d61a55..f232b12 100755 --- a/deepmatch/models/mind.py +++ b/deepmatch/models/mind.py @@ -58,6 +58,7 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 item_feature_column = item_feature_columns[0] item_feature_name = item_feature_column.name item_vocabulary_size = item_feature_columns[0].vocabulary_size + item_embedding_dim = item_feature_columns[0].embedding_dim #item_index = Input(tensor=tf.constant([list(range(item_vocabulary_size))])) history_feature_list = [item_feature_name] @@ -78,7 +79,7 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 history_feature_columns.append(fc) else: sparse_varlen_feature_columns.append(fc) - + seq_max_len = history_feature_columns[0].maxlen inputs_list = list(features.values()) embedding_matrix_dict = create_embedding_matrix(user_feature_columns + item_feature_columns, l2_reg_embedding, init_std, @@ -107,12 +108,12 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 history_emb = PoolingLayer()(NoMask()(keys_emb_list)) target_emb = PoolingLayer()(NoMask()(query_emb_list)) - target_emb_size = target_emb.get_shape()[-1].value - max_len = history_emb.get_shape()[1].value + #target_emb_size = target_emb.get_shape()[-1].value + #max_len = history_emb.get_shape()[1].value hist_len = features['hist_len'] - high_capsule = CapsuleLayer(input_units=target_emb_size, - out_units=target_emb_size, max_len=max_len, + high_capsule = CapsuleLayer(input_units=item_embedding_dim, + out_units=item_embedding_dim, max_len=seq_max_len, k_max=k_max)((history_emb, hist_len)) if len(dnn_input_emb_list) > 0 or len(dense_value_list) > 0: From 3e92043afaba58252af3f33d0422662c5e83d472 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 12:59:45 +0800 Subject: [PATCH 20/31] no message --- deepmatch/models/youtubednn.py | 3 --- tests/models/MIND_test.py | 7 ++----- tests/models/YoutubeDNN_test.py | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 844e743..5e5bb9b 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -4,13 +4,10 @@ Reference: Covington P, Adams J, Sargin E. Deep neural networks for youtube recommendations[C]//Proceedings of the 10th ACM conference on recommender systems. 2016: 191-198. """ -import tensorflow as tf from deepctr.inputs import input_from_feature_columns, build_input_features, combined_dnn_input, create_embedding_matrix from deepctr.layers.core import DNN from deepctr.layers.utils import NoMask -import numpy as np from tensorflow.python.keras.models import Model -from tensorflow.python.keras.layers import Input, Lambda from deepmatch.utils import get_item_embedding, get_item_embeddingv2 from deepmatch.layers import PoolingLayer diff --git a/tests/models/MIND_test.py b/tests/models/MIND_test.py index 4bfec8a..9451630 100644 --- a/tests/models/MIND_test.py +++ b/tests/models/MIND_test.py @@ -1,8 +1,5 @@ -import numpy as np - from deepmatch.models import MIND from deepmatch.utils import sampledsoftmaxloss -from deepctr.inputs import SparseFeat,VarLenSparseFeat,DenseFeat,get_feature_names from tensorflow.python.keras import backend as K from ..utils import check_model,get_xy_fd import tensorflow as tf @@ -18,8 +15,8 @@ def test_MIND(): x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) K.set_learning_phase(True) - # if tf.__version__ >= '2.0.0': - # tf.compat.v1.disable_eager_execution() + if tf.__version__ >= '2.0.0': + tf.compat.v1.disable_eager_execution() model = MIND(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) diff --git a/tests/models/YoutubeDNN_test.py b/tests/models/YoutubeDNN_test.py index 241b2d6..ea7560c 100644 --- a/tests/models/YoutubeDNN_test.py +++ b/tests/models/YoutubeDNN_test.py @@ -16,8 +16,8 @@ def test_YoutubeDNN(): x, y, user_feature_columns, item_feature_columns = get_xy_fd(False) K.set_learning_phase(True) - # if tf.__version__ >= '2.0.0': - # tf.compat.v1.disable_eager_execution() + if tf.__version__ >= '2.0.0': + tf.compat.v1.disable_eager_execution() model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=2, user_dnn_hidden_units=(16, 4)) model.compile('adam', sampledsoftmaxloss) From 7bce6ba9fb9533fdda62eb56dbb2383f0bea8e76 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 13:04:47 +0800 Subject: [PATCH 21/31] no message --- deepmatch/layers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index a08f11a..ad76aa7 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -260,7 +260,7 @@ def get_config(self, ): def squash(inputs): - vec_squared_norm = tf.reduce_sum(tf.square(inputs), axis=-1, keep_dims=True) + vec_squared_norm = reduce_sum(tf.square(inputs), axis=-1, keep_dims=True) scalar_factor = vec_squared_norm / (1 + vec_squared_norm) / tf.sqrt(vec_squared_norm + 1e-8) vec_squashed = scalar_factor * inputs return vec_squashed From 97e365259bc5d144adf8bbce41ed1582f2391386 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 13:09:18 +0800 Subject: [PATCH 22/31] no message --- deepmatch/layers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index ad76aa7..00cbda4 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -241,7 +241,7 @@ def call(self, inputs, **kwargs): behavior_embdding_mapping = tf.tensordot(behavior_embddings, self.bilinear_mapping_matrix, axes=1) Z = tf.matmul(weight, behavior_embdding_mapping) interest_capsules = squash(Z) - delta_routing_logits = tf.reduce_sum( + delta_routing_logits = reduce_sum( tf.matmul(interest_capsules, tf.transpose(behavior_embdding_mapping, perm=[0, 2, 1])), axis=0, keep_dims=True ) From 1bfdc90bf177b9b94e9ad16a753bbd116b600874 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 13:14:15 +0800 Subject: [PATCH 23/31] no message --- deepmatch/layers/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index 00cbda4..bf028b3 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -147,7 +147,7 @@ def build(self, input_shape): def call(self, inputs, training=None, **kwargs): keys = inputs[0] query = inputs[1] - weight = tf.reduce_sum(keys * query, axis=-1, keep_dims=True) + weight = reduce_sum(keys * query, axis=-1, keep_dims=True) weight = tf.pow(weight, self.pow_p) # [x,k_max,1] if len(inputs) == 3: @@ -163,7 +163,7 @@ def call(self, inputs, training=None, **kwargs): weight = tf.where(seq_mask, weight, padding) weight = softmax(weight, dim=1, name="weight") - output = tf.reduce_sum(keys * weight, axis=1) + output = reduce_sum(keys * weight, axis=1) return output From 4816663f2ef37b9a8e6f19fc790254abd2a009f5 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 17:20:39 +0800 Subject: [PATCH 24/31] no message --- deepmatch/__init__.py | 2 +- docs/source/Examples.md | 37 ++++++++++++----------- docs/source/History.md | 1 + docs/source/conf.py | 2 +- examples/run_youtubednn_sampledsoftmax.py | 2 +- setup.py | 2 +- 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/deepmatch/__init__.py b/deepmatch/__init__.py index 5c8ec87..4c4a1c1 100644 --- a/deepmatch/__init__.py +++ b/deepmatch/__init__.py @@ -1,4 +1,4 @@ from .utils import check_version -__version__ = '0.1.1' +__version__ = '0.1.2' check_version(__version__) diff --git a/docs/source/Examples.md b/docs/source/Examples.md index 81cc023..90577c1 100644 --- a/docs/source/Examples.md +++ b/docs/source/Examples.md @@ -1,6 +1,9 @@ # Examples -## DSSM with negative sampling + + + +## YoutubeDNN with sampled softmax The MovieLens data has been used for personalized tag recommendation,which contains 668, 953 tag applications of users on movies. @@ -9,7 +12,7 @@ Here is a small fraction of data include only sparse field. ![image](../pics/movielens_sample.png) -This example shows how to use ``DSSM`` to solve a matching task. You can get the demo data +This example shows how to use ``YoutubeDNN`` to solve a matching task. You can get the demo data [movielens_sample.txt](https://github.com/shenweichen/DeepMatch/tree/master/examples/movielens_sample.txt) and run the following codes. ```python @@ -17,9 +20,11 @@ import pandas as pd from deepctr.inputs import SparseFeat, VarLenSparseFeat from preprocess import gen_data_set, gen_model_input from sklearn.preprocessing import LabelEncoder +from tensorflow.python.keras import backend as K from tensorflow.python.keras.models import Model from deepmatch.models import * +from deepmatch.utils import sampledsoftmaxloss if __name__ == "__main__": @@ -27,7 +32,7 @@ if __name__ == "__main__": sparse_features = ["movie_id", "user_id", "gender", "age", "occupation", "zip", ] SEQ_LEN = 50 - negsample = 3 + negsample = 0 # 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input` @@ -68,9 +73,12 @@ if __name__ == "__main__": # 3.Define Model and train - model = DSSM(user_feature_columns, item_feature_columns) # FM(user_feature_columns,item_feature_columns) + K.set_learning_phase(True) - model.compile(optimizer='adagrad', loss="binary_crossentropy") + model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_hidden_units=(64, 16)) + # model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,p=1,k_max=2,num_sampled=5,user_dnn_hidden_units=(64,16),init_std=0.001) + + model.compile(optimizer="adagrad", loss=sampledsoftmaxloss) # "binary_crossentropy") history = model.fit(train_model_input, train_label, # train_label, batch_size=256, epochs=1, verbose=1, validation_split=0.0, ) @@ -83,6 +91,7 @@ if __name__ == "__main__": item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding) user_embs = user_embedding_model.predict(test_user_model_input, batch_size=2 ** 12) + # user_embs = user_embs[:, i, :] i in [0,k_max) if MIND item_embs = item_embedding_model.predict(all_item_model_input, batch_size=2 ** 12) print(user_embs.shape) @@ -120,7 +129,7 @@ if __name__ == "__main__": ``` -## YoutubeDNN with sampled softmax +## DSSM with negative sampling The MovieLens data has been used for personalized tag recommendation,which contains 668, 953 tag applications of users on movies. @@ -129,7 +138,7 @@ Here is a small fraction of data include only sparse field. ![image](../pics/movielens_sample.png) -This example shows how to use ``YoutubeDNN`` to solve a matching task. You can get the demo data +This example shows how to use ``DSSM`` to solve a matching task. You can get the demo data [movielens_sample.txt](https://github.com/shenweichen/DeepMatch/tree/master/examples/movielens_sample.txt) and run the following codes. ```python @@ -137,11 +146,9 @@ import pandas as pd from deepctr.inputs import SparseFeat, VarLenSparseFeat from preprocess import gen_data_set, gen_model_input from sklearn.preprocessing import LabelEncoder -from tensorflow.python.keras import backend as K from tensorflow.python.keras.models import Model from deepmatch.models import * -from deepmatch.utils import sampledsoftmaxloss if __name__ == "__main__": @@ -149,7 +156,7 @@ if __name__ == "__main__": sparse_features = ["movie_id", "user_id", "gender", "age", "occupation", "zip", ] SEQ_LEN = 50 - negsample = 0 + negsample = 3 # 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input` @@ -190,25 +197,21 @@ if __name__ == "__main__": # 3.Define Model and train - K.set_learning_phase(True) - - model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_hidden_units=(64, 16)) - # model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,p=1,k_max=2,num_sampled=5,user_dnn_hidden_units=(64,16),init_std=0.001) + model = DSSM(user_feature_columns, item_feature_columns) # FM(user_feature_columns,item_feature_columns) - model.compile(optimizer="adagrad", loss=sampledsoftmaxloss) # "binary_crossentropy") + model.compile(optimizer='adagrad', loss="binary_crossentropy") history = model.fit(train_model_input, train_label, # train_label, batch_size=256, epochs=1, verbose=1, validation_split=0.0, ) # 4. Generate user features for testing and full item features for retrieval test_user_model_input = test_model_input - all_item_model_input = {"movie_id": item_profile['movie_id'].values, "movie_idx": item_profile['movie_id'].values} + all_item_model_input = {"movie_id": item_profile['movie_id'].values,} user_embedding_model = Model(inputs=model.user_input, outputs=model.user_embedding) item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding) user_embs = user_embedding_model.predict(test_user_model_input, batch_size=2 ** 12) - # user_embs = user_embs[:, i, :] i in [0,k_max) if MIND item_embs = item_embedding_model.predict(all_item_model_input, batch_size=2 ** 12) print(user_embs.shape) diff --git a/docs/source/History.md b/docs/source/History.md index 5f22951..318d26d 100644 --- a/docs/source/History.md +++ b/docs/source/History.md @@ -1,2 +1,3 @@ # History +- 04/10/2020 : DeepMatch first version is released on [PyPi](https://pypi.org/project/deepmatch/) - 04/06/2020 : DeepMatch first version is released on [PyPi](https://pypi.org/project/deepmatch/) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 65ff099..bbff490 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ # The short X.Y version version = '' # The full version, including alpha/beta/rc tags -release = '0.1.1' +release = '0.1.2' # -- General configuration --------------------------------------------------- diff --git a/examples/run_youtubednn_sampledsoftmax.py b/examples/run_youtubednn_sampledsoftmax.py index 6775f34..a50adda 100644 --- a/examples/run_youtubednn_sampledsoftmax.py +++ b/examples/run_youtubednn_sampledsoftmax.py @@ -63,7 +63,7 @@ model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, user_dnn_hidden_units=(64, 16)) # model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,p=1,k_max=2,num_sampled=5,user_dnn_hidden_units=(64,16),init_std=0.001) - model.compile(optimizer="adagrad", loss=sampledsoftmaxloss) # "binary_crossentropy") + model.compile(optimizer="adam", loss=sampledsoftmaxloss) # "binary_crossentropy") history = model.fit(train_model_input, train_label, # train_label, batch_size=256, epochs=1, verbose=1, validation_split=0.0, ) diff --git a/setup.py b/setup.py index 9ee910d..9f4850d 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setuptools.setup( name="deepmatch", - version="0.1.1", + version="0.1.2", author="Weichen Shen", author_email="wcshen1994@163.com", description="Deep matching model library for recommendations, advertising, and search. It's easy to train models and to **export representation vectors** for user and item which can be used for **ANN search**.", From 832474d9e75f8d6509b34c15e1bb6c58479355bf Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 19:06:44 +0800 Subject: [PATCH 25/31] no message --- deepmatch/layers/__init__.py | 5 ++-- deepmatch/layers/core.py | 50 ++------------------------------ deepmatch/models/dssm.py | 4 +-- deepmatch/models/mind.py | 10 +++---- deepmatch/models/ncf.py | 2 +- deepmatch/models/youtubednn.py | 10 +++---- deepmatch/utils.py | 7 +---- docs/source/History.md | 2 +- docs/source/index.rst | 2 ++ examples/run_dssm_negsampling.py | 2 +- 10 files changed, 23 insertions(+), 71 deletions(-) diff --git a/deepmatch/layers/__init__.py b/deepmatch/layers/__init__.py index c36b462..8978804 100644 --- a/deepmatch/layers/__init__.py +++ b/deepmatch/layers/__init__.py @@ -1,16 +1,15 @@ from deepctr.layers import custom_objects from deepctr.layers.utils import reduce_sum -from .core import PoolingLayer, SampledSoftmaxLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayerv2,EmbeddingIndex +from .core import PoolingLayer, Similarity, LabelAwareAttention, CapsuleLayer,SampledSoftmaxLayer,EmbeddingIndex from ..utils import sampledsoftmaxloss _custom_objects = {'PoolingLayer': PoolingLayer, - 'SampledSoftmaxLayer': SampledSoftmaxLayer, 'Similarity': Similarity, 'LabelAwareAttention': LabelAwareAttention, 'CapsuleLayer': CapsuleLayer, 'reduce_sum':reduce_sum, - 'SampledSoftmaxLayerv2':SampledSoftmaxLayerv2, + 'SampledSoftmaxLayer':SampledSoftmaxLayer, 'sampledsoftmaxloss':sampledsoftmaxloss, 'EmbeddingIndex':EmbeddingIndex } diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index bf028b3..a420a85 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -44,55 +44,11 @@ def get_config(self, ): class SampledSoftmaxLayer(Layer): - def __init__(self, item_embedding, num_sampled=5, **kwargs): - self.num_sampled = num_sampled - self.target_song_size = item_embedding.input_dim - self.item_embedding = item_embedding - super(SampledSoftmaxLayer, self).__init__(**kwargs) - - def build(self, input_shape): - self.zero_bias = self.add_weight(shape=[self.target_song_size], - initializer=Zeros, - dtype=tf.float32, - trainable=False, - name="bias") - if not self.item_embedding.built: - self.item_embedding.build([]) - self.trainable_weights.append(self.item_embedding.embeddings) - super(SampledSoftmaxLayer, self).build(input_shape) - - def call(self, inputs_with_label_idx, training=None, **kwargs): - """ - The first input should be the model as it were, and the second the - target (i.e., a repeat of the training data) to compute the labels - argument - """ - inputs, label_idx = inputs_with_label_idx - - loss = tf.nn.sampled_softmax_loss(weights=self.item_embedding.embeddings, - biases=self.zero_bias, - labels=label_idx, - inputs=inputs, - num_sampled=self.num_sampled, - num_classes=self.target_song_size - ) - return tf.expand_dims(loss, axis=1) - - def compute_output_shape(self, input_shape): - return (None, 1) - - def get_config(self, ): - config = {'item_embedding': self.item_embedding, 'num_sampled': self.num_sampled} - base_config = super(SampledSoftmaxLayer, self).get_config() - return dict(list(base_config.items()) + list(config.items())) - - -class SampledSoftmaxLayerv2(Layer): def __init__(self, num_sampled=5, **kwargs): self.num_sampled = num_sampled # self.target_song_size = item_embedding.input_dim # self.item_embedding = item_embedding - super(SampledSoftmaxLayerv2, self).__init__(**kwargs) + super(SampledSoftmaxLayer, self).__init__(**kwargs) def build(self, input_shape): self.size = input_shape[0][0] @@ -104,7 +60,7 @@ def build(self, input_shape): # if not self.item_embedding.built: # self.item_embedding.build([]) # self.trainable_weights.append(self.item_embedding.embeddings) - super(SampledSoftmaxLayerv2, self).build(input_shape) + super(SampledSoftmaxLayer, self).build(input_shape) def call(self, inputs_with_label_idx, training=None, **kwargs): """ @@ -128,7 +84,7 @@ def compute_output_shape(self, input_shape): def get_config(self, ): config = {'num_sampled': self.num_sampled} - base_config = super(SampledSoftmaxLayerv2, self).get_config() + base_config = super(SampledSoftmaxLayer, self).get_config() return dict(list(base_config.items()) + list(config.items())) diff --git a/deepmatch/models/dssm.py b/deepmatch/models/dssm.py index 496d2b4..74ac3b1 100644 --- a/deepmatch/models/dssm.py +++ b/deepmatch/models/dssm.py @@ -13,8 +13,8 @@ from ..layers.core import Similarity -def DSSM(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(64, 16), - item_dnn_hidden_units=(64, 16), +def DSSM(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(64, 32), + item_dnn_hidden_units=(64, 32), dnn_activation='tanh', dnn_use_bn=False, l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, init_std=0.0001, seed=1024, metric='cos'): """Instantiates the Deep Structured Semantic Model architecture. diff --git a/deepmatch/models/mind.py b/deepmatch/models/mind.py index f232b12..4a8afb2 100755 --- a/deepmatch/models/mind.py +++ b/deepmatch/models/mind.py @@ -12,12 +12,12 @@ combined_dnn_input from deepctr.layers.core import DNN from deepctr.layers.utils import NoMask -from tensorflow.python.keras.layers import Concatenate,Input,Lambda +from tensorflow.python.keras.layers import Concatenate from tensorflow.python.keras.models import Model -from deepmatch.utils import get_item_embedding,get_item_embeddingv2 +from deepmatch.utils import get_item_embedding from ..inputs import create_embedding_matrix -from ..layers.core import CapsuleLayer, SampledSoftmaxLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayerv2,EmbeddingIndex +from ..layers.core import CapsuleLayer, PoolingLayer, LabelAwareAttention,SampledSoftmaxLayer,EmbeddingIndex def shape_target(target_emb_tmp, target_emb_size): @@ -143,7 +143,7 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 else: user_embedding_final = LabelAwareAttention(k_max=k_max, pow_p=p, )((user_embeddings, target_emb)) - output = SampledSoftmaxLayerv2( num_sampled=num_sampled)( + output = SampledSoftmaxLayer(num_sampled=num_sampled)( inputs=(pooling_item_embedding_weight,user_embedding_final, item_features[item_feature_name])) model = Model(inputs=inputs_list + item_inputs_list, outputs=output) @@ -151,6 +151,6 @@ def MIND(user_feature_columns, item_feature_columns, num_sampled=5, k_max=2, p=1 model.__setattr__("user_embedding", user_embeddings) model.__setattr__("item_input", item_inputs_list) - model.__setattr__("item_embedding", get_item_embeddingv2(pooling_item_embedding_weight, item_features[item_feature_name])) + model.__setattr__("item_embedding", get_item_embedding(pooling_item_embedding_weight, item_features[item_feature_name])) return model diff --git a/deepmatch/models/ncf.py b/deepmatch/models/ncf.py index 3484278..2de4c5f 100644 --- a/deepmatch/models/ncf.py +++ b/deepmatch/models/ncf.py @@ -16,7 +16,7 @@ def NCF(user_feature_columns, item_feature_columns, user_gmf_embedding_dim=20, item_gmf_embedding_dim=20, user_mlp_embedding_dim=20, item_mlp_embedding_dim=20, dnn_use_bn=False, - dnn_hidden_units=(64, 16), dnn_activation='relu', l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, + dnn_hidden_units=(64, 32), dnn_activation='relu', l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, init_std=0.0001, seed=1024): """Instantiates the NCF Model architecture. diff --git a/deepmatch/models/youtubednn.py b/deepmatch/models/youtubednn.py index 5e5bb9b..a4ada3c 100644 --- a/deepmatch/models/youtubednn.py +++ b/deepmatch/models/youtubednn.py @@ -9,14 +9,14 @@ from deepctr.layers.utils import NoMask from tensorflow.python.keras.models import Model -from deepmatch.utils import get_item_embedding, get_item_embeddingv2 +from deepmatch.utils import get_item_embedding from deepmatch.layers import PoolingLayer from ..inputs import input_from_feature_columns -from ..layers.core import SampledSoftmaxLayer, SampledSoftmaxLayerv2,EmbeddingIndex +from ..layers.core import SampledSoftmaxLayer,EmbeddingIndex def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, - user_dnn_hidden_units=(64, 16), + user_dnn_hidden_units=(64, 32), dnn_activation='relu', dnn_use_bn=False, l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, init_std=0.0001, seed=1024, ): """Instantiates the YoutubeDNN Model architecture. @@ -66,7 +66,7 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, pooling_item_embedding_weight = PoolingLayer()([item_embedding_weight]) - output = SampledSoftmaxLayerv2(num_sampled=num_sampled)( + output = SampledSoftmaxLayer(num_sampled=num_sampled)( inputs=(pooling_item_embedding_weight, user_dnn_out, item_features[item_feature_name])) model = Model(inputs=user_inputs_list + item_inputs_list , outputs=output) @@ -74,6 +74,6 @@ def YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=5, model.__setattr__("user_embedding", user_dnn_out) model.__setattr__("item_input", item_inputs_list) - model.__setattr__("item_embedding", get_item_embeddingv2(pooling_item_embedding_weight, item_features[item_feature_name])) + model.__setattr__("item_embedding", get_item_embedding(pooling_item_embedding_weight, item_features[item_feature_name])) return model diff --git a/deepmatch/utils.py b/deepmatch/utils.py index 552c51d..cbbe126 100644 --- a/deepmatch/utils.py +++ b/deepmatch/utils.py @@ -29,12 +29,7 @@ def recall_N(y_true, y_pred, N=50): def sampledsoftmaxloss(y_true, y_pred): return K.mean(y_pred) - -def get_item_embedding(item_embedding_layer, item_input_layer): - return Lambda(lambda x: tf.squeeze(tf.gather(item_embedding_layer.embeddings, x), axis=1))( - item_input_layer) - -def get_item_embeddingv2(item_embedding, item_input_layer): +def get_item_embedding(item_embedding, item_input_layer): return Lambda(lambda x: tf.squeeze(tf.gather(item_embedding, x), axis=1))( item_input_layer) diff --git a/docs/source/History.md b/docs/source/History.md index 318d26d..a4279aa 100644 --- a/docs/source/History.md +++ b/docs/source/History.md @@ -1,3 +1,3 @@ # History -- 04/10/2020 : DeepMatch first version is released on [PyPi](https://pypi.org/project/deepmatch/) +- 04/10/2020 : [v0.1.2](https://github.com/shenweichen/DeepMatch/releases/tag/v0.1.2) released.Support [saving and loading model](./FAQ.html#save-or-load-weights-models). - 04/06/2020 : DeepMatch first version is released on [PyPi](https://pypi.org/project/deepmatch/) \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index dac7a89..8d151cf 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,6 +18,8 @@ You can read the latest code at https://github.com/shenweichen/DeepMatch News ----- +04/10/2020 : Support `saving and loading model <./FAQ.html#save-or-load-weights-models>`_ . `Changelog `_ + 04/06/2020 : DeepMatch first version . DisscussionGroup diff --git a/examples/run_dssm_negsampling.py b/examples/run_dssm_negsampling.py index 52c372f..e40a0c1 100644 --- a/examples/run_dssm_negsampling.py +++ b/examples/run_dssm_negsampling.py @@ -38,7 +38,7 @@ # 2.count #unique features for each sparse field and generate feature config for sequence feature - embedding_dim = 16 + embedding_dim = 8 user_feature_columns = [SparseFeat('user_id', feature_max_idx['user_id'], embedding_dim), SparseFeat("gender", feature_max_idx['gender'], embedding_dim), From 1897dbefe66deb49e631759420accab14ef3bd09 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 19:25:58 +0800 Subject: [PATCH 26/31] no message --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9f4850d..d0b822b 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ version="0.1.2", author="Weichen Shen", author_email="wcshen1994@163.com", - description="Deep matching model library for recommendations, advertising, and search. It's easy to train models and to **export representation vectors** for user and item which can be used for **ANN search**.", + description="Deep matching model library for recommendations, advertising. It's easy to train models and to **export representation vectors** for user and item which can be used for **ANN search**.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/shenweichen/deepmatch", From 210f950e5e74d363cd201ba23dec95ed3ef53ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B5=85=E6=A2=A6?= Date: Fri, 10 Apr 2020 19:26:37 +0800 Subject: [PATCH 27/31] =?UTF-8?q?=E4=BD=BF=E7=94=A8=20Colaboratory=20?= =?UTF-8?q?=E5=88=9B=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/colab_MovieLen1M_YoutubeDNN.ipynb | 551 +++++++++++++++++++++ 1 file changed, 551 insertions(+) create mode 100644 examples/colab_MovieLen1M_YoutubeDNN.ipynb diff --git a/examples/colab_MovieLen1M_YoutubeDNN.ipynb b/examples/colab_MovieLen1M_YoutubeDNN.ipynb new file mode 100644 index 0000000..d2514f0 --- /dev/null +++ b/examples/colab_MovieLen1M_YoutubeDNN.ipynb @@ -0,0 +1,551 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "colab_MovieLen1M_YoutubeDNN", + "provenance": [], + "collapsed_sections": [], + "toc_visible": true, + "authorship_tag": "ABX9TyOrqZNeC0DgyPX2JmJid1m7", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rtox72csOQUN", + "colab_type": "text" + }, + "source": [ + "# DeepMatch 样例代码\n", + "- https://github.com/shenweichen/DeepMatch\n", + "- https://deepmatch.readthedocs.io/en/latest/" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bTWHz-heMkyw", + "colab_type": "text" + }, + "source": [ + "# 下载movielens-1M数据 安装依赖包" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "yTl6d6jO1oqf", + "colab_type": "code", + "outputId": "19f1902e-17a6-4a07-ab42-f012bd286eda", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 510 + } + }, + "source": [ + "! wget http://files.grouplens.org/datasets/movielens/ml-1m.zip -O ./ml-1m.zip \n", + "! wget https://raw.githubusercontent.com/shenweichen/DeepMatch/master/examples/preprocess.py -O preprocess.py\n", + "! unzip -o ml-1m.zip \n", + "%tensorflow_version 1.x\n", + "! pip uninstall -y -q tensorflow\n", + "! pip install -q tensorflow-gpu==1.14.0\n", + "! pip install -q deepmatch==0.1.2" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "--2020-04-10 09:06:19-- http://files.grouplens.org/datasets/movielens/ml-1m.zip\n", + "Resolving files.grouplens.org (files.grouplens.org)... 128.101.65.152\n", + "Connecting to files.grouplens.org (files.grouplens.org)|128.101.65.152|:80... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 5917549 (5.6M) [application/zip]\n", + "Saving to: ‘./ml-1m.zip’\n", + "\n", + "\r./ml-1m.zip 0%[ ] 0 --.-KB/s \r./ml-1m.zip 7%[> ] 443.20K 1.85MB/s \r./ml-1m.zip 100%[===================>] 5.64M 14.4MB/s in 0.4s \n", + "\n", + "2020-04-10 09:06:19 (14.4 MB/s) - ‘./ml-1m.zip’ saved [5917549/5917549]\n", + "\n", + "--2020-04-10 09:06:21-- https://raw.githubusercontent.com/shenweichen/DeepMatch/master/examples/preprocess.py\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 2049 (2.0K) [text/plain]\n", + "Saving to: ‘preprocess.py’\n", + "\n", + "preprocess.py 100%[===================>] 2.00K --.-KB/s in 0s \n", + "\n", + "2020-04-10 09:06:21 (34.8 MB/s) - ‘preprocess.py’ saved [2049/2049]\n", + "\n", + "Archive: ml-1m.zip\n", + " inflating: ml-1m/movies.dat \n", + " inflating: ml-1m/ratings.dat \n", + " inflating: ml-1m/README \n", + " inflating: ml-1m/users.dat \n", + "TensorFlow 1.x selected.\n", + "\u001b[33mWARNING: Skipping tensorflow as it is not installed.\u001b[0m\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p9UxNHuPMuW2", + "colab_type": "text" + }, + "source": [ + "# 导入需要的库" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "C_ZR6gzp1E2N", + "colab_type": "code", + "outputId": "8401132a-5090-464a-879d-a27492416f4c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 496 + } + }, + "source": [ + "import pandas as pd\n", + "from deepctr.inputs import SparseFeat, VarLenSparseFeat\n", + "from preprocess import gen_data_set, gen_model_input\n", + "from sklearn.preprocessing import LabelEncoder\n", + "from tensorflow.python.keras import backend as K\n", + "from tensorflow.python.keras.models import Model\n", + "\n", + "from deepmatch.models import *\n", + "from deepmatch.utils import sampledsoftmaxloss\n" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", + "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", + " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:143: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Call initializer instance with the dtype argument instead of passing it to the constructor\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fQq6O9XAMzPF", + "colab_type": "text" + }, + "source": [ + "# 读取数据" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "lcO29zFb21Od", + "colab_type": "code", + "outputId": "97d9023e-cbd5-43dd-d2ef-769e3a34cf2c", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 139 + } + }, + "source": [ + "data_path = \"./\"\n", + "\n", + "unames = ['user_id','gender','age','occupation','zip']\n", + "user = pd.read_csv(data_path+'ml-1m/users.dat',sep='::',header=None,names=unames)\n", + "rnames = ['user_id','movie_id','rating','timestamp']\n", + "ratings = pd.read_csv(data_path+'ml-1m/ratings.dat',sep='::',header=None,names=rnames)\n", + "mnames = ['movie_id','title','genres']\n", + "movies = pd.read_csv(data_path+'ml-1m/movies.dat',sep='::',header=None,names=mnames)\n", + "\n", + "data = pd.merge(pd.merge(ratings,movies),user)#.iloc[:10000]\n" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n", + " after removing the cwd from sys.path.\n", + "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n", + " \n", + "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n", + " \n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L0yCWxQxM3se", + "colab_type": "text" + }, + "source": [ + "# 构建特征列，训练模型，导出embedding" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "BMOvk_de2ML3", + "colab_type": "code", + "outputId": "1e43a5e7-f71c-45a4-bab4-e1d6b1a73669", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + } + }, + "source": [ + "#data = pd.read_csvdata = pd.read_csv(\"./movielens_sample.txt\")\n", + "sparse_features = [\"movie_id\", \"user_id\",\n", + " \"gender\", \"age\", \"occupation\", \"zip\", ]\n", + "SEQ_LEN = 50\n", + "negsample = 0\n", + "\n", + "# 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input`\n", + "\n", + "features = ['user_id', 'movie_id', 'gender', 'age', 'occupation', 'zip']\n", + "feature_max_idx = {}\n", + "for feature in features:\n", + " lbe = LabelEncoder()\n", + " data[feature] = lbe.fit_transform(data[feature]) + 1\n", + " feature_max_idx[feature] = data[feature].max() + 1\n", + "\n", + "user_profile = data[[\"user_id\", \"gender\", \"age\", \"occupation\", \"zip\"]].drop_duplicates('user_id')\n", + "\n", + "item_profile = data[[\"movie_id\"]].drop_duplicates('movie_id')\n", + "\n", + "user_profile.set_index(\"user_id\", inplace=True)\n", + "\n", + "user_item_list = data.groupby(\"user_id\")['movie_id'].apply(list)\n", + "\n", + "train_set, test_set = gen_data_set(data, negsample)\n", + "\n", + "train_model_input, train_label = gen_model_input(train_set, user_profile, SEQ_LEN)\n", + "test_model_input, test_label = gen_model_input(test_set, user_profile, SEQ_LEN)\n", + "\n", + "# 2.count #unique features for each sparse field and generate feature config for sequence feature\n", + "\n", + "embedding_dim = 32\n", + "\n", + "user_feature_columns = [SparseFeat('user_id', feature_max_idx['user_id'], 16),\n", + " SparseFeat(\"gender\", feature_max_idx['gender'], 16),\n", + " SparseFeat(\"age\", feature_max_idx['age'], 16),\n", + " SparseFeat(\"occupation\", feature_max_idx['occupation'], 16),\n", + " SparseFeat(\"zip\", feature_max_idx['zip'], 16),\n", + " VarLenSparseFeat(SparseFeat('hist_movie_id', feature_max_idx['movie_id'], embedding_dim,\n", + " embedding_name=\"movie_id\"), SEQ_LEN, 'mean', 'hist_len'),\n", + " ]\n", + "\n", + "item_feature_columns = [SparseFeat('movie_id', feature_max_idx['movie_id'], embedding_dim)]\n", + "\n", + "# 3.Define Model and train\n", + "\n", + "K.set_learning_phase(True)\n", + "\n", + "model = YoutubeDNN(user_feature_columns, item_feature_columns, num_sampled=200, user_dnn_hidden_units=(128,64, embedding_dim))\n", + "# model = MIND(user_feature_columns,item_feature_columns,dynamic_k=True,p=1,k_max=2,num_sampled=5,user_dnn_hidden_units=(64,16),init_std=0.001)\n", + "\n", + "model.compile(optimizer=\"adam\", loss=sampledsoftmaxloss) # \"binary_crossentropy\")\n", + "\n", + "history = model.fit(train_model_input, train_label, # train_label,\n", + " batch_size=512, epochs=24, verbose=1, validation_split=0.0, )\n", + "\n", + "# 4. Generate user features for testing and full item features for retrieval\n", + "test_user_model_input = test_model_input\n", + "all_item_model_input = {\"movie_id\": item_profile['movie_id'].values,}\n", + "\n", + "user_embedding_model = Model(inputs=model.user_input, outputs=model.user_embedding)\n", + "item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding)\n", + "\n", + "user_embs = user_embedding_model.predict(test_user_model_input, batch_size=2 ** 12)\n", + "# user_embs = user_embs[:, i, :] i in [0,k_max) if MIND\n", + "item_embs = item_embedding_model.predict(all_item_model_input, batch_size=2 ** 12)\n", + "\n", + "print(user_embs.shape)\n", + "print(item_embs.shape)\n", + "\n" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "100%|██████████| 6040/6040 [00:12<00:00, 489.02it/s]\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "6 6\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", + "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/deepctr/layers/utils.py:167: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "keep_dims is deprecated, use keepdims instead\n", + "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/deepctr/layers/utils.py:193: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Deprecated in favor of operator or tf.math.divide.\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1288: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Call initializer instance with the dtype argument instead of passing it to the constructor\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AttributeError: module 'gast' has no attribute 'Num'\n", + "WARNING:tensorflow:Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "WARNING: Entity > could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting >: AssertionError: Bad argument number for Name: 3, expecting 4\n", + "Epoch 1/24\n", + "988129/988129 [==============================] - 22s 23us/sample - loss: 4.0715\n", + "Epoch 2/24\n", + "988129/988129 [==============================] - 21s 21us/sample - loss: 3.7855\n", + "Epoch 3/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 3.5596\n", + "Epoch 4/24\n", + "988129/988129 [==============================] - 21s 21us/sample - loss: 3.4199\n", + "Epoch 5/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 3.3216\n", + "Epoch 6/24\n", + "988129/988129 [==============================] - 21s 21us/sample - loss: 3.2499\n", + "Epoch 7/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 3.1846\n", + "Epoch 8/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 3.1264\n", + "Epoch 9/24\n", + "988129/988129 [==============================] - 22s 22us/sample - loss: 3.0923\n", + "Epoch 10/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 3.0566\n", + "Epoch 11/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 3.0336\n", + "Epoch 12/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 3.0066\n", + "Epoch 13/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9872\n", + "Epoch 14/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9659\n", + "Epoch 15/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9476\n", + "Epoch 16/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9363\n", + "Epoch 17/24\n", + "988129/988129 [==============================] - 21s 21us/sample - loss: 2.9267\n", + "Epoch 18/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9179\n", + "Epoch 19/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.9012\n", + "Epoch 20/24\n", + "988129/988129 [==============================] - 21s 21us/sample - loss: 2.8925\n", + "Epoch 21/24\n", + "988129/988129 [==============================] - 21s 21us/sample - loss: 2.8830\n", + "Epoch 22/24\n", + "988129/988129 [==============================] - 22s 22us/sample - loss: 2.8797\n", + "Epoch 23/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.8711\n", + "Epoch 24/24\n", + "988129/988129 [==============================] - 21s 22us/sample - loss: 2.8619\n", + "(6040, 32)\n", + "(3706, 32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "w_G3KWslKmJo", + "colab_type": "text" + }, + "source": [ + "# 使用faiss进行ANN查找并评估结果" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5SvyQLNVKkcs", + "colab_type": "text" + }, + "source": [ + "" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "j2ZNYNBOOqrN", + "colab_type": "code", + "outputId": "2938673c-ff81-49a2-86d8-4266d2060ea3", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 51 + } + }, + "source": [ + "! pip install faiss-cpu" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.6/dist-packages (1.6.3)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from faiss-cpu) (1.18.2)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6TY1l27iJU8U", + "colab_type": "code", + "outputId": "5316c37f-fef1-44b3-8c31-6600d1e44da5", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 85 + } + }, + "source": [ + "\n", + "\n", + "test_true_label = {line[0]:[line[2]] for line in test_set}\n", + "\n", + "import numpy as np\n", + "import faiss\n", + "from tqdm import tqdm\n", + "from deepmatch.utils import recall_N\n", + "\n", + "index = faiss.IndexFlatIP(embedding_dim)\n", + "# faiss.normalize_L2(item_embs)\n", + "index.add(item_embs)\n", + "# faiss.normalize_L2(user_embs)\n", + "D, I = index.search(user_embs, 50)\n", + "s = []\n", + "hit = 0\n", + "for i, uid in tqdm(enumerate(test_user_model_input['user_id'])):\n", + " try:\n", + " pred = [item_profile['movie_id'].values[x] for x in I[i]]\n", + " filter_item = None\n", + " recall_score = recall_N(test_true_label[uid], pred, N=50)\n", + " s.append(recall_score)\n", + " if test_true_label[uid] in pred:\n", + " hit += 1\n", + " except:\n", + " print(i)\n", + "print(\"\")\n", + "print(\"recall\", np.mean(s))\n", + "print(\"hit rate\", hit / len(test_user_model_input['user_id']))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "6040it [00:01, 4290.79it/s]" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "\n", + "recall 0.26473509933774836\n", + "hit rate 0.26473509933774836\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "a97TB0obOrRe", + "colab_type": "code", + "colab": {} + }, + "source": [ + "" + ], + "execution_count": 0, + "outputs": [] + } + ] +} \ No newline at end of file From 3d5e04c80d54c8030f46cace820b8d14f059f19a Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 19:43:53 +0800 Subject: [PATCH 28/31] no message --- README.md | 3 ++- docs/source/Examples.md | 4 ++++ docs/source/index.rst | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2036b2a..b574fea 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ DeepMatch is a deep matching model library for recommendations & advertising. It's easy to **train models** and to **export representation vectors** for user and item which can be used for **ANN search**.You can use any complex model with `model.fit()`and `model.predict()` . -Let's [**Get Started!**](https://deepmatch.readthedocs.io/en/latest/Quick-Start.html) +Let's [**Get Started!**](https://deepmatch.readthedocs.io/en/latest/Quick-Start.html) or [**Run examples**](./examples/colab_MovieLen1M_YoutubeDNN.ipynb) ! + ## Models List diff --git a/docs/source/Examples.md b/docs/source/Examples.md index 90577c1..113cc54 100644 --- a/docs/source/Examples.md +++ b/docs/source/Examples.md @@ -1,6 +1,10 @@ # Examples +## Run YoutubeDNN on MovieLen1M on Google colab + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg +)](https://colab.research.google.com/github/shenweichen/DeepMatch/blob/dev_shenweichen/examples/colab_MovieLen1M_YoutubeDNN.ipynb) ## YoutubeDNN with sampled softmax diff --git a/docs/source/index.rst b/docs/source/index.rst index 8d151cf..ae002e0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,7 +11,7 @@ Welcome to DeepMatch's documentation! DeepMatch is a deep matching model library for recommendations, advertising, and search. It's easy to **train models** and to **export representation vectors** for user and item which can be used for **ANN search**.You can use any complex model with ``model.fit()`` and ``model.predict()`` . -Let's `Get Started! <./Quick-Start.html>`_ +Let's `Get Started! <./Quick-Start.html>`_ or `Run examples! ` _ You can read the latest code at https://github.com/shenweichen/DeepMatch From 23b0c1bfb0f33a1aecd4a691595ba467ddb22236 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 19:45:53 +0800 Subject: [PATCH 29/31] no message --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index ae002e0..bb0f6ac 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,7 +11,7 @@ Welcome to DeepMatch's documentation! DeepMatch is a deep matching model library for recommendations, advertising, and search. It's easy to **train models** and to **export representation vectors** for user and item which can be used for **ANN search**.You can use any complex model with ``model.fit()`` and ``model.predict()`` . -Let's `Get Started! <./Quick-Start.html>`_ or `Run examples! ` _ +Let's `Get Started! <./Quick-Start.html>`_ or `Run examples! `_ You can read the latest code at https://github.com/shenweichen/DeepMatch From 836782fb4bd501d27c493caa5c273fee50327b87 Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 19:49:30 +0800 Subject: [PATCH 30/31] no message --- docs/source/Quick-Start.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/Quick-Start.md b/docs/source/Quick-Start.md index 6ecb6e0..f370263 100644 --- a/docs/source/Quick-Start.md +++ b/docs/source/Quick-Start.md @@ -16,5 +16,6 @@ $ pip install deepmatch[gpu] ``` ## Run examples !! +- [Run YoutubeDNN on MovieLen1M on Google colab](https://colab.research.google.com/github/shenweichen/DeepMatch/blob/dev_shenweichen/examples/colab_MovieLen1M_YoutubeDNN.ipynb) - [DSSM with negative sampling](./Examples.html#dssm-with-negative-sampling) - [YoutubeDNN with sampled softmax](./Examples.html#youtubednn-with-sampled-softmax) \ No newline at end of file From c36d2c0a4927daa65d5d695be334acabfcc1156e Mon Sep 17 00:00:00 2001 From: shenweichen Date: Fri, 10 Apr 2020 19:56:53 +0800 Subject: [PATCH 31/31] no message --- deepmatch/layers/core.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/deepmatch/layers/core.py b/deepmatch/layers/core.py index a420a85..293b885 100644 --- a/deepmatch/layers/core.py +++ b/deepmatch/layers/core.py @@ -26,7 +26,6 @@ def call(self, seq_value_len_list, mask=None, **kwargs): seq_value_len_list = [seq_value_len_list] if len(seq_value_len_list) == 1: return seq_value_len_list[0] - # seq_value_len_list[1] = tf.squeeze(seq_value_len_list[1],axis=0) expand_seq_value_len_list = list(map(lambda x: tf.expand_dims(x, axis=-1), seq_value_len_list)) a = concat_func(expand_seq_value_len_list) if self.mode == "mean": @@ -46,8 +45,6 @@ def get_config(self, ): class SampledSoftmaxLayer(Layer): def __init__(self, num_sampled=5, **kwargs): self.num_sampled = num_sampled - # self.target_song_size = item_embedding.input_dim - # self.item_embedding = item_embedding super(SampledSoftmaxLayer, self).__init__(**kwargs) def build(self, input_shape): @@ -57,9 +54,6 @@ def build(self, input_shape): dtype=tf.float32, trainable=False, name="bias") - # if not self.item_embedding.built: - # self.item_embedding.build([]) - # self.trainable_weights.append(self.item_embedding.embeddings) super(SampledSoftmaxLayer, self).build(input_shape) def call(self, inputs_with_label_idx, training=None, **kwargs):