Merge pull request #556 from guillaumekln/fix-r1-for-tf1.15

Make r1 branch compatible with TensorFlow 1.15

.travis.yml

@@ -3,13 +3,11 @@ python:
   - "2.7"
 env:
   global:
-    - LATEST_TF_VERSION="1.13.*"
+    - LATEST_TF_VERSION="1.15.*"
   matrix:
-    - TF_VERSION="1.8.*"
-    - TF_VERSION="1.9.*"
-    - TF_VERSION="1.10.*"
-    - TF_VERSION="1.11.*"
     - TF_VERSION="1.12.*"
+    - TF_VERSION="1.13.*"
+    - TF_VERSION="1.14.*"
     - TF_VERSION="$LATEST_TF_VERSION"
 before_install:
   - pip install tensorflow==$TF_VERSION

CHANGELOG.md

@@ -16,6 +16,12 @@ OpenNMT-tf follows [semantic versioning 2.0.0](https://semver.org/). The API cov
 
 ### Fixes and improvements
 
+## [1.25.3](https://github.com/OpenNMT/OpenNMT-tf/releases/tag/v1.25.3) (2019-11-25)
+
+### Fixes and improvements
+
+* Fix compatibility with TensorFlow 1.15
+
 ## [1.25.2](https://github.com/OpenNMT/OpenNMT-tf/releases/tag/v1.25.2) (2019-10-22)
 
 ### Fixes and improvements

docs/conf.py

@@ -12,7 +12,7 @@
 language = "en"
 
 version = "1.25"  # The short X.Y version.
-release = "1.25.2"  # The full version, including alpha/beta/rc tags.
+release = "1.25.3"  # The full version, including alpha/beta/rc tags.
 
 source_suffix = ".rst"
 master_doc = "index"

opennmt/__init__.py

@@ -1,6 +1,6 @@
 """OpenNMT module."""
 
-__version__ = "1.25.2"
+__version__ = "1.25.3"
 
 from opennmt import decoders
 from opennmt import encoders

opennmt/estimator.py

@@ -1,10 +1,9 @@
 """Functions for Estimator API integration."""
 
-import copy
-
 import tensorflow as tf
 
 from opennmt.utils import hooks
+from opennmt.utils import misc
 from opennmt.utils import parallel
 
 
@@ -21,7 +20,7 @@ def make_serving_input_fn(model, metadata=None):
   """
 
   def _fn():
-    local_model = copy.deepcopy(model)
+    local_model = misc.clone_layer(model)
     # This is a hack for SequenceRecordInputter that currently infers the input
     # depth from the data files.
     # TODO: This function should not require the training data.
@@ -89,7 +88,7 @@ def make_input_fn(model,
     batch_size_multiple = 8
 
   def _fn():
-    local_model = copy.deepcopy(model)
+    local_model = misc.clone_layer(model)
 
     if mode == tf.estimator.ModeKeys.PREDICT:
       dataset = local_model.examples_inputter.make_inference_dataset(
@@ -160,7 +159,7 @@ def make_model_fn(model,
 
   def _fn(features, labels, params, mode, config):
     """model_fn implementation."""
-    local_model = copy.deepcopy(model)
+    local_model = misc.clone_layer(model)
 
     if mode == tf.estimator.ModeKeys.TRAIN:
       features_shards = dispatcher.shard(features)

opennmt/runner.py

@@ -493,7 +493,7 @@ def score(self, features_file, predictions_file, checkpoint_path=None, output_fi
     if checkpoint_path is None:
       raise ValueError("could not find a trained model in %s" % self._config["model_dir"])
 
-    model = copy.deepcopy(self._model)
+    model = misc.clone_layer(self._model)
     with tf.Graph().as_default():
       dataset = model.examples_inputter.make_evaluation_dataset(
           features_file,

opennmt/tokenizers/tokenizer.py

@@ -24,6 +24,11 @@ def _make_config_asset_file(config, asset_path):
   with open(asset_path, "w") as asset_file:
     yaml.dump(asset_config, stream=asset_file, default_flow_style=False)
 
+def _autograph_do_not_convert(func):
+  if not compat.tf_supports("autograph.experimental.do_not_convert"):
+    return func
+  return tf.autograph.experimental.do_not_convert(func)
+
 
 @six.add_metaclass(abc.ABCMeta)
 class Tokenizer(object):
@@ -174,6 +179,7 @@ def detokenize(self, tokens, sequence_length=None):
       tokens = [tf.compat.as_text(token) for token in tokens]
       return self._detokenize_string(tokens)
 
+  @_autograph_do_not_convert
   def _tokenize_tensor(self, text):
     """Tokenizes a tensor.
 
@@ -200,6 +206,7 @@ def _python_wrapper(string_t):
     tokens = tf.string_split([text], delimiter="\0").values
     return tokens
 
+  @_autograph_do_not_convert
   def _detokenize_tensor(self, tokens):
     """Detokenizes tokens.
 

opennmt/utils/misc.py

@@ -2,12 +2,16 @@
 
 from __future__ import print_function
 
+import copy
 import os
 import sys
 import inspect
 import heapq
+import threading
 import six
 
+from six.moves import copyreg
+
 import numpy as np
 import tensorflow as tf
 
@@ -174,6 +178,11 @@ def merge_dict(dict1, dict2):
       dict1[key] = value
   return dict1
 
+def clone_layer(layer):
+  """Clones a layer."""
+  copyreg.pickle(threading.local, lambda _: (threading.local, []))
+  return copy.deepcopy(layer)
+
 
 class OrderRestorer(object):
   """Helper class to restore out-of-order elements in order."""

pylintrc

@@ -50,7 +50,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,no-member,no-name-in-module,import-error,unsubscriptable-object,unbalanced-tuple-unpacking,undefined-variable,not-context-manager,no-else-return,useless-object-inheritance,assignment-from-none,useless-return
+disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,no-member,no-name-in-module,import-error,unsubscriptable-object,unbalanced-tuple-unpacking,undefined-variable,not-context-manager,no-else-return,useless-object-inheritance,assignment-from-none,useless-return,C
 #disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call
 
 # Enable the message, report, category or checker with the given id(s). You can

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name="OpenNMT-tf",
-    version="1.25.2",
+    version="1.25.3",
     license="MIT",
     description="Neural machine translation and sequence learning using TensorFlow",
     author="OpenNMT",