diff --git a/Dockerfile-ModelBuilder b/Dockerfile-ModelBuilder
index e34f1cc08..fe3a9a7a0 100644
--- a/Dockerfile-ModelBuilder
+++ b/Dockerfile-ModelBuilder
@@ -6,8 +6,9 @@ COPY . /code
 COPY .git /code/
 
 WORKDIR /code
-RUN python setup.py sdist && \
-    mv /code/dist/$(ls /code/dist | head -1) /code/dist/gordo-components-packed.tar.gz
+RUN rm -rf /code/dist \
+    && python setup.py sdist \
+    && mv /code/dist/$(ls /code/dist | head -1) /code/dist/gordo-components-packed.tar.gz
 
 FROM python:3.6.6-slim-stretch
 
diff --git a/Dockerfile-ModelServer b/Dockerfile-ModelServer
index 7a7000c6b..d602ab28d 100644
--- a/Dockerfile-ModelServer
+++ b/Dockerfile-ModelServer
@@ -6,10 +6,11 @@ COPY . /code
 COPY .git /code/
 
 WORKDIR /code
-RUN python setup.py sdist && \
-    mv /code/dist/$(ls /code/dist | head -1) /code/dist/gordo-components-packed.tar.gz
+RUN rm -rf /code/dist \
+    && python setup.py sdist \
+    && mv /code/dist/$(ls /code/dist | head -1) /code/dist/gordo-components-packed.tar.gz
 
-FROM seldonio/seldon-core-s2i-python3
+FROM seldonio/seldon-core-s2i-python3:0.3
 
 # Install requirements separately for improved docker caching
 COPY requirements.txt /code/
diff --git a/Makefile b/Makefile
index bb4d25450..d2b0220c4 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,7 @@ push-builder: model-builder
 push-dev-images: push-builder push-server
 
 push-prod-images: export GORDO_PROD_MODE:="true"
-push-prod-images:	push-builder push-server
+push-prod-images: push-builder push-server
 
 # Make the python source distribution
 sdist:
diff --git a/gordo_components/model/models.py b/gordo_components/model/models.py
index 0ed8edf81..6befe2465 100644
--- a/gordo_components/model/models.py
+++ b/gordo_components/model/models.py
@@ -4,7 +4,9 @@
 import json
 from typing import Union, Callable, Dict, Any
 from os import path
+from contextlib import contextmanager
 
+import keras.backend as K
 from keras.wrappers.scikit_learn import KerasRegressor
 from keras.models import Model as KerasBaseModel
 from keras.models import load_model
@@ -19,6 +21,22 @@
 logger = logging.getLogger(__name__)
 
 
+@contextmanager
+def possible_tf_mgmt(keras_model):
+    """
+    When serving a Keras model backed by tensorflow, the object is expected
+    to have `_tf_graph` and `_tf_session` stored attrs. Which will be used
+    as the default when calling the Keras model
+    """
+    logger.info(f'Keras backend: {K.backend()}')
+    if K.backend() == 'tensorflow':
+        logger.debug(f'Using keras_model {keras_model} local TF Graph and Session')
+        with keras_model._tf_graph.as_default(), keras_model._tf_session.as_default():
+            yield
+    else:
+        yield
+
+
 class KerasModel(KerasRegressor, GordoBaseModel):
 
     def __init__(self,
@@ -54,6 +72,17 @@ def this_function_returns_a_special_keras_model(n_features, extra_param1, extra_
                         building function and/or any additional args to be passed
                         to Keras' fit() method
         """
+        # Tensorflow requires managed graph/session as to not default to global
+        if K.backend() == 'tensorflow':
+            logger.info(f'Keras backend detected as tensorflow, keeping local graph')
+            import tensorflow as tf
+            self._tf_graph = tf.Graph()
+            self._tf_session = tf.Session(graph=self._tf_graph)
+        else:
+            logger.info(f'Keras backend detected as NOT tensorflow, but: {K.backend()}')
+            self._tf_session = None
+            self._tf_graph = None
+
         self.build_fn = None
         self.kwargs = kwargs
 
@@ -78,19 +107,28 @@ def get_params(self, **params):
 
     def __call__(self):
         build_fn = register_model_builder.factories[self.__class__.__name__][self.kind]
-        return build_fn(**self.sk_params)
+        with possible_tf_mgmt(self):
+            return build_fn(**self.sk_params)
 
     def fit(self, X, y, sample_weight=None, **kwargs):
+        logger.debug(f'Fitting to data of length: {len(X)}')
         self.kwargs.update({'n_features': X.shape[1]})
-        super().fit(X, y, sample_weight=None, **kwargs)
+        with possible_tf_mgmt(self):
+            super().fit(X, y, sample_weight=None, **kwargs)
         return self
 
+    def predict(self, x, **kwargs):
+        logger.debug(f'Predicting data of length: {len(x)}')
+        with possible_tf_mgmt(self):
+            return super().predict(x, **kwargs)
+
     def save_to_dir(self, directory: str):
         params = self.get_params()
         with open(path.join(directory, 'params.json'), 'w') as f:
             json.dump(params, f)
         if self.model is not None:
-            self.model.save(path.join(directory, 'model.h5'))
+            with possible_tf_mgmt(self):
+                self.model.save(path.join(directory, 'model.h5'))
 
     @classmethod
     def load_from_dir(cls, directory: str):
@@ -99,7 +137,9 @@ def load_from_dir(cls, directory: str):
         obj = cls(**params)
         model_file = path.join(directory, 'model.h5')
         if path.isfile(model_file):
-            obj.model = load_model(model_file)
+            with possible_tf_mgmt(obj):
+                K.set_learning_phase(0)
+                obj.model = load_model(model_file)
         return obj
 
 
diff --git a/gordo_components/runtime/SeldonModel.py b/gordo_components/runtime/SeldonModel.py
index 5bbf8e31a..34273d2ac 100644
--- a/gordo_components/runtime/SeldonModel.py
+++ b/gordo_components/runtime/SeldonModel.py
@@ -2,7 +2,6 @@
 
 import os
 import logging
-
 from gordo_components import serializer
 
 logger = logging.getLogger(__name__)
@@ -27,6 +26,7 @@ def __init__(self):
                 f'The supplied directory: "{model_location}" does not exist!')
 
         logger.info(f'Loading up serialized model from dir: {model_location}')
+
         self.model = serializer.load(model_location)
         logger.info(f'Model loaded successfully, ready to serve predictions!')