From 23bec5ef7adc3ce9141864f9e5b33f6134479e70 Mon Sep 17 00:00:00 2001
From: mjaworski <mjaworski@opera.com>
Date: Mon, 10 Apr 2017 17:05:21 +0200
Subject: [PATCH 1/4] serializers: redesigned validation and star-like writable
 fields

* remove `source="*"` handling
* move instance/representation manipulation responsibility
  to field objects to support nested objects and multiple key
  access pattern
* update docs
* fix #44 (writable star-like fields)
* fix #43 (broken resource manipulation on star-like fields)
* fix #42 (wrong field descriptions on validation errors)
* redesign resource/field validation process
---
 docs/guide/serializers.rst     | 157 ++++++++++++++++----
 src/graceful/errors.py         |   2 +-
 src/graceful/fields.py         |  69 +++++++++
 src/graceful/resources/base.py |   8 +-
 src/graceful/serializers.py    | 263 ++++++++++++---------------------
 tests/test_resources.py        |   6 +-
 tests/test_serializers.py      |  72 +--------
 7 files changed, 306 insertions(+), 271 deletions(-)

diff --git a/docs/guide/serializers.rst b/docs/guide/serializers.rst
index 722da38..9b9a3b3 100644
--- a/docs/guide/serializers.rst
+++ b/docs/guide/serializers.rst
@@ -110,10 +110,8 @@ All field classes accept this set of arguments:
   intead of relying on param labels.*
 
 * **source** *(str, optional):* name of internal object key/attribute
-  that will be passed to field's on ``.to_representation(value)`` call.
-  Special ``'*'`` value is allowed that will pass whole object to
-  field when making representation. If not set then default source will
-  be a field name used as a serializer's attribute.
+  that will be passed to field's on ``.to_representation(value)`` call. If not
+  set then default source is a field name used as a serializer's attribute.
 
 * **validators** *(list, optional):* list of validator callables.
 
@@ -121,14 +119,12 @@ All field classes accept this set of arguments:
   of given type objects
 
 
-.. note::
-
-   ``source='*'`` is in fact a dirty workaround and will not work well
-   on validation when new object instances needs to be created/updated
-   using POST/PUT requests. This works quite well with simple retrieve/list
-   type resources but in more sophisticated cases it is better to use
-   custom object properties as sources to encapsulate such fields.
+.. versionchanged:: 1.0.0
 
+   Fields no no longer have special case treatment for ``source='*'`` argument.
+   If you want to access multiple object keys and values within single
+   serializer field please refer to :ref:`guide-field-attribute-access` section
+   of this document.
 
 .. _field-validation:
 
@@ -143,9 +139,9 @@ in order to provide correct HTTP responses each validator shoud raise
 .. note::
 
    Concept of validation for fields is understood here as a process of checking
-   if data of valid type (successfully parsed/processed by
-   ``.from_representation`` handler) does meet some other constraints
-   (lenght, bounds, unique, etc).
+   if data of valid type (i.e. data that was successfully parsed/processed by
+   ``.from_representation()`` handler) does meet some other constraints
+   (lenght, bounds, uniquess, etc).
 
 
 Example of simple validator usage:
@@ -174,31 +170,53 @@ Resource validation
 ~~~~~~~~~~~~~~~~~~~
 
 In most cases field level validation is all that you need but sometimes you
-need to perfom obejct level validation that needs to access multiple fields
-that are already deserialized and validated. Suggested way to do this in
-graceful is to override serializer's ``.validate()`` method and raise
-:class:`graceful.errors.ValidationError` when your validation fails. This
-exception will be then automatically translated to HTTP Bad Request response
-on resource-level handlers. Here is example:
+need to perfom validation on whole resource representation or deserialized
+object. It is possible to access multiple fields that were already deserialized
+and pre-validated directly from serializer class.
+
+You can provide your own object-level serialization handler using serializer's
+``validate()`` method. This method accepts two arguments:
+
+* **object_dict** *(dict):* it is deserialized object dictionary that already
+  passed validation. Field sources instead of their representation names are
+  used as its keys.
+
+* **partial** *(bool):* it is set to ``True`` only on partial object updates
+  (e.g. on ``PATCH`` requests). If you plan to support partial resource
+  modification you should check this field and verify if you object has
+  all the existing keys.
+
+If your validation fails you should raise the
+:class:`graceful.errors.ValidationError` exception. Following is the example
+of resource serializer with custom object-level validation:
 
 
 .. code-block:: python
 
     class DrinkSerializer():
-        alcohol = StringField("main ingredient", required=True)
-        mixed_with = StringField("what makes it tasty", required=True)
+        alcohol = StringField("main ingredient")
+        mixed_with = StringField("what makes it tasty")
+
+        def validate(self, object_dict, partial):
+            # note: always make sure to call super `validate_object()`
+            # to make sure that per-field validation is enabled.
 
-        def validate(self, object_dict, partial=False):
-            # note: always make sure to call super `validate()`
-            # so whole validation of fields works as expected
-            super().validate(object_dict, partial)
+            if partial and any([
+                'alcohol' in object_dict,
+                'mixed_with' in object_dict,
+            ]):
+                raise ValidationError(
+                    "bartender refused to change ingredients"
+                )
 
             # here is a place for your own validation
             if (
                 object_dict['alcohol'] == 'whisky' and
                 object_dict['mixed_with'] == 'cola'
             ):
-                raise ValidationError("bartender refused!')
+                raise ValidationError(
+                    "bartender refused to mix whisky with cola!"
+                )
 
 
 Custom fields
@@ -228,3 +246,88 @@ as a serialized JSON string that we would like to (de)serialize:
         def to_representation(data):
             return json.loads(data)
 
+.. _guide-field-attribute-access:
+
+
+Accessing multiple fields at once
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes you need to access multiple fields of internal object instance at
+once in order to properly represent data in your API. This is very common when
+interacting with legacy services/components that cannot be changed or when
+your storage engine simply does not allow to store nested or structured objects.
+
+Serializers generally work on per-field basis and allow only to translate field
+names between representation and application internal objects. In order to
+manipulate multiple representation or internal object instance keys within the
+single field you need to create custom field class and override one or more
+of following methods:
+
+* ``read_instance(self, instance, key_or_attribute)``: read value from the
+  object instance before serialization. The return value will be later passed
+  as an argument to ``to_representation()`` method. The ``key_or_attribute``
+  argument is field's name or source (if ``source`` explicitly specified).
+  Base implementation defaults to dictionary key lookup or object attribute
+  lookup.
+* ``read_representation(self, representation, key_or_attribute)``: read value
+  from the object instance before deserialization. The return value will be
+  later passed as an argument to ``from_representation()`` method. The
+  ``key_or_attribute`` argument the field's name. Base implementation defaults
+  to dictionary key lookup or object attribute lookup.
+* ``update_instance(self, instance, key_or_attribute, value)``: update the
+  content of object instance after deserialization. The ``value`` argument is
+  the return value of ``from_representation()`` method. The
+  ``key_or_attribute`` argument the field's name or source (if ``source``
+  explicitly specified). Base implementation defaults to dictionary key
+  assignment or object attribute assignment.
+* ``update_representation(self, representation, key_or_attribute, value)``:
+  update the content of representation instance after serialization.
+  The ``value`` argument is the return value of ``to_representation()`` method.
+  The ``key_or_attribute`` argument the field's name. Base implementation
+  defaults to dictionary key assignment or object attribute assignment.
+
+To better explain how to use these methods let's assume that due to some
+storage backend constraints we cannot save nested dictionaries. All of fields
+of some nested object will have to be stored under separate keys but we still
+want to present this to the user as separate nested dictionary. And of course
+we want to support both writes and saves.
+
+.. code-block:: python
+
+    class OwnerField(RawField):
+        def from_representation(self, data):
+            if not isinstance(data, dict):
+                raise ValueError("expected object")
+
+            return {
+                'owner_name': data.get('name'),
+                'owner_age': data.get('age'),
+            }
+
+        def to_representation(self, value):
+            return {
+                'age': value.get('owner_age'),
+                'name': value.get('owner_name'),
+            }
+
+        def validate(self, value):
+            print(value)
+            if 'owner_age' not in value or not isinstance(value['owner_age'], int):
+                raise ValidationError("invalid owner age")
+
+            if 'owner_name' not in value:
+                raise ValidationError("invalid owner name")
+
+        def update_instance(self, instance, attribute_or_key, value):
+            # we assume that instance is always a dictionary so we can
+            # use the .update() method
+            instance.update(value)
+
+        def read_instance(self, instance, attribute_or_key):
+            # .to_representation() method requires acces to whole object
+            # dictionary so we have to return whole object.
+            return instance
+
+
+Similar approach may be used to flatten nested objects into more compact
+representations.
diff --git a/src/graceful/errors.py b/src/graceful/errors.py
index 110095e..55fb5d7 100644
--- a/src/graceful/errors.py
+++ b/src/graceful/errors.py
@@ -34,7 +34,7 @@ def _get_description(self):
                     "forbidden: {}".format(self.forbidden)
                     if self.forbidden else ""
                 ),
-                "invalid: {}:".format(self.invalid) if self.invalid else "",
+                "invalid: {}".format(self.invalid) if self.invalid else "",
                 (
                     "failed to parse: {}".format(self.failed)
                     if self.failed else ""
diff --git a/src/graceful/fields.py b/src/graceful/fields.py
index b306aa2..6a162f2 100644
--- a/src/graceful/fields.py
+++ b/src/graceful/fields.py
@@ -1,4 +1,5 @@
 import inspect
+from collections import Mapping, MutableMapping
 
 from graceful.validators import min_validator, max_validator
 
@@ -60,6 +61,14 @@ def from_representation(self, data):
             def to_representation(self, value):
                 return ["True", "False"][value]
 
+    .. versionchanged:: 1.0.0
+        Field instances no longer support ``source="*"`` to access whole object
+        for the purpose of representation serialization/deserialization. If you
+        want to access multiple fields of object instance and/or its
+        representation you must override ``read_*`` and ``update_*`` methods in
+        your custom field classes (see :ref:`guide-field-attribute-access`
+        section in documentation).
+
     """
 
     #: Two-tuple ``(label, url)`` pointing to represented type specification
@@ -172,6 +181,66 @@ def validate(self, value):
         for validator in self.validators:
             validator(value)
 
+    def update_instance(self, instance, attribute_or_key, value):
+        """Update object instance after deserialization.
+
+        Args:
+            instance (object): dictionary or object after serialization.
+            attribute_or_key (str): field's name or source (if ``source``
+                explicitly specified).
+            value (object): return value from ``from_representation`` method.
+        """
+        if isinstance(instance, MutableMapping):
+            instance[attribute_or_key] = value
+        else:
+            setattr(instance, attribute_or_key, value)
+
+    def read_instance(self, instance, attribute_or_key):
+        """Read value from the object instance before serialization.
+
+        Args:
+            instance (object): dictionary or object before serialization.
+            attribute_or_key (str): field's name or source (if ``source``
+                explicitly specified).
+
+        Returns:
+            The value that will be later passed as an argument to
+            ``to_representation()`` method.
+        """
+        if isinstance(instance, Mapping):
+            return instance.get(attribute_or_key, None)
+
+        return getattr(instance, attribute_or_key, None)
+
+    def update_representation(self, representation, attribute_or_key, value):
+        """Update representation after field serialization.
+
+        Args:
+            instance (object): representation object.
+            attribute_or_key (str): field's name.
+            value (object): return value from ``to_representation`` method.
+        """
+        if isinstance(representation, MutableMapping):
+            representation[attribute_or_key] = value
+        else:
+            setattr(representation, attribute_or_key, value)
+
+    def read_representation(self, representation, attribute_or_key):
+        """Read value from the representation before deserialization.
+
+        Args:
+            instance (object): dictionary or object before deserialization.
+            attribute_or_key (str): field's name.
+
+        Returns:
+            The value that will be later passed as an argument to
+            ``from_representation()`` method.
+        """
+        if isinstance(representation, Mapping):
+            return representation.get(attribute_or_key, None)
+
+        return getattr(representation, attribute_or_key, None)
+
 
 class RawField(BaseField):
     """Represents raw field subtype.
diff --git a/src/graceful/resources/base.py b/src/graceful/resources/base.py
index 5364c7a..f60bce7 100644
--- a/src/graceful/resources/base.py
+++ b/src/graceful/resources/base.py
@@ -424,11 +424,11 @@ def require_validated(self, req, partial=False, bulk=False):
 
         try:
             for representation in representations:
-                object_dict = self.serializer.from_representation(
-                    representation
+                object_dicts.append(
+                    self.serializer.from_representation(
+                        representation, partial
+                    )
                 )
-                self.serializer.validate(object_dict, partial)
-                object_dicts.append(object_dict)
 
         except DeserializationError as err:
             # when working on Resource we know that we can finally raise
diff --git a/src/graceful/serializers.py b/src/graceful/serializers.py
index e83a76a..3754bdf 100644
--- a/src/graceful/serializers.py
+++ b/src/graceful/serializers.py
@@ -1,22 +1,9 @@
 from collections import OrderedDict
-from collections.abc import Mapping, MutableMapping
 
 from graceful.errors import DeserializationError
 from graceful.fields import BaseField
 
 
-def _source(name, field):
-    """Translate field name to instance source name with respect to source=*.
-
-    .. deprecated::
-        This function will be removed in 1.0.0.
-    """
-    if field.source == '*':
-        return name
-    else:
-        return field.source or name
-
-
 class MetaSerializer(type):
     """Metaclass for handling serialization with field objects."""
 
@@ -93,12 +80,15 @@ class CatSerializer(BaseSerializer):
 
     """
 
+    instance_factory = dict
+    representation_factory = dict
+
     @property
     def fields(self):
         """Return dictionary of field definition objects of this serializer."""
         return getattr(self, self.__class__._fields_storage_key)
 
-    def to_representation(self, obj):
+    def to_representation(self, instance):
         """Convert given internal object instance into representation dict.
 
         Representation dict may be later serialized to the content-type
@@ -109,223 +99,164 @@ def to_representation(self, obj):
         one using ``field.to_representation()`` method.
 
         Args:
-            obj (object): internal object that needs to be represented
+            instance (object): internal object that needs to be represented
 
         Returns:
             dict: representation dictionary
 
         """
-        representation = {}
+        representation = self.representation_factory()
 
         for name, field in self.fields.items():
             # note fields do not know their names in source representation
             # but may know what attribute they target from source object
-            attribute = self.get_attribute(obj, field.source or name)
+            attribute = field.read_instance(instance, field.source or name)
 
             if attribute is None:
                 # Skip none attributes so fields do not have to deal with them
-                representation[name] = [] if field.many else None
+                field.update_representation(
+                    representation, name, [] if field.many else None
+                )
+
             elif field.many:
-                representation[name] = [
-                    field.to_representation(item) for item in attribute
-                ]
+                field.update_representation(
+                    representation, name, [
+                        field.to_representation(item) for item in attribute
+                    ]
+                )
             else:
-                representation[name] = field.to_representation(attribute)
+                field.update_representation(
+                    representation, name, field.to_representation(attribute)
+                )
 
         return representation
 
-    def from_representation(self, representation):
+    def from_representation(self, representation, partial=False):
         """Convert given representation dict into internal object.
 
         Internal object is simply a dictionary of values with respect to field
-        sources.
-
-        This does not check if all required fields exist or values are
-        valid in terms of value validation
-        (see: :meth:`BaseField.validate()`) but still requires all of passed
-        representation values to be well formed representation (success call
-        to ``field.from_representation``).
-
-        In case of malformed representation it will run additional validation
-        only to provide a full detailed exception about all that might be
-        wrong with provided representation.
+        sources. This method does not quit on first failure to make sure that
+        as many as possible issues will be presented to the client.
 
         Args:
            representation (dict): dictionary with field representation values
 
         Raises:
-            DeserializationError: when at least one representation field
-                is not formed as expected by field object. Information
-                about additional forbidden/missing/invalid fields is provided
-                as well.
+            DeserializationError: when at least of these issues occurs:
+
+                * if at least one of representation field is not formed as
+                  expected by the field object (``ValueError`` raised by
+                  field's ``from_representation()`` method).
+                * if ``partial=False`` and at least one representation fields
+                  is missing.
+                * if any non-existing or non-writable field is provided in
+                  representation.
+                * if any custom field validator fails (raises
+                  ``ValidationError`` or ``ValueError`` exception)
+
+            ValidationError: on custom user validation checks implemented with
+                ``validate()`` handler.
 
         """
-        object_dict = {}
+        instance = self.instance_factory()
+
         failed = {}
+        invalid = {}
+
+        # note: we need to perform validation on whole representation before
+        #       validation because there is no
+        missing, forbidden = self._validate_representation(
+            representation, partial
+        )
 
         for name, field in self.fields.items():
             if name not in representation:
                 continue
 
             try:
+                raw_entry = field.read_representation(representation, name)
                 if (
                     # note: we cannot check for any sequence or iterable
                     #       because of strings and nested dicts.
-                    not isinstance(representation[name], (list, tuple)) and
+                    not isinstance(raw_entry, (list, tuple)) and
                     field.many
                 ):
                     raise ValueError("field should be sequence")
 
-                source = _source(name, field)
-                value = representation[name]
+                field_values = [
+                    field.from_representation(item)
+                    for item in ([raw_entry] if not field.many else raw_entry)
+                ]
 
-                if field.many:
-                    object_dict[source] = [
-                        field.from_representation(single_value)
-                        for single_value in value
-                    ]
-                else:
-                    object_dict[source] = field.from_representation(value)
+                for value in field_values:
+                    field.validate(value)
+
+                field.update_instance(
+                    instance,
+                    # If field does not have explicit source string then use
+                    # its name.
+                    field.source or name,
+                    # many=True fields require special care
+                    field_values if field.many else field_values[0]
+                )
 
             except ValueError as err:
                 failed[name] = str(err)
 
-        if failed:
-            # if failed to parse we eagerly perform validation so full
-            # information about what is wrong will be returned
-            try:
-                self.validate(object_dict)
-                # note: this exception can be reached with partial==True
-                # since do not support partial updates yet this has 'no cover'
-                raise DeserializationError()  # pragma: no cover
-            except DeserializationError as err:
-                err.failed = failed
-                raise
+        if any([missing, forbidden, invalid, failed]):
+            raise DeserializationError(missing, forbidden, invalid, failed)
 
-        return object_dict
+        # note: expected to raise ValidationError. It is extra feature handle
+        #       so we dont try hard to merge wit previous errors.
+        self.validate(instance, partial)
 
-    def validate(self, object_dict, partial=False):
-        """Validate given internal object returned by ``to_representation()``.
+        return instance
 
-        Internal object is validated against missing/forbidden/invalid fields
-        values using fields definitions defined in serializer.
+    def _validate_representation(self, representation, partial=False):
+        """Validate resource representation fieldwise.
 
-        Args:
-            object_dict (dict): internal object dictionart to perform
-              to validate
-            partial (bool): if set to True then incomplete object_dict
-              is accepter and will not raise any exceptions when one
-              of fields is missing
+        Check if object has all required fields to support full or partial
+        object modification/creation and ensure it does not contain any
+        forbidden fields.
 
-        Raises:
-            DeserializationError:
+        Returns:
 
+            A ``(missing, forbidden)`` tuple with lists indicating fields that
+            failed validation.
         """
-        # we are working on object_dict not an representation so there
-        # is a need to annotate sources differently
-        sources = {
-            _source(name, field): field
-            for name, field in self.fields.items()
-        }
-
-        # note: we are checking for all mising and invalid fields so we can
-        # return exception with all fields that are missing and should
-        # exist instead of single one
         missing = [
-            name for name, field in sources.items()
-            if all((not partial, name not in object_dict, not field.read_only))
+            name for name, field in self.fields.items()
+            if all((
+                not partial,
+                name not in representation,
+                not field.read_only
+            ))
         ]
 
         forbidden = [
-            name for name in object_dict
-            if any((name not in sources, sources[name].read_only))
+            name for name in representation
+            if name not in self.fields or self.fields[name].read_only
         ]
 
-        invalid = {}
-        for name, value in object_dict.items():
-            try:
-                field = sources[name]
-
-                if field.many:
-                    for single_value in value:
-                        field.validate(single_value)
-                else:
-                    field.validate(value)
-
-            except ValueError as err:
-                invalid[name] = str(err)
-
-        if any([missing, forbidden, invalid]):
-            # note: We have validated internal object instance but need to
-            #       inform the user about problems with his representation.
-            #       This is why we have to do this dirty transformation.
-            # note: This will be removed in 1.0.0 where we change how
-            #       validation works and where we remove star-like fields.
-            # refs: #42 (https://github.com/swistakm/graceful/issues/42)
-            sources_to_field_names = {
-                _source(name, field): name
-                for name, field in self.fields.items()
-            }
-
-            def _(names):
-                if isinstance(names, list):
-                    return [
-                        sources_to_field_names.get(name, name)
-                        for name in names
-                    ]
-                elif isinstance(names, dict):
-                    return {
-                        sources_to_field_names.get(name, name): value
-                        for name, value in names.items()
-                    }
-                else:
-                    return names  # pragma: nocover
-
-            raise DeserializationError(_(missing), _(forbidden), _(invalid))
-
-    def get_attribute(self, obj, attr):
-        """Get attribute of given object instance.
+        return missing, forbidden
 
-        Reason for existence of this method is the fact that  'attribute' can
-        be also object's key from if is a dict or any other kind of mapping.
+    def validate(self, instance, partial=False):
+        """Validate given internal object.
 
-        Note: it will return None if attribute key does not exist
+        Internal object is a dictionary that have sucesfully passed general
+        validation against missing/forbidden fields and was checked with
+        per-field custom validators.
 
         Args:
-            obj (object): internal object to retrieve data from
-
-        Returns:
-            internal object's key value or attribute
-
-        """
-        # '*' is a special wildcard character that means whole object
-        # is passed
-        if attr == '*':
-            return obj
-
-        # if this is any mapping then instead of attributes use keys
-        if isinstance(obj, Mapping):
-            return obj.get(attr, None)
-
-        return getattr(obj, attr, None)
-
-    def set_attribute(self, obj, attr, value):
-        """Set value of attribute in given object instance.
-
-        Reason for existence of this method is the fact that 'attribute' can
-        be also a object's key if it is a dict or any other kind of mapping.
-
-        Args:
-            obj (object): object instance to modify
-            attr (str): attribute (or key) to change
-            value: value to set
+            instance (dict): internal object instance to be validated.
+            partial (bool): if set to True then incomplete instance
+              is accepted (e.g. on PATCH requests) so it is possible that
+              not every field is available.
 
+        Raises:
+            ValidationError: raised when deserialized object does not meet some
+                user-defined contraints.
         """
-        # if this is any mutable mapping then instead of attributes use keys
-        if isinstance(obj, MutableMapping):
-            obj[attr] = value
-        else:
-            setattr(obj, attr, value)
 
     def describe(self):
         """Describe all serialized fields.
diff --git a/tests/test_resources.py b/tests/test_resources.py
index 0869a18..4ae492f 100644
--- a/tests/test_resources.py
+++ b/tests/test_resources.py
@@ -398,10 +398,10 @@ class TestSerializer(BaseSerializer):
         one = StringField("one different than two")
         two = StringField("two different than one")
 
-        def validate(self, object_dict, partial=False):
-            super().validate(object_dict, partial)
+        def validate(self, instance, partial=False):
+            super().validate(instance, partial)
             # possible use case: kind of uniqueness relationship
-            if object_dict['one'] == object_dict['two']:
+            if instance['one'] == instance['two']:
                 raise ValidationError("one must be different than two")
 
     class TestResource(Resource):
diff --git a/tests/test_serializers.py b/tests/test_serializers.py
index 5e7b353..ea91ea4 100644
--- a/tests/test_serializers.py
+++ b/tests/test_serializers.py
@@ -158,74 +158,6 @@ class SomeConcreteSerializer(BaseSerializer):
     assert recreated == {"_name": "John", "_address": "US"}
 
 
-def test_serializer_set_attribute():
-    serializer = BaseSerializer()
-
-    # test dict keys are treated as attributes
-    instance = {}
-    serializer.set_attribute(instance, 'foo', 'bar')
-    assert instance == {'foo': 'bar'}
-
-    # test normal objects atrributes are attributes indeed
-    # in scope of this method
-    class SomeObject:
-        def __init__(self):
-            self.foo = None
-
-    instance = SomeObject()
-    serializer.set_attribute(instance, 'foo', 'bar')
-    assert instance.foo == 'bar'
-
-
-def test_serializer_get_attribute():
-    serializer = BaseSerializer()
-
-    # test dict keys are treated as attributes
-    instance = {'foo': 'bar'}
-    assert serializer.get_attribute(instance, 'foo') == 'bar'
-
-    # test normal objects atrributes are attributes indeed
-    # in scope of this method
-    class SomeObject:
-        def __init__(self):
-            self.foo = 'bar'
-
-    instance = SomeObject()
-    assert serializer.get_attribute(instance, 'foo') == 'bar'
-
-    # test that getting non existent attribute returns None
-    assert serializer.get_attribute(instance, 'nonexistens') is None
-
-
-def test_serializer_source_wildcard():
-    """
-    Test that '*' wildcard causes whole instance is returned on get attribute
-    """
-    serializer = BaseSerializer()
-
-    instance = {"foo", "bar"}
-    assert serializer.get_attribute(instance, '*') == instance
-
-
-def test_serializer_source_field_with_wildcard():
-    class ExampleSerializer(BaseSerializer):
-        starfield = ExampleField(
-            details='whole object instance goes here',
-            source='*',
-        )
-
-    serializer = ExampleSerializer()
-    instance = {'foo': 'bar'}
-    representation = {"starfield": "bizbaz"}
-
-    assert serializer.to_representation(
-        instance
-    )['starfield'] == instance
-    assert serializer.from_representation(
-        representation
-    )['starfield'] == representation["starfield"]
-
-
 def test_serializer_describe():
     """ Test that serializers are self-describing
     """
@@ -287,6 +219,6 @@ class ExampleSerializer(BaseSerializer):
     serializer = ExampleSerializer()
 
     with pytest.raises(ValueError):
-        serializer.validate(invalid)
+        serializer.from_representation(invalid)
 
-    serializer.validate(valid)
+    serializer.from_representation(valid)

From cfd239ee84eedecbc1804c84b91e3d9efa20376a Mon Sep 17 00:00:00 2001
From: mjaworski <mjaworski@opera.com>
Date: Wed, 12 Apr 2017 12:29:37 +0200
Subject: [PATCH 2/4] docs: remove obsolete code and comments from serializers
 examples

---
 docs/guide/serializers.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/guide/serializers.rst b/docs/guide/serializers.rst
index 9b9a3b3..c4e3748 100644
--- a/docs/guide/serializers.rst
+++ b/docs/guide/serializers.rst
@@ -198,9 +198,6 @@ of resource serializer with custom object-level validation:
         mixed_with = StringField("what makes it tasty")
 
         def validate(self, object_dict, partial):
-            # note: always make sure to call super `validate_object()`
-            # to make sure that per-field validation is enabled.
-
             if partial and any([
                 'alcohol' in object_dict,
                 'mixed_with' in object_dict,
@@ -311,7 +308,6 @@ we want to support both writes and saves.
             }
 
         def validate(self, value):
-            print(value)
             if 'owner_age' not in value or not isinstance(value['owner_age'], int):
                 raise ValidationError("invalid owner age")
 

From 91bcd1814e6781683a4e67bfabfb5c258044fe8f Mon Sep 17 00:00:00 2001
From: mjaworski <mjaworski@opera.com>
Date: Wed, 12 Apr 2017 13:02:21 +0200
Subject: [PATCH 3/4] serializers and fields: improve test for custom instance
 factories and clarify terminology

---
 docs/guide/serializers.rst  | 41 +++++++++--------
 src/graceful/fields.py      | 44 +++++++++----------
 src/graceful/serializers.py | 13 ++++--
 tests/conftest.py           |  2 +-
 tests/fixtures.py           | 35 +++++++++++++++
 tests/test_serializers.py   | 88 +++++++++++++++++++++++--------------
 6 files changed, 144 insertions(+), 79 deletions(-)

diff --git a/docs/guide/serializers.rst b/docs/guide/serializers.rst
index c4e3748..71f5e61 100644
--- a/docs/guide/serializers.rst
+++ b/docs/guide/serializers.rst
@@ -260,28 +260,31 @@ manipulate multiple representation or internal object instance keys within the
 single field you need to create custom field class and override one or more
 of following methods:
 
-* ``read_instance(self, instance, key_or_attribute)``: read value from the
+* ``read_instance(self, instance, source)``: read value from the
   object instance before serialization. The return value will be later passed
-  as an argument to ``to_representation()`` method. The ``key_or_attribute``
-  argument is field's name or source (if ``source`` explicitly specified).
-  Base implementation defaults to dictionary key lookup or object attribute
-  lookup.
-* ``read_representation(self, representation, key_or_attribute)``: read value
-  from the object instance before deserialization. The return value will be
-  later passed as an argument to ``from_representation()`` method. The
-  ``key_or_attribute`` argument the field's name. Base implementation defaults
-  to dictionary key lookup or object attribute lookup.
-* ``update_instance(self, instance, key_or_attribute, value)``: update the
+  as an argument to ``to_representation()`` method. The ``source`` argument is
+  field's name or configured source name (if custom ``source`` is explicitly
+  specified for that field). Base implementation defaults to dictionary key
+  lookup or object attribute lookup.
+
+* ``update_instance(self, instance, source, value)``: update the
   content of object instance after deserialization. The ``value`` argument is
-  the return value of ``from_representation()`` method. The
-  ``key_or_attribute`` argument the field's name or source (if ``source``
-  explicitly specified). Base implementation defaults to dictionary key
+  the return value of ``from_representation()`` method. The ``source`` argument
+  is the field's name or source name (if custom ``source`` is explicitly
+  specified for that field). Base implementation defaults to dictionary key
   assignment or object attribute assignment.
-* ``update_representation(self, representation, key_or_attribute, value)``:
+
+* ``read_representation(self, representation, field_name)``: read value
+  from the representation before deserialization. The return value will be
+  later passed as an argument to ``from_representation()`` method. The
+  ``field_name`` argument is a field's name. Base implementation defaults
+  to dictionary key lookup or object attribute lookup.
+
+* ``update_representation(self, representation, field_name, value)``:
   update the content of representation instance after serialization.
   The ``value`` argument is the return value of ``to_representation()`` method.
-  The ``key_or_attribute`` argument the field's name. Base implementation
-  defaults to dictionary key assignment or object attribute assignment.
+  The ``field_name`` argument is a field's name. Base implementation
+  defaults to dictionary key assignment.
 
 To better explain how to use these methods let's assume that due to some
 storage backend constraints we cannot save nested dictionaries. All of fields
@@ -314,12 +317,12 @@ we want to support both writes and saves.
             if 'owner_name' not in value:
                 raise ValidationError("invalid owner name")
 
-        def update_instance(self, instance, attribute_or_key, value):
+        def update_instance(self, instance, source, value):
             # we assume that instance is always a dictionary so we can
             # use the .update() method
             instance.update(value)
 
-        def read_instance(self, instance, attribute_or_key):
+        def read_instance(self, instance, source):
             # .to_representation() method requires acces to whole object
             # dictionary so we have to return whole object.
             return instance
diff --git a/src/graceful/fields.py b/src/graceful/fields.py
index 6a162f2..7fa9b90 100644
--- a/src/graceful/fields.py
+++ b/src/graceful/fields.py
@@ -181,65 +181,63 @@ def validate(self, value):
         for validator in self.validators:
             validator(value)
 
-    def update_instance(self, instance, attribute_or_key, value):
+    def update_instance(self, instance, source, value):
         """Update object instance after deserialization.
 
         Args:
             instance (object): dictionary or object after serialization.
-            attribute_or_key (str): field's name or source (if ``source``
-                explicitly specified).
+            source (str): field's name or source if ``source`` was
+                explicitly specified for that field.
             value (object): return value from ``from_representation`` method.
         """
         if isinstance(instance, MutableMapping):
-            instance[attribute_or_key] = value
+            instance[source] = value
         else:
-            setattr(instance, attribute_or_key, value)
+            setattr(instance, source, value)
 
-    def read_instance(self, instance, attribute_or_key):
+    def read_instance(self, instance, source):
         """Read value from the object instance before serialization.
 
         Args:
             instance (object): dictionary or object before serialization.
-            attribute_or_key (str): field's name or source (if ``source``
-                explicitly specified).
+            source (str): field's name or source if ``source`` was
+                explicitly specified for that field.
 
         Returns:
             The value that will be later passed as an argument to
             ``to_representation()`` method.
         """
         if isinstance(instance, Mapping):
-            return instance.get(attribute_or_key, None)
+            return instance.get(source, None)
 
-        return getattr(instance, attribute_or_key, None)
+        return getattr(instance, source, None)
 
-    def update_representation(self, representation, attribute_or_key, value):
+    def update_representation(self, representation, field_name, value):
         """Update representation after field serialization.
 
         Args:
-            instance (object): representation object.
+            representation (dict): representation dictionary.
             attribute_or_key (str): field's name.
             value (object): return value from ``to_representation`` method.
         """
-        if isinstance(representation, MutableMapping):
-            representation[attribute_or_key] = value
-        else:
-            setattr(representation, attribute_or_key, value)
+        # note: Unlike instances, representations can only be dictionaries
+        #       and their type cannot be overriden inside of serializer's class
+        representation[field_name] = value
 
-    def read_representation(self, representation, attribute_or_key):
+    def read_representation(self, representation, field_name):
         """Read value from the representation before deserialization.
 
         Args:
-            instance (object): dictionary or object before deserialization.
-            attribute_or_key (str): field's name.
+            representation (dict): representation dictionary.
+            field_name (str): field's name.
 
         Returns:
             The value that will be later passed as an argument to
             ``from_representation()`` method.
         """
-        if isinstance(representation, Mapping):
-            return representation.get(attribute_or_key, None)
-
-        return getattr(representation, attribute_or_key, None)
+        # note: Unlike instances, representations can only be dictionaries
+        #       and their type cannot be overriden inside of serializer's class
+        return representation.get(field_name, None)
 
 
 class RawField(BaseField):
diff --git a/src/graceful/serializers.py b/src/graceful/serializers.py
index 3754bdf..fdb11f7 100644
--- a/src/graceful/serializers.py
+++ b/src/graceful/serializers.py
@@ -80,8 +80,9 @@ class CatSerializer(BaseSerializer):
 
     """
 
+    #: Allows to override instance object constructon with custom type
+    #: like defaultdict, SimpleNamespace or database model class.
     instance_factory = dict
-    representation_factory = dict
 
     @property
     def fields(self):
@@ -105,11 +106,15 @@ def to_representation(self, instance):
             dict: representation dictionary
 
         """
-        representation = self.representation_factory()
+        # note: representations does not have their custom facotries like
+        #       instances because they as only used during content-type
+        #       serialization and deserialization and cannot be manipulated
+        #       inside of resource classes.
+        representation = {}
 
         for name, field in self.fields.items():
-            # note fields do not know their names in source representation
-            # but may know what attribute they target from source object
+            # note: fields do not know their names in source representation
+            #        but may know what attribute they target from instance
             attribute = field.read_instance(instance, field.source or name)
 
             if attribute is None:
diff --git a/tests/conftest.py b/tests/conftest.py
index 023bbae..38062e9 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,2 +1,2 @@
 # those fixtures will be available for whole tests package
-from .fixtures import req, resp  # noqa
+from .fixtures import req, resp, instance_class  # noqa
diff --git a/tests/fixtures.py b/tests/fixtures.py
index 532a695..c7c878c 100644
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -1,8 +1,27 @@
+from types import SimpleNamespace as BaseSimpleNamespace
+from collections import defaultdict
+from functools import partial
+
 import pytest
 
 from falcon import Request, Response
 from falcon.testing import create_environ
 
+# compat: SimpleNamespace depends on of built-in sys module (the type of
+#         type of sys.implementation variable) and may be implementation
+#         dependent. For instance, in Python3.3 it does not have its own
+#         __eq__ method implementation so two equivalent namespaces do not
+#         compare as equal ones.
+if BaseSimpleNamespace() == BaseSimpleNamespace():
+    SimpleNamespace = BaseSimpleNamespace
+else:
+    class SimpleNamespace(BaseSimpleNamespace):
+        """Extended ``SimpleNamespace`` implementation for Python3.3."""
+
+        def __eq__(self, other):
+            """Check if two namesapces have equal content."""
+            return self.__dict__ == other.__dict__
+
 
 @pytest.fixture
 def req():
@@ -15,3 +34,19 @@ def req():
 def resp():
     """Simple empty Response fixture."""
     return Response()
+
+
+@pytest.fixture(
+    params=[
+        dict,
+        SimpleNamespace,
+        # note: this is not a class per-se but acts like a one
+        partial(defaultdict, str)
+    ],
+)
+def instance_class(request):
+    """Instance class that may be used as a instance_factory attribute.
+
+    This fixture helps testing serializers.
+    """
+    return request.param
diff --git a/tests/test_serializers.py b/tests/test_serializers.py
index ea91ea4..41a0bc5 100644
--- a/tests/test_serializers.py
+++ b/tests/test_serializers.py
@@ -20,11 +20,13 @@ def to_representation(self, value):
         return value
 
 
-def test_simple_serializer_definition():
+def test_simple_serializer_definition(instance_class):
     """
     Test if serializers can be defined in declarative way
     """
     class TestSimpleSerializer(BaseSerializer):
+        instance_factory = instance_class
+
         foo = ExampleField(
             details="first field for testing"
         )
@@ -40,11 +42,12 @@ class TestSimpleSerializer(BaseSerializer):
     assert 'foo' in serializer.fields
 
 
-def test_empty_serializer_definition():
+def test_empty_serializer_definition(instance_class):
     """
     Make sure even empty serializer has the same interface
     """
     class TestEmptySerializer(BaseSerializer):
+        instance_factory = instance_class
         pass
 
     serializer = TestEmptySerializer()
@@ -52,8 +55,9 @@ class TestEmptySerializer(BaseSerializer):
     assert len(serializer.fields) == 0
 
 
-def test_serializer_inheritance():
+def test_serializer_inheritance(instance_class):
     class TestParentSerializer(BaseSerializer):
+        instance_factory = instance_class
         foo = ExampleField(
             details="first field for testing"
         )
@@ -62,6 +66,7 @@ class TestParentSerializer(BaseSerializer):
         )
 
     class TestDerivedSerializer(TestParentSerializer):
+        instance_factory = instance_class
         baz = ExampleField(
             details="this is additional field added as an extension"
         )
@@ -76,7 +81,7 @@ class TestDerivedSerializer(TestParentSerializer):
     assert 'baz' in serializer.fields
 
 
-def test_serializer_field_overriding():
+def test_serializer_field_overriding(instance_class):
     """
     Test serializer fields can be overriden
     """
@@ -84,10 +89,14 @@ def test_serializer_field_overriding():
     parent_label = "parent"
 
     class TestParentSerializer(BaseSerializer):
+        instance_factory = instance_class
+
         foo = ExampleField(label=parent_label, details="parent foo field")
         bar = ExampleField(label=parent_label, details="parent bar field")
 
     class TestOverridingSerializer(TestParentSerializer):
+        instance_factory = instance_class
+
         foo = ExampleField(label=override_label, details='overriden foo field')
 
     serializer = TestOverridingSerializer()
@@ -100,36 +109,39 @@ class TestOverridingSerializer(TestParentSerializer):
     assert serializer.fields['bar'].label == parent_label
 
 
-def test_serialiser_simple_representation():
+def test_serializer_simple_representation(instance_class):
     class SomeConcreteSerializer(BaseSerializer):
+        instance_factory = instance_class
+
         name = ExampleField(details="name of instance object")
         address = ExampleField(details="instance address")
 
-    object_instance = {
-        "name": "John",
-        "address": "US",
+    instance = instance_class(
+        name="John",
+        address="US",
         # note: gender is not included in serializer
         #   fields so it will be dropped
-        "gender": "male",
-    }
+        gender="male",
+    )
 
     serializer = SomeConcreteSerializer()
 
     # test creating representation
-    representation = serializer.to_representation(object_instance)
+    representation = serializer.to_representation(instance)
     assert representation == {"name": "John", "address": "US"}
 
     # test recreating instance
     recreated = serializer.from_representation(representation)
-    assert recreated == {"name": "John", "address": "US"}
+    assert recreated == instance_class(name="John", address="US")
 
 
-def test_serialiser_sources_representation():
+def test_serializer_sources_representation(instance_class):
     """
     Test representing objects with sources of fields different that their names
     """
-
     class SomeConcreteSerializer(BaseSerializer):
+        instance_factory = instance_class
+
         name = ExampleField(
             details="name of instance object (taken from _name)",
             source="_name",
@@ -139,29 +151,31 @@ class SomeConcreteSerializer(BaseSerializer):
             source="_address"
         )
 
-    object_instance = {
-        "_name": "John",
-        "_address": "US",
+    instance = instance_class(
+        _name="John",
+        _address="US",
         # note: gender is not included in serializer
         #   fields so it will be dropped
-        "gender": "male",
-    }
+        gender="male",
+    )
 
     serializer = SomeConcreteSerializer()
 
     # test creating representation
-    representation = serializer.to_representation(object_instance)
+    representation = serializer.to_representation(instance)
     assert representation == {"name": "John", "address": "US"}
 
     # test recreating instance
     recreated = serializer.from_representation(representation)
-    assert recreated == {"_name": "John", "_address": "US"}
+    assert recreated == instance_class(_name="John", _address="US")
 
 
-def test_serializer_describe():
+def test_serializer_describe(instance_class):
     """ Test that serializers are self-describing
     """
     class ExampleSerializer(BaseSerializer):
+        instance_factory = instance_class
+
         foo = ExampleField(label='foo', details='foo foo')
         bar = ExampleField(label='bar', details='bar bar')
 
@@ -179,34 +193,44 @@ class ExampleSerializer(BaseSerializer):
     ])
 
 
-def test_serialiser_with_field_many():
-    class UpperField(BaseField):
+def test_serializer_with_field_many(instance_class):
+    class CaseSwitchField(BaseField):
         def to_representation(self, value):
             return value.upper()
 
         def from_representation(self, data):
-            return data.upper()
+            return data.lower()
 
     class ExampleSerializer(BaseSerializer):
-        up = UpperField(details='multiple values field', many=True)
+        instance_factory = instance_class
+
+        case_switch = CaseSwitchField(
+            details='multiple values field', many=True
+        )
 
     serializer = ExampleSerializer()
-    obj = {'up': ["aa", "bb", "cc"]}
-    desired = {'up': ["AA", "BB", "CC"]}
 
-    assert serializer.to_representation(obj) == desired
-    assert serializer.from_representation(obj) == desired
+    # note: it can be any object type, not only a dictionary
+    instance = instance_class(case_switch=["aa", "bb", "cc"])
+    desired = {'case_switch': ["AA", "BB", "CC"]}
+
+    assert serializer.to_representation(instance) == desired
+    assert serializer.from_representation(desired) == instance
 
     with pytest.raises(ValueError):
-        serializer.from_representation({"up": "definitely not a sequence"})
+        serializer.from_representation(
+            {"case_switch": "definitely not a sequence"}
+        )
 
 
-def test_serializer_many_validation():
+def test_serializer_many_validation(instance_class):
     def is_upper(value):
         if value.upper() != value:
             raise ValueError("should be upper")
 
     class ExampleSerializer(BaseSerializer):
+        instance_factory = instance_class
+
         up = StringField(
             details='multiple values field',
             many=True,

From 656db47d235cc5afd57a5cf13853b50304dd4bf3 Mon Sep 17 00:00:00 2001
From: mjaworski <mjaworski@opera.com>
Date: Mon, 17 Apr 2017 20:17:02 +0200
Subject: [PATCH 4/4] serializers: improve serialization time by caching fields
 items

---
 src/graceful/serializers.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/graceful/serializers.py b/src/graceful/serializers.py
index fdb11f7..8a0638e 100644
--- a/src/graceful/serializers.py
+++ b/src/graceful/serializers.py
@@ -84,6 +84,11 @@ class CatSerializer(BaseSerializer):
     #: like defaultdict, SimpleNamespace or database model class.
     instance_factory = dict
 
+    def __init__(self):
+        # perf: create cached list to save some processing
+        #       time during serialization and deserialization
+        self._fields_items = list(self.fields.items())
+
     @property
     def fields(self):
         """Return dictionary of field definition objects of this serializer."""
@@ -112,7 +117,7 @@ def to_representation(self, instance):
         #       inside of resource classes.
         representation = {}
 
-        for name, field in self.fields.items():
+        for name, field in self._fields_items:
             # note: fields do not know their names in source representation
             #        but may know what attribute they target from instance
             attribute = field.read_instance(instance, field.source or name)
@@ -174,7 +179,7 @@ def from_representation(self, representation, partial=False):
             representation, partial
         )
 
-        for name, field in self.fields.items():
+        for name, field in self._fields_items:
             if name not in representation:
                 continue
 
@@ -230,7 +235,7 @@ def _validate_representation(self, representation, partial=False):
             failed validation.
         """
         missing = [
-            name for name, field in self.fields.items()
+            name for name, field in self._fields_items
             if all((
                 not partial,
                 name not in representation,