Implement hooks for a JIT to integrate into marshmallow.

marshmallow/__init__.py

@@ -3,6 +3,7 @@
 
 from marshmallow.schema import (
     Schema,
+    SchemaJit,
     SchemaOpts,
     MarshalResult,
     UnmarshalResult,
@@ -19,6 +20,7 @@
 
 __all__ = [
     'Schema',
+    'SchemaJit',
     'SchemaOpts',
     'fields',
     'validates',

marshmallow/fields.py

@@ -430,6 +430,8 @@ def schema(self):
                 raise ValueError('Nested fields must be passed a '
                                  'Schema, not {0}.'.format(self.nested.__class__))
             self.__schema.ordered = getattr(self.parent, 'ordered', False)
+            # If the parent has a jit specified we should use that same jit.
+            self.__schema.jit = getattr(self.parent, 'jit', None)
         return self.__schema
 
     def _nested_normalized_option(self, option_name):

marshmallow/schema.py

@@ -2,12 +2,15 @@
 """The :class:`Schema` class, including its metaclass and options (class Meta)."""
 from __future__ import absolute_import, unicode_literals
 
+from abc import ABCMeta, abstractmethod, abstractproperty
 from collections import defaultdict, Mapping
 import copy
 import datetime as dt
 import decimal
 import inspect
+import importlib
 import json
+import os
 import uuid
 import warnings
 from collections import namedtuple, OrderedDict
@@ -19,7 +22,31 @@
 from marshmallow.orderedset import OrderedSet
 from marshmallow.decorators import (PRE_DUMP, POST_DUMP, PRE_LOAD, POST_LOAD,
                                     VALIDATES, VALIDATES_SCHEMA)
-from marshmallow.utils import missing
+from marshmallow.utils import missing, suppress
+
+DEFAULT_JIT_ENVIRONMENT_VARIABLE = 'MARSHMALLOW_SCHEMA_DEFAULT_JIT'
+DEFAULT_JIT = missing
+
+
+def get_default_jit():
+    """Allows overriding the default JIT to use from the environment.
+
+    This is useful for running tests as well as rolling out a Marshmallow JIT to
+    every schema in a process.
+    """
+    global DEFAULT_JIT
+    if DEFAULT_JIT == missing:
+        DEFAULT_JIT = None
+        with suppress(Exception):
+            default_jit_path = os.getenv(DEFAULT_JIT_ENVIRONMENT_VARIABLE)
+
+            if default_jit_path:
+                parts = default_jit_path.split('.')
+                module_name = '.'.join(parts[0:-1])
+                class_name = parts[-1]
+                jit_module = importlib.import_module(module_name)
+                DEFAULT_JIT = getattr(jit_module, class_name, None)
+    return DEFAULT_JIT
 
 
 #: Return type of :meth:`Schema.dump` including serialized data and errors
@@ -174,6 +201,37 @@ def _resolve_processors(self):
                 self.__processors__[tag].append(attr_name)
 
 
+class SchemaJit(with_metaclass(ABCMeta)):
+    """Abstract Base Class for implementing a jit for Marshmallow.
+
+    A SchemaJit should create methods to marshal or unmarshal an object for a
+    given schema.  These methods should, given an object, optimistically perform
+    marshalling or unmarshalling.  If they fail, Marshmallow will fallback to the existing
+    reflection-based methods.
+    """
+    @abstractmethod
+    def __init__(self, schema):
+        pass
+
+    @abstractproperty
+    def jitted_marshal_method(self):
+        """Returns the jitted marshal method.
+
+        :return: A method that accepts an object and returns the marshalling result or None
+            if there is no applicable marshal method.
+        """
+        pass
+
+    @abstractproperty
+    def jitted_unmarshal_method(self):
+        """Returns the jitted unmarshal method.
+
+        :return: A method that accepts an object and returns the unmarshalling result or None
+            if there is no applicable unmarshal method.
+        """
+        pass
+
+
 class SchemaOpts(object):
     """class Meta options for the :class:`Schema`. Defines defaults."""
 
@@ -206,6 +264,8 @@ def __init__(self, meta, ordered=False):
         self.include = getattr(meta, 'include', {})
         self.load_only = getattr(meta, 'load_only', ())
         self.dump_only = getattr(meta, 'dump_only', ())
+        self.jit = getattr(meta, 'jit', missing)
+        self.jit_options = getattr(meta, 'jit_options', {})
 
 
 class BaseSchema(base.SchemaABC):
@@ -345,13 +405,29 @@ def __init__(self, only=(), exclude=(), prefix='', strict=None,
         self.context = context or {}
         self._normalize_nested_options()
         self._types_seen = set()
+        self._jit_class = get_default_jit()
+        if self.opts.jit is not missing:
+            self._jit_class = self.opts.jit
+        self._jit_instance = None
+        self._current_name_to_type = None
         self._update_fields(many=many)
 
     def __repr__(self):
         return '<{ClassName}(many={self.many}, strict={self.strict})>'.format(
             ClassName=self.__class__.__name__, self=self
         )
 
+    @property
+    def jit(self):
+        return self._jit_class
+
+    @jit.setter
+    def jit(self, value):
+        self._jit_instance = None
+        self._current_name_to_type = None
+        self._jit_class = value
+        self._update_fields(many=self.many)
+
     @property
     def dict_class(self):
         return OrderedDict if self.ordered else dict
@@ -424,20 +500,13 @@ def dump(self, obj, many=None, update_fields=True, **kwargs):
                     self._update_fields(processed_obj, many=many)
                     if not isinstance(processed_obj, Mapping):
                         self._types_seen.add(obj_type)
-
-            try:
-                result = self._marshal(
-                    processed_obj,
-                    self.fields,
-                    many=many,
-                    accessor=self.get_attribute,
-                    dict_class=self.dict_class,
-                    index_errors=self.opts.index_errors,
-                    **kwargs
-                )
-            except ValidationError as error:
-                errors = self._marshal.errors
-                result = error.data
+            jitted_method = (self._jit_instance.jitted_marshal_method
+                             if self._jit_instance else None)
+            errors, result = self._transform(jitted_method,
+                                             self._marshal,
+                                             many, obj,
+                                             accessor=self.get_attribute,
+                                             **kwargs)
 
         if not errors and self._has_processors:
             try:
@@ -464,6 +533,49 @@ def dump(self, obj, many=None, update_fields=True, **kwargs):
 
         return MarshalResult(result, errors)
 
+    def _transform(self, jitted_method, marshalling_method,
+                   many, obj, **kwargs):
+        """Transforms an object by serializing or deserializing it, returning
+        any errors that occurred as well as the result.
+
+        This method allows for a single place for the jitted
+        serialize/deserialize methods to be invoked.  First it willa attempt to
+        use the `jitted_method` if available and, should that fail, will fall
+        back to the normal `marshmallow.marshalling`-based methods.
+
+        :param jitted_method: The jitted method to first attempt to invoke when
+            serializing or deserializng `obj`
+        :param marshalling_method: The marshalling method to call should the
+            jitted_method be absent or fail.  Should be either `self._marshal`
+            or `self._unmarshal`.
+        :param bool many: Whether to serialize `obj` as a collection.
+        :param obj:  The object to serialize or deserialize.
+        :return tuple: A tuple of errors:result.
+        """
+        result = None
+        errors = {}
+        marshalling_method.reset_errors()
+        if jitted_method:
+            try:
+                result = jitted_method(obj, many=many)
+            except (ValidationError, KeyError, AttributeError, ValueError, TypeError):
+                # Fall through to slow path
+                    pass
+        if not result:
+            try:
+                result = marshalling_method(
+                    obj,
+                    self.fields,
+                    many=many,
+                    dict_class=self.dict_class,
+                    index_errors=self.opts.index_errors,
+                    **kwargs
+                )
+            except ValidationError as error:
+                errors = marshalling_method.errors
+                result = error.data
+        return errors, result
+
     def dumps(self, obj, many=None, update_fields=True, *args, **kwargs):
         """Same as :meth:`dump`, except return a JSON-encoded string.
 
@@ -569,18 +681,13 @@ def _do_load(self, data, many=None, partial=None, postprocess=True):
         except ValidationError as err:
             errors = err.normalized_messages()
             result = None
-        if not errors:
-            try:
-                result = self._unmarshal(
-                    processed_data,
-                    self.fields,
-                    many=many,
-                    partial=partial,
-                    dict_class=self.dict_class,
-                    index_errors=self.opts.index_errors,
-                )
-            except ValidationError as error:
-                result = error.data
+        else:
+            jitted_method = (self._jit_instance.jitted_unmarshal_method
+                             if self._jit_instance else None)
+            _, result = self._transform(jitted_method,
+                                        self._unmarshal,
+                                        many, processed_data,
+                                        partial=partial)
             self._invoke_field_validators(data=result, many=many)
             errors = self._unmarshal.errors
             field_errors = bool(errors)
@@ -675,10 +782,17 @@ def _update_fields(self, obj=None, many=False):
         excludes = set(self.opts.exclude) | set(self.exclude)
         if excludes:
             field_names = field_names - excludes
-        ret = self.__filter_fields(field_names, obj, many=many)
+        ret, name_to_type = self.__filter_fields(field_names, obj, many=many)
         # Set parents
         self.__set_field_attrs(ret)
         self.fields = ret
+        # If this schema has a JIT attached and we've changed the types of
+        # the current set of fields we need to create a new JIT instance to
+        # recompile the backing code.
+        if self._jit_class and (not self._jit_instance or
+                                self._current_name_to_type != name_to_type):
+            self._jit_instance = self._jit_class(self)
+            self._current_name_to_type = name_to_type
         return self.fields
 
     def on_bind_field(self, field_name, field_obj):
@@ -717,10 +831,13 @@ def __filter_fields(self, field_names, obj, many=False):
 
         :param set field_names: Field names to include in the final
             return dictionary.
-        :returns: An dict of field_name:field_obj pairs.
+        :returns: A tuple of a dict of field_name:field_obj pairs and a dict of
+            field_name:type pairs.
         """
+        name_to_type = {}
         if obj and many:
-            try:  # Homogeneous collection
+            try:
+                # Homogeneous collection
                 # Prefer getitem over iter to prevent breaking serialization
                 # of objects for which iter will modify position in the collection
                 # e.g. Pymongo cursors
@@ -735,6 +852,7 @@ def __filter_fields(self, field_names, obj, many=False):
         for key in field_names:
             if key in self.declared_fields:
                 ret[key] = self.declared_fields[key]
+                name_to_type[key] = type(self.declared_fields[key])
             else:  # Implicit field creation (class Meta 'fields' or 'additional')
                 if obj:
                     attribute_type = None
@@ -748,11 +866,13 @@ def __filter_fields(self, field_names, obj, many=False):
                         raise err_type(
                             '"{0}" is not a valid field for {1}.'.format(key, obj))
                     field_obj = self.TYPE_MAPPING.get(attribute_type, fields.Field)()
+                    name_to_type[key] = attribute_type
                 else:  # Object is None
                     field_obj = fields.Field()
+                    name_to_type[key] = None
                 # map key -> field (default to Raw)
                 ret[key] = field_obj
-        return ret
+        return ret, name_to_type
 
     def _invoke_dump_processors(self, tag_name, data, many, original_data=None):
         # The pass_many post-dump processors may do things like add an envelope, so

marshmallow/utils.py

@@ -11,6 +11,7 @@
 import time
 import types
 from calendar import timegm
+from contextlib import contextmanager
 from decimal import Decimal, ROUND_HALF_EVEN, Context, Inexact
 from email.utils import formatdate, parsedate
 from pprint import pprint as py_pprint
@@ -394,3 +395,10 @@ def get_func_args(func):
 
 def if_none(value, default):
     return value if value is not None else default
+
+@contextmanager
+def suppress(*exceptions):
+    try:
+        yield
+    except exceptions:
+        pass