314 lines
14 KiB
Python
314 lines
14 KiB
Python
|
# This file is part of Patsy
|
||
|
|
# Copyright (C) 2011-2013 Nathaniel Smith <njs@pobox.com>
|
||
|
|
# See file LICENSE.txt for license information.
|
||
|
|
|
||
|
|
# These are made available in the patsy.* namespace:
|
||
|
|
__all__ = ["dmatrix", "dmatrices",
|
||
|
|
"incr_dbuilder", "incr_dbuilders"]
|
||
|
|
|
||
|
|
# problems:
|
||
|
|
# statsmodels reluctant to pass around separate eval environment, suggesting
|
||
|
|
# that design_and_matrices-equivalent should return a formula_like
|
||
|
|
# is ModelDesc really the high-level thing?
|
||
|
|
# ModelDesign doesn't work -- need to work with the builder set
|
||
|
|
# want to be able to return either a matrix or a pandas dataframe
|
||
|
|
|
||
|
|
import six
|
||
|
|
import numpy as np
|
||
|
|
from patsy import PatsyError
|
||
|
|
from patsy.design_info import DesignMatrix, DesignInfo
|
||
|
|
from patsy.eval import EvalEnvironment
|
||
|
|
from patsy.desc import ModelDesc
|
||
|
|
from patsy.build import (design_matrix_builders,
|
||
|
|
build_design_matrices)
|
||
|
|
from patsy.util import (have_pandas, asarray_or_pandas,
|
||
|
|
atleast_2d_column_default)
|
||
|
|
|
||
|
|
if have_pandas:
|
||
|
|
import pandas
|
||
|
|
|
||
|
|
# Tries to build a (lhs, rhs) design given a formula_like and an incremental
|
||
|
|
# data source. If formula_like is not capable of doing this, then returns
|
||
|
|
# None.
|
||
|
|
def _try_incr_builders(formula_like, data_iter_maker, eval_env,
|
||
|
|
NA_action):
|
||
|
|
if isinstance(formula_like, DesignInfo):
|
||
|
|
return (design_matrix_builders([[]], data_iter_maker, eval_env, NA_action)[0],
|
||
|
|
formula_like)
|
||
|
|
if (isinstance(formula_like, tuple)
|
||
|
|
and len(formula_like) == 2
|
||
|
|
and isinstance(formula_like[0], DesignInfo)
|
||
|
|
and isinstance(formula_like[1], DesignInfo)):
|
||
|
|
return formula_like
|
||
|
|
if hasattr(formula_like, "__patsy_get_model_desc__"):
|
||
|
|
formula_like = formula_like.__patsy_get_model_desc__(eval_env)
|
||
|
|
if not isinstance(formula_like, ModelDesc):
|
||
|
|
raise PatsyError("bad value from %r.__patsy_get_model_desc__"
|
||
|
|
% (formula_like,))
|
||
|
|
# fallthrough
|
||
|
|
if not six.PY3 and isinstance(formula_like, unicode):
|
||
|
|
# Included for the convenience of people who are using py2 with
|
||
|
|
# __future__.unicode_literals.
|
||
|
|
try:
|
||
|
|
formula_like = formula_like.encode("ascii")
|
||
|
|
except UnicodeEncodeError:
|
||
|
|
raise PatsyError(
|
||
|
|
"On Python 2, formula strings must be either 'str' objects, "
|
||
|
|
"or else 'unicode' objects containing only ascii "
|
||
|
|
"characters. You passed a unicode string with non-ascii "
|
||
|
|
"characters. I'm afraid you'll have to either switch to "
|
||
|
|
"ascii-only, or else upgrade to Python 3.")
|
||
|
|
if isinstance(formula_like, str):
|
||
|
|
formula_like = ModelDesc.from_formula(formula_like)
|
||
|
|
# fallthrough
|
||
|
|
if isinstance(formula_like, ModelDesc):
|
||
|
|
assert isinstance(eval_env, EvalEnvironment)
|
||
|
|
return design_matrix_builders([formula_like.lhs_termlist,
|
||
|
|
formula_like.rhs_termlist],
|
||
|
|
data_iter_maker,
|
||
|
|
eval_env,
|
||
|
|
NA_action)
|
||
|
|
else:
|
||
|
|
return None
|
||
|
|
|
||
|
|
def incr_dbuilder(formula_like, data_iter_maker, eval_env=0, NA_action="drop"):
|
||
|
|
"""Construct a design matrix builder incrementally from a large data set.
|
||
|
|
|
||
|
|
:arg formula_like: Similar to :func:`dmatrix`, except that explicit
|
||
|
|
matrices are not allowed. Must be a formula string, a
|
||
|
|
:class:`ModelDesc`, a :class:`DesignInfo`, or an object with a
|
||
|
|
``__patsy_get_model_desc__`` method.
|
||
|
|
:arg data_iter_maker: A zero-argument callable which returns an iterator
|
||
|
|
over dict-like data objects. This must be a callable rather than a
|
||
|
|
simple iterator because sufficiently complex formulas may require
|
||
|
|
multiple passes over the data (e.g. if there are nested stateful
|
||
|
|
transforms).
|
||
|
|
:arg eval_env: Either a :class:`EvalEnvironment` which will be used to
|
||
|
|
look up any variables referenced in `formula_like` that cannot be
|
||
|
|
found in `data`, or else a depth represented as an
|
||
|
|
integer which will be passed to :meth:`EvalEnvironment.capture`.
|
||
|
|
``eval_env=0`` means to use the context of the function calling
|
||
|
|
:func:`incr_dbuilder` for lookups. If calling this function from a
|
||
|
|
library, you probably want ``eval_env=1``, which means that variables
|
||
|
|
should be resolved in *your* caller's namespace.
|
||
|
|
:arg NA_action: An :class:`NAAction` object or string, used to determine
|
||
|
|
what values count as 'missing' for purposes of determining the levels of
|
||
|
|
categorical factors.
|
||
|
|
:returns: A :class:`DesignInfo`
|
||
|
|
|
||
|
|
Tip: for `data_iter_maker`, write a generator like::
|
||
|
|
|
||
|
|
def iter_maker():
|
||
|
|
for data_chunk in my_data_store:
|
||
|
|
yield data_chunk
|
||
|
|
|
||
|
|
and pass `iter_maker` (*not* `iter_maker()`).
|
||
|
|
|
||
|
|
.. versionadded:: 0.2.0
|
||
|
|
The ``NA_action`` argument.
|
||
|
|
"""
|
||
|
|
eval_env = EvalEnvironment.capture(eval_env, reference=1)
|
||
|
|
design_infos = _try_incr_builders(formula_like, data_iter_maker, eval_env,
|
||
|
|
NA_action)
|
||
|
|
if design_infos is None:
|
||
|
|
raise PatsyError("bad formula-like object")
|
||
|
|
if len(design_infos[0].column_names) > 0:
|
||
|
|
raise PatsyError("encountered outcome variables for a model "
|
||
|
|
"that does not expect them")
|
||
|
|
return design_infos[1]
|
||
|
|
|
||
|
|
def incr_dbuilders(formula_like, data_iter_maker, eval_env=0,
|
||
|
|
NA_action="drop"):
|
||
|
|
"""Construct two design matrix builders incrementally from a large data
|
||
|
|
set.
|
||
|
|
|
||
|
|
:func:`incr_dbuilders` is to :func:`incr_dbuilder` as :func:`dmatrices` is
|
||
|
|
to :func:`dmatrix`. See :func:`incr_dbuilder` for details.
|
||
|
|
"""
|
||
|
|
eval_env = EvalEnvironment.capture(eval_env, reference=1)
|
||
|
|
design_infos = _try_incr_builders(formula_like, data_iter_maker, eval_env,
|
||
|
|
NA_action)
|
||
|
|
if design_infos is None:
|
||
|
|
raise PatsyError("bad formula-like object")
|
||
|
|
if len(design_infos[0].column_names) == 0:
|
||
|
|
raise PatsyError("model is missing required outcome variables")
|
||
|
|
return design_infos
|
||
|
|
|
||
|
|
# This always returns a length-two tuple,
|
||
|
|
# response, predictors
|
||
|
|
# where
|
||
|
|
# response is a DesignMatrix (possibly with 0 columns)
|
||
|
|
# predictors is a DesignMatrix
|
||
|
|
# The input 'formula_like' could be like:
|
||
|
|
# (np.ndarray, np.ndarray)
|
||
|
|
# (DesignMatrix, DesignMatrix)
|
||
|
|
# (None, DesignMatrix)
|
||
|
|
# np.ndarray # for predictor-only models
|
||
|
|
# DesignMatrix
|
||
|
|
# (None, np.ndarray)
|
||
|
|
# "y ~ x"
|
||
|
|
# ModelDesc(...)
|
||
|
|
# DesignInfo
|
||
|
|
# (DesignInfo, DesignInfo)
|
||
|
|
# any object with a special method __patsy_get_model_desc__
|
||
|
|
def _do_highlevel_design(formula_like, data, eval_env,
|
||
|
|
NA_action, return_type):
|
||
|
|
if return_type == "dataframe" and not have_pandas:
|
||
|
|
raise PatsyError("pandas.DataFrame was requested, but pandas "
|
||
|
|
"is not installed")
|
||
|
|
if return_type not in ("matrix", "dataframe"):
|
||
|
|
raise PatsyError("unrecognized output type %r, should be "
|
||
|
|
"'matrix' or 'dataframe'" % (return_type,))
|
||
|
|
def data_iter_maker():
|
||
|
|
return iter([data])
|
||
|
|
design_infos = _try_incr_builders(formula_like, data_iter_maker, eval_env,
|
||
|
|
NA_action)
|
||
|
|
if design_infos is not None:
|
||
|
|
return build_design_matrices(design_infos, data,
|
||
|
|
NA_action=NA_action,
|
||
|
|
return_type=return_type)
|
||
|
|
else:
|
||
|
|
# No builders, but maybe we can still get matrices
|
||
|
|
if isinstance(formula_like, tuple):
|
||
|
|
if len(formula_like) != 2:
|
||
|
|
raise PatsyError("don't know what to do with a length %s "
|
||
|
|
"matrices tuple"
|
||
|
|
% (len(formula_like),))
|
||
|
|
(lhs, rhs) = formula_like
|
||
|
|
else:
|
||
|
|
# subok=True is necessary here to allow DesignMatrixes to pass
|
||
|
|
# through
|
||
|
|
(lhs, rhs) = (None, asarray_or_pandas(formula_like, subok=True))
|
||
|
|
# some sort of explicit matrix or matrices were given. Currently we
|
||
|
|
# have them in one of these forms:
|
||
|
|
# -- an ndarray or subclass
|
||
|
|
# -- a DesignMatrix
|
||
|
|
# -- a pandas.Series
|
||
|
|
# -- a pandas.DataFrame
|
||
|
|
# and we have to produce a standard output format.
|
||
|
|
def _regularize_matrix(m, default_column_prefix):
|
||
|
|
di = DesignInfo.from_array(m, default_column_prefix)
|
||
|
|
if have_pandas and isinstance(m, (pandas.Series, pandas.DataFrame)):
|
||
|
|
orig_index = m.index
|
||
|
|
else:
|
||
|
|
orig_index = None
|
||
|
|
if return_type == "dataframe":
|
||
|
|
m = atleast_2d_column_default(m, preserve_pandas=True)
|
||
|
|
m = pandas.DataFrame(m)
|
||
|
|
m.columns = di.column_names
|
||
|
|
m.design_info = di
|
||
|
|
return (m, orig_index)
|
||
|
|
else:
|
||
|
|
return (DesignMatrix(m, di), orig_index)
|
||
|
|
rhs, rhs_orig_index = _regularize_matrix(rhs, "x")
|
||
|
|
if lhs is None:
|
||
|
|
lhs = np.zeros((rhs.shape[0], 0), dtype=float)
|
||
|
|
lhs, lhs_orig_index = _regularize_matrix(lhs, "y")
|
||
|
|
|
||
|
|
assert isinstance(getattr(lhs, "design_info", None), DesignInfo)
|
||
|
|
assert isinstance(getattr(rhs, "design_info", None), DesignInfo)
|
||
|
|
if lhs.shape[0] != rhs.shape[0]:
|
||
|
|
raise PatsyError("shape mismatch: outcome matrix has %s rows, "
|
||
|
|
"predictor matrix has %s rows"
|
||
|
|
% (lhs.shape[0], rhs.shape[0]))
|
||
|
|
if rhs_orig_index is not None and lhs_orig_index is not None:
|
||
|
|
if not rhs_orig_index.equals(lhs_orig_index):
|
||
|
|
raise PatsyError("index mismatch: outcome and "
|
||
|
|
"predictor have incompatible indexes")
|
||
|
|
if return_type == "dataframe":
|
||
|
|
if rhs_orig_index is not None and lhs_orig_index is None:
|
||
|
|
lhs.index = rhs.index
|
||
|
|
if rhs_orig_index is None and lhs_orig_index is not None:
|
||
|
|
rhs.index = lhs.index
|
||
|
|
return (lhs, rhs)
|
||
|
|
|
||
|
|
def dmatrix(formula_like, data={}, eval_env=0,
|
||
|
|
NA_action="drop", return_type="matrix"):
|
||
|
|
"""Construct a single design matrix given a formula_like and data.
|
||
|
|
|
||
|
|
:arg formula_like: An object that can be used to construct a design
|
||
|
|
matrix. See below.
|
||
|
|
:arg data: A dict-like object that can be used to look up variables
|
||
|
|
referenced in `formula_like`.
|
||
|
|
:arg eval_env: Either a :class:`EvalEnvironment` which will be used to
|
||
|
|
look up any variables referenced in `formula_like` that cannot be
|
||
|
|
found in `data`, or else a depth represented as an
|
||
|
|
integer which will be passed to :meth:`EvalEnvironment.capture`.
|
||
|
|
``eval_env=0`` means to use the context of the function calling
|
||
|
|
:func:`dmatrix` for lookups. If calling this function from a library,
|
||
|
|
you probably want ``eval_env=1``, which means that variables should be
|
||
|
|
resolved in *your* caller's namespace.
|
||
|
|
:arg NA_action: What to do with rows that contain missing values. You can
|
||
|
|
``"drop"`` them, ``"raise"`` an error, or for customization, pass an
|
||
|
|
:class:`NAAction` object. See :class:`NAAction` for details on what
|
||
|
|
values count as 'missing' (and how to alter this).
|
||
|
|
:arg return_type: Either ``"matrix"`` or ``"dataframe"``. See below.
|
||
|
|
|
||
|
|
The `formula_like` can take a variety of forms. You can use any of the
|
||
|
|
following:
|
||
|
|
|
||
|
|
* (The most common option) A formula string like ``"x1 + x2"`` (for
|
||
|
|
:func:`dmatrix`) or ``"y ~ x1 + x2"`` (for :func:`dmatrices`). For
|
||
|
|
details see :ref:`formulas`.
|
||
|
|
* A :class:`ModelDesc`, which is a Python object representation of a
|
||
|
|
formula. See :ref:`formulas` and :ref:`expert-model-specification` for
|
||
|
|
details.
|
||
|
|
* A :class:`DesignInfo`.
|
||
|
|
* An object that has a method called :meth:`__patsy_get_model_desc__`.
|
||
|
|
For details see :ref:`expert-model-specification`.
|
||
|
|
* A numpy array_like (for :func:`dmatrix`) or a tuple
|
||
|
|
(array_like, array_like) (for :func:`dmatrices`). These will have
|
||
|
|
metadata added, representation normalized, and then be returned
|
||
|
|
directly. In this case `data` and `eval_env` are
|
||
|
|
ignored. There is special handling for two cases:
|
||
|
|
|
||
|
|
* :class:`DesignMatrix` objects will have their :class:`DesignInfo`
|
||
|
|
preserved. This allows you to set up custom column names and term
|
||
|
|
information even if you aren't using the rest of the patsy
|
||
|
|
machinery.
|
||
|
|
* :class:`pandas.DataFrame` or :class:`pandas.Series` objects will have
|
||
|
|
their (row) indexes checked. If two are passed in, their indexes must
|
||
|
|
be aligned. If ``return_type="dataframe"``, then their indexes will be
|
||
|
|
preserved on the output.
|
||
|
|
|
||
|
|
Regardless of the input, the return type is always either:
|
||
|
|
|
||
|
|
* A :class:`DesignMatrix`, if ``return_type="matrix"`` (the default)
|
||
|
|
* A :class:`pandas.DataFrame`, if ``return_type="dataframe"``.
|
||
|
|
|
||
|
|
The actual contents of the design matrix is identical in both cases, and
|
||
|
|
in both cases a :class:`DesignInfo` object will be available in a
|
||
|
|
``.design_info`` attribute on the return value. However, for
|
||
|
|
``return_type="dataframe"``, any pandas indexes on the input (either in
|
||
|
|
`data` or directly passed through `formula_like`) will be preserved, which
|
||
|
|
may be useful for e.g. time-series models.
|
||
|
|
|
||
|
|
.. versionadded:: 0.2.0
|
||
|
|
The ``NA_action`` argument.
|
||
|
|
"""
|
||
|
|
eval_env = EvalEnvironment.capture(eval_env, reference=1)
|
||
|
|
(lhs, rhs) = _do_highlevel_design(formula_like, data, eval_env,
|
||
|
|
NA_action, return_type)
|
||
|
|
if lhs.shape[1] != 0:
|
||
|
|
raise PatsyError("encountered outcome variables for a model "
|
||
|
|
"that does not expect them")
|
||
|
|
return rhs
|
||
|
|
|
||
|
|
def dmatrices(formula_like, data={}, eval_env=0,
|
||
|
|
NA_action="drop", return_type="matrix"):
|
||
|
|
"""Construct two design matrices given a formula_like and data.
|
||
|
|
|
||
|
|
This function is identical to :func:`dmatrix`, except that it requires
|
||
|
|
(and returns) two matrices instead of one. By convention, the first matrix
|
||
|
|
is the "outcome" or "y" data, and the second is the "predictor" or "x"
|
||
|
|
data.
|
||
|
|
|
||
|
|
See :func:`dmatrix` for details.
|
||
|
|
"""
|
||
|
|
eval_env = EvalEnvironment.capture(eval_env, reference=1)
|
||
|
|
(lhs, rhs) = _do_highlevel_design(formula_like, data, eval_env,
|
||
|
|
NA_action, return_type)
|
||
|
|
if lhs.shape[1] == 0:
|
||
|
|
raise PatsyError("model is missing required outcome variables")
|
||
|
|
return (lhs, rhs)
|