404 lines
15 KiB
Python
404 lines
15 KiB
Python
|
"""
|
|||
|
|
Created on Wed May 30 15:11:09 2018
|
|||
|
|
|
|||
|
|
@author: josef
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import numpy as np
|
|||
|
|
from scipy import stats
|
|||
|
|
|
|||
|
|
|
|||
|
|
# this is a copy from stats._diagnostic_other to avoid circular imports
|
|||
|
|
def _lm_robust(score, constraint_matrix, score_deriv_inv, cov_score,
|
|||
|
|
cov_params=None):
|
|||
|
|
'''general formula for score/LM test
|
|||
|
|
|
|||
|
|
generalized score or lagrange multiplier test for implicit constraints
|
|||
|
|
|
|||
|
|
`r(params) = 0`, with gradient `R = d r / d params`
|
|||
|
|
|
|||
|
|
linear constraints are given by `R params - q = 0`
|
|||
|
|
|
|||
|
|
It is assumed that all arrays are evaluated at the constrained estimates.
|
|||
|
|
|
|||
|
|
|
|||
|
|
Parameters
|
|||
|
|
----------
|
|||
|
|
score : ndarray, 1-D
|
|||
|
|
derivative of objective function at estimated parameters
|
|||
|
|
of constrained model
|
|||
|
|
constraint_matrix R : ndarray
|
|||
|
|
Linear restriction matrix or Jacobian of nonlinear constraints
|
|||
|
|
score_deriv_inv, Ainv : ndarray, symmetric, square
|
|||
|
|
inverse of second derivative of objective function
|
|||
|
|
TODO: could be inverse of OPG or any other estimator if information
|
|||
|
|
matrix equality holds
|
|||
|
|
cov_score B : ndarray, symmetric, square
|
|||
|
|
covariance matrix of the score. This is the inner part of a sandwich
|
|||
|
|
estimator.
|
|||
|
|
cov_params V : ndarray, symmetric, square
|
|||
|
|
covariance of full parameter vector evaluated at constrained parameter
|
|||
|
|
estimate. This can be specified instead of cov_score B.
|
|||
|
|
|
|||
|
|
Returns
|
|||
|
|
-------
|
|||
|
|
lm_stat : float
|
|||
|
|
score/lagrange multiplier statistic
|
|||
|
|
p-value : float
|
|||
|
|
p-value of the LM test based on chisquare distribution
|
|||
|
|
|
|||
|
|
Notes
|
|||
|
|
-----
|
|||
|
|
|
|||
|
|
'''
|
|||
|
|
# shorthand alias
|
|||
|
|
R, Ainv, B, V = constraint_matrix, score_deriv_inv, cov_score, cov_params
|
|||
|
|
|
|||
|
|
k_constraints = np.linalg.matrix_rank(R)
|
|||
|
|
tmp = R.dot(Ainv)
|
|||
|
|
wscore = tmp.dot(score) # C Ainv score
|
|||
|
|
|
|||
|
|
if B is None and V is None:
|
|||
|
|
# only Ainv is given, so we assume information matrix identity holds
|
|||
|
|
# computational short cut, should be same if Ainv == inv(B)
|
|||
|
|
lm_stat = score.dot(Ainv.dot(score))
|
|||
|
|
else:
|
|||
|
|
# information matrix identity does not hold
|
|||
|
|
if V is None:
|
|||
|
|
inner = tmp.dot(B).dot(tmp.T)
|
|||
|
|
else:
|
|||
|
|
inner = R.dot(V).dot(R.T)
|
|||
|
|
|
|||
|
|
#lm_stat2 = wscore.dot(np.linalg.pinv(inner).dot(wscore))
|
|||
|
|
# Let's assume inner is invertible, TODO: check if usecase for pinv exists
|
|||
|
|
lm_stat = wscore.dot(np.linalg.solve(inner, wscore))
|
|||
|
|
pval = stats.chi2.sf(lm_stat, k_constraints)
|
|||
|
|
return lm_stat, pval, k_constraints
|
|||
|
|
|
|||
|
|
|
|||
|
|
def score_test(self, exog_extra=None, params_constrained=None,
|
|||
|
|
hypothesis='joint', cov_type=None, cov_kwds=None,
|
|||
|
|
k_constraints=None, r_matrix=None, scale=None, observed=True):
|
|||
|
|
"""score test for restrictions or for omitted variables
|
|||
|
|
|
|||
|
|
Null Hypothesis : constraints are satisfied
|
|||
|
|
|
|||
|
|
Alternative Hypothesis : at least one of the constraints does not hold
|
|||
|
|
|
|||
|
|
This allows to specify restricted and unrestricted model properties in
|
|||
|
|
three different ways
|
|||
|
|
|
|||
|
|
- fit_constrained result: model contains score and hessian function for
|
|||
|
|
the full, unrestricted model, but the parameter estimate in the results
|
|||
|
|
instance is for the restricted model. This is the case if the model
|
|||
|
|
was estimated with fit_constrained.
|
|||
|
|
- restricted model with variable addition: If exog_extra is not None, then
|
|||
|
|
it is assumed that the current model is a model with zero restrictions
|
|||
|
|
and the unrestricted model is given by adding exog_extra as additional
|
|||
|
|
explanatory variables.
|
|||
|
|
- unrestricted model with restricted parameters explicitly provided. If
|
|||
|
|
params_constrained is not None, then the model is assumed to be for the
|
|||
|
|
unrestricted model, but the provided parameters are for the restricted
|
|||
|
|
model.
|
|||
|
|
TODO: This case will currently only work for `nonrobust` cov_type,
|
|||
|
|
otherwise we will also need the restriction matrix provided by the user.
|
|||
|
|
|
|||
|
|
|
|||
|
|
Parameters
|
|||
|
|
----------
|
|||
|
|
exog_extra : None or array_like
|
|||
|
|
Explanatory variables that are jointly tested for inclusion in the
|
|||
|
|
model, i.e. omitted variables.
|
|||
|
|
params_constrained : array_like
|
|||
|
|
estimated parameter of the restricted model. This can be the
|
|||
|
|
parameter estimate for the current when testing for omitted
|
|||
|
|
variables.
|
|||
|
|
hypothesis : str, 'joint' (default) or 'separate'
|
|||
|
|
If hypothesis is 'joint', then the chisquare test results for the
|
|||
|
|
joint hypothesis that all constraints hold is returned.
|
|||
|
|
If hypothesis is 'joint', then z-test results for each constraint
|
|||
|
|
is returned.
|
|||
|
|
This is currently only implemented for cov_type="nonrobust".
|
|||
|
|
cov_type : str
|
|||
|
|
Warning: only partially implemented so far, currently only "nonrobust"
|
|||
|
|
and "HC0" are supported.
|
|||
|
|
If cov_type is None, then the cov_type specified in fit for the Wald
|
|||
|
|
tests is used.
|
|||
|
|
If the cov_type argument is not None, then it will be used instead of
|
|||
|
|
the Wald cov_type given in fit.
|
|||
|
|
k_constraints : int or None
|
|||
|
|
Number of constraints that were used in the estimation of params
|
|||
|
|
restricted relative to the number of exog in the model.
|
|||
|
|
This must be provided if no exog_extra are given. If exog_extra is
|
|||
|
|
not None, then k_constraints is assumed to be zero if it is None.
|
|||
|
|
observed : bool
|
|||
|
|
If True, then the observed Hessian is used in calculating the
|
|||
|
|
covariance matrix of the score. If false then the expected
|
|||
|
|
information matrix is used. This currently only applies to GLM where
|
|||
|
|
EIM is available.
|
|||
|
|
Warning: This option might still change.
|
|||
|
|
|
|||
|
|
Returns
|
|||
|
|
-------
|
|||
|
|
chi2_stat : float
|
|||
|
|
chisquare statistic for the score test
|
|||
|
|
p-value : float
|
|||
|
|
P-value of the score test based on the chisquare distribution.
|
|||
|
|
df : int
|
|||
|
|
Degrees of freedom used in the p-value calculation. This is equal
|
|||
|
|
to the number of constraints.
|
|||
|
|
|
|||
|
|
Notes
|
|||
|
|
-----
|
|||
|
|
Status: experimental, several options are not implemented yet or are not
|
|||
|
|
verified yet. Currently available ptions might also still change.
|
|||
|
|
|
|||
|
|
cov_type is 'nonrobust':
|
|||
|
|
|
|||
|
|
The covariance matrix for the score is based on the Hessian, i.e.
|
|||
|
|
observed information matrix or optionally on the expected information
|
|||
|
|
matrix.
|
|||
|
|
|
|||
|
|
cov_type is 'HC0'
|
|||
|
|
|
|||
|
|
The covariance matrix of the score is the simple empirical covariance of
|
|||
|
|
score_obs without degrees of freedom correction.
|
|||
|
|
"""
|
|||
|
|
# TODO: we are computing unnecessary things for cov_type nonrobust
|
|||
|
|
if hasattr(self, "_results"):
|
|||
|
|
# use numpy if we have wrapper, not relevant if method
|
|||
|
|
self = self._results
|
|||
|
|
model = self.model
|
|||
|
|
nobs = model.endog.shape[0] # model.nobs
|
|||
|
|
# discrete Poisson does not have nobs
|
|||
|
|
if params_constrained is None:
|
|||
|
|
params_constrained = self.params
|
|||
|
|
cov_type = cov_type if cov_type is not None else self.cov_type
|
|||
|
|
|
|||
|
|
if observed is False:
|
|||
|
|
hess_kwd = {'observed': False}
|
|||
|
|
else:
|
|||
|
|
hess_kwd = {}
|
|||
|
|
|
|||
|
|
if exog_extra is None:
|
|||
|
|
|
|||
|
|
if hasattr(self, 'constraints'):
|
|||
|
|
if isinstance(self.constraints, tuple):
|
|||
|
|
r_matrix = self.constraints[0]
|
|||
|
|
else:
|
|||
|
|
r_matrix = self.constraints.coefs
|
|||
|
|
k_constraints = r_matrix.shape[0]
|
|||
|
|
|
|||
|
|
else:
|
|||
|
|
if k_constraints is None:
|
|||
|
|
raise ValueError('if exog_extra is None, then k_constraints'
|
|||
|
|
'needs to be given')
|
|||
|
|
|
|||
|
|
# we need to use results scale as additional parameter
|
|||
|
|
if scale is not None:
|
|||
|
|
# we need to use results scale as additional parameter, gh #7840
|
|||
|
|
score_kwd = {'scale': scale}
|
|||
|
|
hess_kwd['scale'] = scale
|
|||
|
|
else:
|
|||
|
|
score_kwd = {}
|
|||
|
|
|
|||
|
|
# duplicate computation of score, might not be needed
|
|||
|
|
score = model.score(params_constrained, **score_kwd)
|
|||
|
|
score_obs = model.score_obs(params_constrained, **score_kwd)
|
|||
|
|
hessian = model.hessian(params_constrained, **hess_kwd)
|
|||
|
|
|
|||
|
|
else:
|
|||
|
|
if cov_type == 'V':
|
|||
|
|
raise ValueError('if exog_extra is not None, then cov_type cannot '
|
|||
|
|
'be V')
|
|||
|
|
if hasattr(self, 'constraints'):
|
|||
|
|
raise NotImplementedError('if exog_extra is not None, then self'
|
|||
|
|
'should not be a constrained fit result')
|
|||
|
|
|
|||
|
|
if isinstance(exog_extra, tuple):
|
|||
|
|
sh = _scorehess_extra(self, params_constrained, *exog_extra,
|
|||
|
|
hess_kwds=hess_kwd)
|
|||
|
|
score_obs, hessian, k_constraints, r_matrix = sh
|
|||
|
|
score = score_obs.sum(0)
|
|||
|
|
else:
|
|||
|
|
exog_extra = np.asarray(exog_extra)
|
|||
|
|
k_constraints = 0
|
|||
|
|
ex = np.column_stack((model.exog, exog_extra))
|
|||
|
|
# this uses shape not matrix rank to determine k_constraints
|
|||
|
|
# requires nonsingular (no added perfect collinearity)
|
|||
|
|
k_constraints += ex.shape[1] - model.exog.shape[1]
|
|||
|
|
# TODO use diag instead of full np.eye
|
|||
|
|
r_matrix = np.eye(len(self.params) + k_constraints
|
|||
|
|
)[-k_constraints:]
|
|||
|
|
|
|||
|
|
score_factor = model.score_factor(params_constrained)
|
|||
|
|
if score_factor.ndim == 1:
|
|||
|
|
score_obs = (score_factor[:, None] * ex)
|
|||
|
|
else:
|
|||
|
|
sf = score_factor
|
|||
|
|
score_obs = np.column_stack((sf[:, :1] * ex, sf[:, 1:]))
|
|||
|
|
score = score_obs.sum(0)
|
|||
|
|
hessian_factor = model.hessian_factor(params_constrained,
|
|||
|
|
**hess_kwd)
|
|||
|
|
# see #4714
|
|||
|
|
from statsmodels.genmod.generalized_linear_model import GLM
|
|||
|
|
if isinstance(model, GLM):
|
|||
|
|
hessian_factor *= -1
|
|||
|
|
hessian = np.dot(ex.T * hessian_factor, ex)
|
|||
|
|
|
|||
|
|
if cov_type == 'nonrobust':
|
|||
|
|
cov_score_test = -hessian
|
|||
|
|
elif cov_type.upper() == 'HC0':
|
|||
|
|
hinv = -np.linalg.inv(hessian)
|
|||
|
|
cov_score = nobs * np.cov(score_obs.T)
|
|||
|
|
# temporary to try out
|
|||
|
|
lm = _lm_robust(score, r_matrix, hinv, cov_score, cov_params=None)
|
|||
|
|
return lm
|
|||
|
|
# alternative is to use only the center, but it is singular
|
|||
|
|
# https://github.com/statsmodels/statsmodels/pull/2096#issuecomment-393646205
|
|||
|
|
# cov_score_test_inv = cov_lm_robust(score, r_matrix, hinv,
|
|||
|
|
# cov_score, cov_params=None)
|
|||
|
|
elif cov_type.upper() == 'V':
|
|||
|
|
# TODO: this does not work, V in fit_constrained results is singular
|
|||
|
|
# we need cov_params without the zeros in it
|
|||
|
|
hinv = -np.linalg.inv(hessian)
|
|||
|
|
cov_score = nobs * np.cov(score_obs.T)
|
|||
|
|
V = self.cov_params_default
|
|||
|
|
# temporary to try out
|
|||
|
|
chi2stat = _lm_robust(score, r_matrix, hinv, cov_score, cov_params=V)
|
|||
|
|
pval = stats.chi2.sf(chi2stat, k_constraints)
|
|||
|
|
return chi2stat, pval
|
|||
|
|
else:
|
|||
|
|
msg = 'Only cov_type "nonrobust" and "HC0" are available.'
|
|||
|
|
raise NotImplementedError(msg)
|
|||
|
|
|
|||
|
|
if hypothesis == 'joint':
|
|||
|
|
chi2stat = score.dot(np.linalg.solve(cov_score_test, score[:, None]))
|
|||
|
|
pval = stats.chi2.sf(chi2stat, k_constraints)
|
|||
|
|
# return a stats results instance instead? Contrast?
|
|||
|
|
return chi2stat, pval, k_constraints
|
|||
|
|
elif hypothesis == 'separate':
|
|||
|
|
diff = score
|
|||
|
|
bse = np.sqrt(np.diag(cov_score_test))
|
|||
|
|
stat = diff / bse
|
|||
|
|
pval = stats.norm.sf(np.abs(stat))*2
|
|||
|
|
return stat, pval
|
|||
|
|
else:
|
|||
|
|
raise NotImplementedError('only hypothesis "joint" is available')
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _scorehess_extra(self, params=None, exog_extra=None,
|
|||
|
|
exog2_extra=None, hess_kwds=None):
|
|||
|
|
"""Experimental helper function for variable addition score test.
|
|||
|
|
|
|||
|
|
This uses score and hessian factor at the params which should be the
|
|||
|
|
params of the restricted model.
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
if hess_kwds is None:
|
|||
|
|
hess_kwds = {}
|
|||
|
|
# this corresponds to a model methods, so we need only the model
|
|||
|
|
model = self.model
|
|||
|
|
# as long as we have results instance, we can take params from it
|
|||
|
|
if params is None:
|
|||
|
|
params = self.params
|
|||
|
|
|
|||
|
|
# get original exog from model, currently only if exactly 2
|
|||
|
|
exog_o1, exog_o2 = model._get_exogs()
|
|||
|
|
|
|||
|
|
if exog_o2 is None:
|
|||
|
|
# if extra params is scalar, as in NB, GPP
|
|||
|
|
exog_o2 = np.ones((exog_o1.shape[0], 1))
|
|||
|
|
|
|||
|
|
k_mean = exog_o1.shape[1]
|
|||
|
|
k_prec = exog_o2.shape[1]
|
|||
|
|
if exog_extra is not None:
|
|||
|
|
exog = np.column_stack((exog_o1, exog_extra))
|
|||
|
|
else:
|
|||
|
|
exog = exog_o1
|
|||
|
|
|
|||
|
|
if exog2_extra is not None:
|
|||
|
|
exog2 = np.column_stack((exog_o2, exog2_extra))
|
|||
|
|
else:
|
|||
|
|
exog2 = exog_o2
|
|||
|
|
|
|||
|
|
k_mean_new = exog.shape[1]
|
|||
|
|
k_prec_new = exog2.shape[1]
|
|||
|
|
k_cm = k_mean_new - k_mean
|
|||
|
|
k_cp = k_prec_new - k_prec
|
|||
|
|
k_constraints = k_cm + k_cp
|
|||
|
|
|
|||
|
|
index_mean = np.arange(k_mean, k_mean_new)
|
|||
|
|
index_prec = np.arange(k_mean_new + k_prec, k_mean_new + k_prec_new)
|
|||
|
|
|
|||
|
|
r_matrix = np.zeros((k_constraints, len(params) + k_constraints))
|
|||
|
|
# print(exog.shape, exog2.shape)
|
|||
|
|
# print(r_matrix.shape, k_cm, k_cp, k_mean_new, k_prec_new)
|
|||
|
|
# print(index_mean, index_prec)
|
|||
|
|
r_matrix[:k_cm, index_mean] = np.eye(k_cm)
|
|||
|
|
r_matrix[k_cm: k_cm + k_cp, index_prec] = np.eye(k_cp)
|
|||
|
|
|
|||
|
|
if hasattr(model, "score_hessian_factor"):
|
|||
|
|
sf, hf = model.score_hessian_factor(params, return_hessian=True,
|
|||
|
|
**hess_kwds)
|
|||
|
|
else:
|
|||
|
|
sf = model.score_factor(params)
|
|||
|
|
hf = model.hessian_factor(params, **hess_kwds)
|
|||
|
|
|
|||
|
|
sf1, sf2 = sf
|
|||
|
|
hf11, hf12, hf22 = hf
|
|||
|
|
|
|||
|
|
# elementwise product for each row (observation)
|
|||
|
|
d1 = sf1[:, None] * exog
|
|||
|
|
d2 = sf2[:, None] * exog2
|
|||
|
|
score_obs = np.column_stack((d1, d2))
|
|||
|
|
|
|||
|
|
# elementwise product for each row (observation)
|
|||
|
|
d11 = (exog.T * hf11).dot(exog)
|
|||
|
|
d12 = (exog.T * hf12).dot(exog2)
|
|||
|
|
d22 = (exog2.T * hf22).dot(exog2)
|
|||
|
|
hessian = np.block([[d11, d12], [d12.T, d22]])
|
|||
|
|
return score_obs, hessian, k_constraints, r_matrix
|
|||
|
|
|
|||
|
|
|
|||
|
|
def im_ratio(results):
|
|||
|
|
res = getattr(results, "_results", results) # shortcut
|
|||
|
|
hess = res.model.hessian(res.params)
|
|||
|
|
if res.cov_type == "nonrobust":
|
|||
|
|
score_obs = res.model.score_obs(res.params)
|
|||
|
|
cov_score = score_obs.T @ score_obs
|
|||
|
|
hessneg_inv = np.linalg.inv(-hess)
|
|||
|
|
im_ratio = hessneg_inv @ cov_score
|
|||
|
|
else:
|
|||
|
|
im_ratio = res.cov_params() @ (-hess)
|
|||
|
|
return im_ratio
|
|||
|
|
|
|||
|
|
|
|||
|
|
def tic(results):
|
|||
|
|
"""Takeuchi information criterion for misspecified models
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
imr = getattr(results, "im_ratio", im_ratio(results))
|
|||
|
|
tic = - 2 * results.llf + 2 * np.trace(imr)
|
|||
|
|
return tic
|
|||
|
|
|
|||
|
|
|
|||
|
|
def gbic(results, gbicp=False):
|
|||
|
|
"""generalized BIC for misspecified models
|
|||
|
|
|
|||
|
|
References
|
|||
|
|
----------
|
|||
|
|
Lv, Jinchi, and Jun S. Liu. 2014. "Model Selection Principles in
|
|||
|
|
Misspecified Models." Journal of the Royal Statistical Society.
|
|||
|
|
Series B (Statistical Methodology) 76 (1): 141–67.
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
self = getattr(results, "_results", results)
|
|||
|
|
k_params = self.df_model + 1
|
|||
|
|
nobs = k_params + self.df_resid
|
|||
|
|
imr = getattr(results, "im_ratio", im_ratio(results))
|
|||
|
|
imr_logdet = np.linalg.slogdet(imr)[1]
|
|||
|
|
gbic = -2 * self.llf + k_params * np.log(nobs) - imr_logdet # LL equ. (20)
|
|||
|
|
gbicp = gbic + np.trace(imr) # LL equ. (23)
|
|||
|
|
return gbic, gbicp
|