some new features
This commit is contained in:
ilgazca
@ -0,0 +1,832 @@
|
||||
# pylint: disable=W0231, W0142
|
||||
"""Tests for statistical power calculations
|
||||
|
||||
Note:
|
||||
tests for chisquare power are in test_gof.py
|
||||
|
||||
Created on Sat Mar 09 08:44:49 2013
|
||||
|
||||
Author: Josef Perktold
|
||||
"""
|
||||
import copy
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
from numpy.testing import (assert_almost_equal, assert_allclose, assert_raises,
|
||||
assert_equal, assert_warns, assert_array_equal)
|
||||
import pytest
|
||||
|
||||
import statsmodels.stats.power as smp
|
||||
from statsmodels.stats.tests.test_weightstats import Holder
|
||||
from statsmodels.tools.sm_exceptions import HypothesisTestWarning
|
||||
|
||||
try:
|
||||
import matplotlib.pyplot as plt # noqa:F401
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class CheckPowerMixin:
|
||||
|
||||
def test_power(self):
|
||||
#test against R results
|
||||
kwds = copy.copy(self.kwds)
|
||||
del kwds['power']
|
||||
kwds.update(self.kwds_extra)
|
||||
if hasattr(self, 'decimal'):
|
||||
decimal = self.decimal
|
||||
else:
|
||||
decimal = 6
|
||||
res1 = self.cls()
|
||||
assert_almost_equal(res1.power(**kwds), self.res2.power, decimal=decimal)
|
||||
|
||||
#@pytest.mark.xfail(strict=True)
|
||||
def test_positional(self):
|
||||
|
||||
res1 = self.cls()
|
||||
|
||||
|
||||
kwds = copy.copy(self.kwds)
|
||||
del kwds['power']
|
||||
kwds.update(self.kwds_extra)
|
||||
|
||||
# positional args
|
||||
if hasattr(self, 'args_names'):
|
||||
args_names = self.args_names
|
||||
else:
|
||||
nobs_ = 'nobs' if 'nobs' in kwds else 'nobs1'
|
||||
args_names = ['effect_size', nobs_, 'alpha']
|
||||
|
||||
# pop positional args
|
||||
args = [kwds.pop(arg) for arg in args_names]
|
||||
|
||||
if hasattr(self, 'decimal'):
|
||||
decimal = self.decimal
|
||||
else:
|
||||
decimal = 6
|
||||
|
||||
res = res1.power(*args, **kwds)
|
||||
assert_almost_equal(res, self.res2.power, decimal=decimal)
|
||||
|
||||
def test_roots(self):
|
||||
kwds = copy.copy(self.kwds)
|
||||
kwds.update(self.kwds_extra)
|
||||
|
||||
# kwds_extra are used as argument, but not as target for root
|
||||
for key in self.kwds:
|
||||
# keep print to check whether tests are really executed
|
||||
#print 'testing roots', key
|
||||
value = kwds[key]
|
||||
kwds[key] = None
|
||||
|
||||
result = self.cls().solve_power(**kwds)
|
||||
assert_allclose(result, value, rtol=0.001, err_msg=key+' failed')
|
||||
# yield can be used to investigate specific errors
|
||||
#yield assert_allclose, result, value, 0.001, 0, key+' failed'
|
||||
kwds[key] = value # reset dict
|
||||
|
||||
@pytest.mark.matplotlib
|
||||
def test_power_plot(self, close_figures):
|
||||
if self.cls in [smp.FTestPower, smp.FTestPowerF2]:
|
||||
pytest.skip('skip FTestPower plot_power')
|
||||
fig = plt.figure()
|
||||
ax = fig.add_subplot(2,1,1)
|
||||
fig = self.cls().plot_power(dep_var='nobs',
|
||||
nobs= np.arange(2, 100),
|
||||
effect_size=np.array([0.1, 0.2, 0.3, 0.5, 1]),
|
||||
#alternative='larger',
|
||||
ax=ax, title='Power of t-Test',
|
||||
**self.kwds_extra)
|
||||
ax = fig.add_subplot(2,1,2)
|
||||
self.cls().plot_power(dep_var='es',
|
||||
nobs=np.array([10, 20, 30, 50, 70, 100]),
|
||||
effect_size=np.linspace(0.01, 2, 51),
|
||||
#alternative='larger',
|
||||
ax=ax, title='',
|
||||
**self.kwds_extra)
|
||||
|
||||
#''' test cases
|
||||
#one sample
|
||||
# two-sided one-sided
|
||||
#large power OneS1 OneS3
|
||||
#small power OneS2 OneS4
|
||||
#
|
||||
#two sample
|
||||
# two-sided one-sided
|
||||
#large power TwoS1 TwoS3
|
||||
#small power TwoS2 TwoS4
|
||||
#small p, ratio TwoS4 TwoS5
|
||||
#'''
|
||||
|
||||
class TestTTPowerOneS1(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
#> p = pwr.t.test(d=1,n=30,sig.level=0.05,type="two.sample",alternative="two.sided")
|
||||
#> cat_items(p, prefix='tt_power2_1.')
|
||||
res2 = Holder()
|
||||
res2.n = 30
|
||||
res2.d = 1
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.9995636009612725
|
||||
res2.alternative = 'two.sided'
|
||||
res2.note = 'NULL'
|
||||
res2.method = 'One-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power}
|
||||
cls.kwds_extra = {}
|
||||
cls.cls = smp.TTestPower
|
||||
|
||||
class TestTTPowerOneS2(CheckPowerMixin):
|
||||
# case with small power
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=0.2,n=20,sig.level=0.05,type="one.sample",alternative="two.sided")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n = 20
|
||||
res2.d = 0.2
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.1359562887679666
|
||||
res2.alternative = 'two.sided'
|
||||
res2.note = '''NULL'''
|
||||
res2.method = 'One-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power}
|
||||
cls.kwds_extra = {}
|
||||
cls.cls = smp.TTestPower
|
||||
|
||||
class TestTTPowerOneS3(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=1,n=30,sig.level=0.05,type="one.sample",alternative="greater")
|
||||
#> cat_items(p, prefix='tt_power1_1g.')
|
||||
res2.n = 30
|
||||
res2.d = 1
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.999892010204909
|
||||
res2.alternative = 'greater'
|
||||
res2.note = 'NULL'
|
||||
res2.method = 'One-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs': res2.n,
|
||||
'alpha': res2.sig_level, 'power': res2.power}
|
||||
cls.kwds_extra = {'alternative': 'larger'}
|
||||
cls.cls = smp.TTestPower
|
||||
|
||||
class TestTTPowerOneS4(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=0.05,n=20,sig.level=0.05,type="one.sample",alternative="greater")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n = 20
|
||||
res2.d = 0.05
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.0764888785042198
|
||||
res2.alternative = 'greater'
|
||||
res2.note = '''NULL'''
|
||||
res2.method = 'One-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs': res2.n,
|
||||
'alpha': res2.sig_level, 'power': res2.power}
|
||||
cls.kwds_extra = {'alternative': 'larger'}
|
||||
cls.cls = smp.TTestPower
|
||||
|
||||
class TestTTPowerOneS5(CheckPowerMixin):
|
||||
# case one-sided less, not implemented yet
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=0.2,n=20,sig.level=0.05,type="one.sample",alternative="less")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n = 20
|
||||
res2.d = 0.2
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.006063932667926375
|
||||
res2.alternative = 'less'
|
||||
res2.note = '''NULL'''
|
||||
res2.method = 'One-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs': res2.n,
|
||||
'alpha': res2.sig_level, 'power': res2.power}
|
||||
cls.kwds_extra = {'alternative': 'smaller'}
|
||||
cls.cls = smp.TTestPower
|
||||
|
||||
class TestTTPowerOneS6(CheckPowerMixin):
|
||||
# case one-sided less, negative effect size, not implemented yet
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=-0.2,n=20,sig.level=0.05,type="one.sample",alternative="less")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n = 20
|
||||
res2.d = -0.2
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.21707518167191
|
||||
res2.alternative = 'less'
|
||||
res2.note = '''NULL'''
|
||||
res2.method = 'One-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs': res2.n,
|
||||
'alpha': res2.sig_level, 'power': res2.power}
|
||||
cls.kwds_extra = {'alternative': 'smaller'}
|
||||
cls.cls = smp.TTestPower
|
||||
|
||||
|
||||
class TestTTPowerTwoS1(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
#> p = pwr.t.test(d=1,n=30,sig.level=0.05,type="two.sample",alternative="two.sided")
|
||||
#> cat_items(p, prefix='tt_power2_1.')
|
||||
res2 = Holder()
|
||||
res2.n = 30
|
||||
res2.d = 1
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.967708258242517
|
||||
res2.alternative = 'two.sided'
|
||||
res2.note = 'n is number in *each* group'
|
||||
res2.method = 'Two-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power': res2.power, 'ratio': 1}
|
||||
cls.kwds_extra = {}
|
||||
cls.cls = smp.TTestIndPower
|
||||
|
||||
class TestTTPowerTwoS2(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=0.1,n=20,sig.level=0.05,type="two.sample",alternative="two.sided")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n = 20
|
||||
res2.d = 0.1
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.06095912465411235
|
||||
res2.alternative = 'two.sided'
|
||||
res2.note = 'n is number in *each* group'
|
||||
res2.method = 'Two-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power': res2.power, 'ratio': 1}
|
||||
cls.kwds_extra = {}
|
||||
cls.cls = smp.TTestIndPower
|
||||
|
||||
class TestTTPowerTwoS3(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=1,n=30,sig.level=0.05,type="two.sample",alternative="greater")
|
||||
#> cat_items(p, prefix='tt_power2_1g.')
|
||||
res2.n = 30
|
||||
res2.d = 1
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.985459690251624
|
||||
res2.alternative = 'greater'
|
||||
res2.note = 'n is number in *each* group'
|
||||
res2.method = 'Two-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power, 'ratio': 1}
|
||||
cls.kwds_extra = {'alternative': 'larger'}
|
||||
cls.cls = smp.TTestIndPower
|
||||
|
||||
class TestTTPowerTwoS4(CheckPowerMixin):
|
||||
# case with small power
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t.test(d=0.01,n=30,sig.level=0.05,type="two.sample",alternative="greater")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n = 30
|
||||
res2.d = 0.01
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.0540740302835667
|
||||
res2.alternative = 'greater'
|
||||
res2.note = 'n is number in *each* group'
|
||||
res2.method = 'Two-sample t test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power}
|
||||
cls.kwds_extra = {'alternative': 'larger'}
|
||||
cls.cls = smp.TTestIndPower
|
||||
|
||||
class TestTTPowerTwoS5(CheckPowerMixin):
|
||||
# case with unequal n, ratio>1
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t2n.test(d=0.1,n1=20, n2=30,sig.level=0.05,alternative="two.sided")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n1 = 20
|
||||
res2.n2 = 30
|
||||
res2.d = 0.1
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.0633081832564667
|
||||
res2.alternative = 'two.sided'
|
||||
res2.method = 't test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n1,
|
||||
'alpha': res2.sig_level, 'power':res2.power, 'ratio': 1.5}
|
||||
cls.kwds_extra = {'alternative': 'two-sided'}
|
||||
cls.cls = smp.TTestIndPower
|
||||
|
||||
class TestTTPowerTwoS6(CheckPowerMixin):
|
||||
# case with unequal n, ratio>1
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
|
||||
res2 = Holder()
|
||||
#> p = pwr.t2n.test(d=0.1,n1=20, n2=30,sig.level=0.05,alternative="greater")
|
||||
#> cat_items(p, "res2.")
|
||||
res2.n1 = 20
|
||||
res2.n2 = 30
|
||||
res2.d = 0.1
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.09623589080917805
|
||||
res2.alternative = 'greater'
|
||||
res2.method = 't test power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n1,
|
||||
'alpha': res2.sig_level, 'power':res2.power, 'ratio': 1.5}
|
||||
cls.kwds_extra = {'alternative': 'larger'}
|
||||
cls.cls = smp.TTestIndPower
|
||||
|
||||
|
||||
|
||||
def test_normal_power_explicit():
|
||||
# a few initial test cases for NormalIndPower
|
||||
sigma = 1
|
||||
d = 0.3
|
||||
nobs = 80
|
||||
alpha = 0.05
|
||||
res1 = smp.normal_power(d, nobs/2., 0.05)
|
||||
res2 = smp.NormalIndPower().power(d, nobs, 0.05)
|
||||
res3 = smp.NormalIndPower().solve_power(effect_size=0.3, nobs1=80, alpha=0.05, power=None)
|
||||
res_R = 0.475100870572638
|
||||
assert_almost_equal(res1, res_R, decimal=13)
|
||||
assert_almost_equal(res2, res_R, decimal=13)
|
||||
assert_almost_equal(res3, res_R, decimal=13)
|
||||
|
||||
|
||||
norm_pow = smp.normal_power(-0.01, nobs/2., 0.05)
|
||||
norm_pow_R = 0.05045832927039234
|
||||
#value from R: >pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="two.sided")
|
||||
assert_almost_equal(norm_pow, norm_pow_R, decimal=11)
|
||||
|
||||
norm_pow = smp.NormalIndPower().power(0.01, nobs, 0.05,
|
||||
alternative="larger")
|
||||
norm_pow_R = 0.056869534873146124
|
||||
#value from R: >pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="greater")
|
||||
assert_almost_equal(norm_pow, norm_pow_R, decimal=11)
|
||||
|
||||
# Note: negative effect size is same as switching one-sided alternative
|
||||
# TODO: should I switch to larger/smaller instead of "one-sided" options
|
||||
norm_pow = smp.NormalIndPower().power(-0.01, nobs, 0.05,
|
||||
alternative="larger")
|
||||
norm_pow_R = 0.0438089705093578
|
||||
#value from R: >pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="less")
|
||||
assert_almost_equal(norm_pow, norm_pow_R, decimal=11)
|
||||
|
||||
class TestNormalIndPower1(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
#> example from above
|
||||
# results copied not directly from R
|
||||
res2 = Holder()
|
||||
res2.n = 80
|
||||
res2.d = 0.3
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.475100870572638
|
||||
res2.alternative = 'two.sided'
|
||||
res2.note = 'NULL'
|
||||
res2.method = 'two sample power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power, 'ratio': 1}
|
||||
cls.kwds_extra = {}
|
||||
cls.cls = smp.NormalIndPower
|
||||
|
||||
class TestNormalIndPower2(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
res2 = Holder()
|
||||
#> np = pwr.2p.test(h=0.01,n=80,sig.level=0.05,alternative="less")
|
||||
#> cat_items(np, "res2.")
|
||||
res2.h = 0.01
|
||||
res2.n = 80
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.0438089705093578
|
||||
res2.alternative = 'less'
|
||||
res2.method = ('Difference of proportion power calculation for' +
|
||||
' binomial distribution (arcsine transformation)')
|
||||
res2.note = 'same sample sizes'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.h, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power, 'ratio': 1}
|
||||
cls.kwds_extra = {'alternative':'smaller'}
|
||||
cls.cls = smp.NormalIndPower
|
||||
|
||||
|
||||
class TestNormalIndPower_onesamp1(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
# forcing one-sample by using ratio=0
|
||||
#> example from above
|
||||
# results copied not directly from R
|
||||
res2 = Holder()
|
||||
res2.n = 40
|
||||
res2.d = 0.3
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.475100870572638
|
||||
res2.alternative = 'two.sided'
|
||||
res2.note = 'NULL'
|
||||
res2.method = 'two sample power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power}
|
||||
# keyword for which we do not look for root:
|
||||
cls.kwds_extra = {'ratio': 0}
|
||||
|
||||
cls.cls = smp.NormalIndPower
|
||||
|
||||
class TestNormalIndPower_onesamp2(CheckPowerMixin):
|
||||
# Note: same power as two sample case with twice as many observations
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
# forcing one-sample by using ratio=0
|
||||
res2 = Holder()
|
||||
#> np = pwr.norm.test(d=0.01,n=40,sig.level=0.05,alternative="less")
|
||||
#> cat_items(np, "res2.")
|
||||
res2.d = 0.01
|
||||
res2.n = 40
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.0438089705093578
|
||||
res2.alternative = 'less'
|
||||
res2.method = 'Mean power calculation for normal distribution with known variance'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.d, 'nobs1': res2.n,
|
||||
'alpha': res2.sig_level, 'power':res2.power}
|
||||
# keyword for which we do not look for root:
|
||||
cls.kwds_extra = {'ratio': 0, 'alternative':'smaller'}
|
||||
|
||||
cls.cls = smp.NormalIndPower
|
||||
|
||||
|
||||
|
||||
class TestChisquarePower(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
# one example from test_gof, results_power
|
||||
res2 = Holder()
|
||||
res2.w = 0.1
|
||||
res2.N = 5
|
||||
res2.df = 4
|
||||
res2.sig_level = 0.05
|
||||
res2.power = 0.05246644635810126
|
||||
res2.method = 'Chi squared power calculation'
|
||||
res2.note = 'N is the number of observations'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.w, 'nobs': res2.N,
|
||||
'alpha': res2.sig_level, 'power':res2.power}
|
||||
# keyword for which we do not look for root:
|
||||
# solving for n_bins does not work, will not be used in regular usage
|
||||
cls.kwds_extra = {'n_bins': res2.df + 1}
|
||||
|
||||
cls.cls = smp.GofChisquarePower
|
||||
|
||||
def test_positional(self):
|
||||
|
||||
res1 = self.cls()
|
||||
args_names = ['effect_size','nobs', 'alpha', 'n_bins']
|
||||
kwds = copy.copy(self.kwds)
|
||||
del kwds['power']
|
||||
kwds.update(self.kwds_extra)
|
||||
args = [kwds[arg] for arg in args_names]
|
||||
if hasattr(self, 'decimal'):
|
||||
decimal = self.decimal #pylint: disable-msg=E1101
|
||||
else:
|
||||
decimal = 6
|
||||
assert_almost_equal(res1.power(*args), self.res2.power, decimal=decimal)
|
||||
|
||||
|
||||
|
||||
def test_ftest_power():
|
||||
#equivalence ftest, ttest
|
||||
|
||||
for alpha in [0.01, 0.05, 0.1, 0.20, 0.50]:
|
||||
res0 = smp.ttest_power(0.01, 200, alpha)
|
||||
res1 = smp.ftest_power(0.01, 199, 1, alpha=alpha, ncc=0)
|
||||
assert_almost_equal(res1, res0, decimal=6)
|
||||
|
||||
|
||||
#example from Gplus documentation F-test ANOVA
|
||||
#Total sample size:200
|
||||
#Effect size "f":0.25
|
||||
#Beta/alpha ratio:1
|
||||
#Result:
|
||||
#Alpha:0.1592
|
||||
#Power (1-beta):0.8408
|
||||
#Critical F:1.4762
|
||||
#Lambda: 12.50000
|
||||
res1 = smp.ftest_anova_power(0.25, 200, 0.1592, k_groups=10)
|
||||
res0 = 0.8408
|
||||
assert_almost_equal(res1, res0, decimal=4)
|
||||
|
||||
|
||||
# TODO: no class yet
|
||||
# examples against R::pwr
|
||||
res2 = Holder()
|
||||
#> rf = pwr.f2.test(u=5, v=199, f2=0.1**2, sig.level=0.01)
|
||||
#> cat_items(rf, "res2.")
|
||||
res2.u = 5
|
||||
res2.v = 199
|
||||
res2.f2 = 0.01
|
||||
res2.sig_level = 0.01
|
||||
res2.power = 0.0494137732920332
|
||||
res2.method = 'Multiple regression power calculation'
|
||||
|
||||
res1 = smp.ftest_power(np.sqrt(res2.f2), res2.v, res2.u,
|
||||
alpha=res2.sig_level, ncc=1)
|
||||
assert_almost_equal(res1, res2.power, decimal=5)
|
||||
|
||||
res2 = Holder()
|
||||
#> rf = pwr.f2.test(u=5, v=199, f2=0.3**2, sig.level=0.01)
|
||||
#> cat_items(rf, "res2.")
|
||||
res2.u = 5
|
||||
res2.v = 199
|
||||
res2.f2 = 0.09
|
||||
res2.sig_level = 0.01
|
||||
res2.power = 0.7967191006290872
|
||||
res2.method = 'Multiple regression power calculation'
|
||||
|
||||
res1 = smp.ftest_power(np.sqrt(res2.f2), res2.v, res2.u,
|
||||
alpha=res2.sig_level, ncc=1)
|
||||
assert_almost_equal(res1, res2.power, decimal=5)
|
||||
|
||||
res2 = Holder()
|
||||
#> rf = pwr.f2.test(u=5, v=19, f2=0.3**2, sig.level=0.1)
|
||||
#> cat_items(rf, "res2.")
|
||||
res2.u = 5
|
||||
res2.v = 19
|
||||
res2.f2 = 0.09
|
||||
res2.sig_level = 0.1
|
||||
res2.power = 0.235454222377575
|
||||
res2.method = 'Multiple regression power calculation'
|
||||
|
||||
res1 = smp.ftest_power(np.sqrt(res2.f2), res2.v, res2.u,
|
||||
alpha=res2.sig_level, ncc=1)
|
||||
assert_almost_equal(res1, res2.power, decimal=5)
|
||||
|
||||
# class based version of two above test for Ftest
|
||||
class TestFtestAnovaPower(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
res2 = Holder()
|
||||
#example from Gplus documentation F-test ANOVA
|
||||
#Total sample size:200
|
||||
#Effect size "f":0.25
|
||||
#Beta/alpha ratio:1
|
||||
#Result:
|
||||
#Alpha:0.1592
|
||||
#Power (1-beta):0.8408
|
||||
#Critical F:1.4762
|
||||
#Lambda: 12.50000
|
||||
#converted to res2 by hand
|
||||
res2.f = 0.25
|
||||
res2.n = 200
|
||||
res2.k = 10
|
||||
res2.alpha = 0.1592
|
||||
res2.power = 0.8408
|
||||
res2.method = 'Multiple regression power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.f, 'nobs': res2.n,
|
||||
'alpha': res2.alpha, 'power': res2.power}
|
||||
# keyword for which we do not look for root:
|
||||
# solving for n_bins does not work, will not be used in regular usage
|
||||
cls.kwds_extra = {'k_groups': res2.k} # rootfinding does not work
|
||||
#cls.args_names = ['effect_size','nobs', 'alpha']#, 'k_groups']
|
||||
cls.cls = smp.FTestAnovaPower
|
||||
# precision for test_power
|
||||
cls.decimal = 4
|
||||
|
||||
|
||||
|
||||
class TestFtestPower(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
res2 = Holder()
|
||||
#> rf = pwr.f2.test(u=5, v=19, f2=0.3**2, sig.level=0.1)
|
||||
#> cat_items(rf, "res2.")
|
||||
res2.u = 5
|
||||
res2.v = 19
|
||||
res2.f2 = 0.09
|
||||
res2.sig_level = 0.1
|
||||
res2.power = 0.235454222377575
|
||||
res2.method = 'Multiple regression power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': np.sqrt(res2.f2), 'df_num': res2.v,
|
||||
'df_denom': res2.u, 'alpha': res2.sig_level,
|
||||
'power': res2.power}
|
||||
# keyword for which we do not look for root:
|
||||
# solving for n_bins does not work, will not be used in regular usage
|
||||
cls.kwds_extra = {}
|
||||
cls.args_names = ['effect_size', 'df_num', 'df_denom', 'alpha']
|
||||
cls.cls = smp.FTestPower
|
||||
# precision for test_power
|
||||
cls.decimal = 5
|
||||
|
||||
def test_kwargs(self):
|
||||
|
||||
with pytest.warns(UserWarning):
|
||||
smp.FTestPower().solve_power(
|
||||
effect_size=0.3, alpha=0.1, power=0.9, df_denom=2,
|
||||
nobs=None)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
smp.FTestPower().solve_power(
|
||||
effect_size=0.3, alpha=0.1, power=0.9, df_denom=2,
|
||||
junk=3)
|
||||
|
||||
|
||||
class TestFtestPowerF2(CheckPowerMixin):
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
res2 = Holder()
|
||||
#> rf = pwr.f2.test(u=5, v=19, f2=0.3**2, sig.level=0.1)
|
||||
#> cat_items(rf, "res2.")
|
||||
res2.u = 5
|
||||
res2.v = 19
|
||||
res2.f2 = 0.09
|
||||
res2.sig_level = 0.1
|
||||
res2.power = 0.235454222377575
|
||||
res2.method = 'Multiple regression power calculation'
|
||||
|
||||
cls.res2 = res2
|
||||
cls.kwds = {'effect_size': res2.f2, 'df_num': res2.u,
|
||||
'df_denom': res2.v, 'alpha': res2.sig_level,
|
||||
'power': res2.power}
|
||||
# keyword for which we do not look for root:
|
||||
# solving for n_bins does not work, will not be used in regular usage
|
||||
cls.kwds_extra = {}
|
||||
cls.args_names = ['effect_size', 'df_num', 'df_denom', 'alpha']
|
||||
cls.cls = smp.FTestPowerF2
|
||||
# precision for test_power
|
||||
cls.decimal = 5
|
||||
|
||||
|
||||
|
||||
def test_power_solver():
|
||||
# messing up the solver to trigger backup
|
||||
|
||||
nip = smp.NormalIndPower()
|
||||
|
||||
# check result
|
||||
es0 = 0.1
|
||||
pow_ = nip.solve_power(es0, nobs1=1600, alpha=0.01, power=None, ratio=1,
|
||||
alternative='larger')
|
||||
# value is regression test
|
||||
assert_almost_equal(pow_, 0.69219411243824214, decimal=5)
|
||||
es = nip.solve_power(None, nobs1=1600, alpha=0.01, power=pow_, ratio=1,
|
||||
alternative='larger')
|
||||
assert_almost_equal(es, es0, decimal=4)
|
||||
assert_equal(nip.cache_fit_res[0], 1)
|
||||
assert_equal(len(nip.cache_fit_res), 2)
|
||||
|
||||
# cause first optimizer to fail
|
||||
nip.start_bqexp['effect_size'] = {'upp': -10, 'low': -20}
|
||||
nip.start_ttp['effect_size'] = 0.14
|
||||
es = nip.solve_power(None, nobs1=1600, alpha=0.01, power=pow_, ratio=1,
|
||||
alternative='larger')
|
||||
assert_almost_equal(es, es0, decimal=4)
|
||||
assert_equal(nip.cache_fit_res[0], 1)
|
||||
assert_equal(len(nip.cache_fit_res), 3, err_msg=repr(nip.cache_fit_res))
|
||||
|
||||
nip.start_ttp['effect_size'] = np.nan
|
||||
es = nip.solve_power(None, nobs1=1600, alpha=0.01, power=pow_, ratio=1,
|
||||
alternative='larger')
|
||||
assert_almost_equal(es, es0, decimal=4)
|
||||
assert_equal(nip.cache_fit_res[0], 1)
|
||||
assert_equal(len(nip.cache_fit_res), 4)
|
||||
|
||||
# Test our edge-case where effect_size = 0
|
||||
es = nip.solve_power(nobs1=1600, alpha=0.01, effect_size=0, power=None)
|
||||
assert_almost_equal(es, 0.01)
|
||||
|
||||
# I let this case fail, could be fixed for some statistical tests
|
||||
# (we should not get here in the first place)
|
||||
# effect size is negative, but last stage brentq uses [1e-8, 1-1e-8]
|
||||
assert_raises(ValueError, nip.solve_power, None, nobs1=1600, alpha=0.01,
|
||||
power=0.005, ratio=1, alternative='larger')
|
||||
|
||||
with pytest.warns(HypothesisTestWarning):
|
||||
with pytest.raises(ValueError):
|
||||
nip.solve_power(nobs1=None, effect_size=0, alpha=0.01,
|
||||
power=0.005, ratio=1, alternative='larger')
|
||||
|
||||
|
||||
# TODO: can something useful be made from this?
|
||||
@pytest.mark.xfail(reason='Known failure on modern SciPy >= 0.10', strict=True)
|
||||
def test_power_solver_warn():
|
||||
# messing up the solver to trigger warning
|
||||
# I wrote this with scipy 0.9,
|
||||
# convergence behavior of scipy 0.11 is different,
|
||||
# fails at a different case, but is successful where it failed before
|
||||
|
||||
pow_ = 0.69219411243824214 # from previous function
|
||||
nip = smp.NormalIndPower()
|
||||
# using nobs, has one backup (fsolve)
|
||||
nip.start_bqexp['nobs1'] = {'upp': 50, 'low': -20}
|
||||
val = nip.solve_power(0.1, nobs1=None, alpha=0.01, power=pow_, ratio=1,
|
||||
alternative='larger')
|
||||
|
||||
assert_almost_equal(val, 1600, decimal=4)
|
||||
assert_equal(nip.cache_fit_res[0], 1)
|
||||
assert_equal(len(nip.cache_fit_res), 3)
|
||||
|
||||
# case that has convergence failure, and should warn
|
||||
nip.start_ttp['nobs1'] = np.nan
|
||||
|
||||
from statsmodels.tools.sm_exceptions import ConvergenceWarning
|
||||
assert_warns(ConvergenceWarning, nip.solve_power, 0.1, nobs1=None,
|
||||
alpha=0.01, power=pow_, ratio=1, alternative='larger')
|
||||
# this converges with scipy 0.11 ???
|
||||
# nip.solve_power(0.1, nobs1=None, alpha=0.01, power=pow_, ratio=1, alternative='larger')
|
||||
|
||||
with warnings.catch_warnings(): # python >= 2.6
|
||||
warnings.simplefilter("ignore")
|
||||
val = nip.solve_power(0.1, nobs1=None, alpha=0.01, power=pow_, ratio=1,
|
||||
alternative='larger')
|
||||
assert_equal(nip.cache_fit_res[0], 0)
|
||||
assert_equal(len(nip.cache_fit_res), 3)
|
||||
|
||||
|
||||
def test_normal_sample_size_one_tail():
|
||||
# Test that using default value of std_alternative does not raise an
|
||||
# exception. A power of 0.8 and alpha of 0.05 were chosen to reflect
|
||||
# commonly used values in hypothesis testing. Difference in means and
|
||||
# standard deviation of null population were chosen somewhat arbitrarily --
|
||||
# there's nothing special about those values. Return value doesn't matter
|
||||
# for this "test", so long as an exception is not raised.
|
||||
smp.normal_sample_size_one_tail(5, 0.8, 0.05, 2, std_alternative=None)
|
||||
|
||||
# Test that zero is returned in the correct elements if power is less
|
||||
# than alpha.
|
||||
alphas = np.asarray([0.01, 0.05, 0.1, 0.5, 0.8])
|
||||
powers = np.asarray([0.99, 0.95, 0.9, 0.5, 0.2])
|
||||
# zero_mask = np.where(alphas - powers > 0, 0.0, alphas - powers)
|
||||
nobs_with_zeros = smp.normal_sample_size_one_tail(5, powers, alphas, 2, 2)
|
||||
# check_nans = np.isnan(zero_mask) == np.isnan(nobs_with_nans)
|
||||
assert_array_equal(nobs_with_zeros[powers <= alphas], 0)
|
||||
Reference in New Issue
Block a user