Package mvpa :: Package tests :: Module test_svm
[hide private]
[frames] | no frames]

Source Code for Module mvpa.tests.test_svm

  1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  # vi: set ft=python sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Unit tests for SVM classifier""" 
 10   
 11  from sets import Set 
 12   
 13  from mvpa.datasets.splitters import NFoldSplitter 
 14  from mvpa.clfs.meta import ProxyClassifier 
 15  from mvpa.clfs.transerror import TransferError 
 16  from mvpa.algorithms.cvtranserror import CrossValidatedTransferError 
 17   
 18  from tests_warehouse import pureMultivariateSignal 
 19  from tests_warehouse import * 
 20  from tests_warehouse_clfs import * 
21 22 -class SVMTests(unittest.TestCase):
23 24 # @sweepargs(nl_clf=clfswh['non-linear', 'svm'] ) 25 # @sweepargs(nl_clf=clfswh['non-linear', 'svm'] )
26 - def testMultivariate(self):
27 mv_perf = [] 28 mv_lin_perf = [] 29 uv_perf = [] 30 31 l_clf = clfswh['linear', 'svm'][0] 32 nl_clf = clfswh['non-linear', 'svm'][0] 33 34 #orig_keys = nl_clf.param._params.keys() 35 #nl_param_orig = nl_clf.param._params.copy() 36 37 # l_clf = LinearNuSVMC() 38 39 # XXX ??? not sure what below meant and it is obsolete if 40 # using SG... commenting out for now 41 # for some reason order is not preserved thus dictionaries are not 42 # the same any longer -- lets compare values 43 #self.failUnlessEqual([nl_clf.param._params[k] for k in orig_keys], 44 # [nl_param_orig[k] for k in orig_keys], 45 # msg="New instance mustn't override values in previously created") 46 ## and keys separately 47 #self.failUnlessEqual(Set(nl_clf.param._params.keys()), 48 # Set(orig_keys), 49 # msg="New instance doesn't change set of parameters in original") 50 51 # We must be able to deepcopy not yet trained SVMs now 52 import mvpa.support.copy as copy 53 try: 54 nl_clf.untrain() 55 nl_clf_copy = copy.deepcopy(nl_clf) 56 except: 57 self.fail(msg="Failed to deepcopy not-yet trained SVM %s" % nl_clf) 58 59 for i in xrange(20): 60 train = pureMultivariateSignal( 20, 3 ) 61 test = pureMultivariateSignal( 20, 3 ) 62 63 # use non-linear CLF on 2d data 64 nl_clf.train(train) 65 p_mv = nl_clf.predict(test.samples) 66 mv_perf.append(N.mean(p_mv==test.labels)) 67 68 # use linear CLF on 2d data 69 l_clf.train(train) 70 p_lin_mv = l_clf.predict(test.samples) 71 mv_lin_perf.append(N.mean(p_lin_mv==test.labels)) 72 73 # use non-linear CLF on 1d data 74 nl_clf.train(train.selectFeatures([0])) 75 p_uv = nl_clf.predict(test.selectFeatures([0]).samples) 76 uv_perf.append(N.mean(p_uv==test.labels)) 77 78 mean_mv_perf = N.mean(mv_perf) 79 mean_mv_lin_perf = N.mean(mv_lin_perf) 80 mean_uv_perf = N.mean(uv_perf) 81 82 # non-linear CLF has to be close to perfect 83 self.failUnless( mean_mv_perf > 0.9 ) 84 # linear CLF cannot learn this problem! 85 self.failUnless( mean_mv_perf > mean_mv_lin_perf ) 86 # univariate has insufficient information 87 self.failUnless( mean_uv_perf < mean_mv_perf )
88 89 90 # XXX for now works only with linear... think it through -- should 91 # work non-linear, shouldn't it? 92 # now all non-linear have C>0 thus skipped anyways 93 94 # TODO: For some reason libsvm's weight assignment has no effect 95 # as well -- need to be fixed :-/ 96 @sweepargs(clf=clfswh['svm', 'sg', '!regression', '!gnpp', '!meta'])
97 - def testCperClass(self, clf):
98 try: 99 if clf.C > 0: 100 # skip those with fixed C 101 return 102 except: 103 # classifier has no C 104 return 105 106 if clf.C < -5: 107 # too soft margin helps to fight disbalance, thus skip 108 # it in testing 109 return 110 #print clf 111 ds = datasets['uni2small'].copy() 112 ds__ = datasets['uni2small'].copy() 113 # 114 # ballanced set 115 # Lets add a bit of noise to drive classifier nuts. same 116 # should be done for disballanced set 117 ds__.samples = ds__.samples + 0.5 * N.random.normal(size=(ds__.samples.shape)) 118 # 119 # disballanced set 120 # lets overpopulate label 0 121 times = 10 122 ds_ = ds.selectSamples(range(ds.nsamples) + range(ds.nsamples/2) * times) 123 ds_.samples = ds_.samples + 0.7 * N.random.normal(size=(ds_.samples.shape)) 124 spl = ds_.samplesperlabel 125 #print ds_.labels, ds_.chunks 126 127 cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(), 128 enable_states='confusion') 129 e = cve(ds__) 130 if cfg.getboolean('tests', 'labile', default='yes'): 131 # without disballance we should already have some hits 132 self.failUnless(cve.confusion.stats["P'"][1] > 0) 133 134 e = cve(ds_) 135 if cfg.getboolean('tests', 'labile', default='yes'): 136 self.failUnless(cve.confusion.stats["P'"][1] < 5, 137 msg="With disballance we should have almost no " 138 "hits. Got %f" % cve.confusion.stats["P'"][1]) 139 #print "D:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1] 140 141 # Set '1 C per label' 142 # recreate cvte since previous might have operated on copies 143 cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(), 144 enable_states='confusion') 145 oldC = clf.C 146 ratio = N.sqrt(float(spl[0])/spl[1]) 147 clf.C = (-1/ratio, -1*ratio) 148 try: 149 e_ = cve(ds_) 150 # reassign C 151 clf.C = oldC 152 except: 153 clf.C = oldC 154 raise 155 #print "B:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1] 156 if cfg.getboolean('tests', 'labile', default='yes'): 157 # Finally test if we get any 'hit' for minor category. In the 158 # classifier, which has way to 'ballance' should be non-0 159 self.failUnless(cve.confusion.stats["P'"][1] > 0)
160 161
162 - def testSillyness(self):
163 """Test if we raise exceptions on incorrect specifications 164 """ 165 166 if externals.exists('libsvm') or externals.exists('shogun'): 167 self.failUnlessRaises(TypeError, SVM, C=1.0, nu=2.3) 168 169 if externals.exists('libsvm'): 170 self.failUnlessRaises(TypeError, libsvm.SVM, C=1.0, nu=2.3) 171 self.failUnlessRaises(TypeError, LinearNuSVMC, C=2.3) 172 self.failUnlessRaises(TypeError, LinearCSVMC, nu=2.3) 173 174 if externals.exists('shogun'): 175 self.failUnlessRaises(TypeError, sg.SVM, C=10, kernel_type='RBF', 176 coef0=3)
177
178 -def suite():
179 return unittest.makeSuite(SVMTests)
180 181 182 if __name__ == '__main__': 183 import runner 184