Package mvpa :: Package clfs :: Package libsvmc :: Module svm
[hide private]
[frames] | no frames]

Source Code for Module mvpa.clfs.libsvmc.svm

  1  #emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  #ex: set sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Wrap the libsvm package into a very simple class interface.""" 
 10   
 11  __docformat__ = 'restructuredtext' 
 12   
 13  import numpy as N 
 14   
 15  import operator 
 16   
 17  from mvpa.misc.param import Parameter 
 18  from mvpa.base import warning 
 19  from mvpa.misc.state import StateVariable 
 20   
 21  from mvpa.clfs.base import Classifier 
 22  from mvpa.clfs._svmbase import _SVM 
 23  from mvpa.measures.base import Sensitivity 
 24   
 25  from mvpa.clfs.libsvmc import _svm as svm 
 26  from sens import * 
 27   
 28  if __debug__: 
 29      from mvpa.base import debug 
 30   
 31  # we better expose those since they are mentioned in docstrings 
 32  from mvpa.clfs.libsvmc._svmc import \ 
 33       C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, \ 
 34       NU_SVR, LINEAR, POLY, RBF, SIGMOID, \ 
 35       PRECOMPUTED 
 36   
 37   
38 -class SVM(_SVM):
39 """Support Vector Machine Classifier. 40 41 This is a simple interface to the libSVM package. 42 """ 43 44 # Since this is internal feature of LibSVM, this state variable is present 45 # here 46 probabilities = StateVariable(enabled=False, 47 doc="Estimates of samples probabilities as provided by LibSVM") 48 49 _KERNELS = { "linear": (svm.svmc.LINEAR, None, LinearSVMWeights), 50 "rbf" : (svm.svmc.RBF, ('gamma',), None), 51 "poly": (svm.svmc.POLY, ('gamma', 'degree', 'coef0'), None), 52 "sigmoid": (svm.svmc.SIGMOID, ('gamma', 'coef0'), None), 53 } 54 # TODO: Complete the list ;-) 55 56 # TODO p is specific for SVR 57 _KNOWN_PARAMS = [ 'epsilon', 'probability', 'shrinking', 58 'weight_label', 'weight'] 59 60 _KNOWN_KERNEL_PARAMS = [ 'cache_size' ] 61 62 _KNOWN_IMPLEMENTATIONS = { 63 'C_SVC' : (svm.svmc.C_SVC, ('C',), 64 ('binary', 'multiclass'), 'C-SVM classification'), 65 'NU_SVC' : (svm.svmc.NU_SVC, ('nu',), 66 ('binary', 'multiclass'), 'nu-SVM classification'), 67 'ONE_CLASS' : (svm.svmc.ONE_CLASS, (), 68 ('oneclass',), 'one-class-SVM'), 69 'EPSILON_SVR' : (svm.svmc.EPSILON_SVR, ('tube_epsilon',), 70 ('regression',), 'epsilon-SVM regression'), 71 'NU_SVR' : (svm.svmc.NU_SVR, ('nu',), 72 ('regression',), 'nu-SVM regression') 73 } 74 75 _clf_internals = _SVM._clf_internals + [ 'libsvm' ] 76
77 - def __init__(self, 78 kernel_type='linear', 79 **kwargs):
80 # XXX Determine which parameters depend on each other and implement 81 # safety/simplifying logic around them 82 # already done for: nr_weight 83 # thought: weight and weight_label should be a dict 84 """This is the base class of all classifier that utilize the libSVM 85 package underneath. It is not really meant to be used directly. Unless 86 you know what you are doing it is most likely better to use one of the 87 subclasses. 88 89 Here is the explaination for some of the parameters from the libSVM 90 documentation: 91 92 svm_type can be one of C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR. 93 94 - `C_SVC`: C-SVM classification 95 - `NU_SVC`: nu-SVM classification 96 - `ONE_CLASS`: one-class-SVM 97 - `EPSILON_SVR`: epsilon-SVM regression 98 - `NU_SVR`: nu-SVM regression 99 100 kernel_type can be one of LINEAR, POLY, RBF, SIGMOID. 101 102 - `LINEAR`: ``u'*v`` 103 - `POLY`: ``(gamma*u'*v + coef0)^degree`` 104 - `RBF`: ``exp(-gamma*|u-v|^2)`` 105 - `SIGMOID`: ``tanh(gamma*u'*v + coef0)`` 106 - `PRECOMPUTED`: kernel values in training_set_file 107 108 cache_size is the size of the kernel cache, specified in megabytes. 109 C is the cost of constraints violation. (we usually use 1 to 1000) 110 eps is the stopping criterion. (we usually use 0.00001 in nu-SVC, 111 0.001 in others). nu is the parameter in nu-SVM, nu-SVR, and 112 one-class-SVM. p is the epsilon in epsilon-insensitive loss function 113 of epsilon-SVM regression. shrinking = 1 means shrinking is conducted; 114 = 0 otherwise. probability = 1 means model with probability 115 information is obtained; = 0 otherwise. 116 117 nr_weight, weight_label, and weight are used to change the penalty 118 for some classes (If the weight for a class is not changed, it is 119 set to 1). This is useful for training classifier using unbalanced 120 input data or with asymmetric misclassification cost. 121 122 Each weight[i] corresponds to weight_label[i], meaning that 123 the penalty of class weight_label[i] is scaled by a factor of weight[i]. 124 125 If you do not want to change penalty for any of the classes, 126 just set nr_weight to 0. 127 """ 128 129 svm_impl = kwargs.get('svm_impl', None) 130 # Depending on given arguments, figure out desired SVM 131 # implementation 132 if svm_impl is None: 133 for arg, impl in [ ('tube_epsilon', 'EPSILON_SVR'), 134 ('C', 'C_SVC'), 135 ('nu', 'NU_SVC') ]: 136 if kwargs.has_key(arg): 137 svm_impl = impl 138 if __debug__: 139 debug('SVM', 'No implementation was specified. Since ' 140 '%s is given among arguments, assume %s' % 141 (arg, impl)) 142 break 143 if svm_impl is None: 144 svm_impl = 'C_SVC' 145 if __debug__: 146 debug('SVM', 'Assign C_SVC "by default"') 147 kwargs['svm_impl'] = svm_impl 148 149 # init base class 150 _SVM.__init__(self, kernel_type, **kwargs) 151 152 self._svm_type = self._KNOWN_IMPLEMENTATIONS[svm_impl][0] 153 154 if 'nu' in self._KNOWN_PARAMS and 'epsilon' in self._KNOWN_PARAMS: 155 # overwrite eps param with new default value (information taken from libSVM 156 # docs 157 self.params['epsilon'].setDefault(0.001) 158 159 self.__model = None 160 """Holds the trained SVM."""
161 162 163
164 - def _train(self, dataset):
165 """Train SVM 166 """ 167 # libsvm needs doubles 168 if dataset.samples.dtype == 'float64': 169 src = dataset.samples 170 else: 171 src = dataset.samples.astype('double') 172 173 svmprob = svm.SVMProblem( dataset.labels.tolist(), src ) 174 175 # Translate few params 176 TRANSLATEDICT={'epsilon': 'eps', 177 'tube_epsilon': 'p'} 178 args = [] 179 for paramname, param in self.params.items.items() \ 180 + self.kernel_params.items.items(): 181 if paramname in TRANSLATEDICT: 182 argname = TRANSLATEDICT[paramname] 183 elif paramname in svm.SVMParameter.default_parameters: 184 argname = paramname 185 else: 186 if __debug__: 187 debug("SVM_", "Skipping parameter %s since it is not known" 188 "to libsvm" % paramname) 189 continue 190 args.append( (argname, param.value) ) 191 192 # XXX All those parameters should be fetched if present from 193 # **kwargs and create appropriate parameters within .params or .kernel_params 194 libsvm_param = svm.SVMParameter( 195 kernel_type=self._kernel_type, 196 svm_type=self._svm_type, 197 **dict(args)) 198 """Store SVM parameters in libSVM compatible format.""" 199 200 if self.params.isKnown('C'):#svm_type in [svm.svmc.C_SVC]: 201 C = self.params.C 202 if not operator.isSequenceType(C): 203 # we were not given a tuple for balancing between classes 204 C = [C] 205 206 Cs = list(C[:]) # copy 207 for i in xrange(len(Cs)): 208 if Cs[i]<0: 209 Cs[i] = self._getDefaultC(dataset.samples)*abs(Cs[i]) 210 if __debug__: 211 debug("SVM", "Default C for %s was computed to be %s" % 212 (C[i], Cs[i])) 213 214 libsvm_param._setParameter('C', Cs[0]) 215 216 if len(Cs)>1: 217 C0 = abs(C[0]) 218 scale = 1.0/(C0)#*N.sqrt(C0)) 219 # so we got 1 C per label 220 if len(Cs) != len(dataset.uniquelabels): 221 raise ValueError, "SVM was parametrized with %d Cs but " \ 222 "there are %d labels in the dataset" % \ 223 (len(Cs), len(dataset.uniquelabels)) 224 weight = [ c*scale for c in Cs ] 225 libsvm_param._setParameter('weight', weight) 226 227 self.__model = svm.SVMModel(svmprob, libsvm_param)
228 229
230 - def _predict(self, data):
231 """Predict values for the data 232 """ 233 # libsvm needs doubles 234 if data.dtype == 'float64': 235 src = data 236 else: 237 src = data.astype('double') 238 states = self.states 239 240 predictions = [ self.model.predict(p) for p in src ] 241 242 if states.isEnabled("values"): 243 if self.regression: 244 values = [ self.model.predictValuesRaw(p)[0] for p in src ] 245 else: 246 trained_labels = self.trained_labels 247 nlabels = len(trained_labels) 248 # XXX We do duplicate work. model.predict calls predictValuesRaw 249 # internally and then does voting or thresholding. So if speed becomes 250 # a factor we might want to move out logic from libsvm over here to base 251 # predictions on obtined values, or adjust libsvm to spit out values from 252 # predict() as well 253 if nlabels == 2: 254 # Apperently libsvm reorders labels so we need to track (1,0) 255 # values instead of (0,1) thus just lets take negative reverse 256 values = [ self.model.predictValues(p)[(trained_labels[1], trained_labels[0])] for p in src ] 257 if len(values)>0: 258 if __debug__: 259 debug("SVM","Forcing values to be ndarray and reshaping " + 260 "them to be 1D vector") 261 values = N.asarray(values).reshape(len(values)) 262 else: 263 # In multiclass we return dictionary for all pairs of labels, 264 # since libsvm does 1-vs-1 pairs 265 values = [ self.model.predictValues(p) for p in src ] 266 states.values = values 267 268 if states.isEnabled("probabilities"): 269 self.probabilities = [ self.model.predictProbability(p) for p in src ] 270 try: 271 states.probabilities = [ self.model.predictProbability(p) for p in src ] 272 except TypeError: 273 warning("Current SVM %s doesn't support probability estimation," % 274 self + " thus no 'values' state") 275 return predictions
276 277
278 - def summary(self):
279 """Provide quick summary over the SVM classifier""" 280 s = super(SVM, self).summary() 281 if self.trained: 282 s += '\n # of SVs: %d' % self.__model.getTotalNSV() 283 try: 284 prm = svm.svmc.svm_model_param_get(self.__model.model) 285 C = svm.svmc.svm_parameter_C_get(prm) 286 # extract information of how many SVs sit inside the margin, 287 # i.e. so called 'bounded SVs' 288 inside_margin = N.sum( 289 # take 0.99 to avoid rounding issues 290 N.abs(self.__model.getSVCoef()) >= 0.99*svm.svmc.svm_parameter_C_get(prm)) 291 s += ' #bounded SVs:%d' % inside_margin 292 s += ' used C:%5g' % C 293 except: 294 pass 295 return s
296 297
298 - def untrain(self):
299 if __debug__: 300 debug("SVM", "Untraining %s and destroying libsvm model" % self) 301 super(SVM, self).untrain() 302 del self.__model 303 self.__model = None
304 305 model = property(fget=lambda self: self.__model) 306 """Access to the SVM model."""
307 308 309 310 #class LinearSVM(SVM): 311 # """Base class of all linear SVM classifiers that make use of the libSVM 312 # package. Still not meant to be used directly. 313 # """ 314 # 315 # def __init__(self, svm_impl, **kwargs): 316 # """The constructor arguments are virtually identical to the ones of 317 # the SVM class, except that 'kernel_type' is set to LINEAR. 318 # """ 319 # # init base class 320 # SVM.__init__(self, kernel_type='linear', 321 # svm_impl=svm_impl, **kwargs) 322 # 323 # 324 # def getSensitivityAnalyzer(self, **kwargs): 325 # """Returns an appropriate SensitivityAnalyzer.""" 326 # return LibSVMLinearSVMWeights(self, **kwargs) 327 # 328 # 329 330 #class LinearNuSVMC(LinearSVM): 331 # """Classifier for linear Nu-SVM classification. 332 # """ 333 # 334 # def __init__(self, **kwargs): 335 # """ 336 # """ 337 # # init base class 338 # LinearSVM.__init__(self, svm_impl='NU_SVC', **kwargs) 339 # 340 # 341 #class LinearCSVMC(LinearSVM): 342 # """Classifier for linear C-SVM classification. 343 # """ 344 # 345 # def __init__(self, **kwargs): 346 # """ 347 # """ 348 # # init base class 349 # LinearSVM.__init__(self, svm_impl='C_SVC', **kwargs) 350 # 351 # 352 # 353 #class RbfNuSVMC(SVM): 354 # """Nu-SVM classifier using a radial basis function kernel. 355 # """ 356 # 357 # def __init__(self, **kwargs): 358 # """ 359 # """ 360 # # init base class 361 # SVM.__init__(self, kernel_type='rbf', 362 # svm_impl='NU_SVC', **kwargs) 363 # 364 # 365 #class RbfCSVMC(SVM): 366 # """C-SVM classifier using a radial basis function kernel. 367 # """ 368 # 369 # def __init__(self, **kwargs): 370 # """ 371 # """ 372 # # init base class 373 # SVM.__init__(self, kernel_type='rbf', 374 # svm_impl='C_SVC', **kwargs) 375 # 376 # 377 # check if there is a libsvm version with configurable 378 # noise reduction ;) 379 if hasattr(svm.svmc, 'svm_set_verbosity'): 380 if __debug__ and "LIBSVM" in debug.active: 381 debug("LIBSVM", "Setting verbosity for libsvm to 255") 382 svm.svmc.svm_set_verbosity(255) 383 else: 384 svm.svmc.svm_set_verbosity(0) 385 386 387
388 -class LinearSVMWeights(Sensitivity):
389 """`SensitivityAnalyzer` for the LIBSVM implementation of a linear SVM. 390 """ 391 392 biases = StateVariable(enabled=True, 393 doc="Offsets of separating hyperplanes") 394 395 396 _LEGAL_CLFS = [ SVM ] 397 398
399 - def __init__(self, clf, **kwargs):
400 """Initialize the analyzer with the classifier it shall use. 401 402 :Parameters: 403 clf: LinearSVM 404 classifier to use. Only classifiers sub-classed from 405 `LinearSVM` may be used. 406 """ 407 # init base classes first 408 Sensitivity.__init__(self, clf, **kwargs)
409 410
411 - def _call(self, dataset, callables=[]):
412 if self.clf.model.nr_class != 2: 413 warning("You are estimating sensitivity for SVM %s trained on %d" % 414 (str(self.clf), self.clf.model.nr_class) + 415 " classes. Make sure that it is what you intended to do" ) 416 417 svcoef = N.matrix(self.clf.model.getSVCoef()) 418 svs = N.matrix(self.clf.model.getSV()) 419 rhos = N.asarray(self.clf.model.getRho()) 420 421 self.biases = rhos 422 # XXX yoh: .mean() is effectively 423 # averages across "sensitivities" of all paired classifiers (I 424 # think). See more info on this topic in svm.py on how sv_coefs 425 # are stored 426 # 427 # First multiply SV coefficients with the actuall SVs to get 428 # weighted impact of SVs on decision, then for each feature 429 # take mean across SVs to get a single weight value 430 # per feature 431 weights = svcoef * svs 432 433 if __debug__: 434 debug('SVM', 435 "Extracting weights for %d-class SVM: #SVs=%s, " % \ 436 (self.clf.model.nr_class, str(self.clf.model.getNSV())) + \ 437 " SVcoefshape=%s SVs.shape=%s Rhos=%s." % \ 438 (svcoef.shape, svs.shape, rhos) + \ 439 " Result: min=%f max=%f" % (N.min(weights), N.max(weights))) 440 441 return N.asarray(weights.T)
442