Package mvpa :: Package clfs :: Package libsvm :: Module svm
[hide private]
[frames] | no frames]

Source Code for Module mvpa.clfs.libsvm.svm

  1  #emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  #ex: set sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Wrap the libsvm package into a very simple class interface.""" 
 10   
 11  __docformat__ = 'restructuredtext' 
 12   
 13  import numpy as N 
 14   
 15  from mvpa.misc.param import Parameter 
 16  from mvpa.misc import warning 
 17  from mvpa.misc.state import StateVariable 
 18   
 19  from mvpa.clfs.base import Classifier 
 20  from mvpa.clfs._svmbase import _SVM 
 21  from mvpa.measures.base import Sensitivity 
 22   
 23  import _svm as svm 
 24  from sens import * 
 25   
 26  if __debug__: 
 27      from mvpa.misc import debug 
 28   
 29  # we better expose those since they are mentioned in docstrings 
 30  from svmc import \ 
 31       C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, \ 
 32       NU_SVR, LINEAR, POLY, RBF, SIGMOID, \ 
 33       PRECOMPUTED 
 34   
 35  known_svm_impl = { 'C_SVC' : (svm.svmc.C_SVC, 'C-SVM classification'), 
 36                     'NU_SVC' : (svm.svmc.NU_SVC, 'nu-SVM classification'), 
 37                     'ONE_CLASS' : (svm.svmc.ONE_CLASS, 'one-class-SVM'), 
 38                     'EPSILON_SVR' : (svm.svmc.EPSILON_SVR, 'epsilon-SVM regression'), 
 39                     'NU_SVR' : (svm.svmc.NU_SVR, 'nu-SVM regression') } 
 40   
 41   
42 -class SVM(_SVM):
43 """Support Vector Machine Classifier. 44 45 This is a simple interface to the libSVM package. 46 """ 47 48 # Since this is internal feature of LibSVM, this state variable is present 49 # here 50 probabilities = StateVariable(enabled=False, 51 doc="Estimates of samples probabilities as provided by LibSVM") 52 53 _KERNELS = { "linear": (svm.svmc.LINEAR, None, LinearSVMWeights), 54 "rbf" : (svm.svmc.RBF, ('gamma',), None), 55 "poly": (svm.svmc.POLY, ('gamma', 'degree', 'coef0'), None), 56 "sigmoid": (svm.svmc.SIGMOID, ('gamma', 'coef0'), None), 57 } 58 # TODO: Complete the list ;-) 59 60 # TODO p is specific for SVR 61 _KNOWN_PARAMS = [ 'epsilon', 'probability', 'shrinking', 'weight_label', 'weight'] 62 63 _KNOWN_KERNEL_PARAMS = [ 'cache_size' ] 64 65
66 - def __init__(self, 67 kernel_type='linear', 68 svm_impl=None, 69 **kwargs):
70 # XXX Determine which parameters depend on each other and implement 71 # safety/simplifying logic around them 72 # already done for: nr_weight 73 # thought: weight and weight_label should be a dict 74 """This is the base class of all classifier that utilize the libSVM 75 package underneath. It is not really meant to be used directly. Unless 76 you know what you are doing it is most likely better to use one of the 77 subclasses. 78 79 Here is the explaination for some of the parameters from the libSVM 80 documentation: 81 82 svm_type can be one of C_SVC, NU_SVC, ONE_CLASS, EPSILON_SVR, NU_SVR. 83 84 - `C_SVC`: C-SVM classification 85 - `NU_SVC`: nu-SVM classification 86 - `ONE_CLASS`: one-class-SVM 87 - `EPSILON_SVR`: epsilon-SVM regression 88 - `NU_SVR`: nu-SVM regression 89 90 kernel_type can be one of LINEAR, POLY, RBF, SIGMOID. 91 92 - `LINEAR`: ``u'*v`` 93 - `POLY`: ``(gamma*u'*v + coef0)^degree`` 94 - `RBF`: ``exp(-gamma*|u-v|^2)`` 95 - `SIGMOID`: ``tanh(gamma*u'*v + coef0)`` 96 - `PRECOMPUTED`: kernel values in training_set_file 97 98 cache_size is the size of the kernel cache, specified in megabytes. 99 C is the cost of constraints violation. (we usually use 1 to 1000) 100 eps is the stopping criterion. (we usually use 0.00001 in nu-SVC, 101 0.001 in others). nu is the parameter in nu-SVM, nu-SVR, and 102 one-class-SVM. p is the epsilon in epsilon-insensitive loss function 103 of epsilon-SVM regression. shrinking = 1 means shrinking is conducted; 104 = 0 otherwise. probability = 1 means model with probability 105 information is obtained; = 0 otherwise. 106 107 nr_weight, weight_label, and weight are used to change the penalty 108 for some classes (If the weight for a class is not changed, it is 109 set to 1). This is useful for training classifier using unbalanced 110 input data or with asymmetric misclassification cost. 111 112 Each weight[i] corresponds to weight_label[i], meaning that 113 the penalty of class weight_label[i] is scaled by a factor of weight[i]. 114 115 If you do not want to change penalty for any of the classes, 116 just set nr_weight to 0. 117 """ 118 self._KNOWN_PARAMS = SVM._KNOWN_PARAMS[:] 119 self._KNOWN_KERNEL_PARAMS = SVM._KNOWN_KERNEL_PARAMS[:] 120 121 # Depending on given arguments, figure out desired SVM 122 # implementation 123 if svm_impl is None: 124 for arg, impl in [ ('tube_epsilon', 'EPSILON_SVR'), 125 ('C', 'C_SVC'), 126 ('nu', 'NU_SVC') ]: 127 if kwargs.has_key(arg): 128 svm_impl = impl 129 if __debug__: 130 debug('SVM', 'No implementation was specified. Since ' 131 '%s is given among arguments, assume %s' % 132 (arg, impl)) 133 break 134 if svm_impl is None: 135 svm_impl = 'C_SVC' 136 if __debug__: 137 debug('SVM', 'Assign C_SVC "by default"') 138 139 svm_type_ = known_svm_impl.get(svm_impl, None) 140 if svm_type_ is None: 141 raise ValueError, \ 142 "Unknown SVM implementation '%s' is requested for libsvm." \ 143 "Known are: %s" % (svm_impl, known_svm_impl.keys()) 144 svm_type = svm_type_[0] # just implementation 145 146 if svm_type in [svm.svmc.C_SVC]: 147 self._KNOWN_PARAMS += ['C'] 148 elif svm_type in [svm.svmc.NU_SVC, svm.svmc.NU_SVR]: 149 self._KNOWN_PARAMS += ['nu'] 150 151 if svm_type in [svm.svmc.EPSILON_SVR]: 152 self._KNOWN_PARAMS += ['tube_epsilon'] 153 154 155 # init base class 156 _SVM.__init__(self, kernel_type, **kwargs) 157 158 # Set internal flags to signal abilities 159 self._clf_internals += [ 'multiclass', 'libsvm' ] # we can do multiclass internally 160 161 if svm_type in [svm.svmc.EPSILON_SVR, svm.svmc.NU_SVR]: 162 self._clf_internals += [ 'regression' ] 163 164 if 'nu' in self._KNOWN_PARAMS: 165 # overwrite eps param with new default value (information taken from libSVM 166 # docs 167 self.params['epsilon'].setDefault(0.001) 168 169 self._svm_type = svm_type 170 171 self.__model = None 172 """Holds the trained SVM."""
173 174 175
176 - def _train(self, dataset):
177 """Train SVM 178 """ 179 # libsvm needs doubles 180 if dataset.samples.dtype == 'float64': 181 src = dataset.samples 182 else: 183 src = dataset.samples.astype('double') 184 185 svmprob = svm.SVMProblem( dataset.labels.tolist(), src ) 186 187 # Translate few params 188 TRANSLATEDICT={'epsilon': 'eps', 189 'tube_epsilon': 'p'} 190 args = [] 191 for paramname, param in self.params.items.items() + self.kernel_params.items.items(): 192 if paramname in TRANSLATEDICT: 193 argname = TRANSLATEDICT[paramname] 194 else: 195 argname = paramname 196 args.append( (argname, param.value) ) 197 198 # XXX All those parameters should be fetched if present from 199 # **kwargs and create appropriate parameters within .params or .kernel_params 200 libsvm_param = svm.SVMParameter( 201 kernel_type=self._kernel_type, 202 svm_type=self._svm_type, 203 **dict(args)) 204 """Store SVM parameters in libSVM compatible format.""" 205 206 if self.params.isKnown('C'):#svm_type in [svm.svmc.C_SVC]: 207 if self.C < 0: 208 newC = self._getDefaultC(dataset.samples)*abs(self.C) 209 if __debug__: 210 debug("SVM", "Computed C to be %s for C=%s" % (newC, self.C)) 211 libsvm_param._setParameter('C', newC) 212 213 self.__model = svm.SVMModel(svmprob, libsvm_param)
214 215
216 - def _predict(self, data):
217 """Predict values for the data 218 """ 219 # libsvm needs doubles 220 if data.dtype == 'float64': 221 src = data 222 else: 223 src = data.astype('double') 224 225 predictions = [ self.model.predict(p) for p in src ] 226 227 if self.states.isEnabled("values"): 228 if len(self.trained_labels) > 2: 229 warning("'Values' for multiclass SVM classifier are ambiguous. You " + 230 "are adviced to wrap your classifier with " + 231 "MulticlassClassifier for explicit handling of " + 232 "separate binary classifiers and corresponding " + 233 "'values'") 234 # XXX We do duplicate work. model.predict calls predictValuesRaw 235 # internally and then does voting or thresholding. So if speed becomes 236 # a factor we might want to move out logic from libsvm over here to base 237 # predictions on obtined values, or adjust libsvm to spit out values from 238 # predict() as well 239 # 240 #try: 241 values = [ self.model.predictValuesRaw(p) for p in src ] 242 if len(values)>0 and len(self.trained_labels) == 2: 243 if __debug__: 244 debug("SVM","Forcing values to be ndarray and reshaping " + 245 "them to be 1D vector") 246 values = N.asarray(values).reshape(len(values)) 247 self.values = values 248 # XXX we should probably do the same as shogun for 249 # multiclass -- just spit out warning without 250 # providing actual values 'per pair' or whatever internal multiclass 251 # implementation it was 252 #except TypeError: 253 # warning("Current SVM doesn't support probability estimation," + 254 # " thus no 'values' state") 255 256 if self.states.isEnabled("probabilities"): 257 self.probabilities = [ self.model.predictProbability(p) for p in src ] 258 try: 259 self.probabilities = [ self.model.predictProbability(p) for p in src ] 260 except TypeError: 261 warning("Current SVM %s doesn't support probability estimation," % 262 self + " thus no 'values' state") 263 return predictions
264 265
266 - def untrain(self):
267 if __debug__: 268 debug("SVM", "Untraining %s and destroying libsvm model" % self) 269 super(SVM, self).untrain() 270 del self.__model 271 self.__model = None
272 273 model = property(fget=lambda self: self.__model) 274 """Access to the SVM model."""
275 276 277 278 #class LinearSVM(SVM): 279 # """Base class of all linear SVM classifiers that make use of the libSVM 280 # package. Still not meant to be used directly. 281 # """ 282 # 283 # def __init__(self, svm_impl, **kwargs): 284 # """The constructor arguments are virtually identical to the ones of 285 # the SVM class, except that 'kernel_type' is set to LINEAR. 286 # """ 287 # # init base class 288 # SVM.__init__(self, kernel_type='linear', 289 # svm_impl=svm_impl, **kwargs) 290 # 291 # 292 # def getSensitivityAnalyzer(self, **kwargs): 293 # """Returns an appropriate SensitivityAnalyzer.""" 294 # return LibSVMLinearSVMWeights(self, **kwargs) 295 # 296 # 297 298 #class LinearNuSVMC(LinearSVM): 299 # """Classifier for linear Nu-SVM classification. 300 # """ 301 # 302 # def __init__(self, **kwargs): 303 # """ 304 # """ 305 # # init base class 306 # LinearSVM.__init__(self, svm_impl='NU_SVC', **kwargs) 307 # 308 # 309 #class LinearCSVMC(LinearSVM): 310 # """Classifier for linear C-SVM classification. 311 # """ 312 # 313 # def __init__(self, **kwargs): 314 # """ 315 # """ 316 # # init base class 317 # LinearSVM.__init__(self, svm_impl='C_SVC', **kwargs) 318 # 319 # 320 # 321 #class RbfNuSVMC(SVM): 322 # """Nu-SVM classifier using a radial basis function kernel. 323 # """ 324 # 325 # def __init__(self, **kwargs): 326 # """ 327 # """ 328 # # init base class 329 # SVM.__init__(self, kernel_type='rbf', 330 # svm_impl='NU_SVC', **kwargs) 331 # 332 # 333 #class RbfCSVMC(SVM): 334 # """C-SVM classifier using a radial basis function kernel. 335 # """ 336 # 337 # def __init__(self, **kwargs): 338 # """ 339 # """ 340 # # init base class 341 # SVM.__init__(self, kernel_type='rbf', 342 # svm_impl='C_SVC', **kwargs) 343 # 344 # 345 # check if there is a libsvm version with configurable 346 # noise reduction ;) 347 if hasattr(svm.svmc, 'svm_set_verbosity'): 348 if __debug__ and "LIBSVM" in debug.active: 349 debug("LIBSVM", "Setting verbosity for libsvm to 255") 350 svm.svmc.svm_set_verbosity(255) 351 else: 352 svm.svmc.svm_set_verbosity(0) 353 354 355
356 -class LinearSVMWeights(Sensitivity):
357 """`SensitivityAnalyzer` for the LIBSVM implementation of a linear SVM. 358 """ 359 360 biases = StateVariable(enabled=True, 361 doc="Offsets of separating hyperplanes") 362
363 - def __init__(self, clf, **kwargs):
364 """Initialize the analyzer with the classifier it shall use. 365 366 :Parameters: 367 clf: LinearSVM 368 classifier to use. Only classifiers sub-classed from 369 `LinearSVM` may be used. 370 """ 371 # init base classes first 372 Sensitivity.__init__(self, clf, **kwargs)
373 374
375 - def _call(self, dataset, callables=[]):
376 if self.clf.model.nr_class != 2: 377 warning("You are estimating sensitivity for SVM %s trained on %d" % 378 (str(self.clf), self.clf.model.nr_class) + 379 " classes. Make sure that it is what you intended to do" ) 380 381 svcoef = N.matrix(self.clf.model.getSVCoef()) 382 svs = N.matrix(self.clf.model.getSV()) 383 rhos = N.asarray(self.clf.model.getRho()) 384 385 self.biases = rhos 386 # XXX yoh: .mean() is effectively 387 # averages across "sensitivities" of all paired classifiers (I 388 # think). See more info on this topic in svm.py on how sv_coefs 389 # are stored 390 # 391 # First multiply SV coefficients with the actuall SVs to get 392 # weighted impact of SVs on decision, then for each feature 393 # take mean across SVs to get a single weight value 394 # per feature 395 weights = svcoef * svs 396 397 if __debug__: 398 debug('SVM', 399 "Extracting weights for %d-class SVM: #SVs=%s, " % \ 400 (self.clf.model.nr_class, str(self.clf.model.getNSV())) + \ 401 " SVcoefshape=%s SVs.shape=%s Rhos=%s." % \ 402 (svcoef.shape, svs.shape, rhos) + \ 403 " Result: min=%f max=%f" % (N.min(weights), N.max(weights))) 404 405 return N.asarray(weights.T)
406