Package mvpa :: Package clfs :: Module transerror
[hide private]
[frames] | no frames]

Source Code for Module mvpa.clfs.transerror

  1  #emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  #ex: set sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Utility class to compute the transfer error of classifiers.""" 
 10   
 11  __docformat__ = 'restructuredtext' 
 12   
 13  import copy 
 14   
 15  import numpy as N 
 16   
 17  from sets import Set 
 18  from StringIO import StringIO 
 19  from math import log10, ceil 
 20   
 21  from mvpa.misc.errorfx import MeanMismatchErrorFx 
 22  from mvpa.misc import warning 
 23  from mvpa.misc.state import StateVariable, Stateful 
 24  from mvpa.base.dochelpers import enhancedDocString 
 25   
 26  if __debug__: 
 27      from mvpa.misc import debug 
28 29 30 -class ConfusionMatrix(object):
31 """Simple class for confusion matrix computation / display. 32 33 Implementation is aimed to be simple, thus it delays actual 34 computation of confusion matrix untill all data is acquired (to 35 figure out complete set of labels. If testing data doesn't have a 36 complete set of labels, but you like to include all labels, 37 provide them as a parameter to constructor. 38 """ 39 40 41 _STATS_DESCRIPTION = ( 42 ('TP', 'true positive (AKA hit)', None), 43 ('TN', 'true negative (AKA correct rejection)', None), 44 ('FP', 'false positive (AKA false alarm, Type I error)', None), 45 ('FN', 'false negative (AKA miss, Type II error)', None), 46 ('TPR', 'true positive rate (AKA hit rate, recall, sensitivity)', 47 'TPR = TP / P = TP / (TP + FN)'), 48 ('FPR', 'false positive rate (AKA false alarm rate, fall-out)', 49 'FPR = FP / N = FP / (FP + TN)'), 50 ('ACC', 'accuracy', 'ACC = (TP + TN) / (P + N)'), 51 ('SPC', 'specificity', 'SPC = TN / (FP + TN) = 1 - FPR'), 52 ('PPV', 'positive predictive value (AKA precision)', 53 'PPV = TP / (TP + FP)'), 54 ('NPV', 'negative predictive value', 'NPV = TN / (TN + FN)'), 55 ('FDR', 'false discovery rate', 'FDR = FP / (FP + TP)'), 56 ('MCC', "Matthews Correlation Coefficient", 57 "MCC = (TP*TN - FP*FN)/sqrt(P N P' N')"), 58 ) 59 60 # XXX Michael: - How do multiple sets work and what are they there for? 61 # YYY Yarik: - Set is just a tuple (targets, predictions). While 62 # 'computing' the matrix, all sets are considered together.
63 - def __init__(self, labels=None, targets=None, predictions=None):
64 """Initialize ConfusionMatrix with optional list of `labels` 65 66 :Parameters: 67 labels : list 68 Optional set of labels to include in the matrix 69 targets 70 Optional set of targets 71 predictions 72 Optional set of predictions 73 """ 74 if labels == None: 75 labels = [] 76 self.__labels = labels 77 """List of known labels""" 78 self.__computed = False 79 """Flag either it was computed for a given set of data""" 80 self.__sets = [] 81 """Datasets (target, prediction) to compute confusion matrix on""" 82 self.__matrix = None 83 """Resultant confusion matrix""" 84 85 if not targets is None or not predictions is None: 86 if not targets is None and not predictions is None: 87 self.add(targets=targets, predictions=predictions) 88 else: 89 raise ValueError, \ 90 "Please provide none or both targets and predictions"
91 92
93 - def add(self, targets, predictions):
94 """Add new results to the set of known results""" 95 if len(targets) != len(predictions): 96 raise ValueError, \ 97 "Targets[%d] and predictions[%d]" % (len(targets), 98 len(predictions)) + \ 99 " have different number of samples" 100 101 # enforce labels in predictions to be of the same datatype as in 102 # targets, since otherwise we are getting doubles for unknown at a 103 # given moment labels 104 nonetype = type(None) 105 for i in xrange(len(targets)): 106 t1, t2 = type(targets[i]), type(predictions[i]) 107 # if there were no prediction made - leave None, otherwise 108 # convert to appropriate type 109 if t1 != t2 and t2 != nonetype: 110 #warning("Obtained target %s and prediction %s are of " % 111 # (t1, t2) + "different datatypes.") 112 if isinstance(predictions, tuple): 113 predictions = list(predictions) 114 predictions[i] = t1(predictions[i]) 115 116 self.__sets.append( (targets, predictions) ) 117 self.__computed = False
118 119
120 - def _compute(self):
121 """Actually compute the confusion matrix based on all the sets""" 122 if self.__computed: 123 return 124 125 if __debug__: 126 if not self.__matrix is None: 127 debug("LAZY", "Have to recompute ConfusionMatrix %s" % `self`) 128 129 # TODO: BinaryClassifier might spit out a list of predictions for each 130 # value need to handle it... for now just keep original labels 131 try: 132 # figure out what labels we have 133 labels = \ 134 list(reduce(lambda x, y: x.union(Set(y[0]).union(Set(y[1]))), 135 self.__sets, 136 Set(self.__labels))) 137 except: 138 labels = self.__labels 139 140 labels.sort() 141 self.__labels = labels # store the recomputed labels 142 143 Nlabels, Nsets = len(labels), len(self.__sets) 144 145 if __debug__: 146 debug("CM", "Got labels %s" % labels) 147 148 # Create a matrix for all votes 149 mat_all = N.zeros( (Nsets, Nlabels, Nlabels), dtype=int ) 150 151 # create total number of samples of each label counts 152 # just for convinience I guess since it can always be 153 # computed from mat_all 154 counts_all = N.zeros( (Nsets, Nlabels) ) 155 156 # reverse mapping from label into index in the list of labels 157 rev_map = dict([ (x[1], x[0]) for x in enumerate(labels)]) 158 for iset, (targets, predictions) in enumerate(self.__sets): 159 for t,p in zip(targets, predictions): 160 mat_all[iset, rev_map[p], rev_map[t]] += 1 161 162 163 # for now simply compute a sum of votes across different sets 164 # we might do something more sophisticated later on, and this setup 165 # should easily allow it 166 self.__matrix = N.sum(mat_all, axis=0) 167 self.__Nsamples = N.sum(self.__matrix, axis=0) 168 self.__Ncorrect = sum(N.diag(self.__matrix)) 169 170 TP = N.diag(self.__matrix) 171 offdiag = self.__matrix - N.diag(TP) 172 stats = {'TP' : TP, 173 'FP' : N.sum(offdiag, axis=1), 174 'FN' : N.sum(offdiag, axis=0)} 175 176 stats['CORR'] = N.sum(TP) 177 stats['TN'] = stats['CORR'] - stats['TP'] 178 stats['P'] = stats['TP'] + stats['FN'] 179 stats['N'] = N.sum(stats['P']) - stats['P'] 180 stats["P'"] = stats['TP'] + stats['FP'] 181 stats["N'"] = stats['TN'] + stats['FN'] 182 stats['TPR'] = stats['TP'] / (1.0*stats['P']) 183 stats['PPV'] = stats['TP'] / (1.0*stats["P'"]) 184 stats['NPV'] = stats['TN'] / (1.0*stats["N'"]) 185 stats['FDR'] = stats['FP'] / (1.0*stats["P'"]) 186 stats['SPC'] = (stats['TN']) / (1.0*stats['FP'] + stats['TN']) 187 stats['MCC'] = \ 188 (stats['TP'] * stats['TN'] - stats['FP'] * stats['FN']) \ 189 / N.sqrt(1.0*stats['P']*stats['N']*stats["P'"]*stats["N'"]) 190 191 stats['ACC'] = N.sum(TP)/(1.0*N.sum(stats['P'])) 192 stats['ACC%'] = stats['ACC'] * 100.0 193 194 self.__stats = stats 195 self.__computed = True
196 197
198 - def asstring(self, header=True, percents=True, summary=True, 199 print_empty=False, description=False):
200 """'Pretty print' the matrix""" 201 self._compute() 202 203 # some shortcuts 204 labels = self.__labels 205 matrix = self.__matrix 206 207 out = StringIO() 208 # numbers of different entries 209 Nlabels = len(labels) 210 Nsamples = self.__Nsamples.astype(int) 211 212 if len(self.__sets) == 0: 213 return "Empty confusion matrix" 214 215 Ndigitsmax = int(ceil(log10(max(Nsamples)))) 216 Nlabelsmax = max( [len(str(x)) for x in labels] ) 217 218 # length of a single label/value 219 L = max(Ndigitsmax+2, Nlabelsmax) #, len("100.00%")) 220 res = "" 221 222 stats_perpredict = ["P'", "N'", 'FP', 'FN', 'PPV', 'NPV', 'TPR', 223 'SPC', 'FDR', 'MCC'] 224 stats_pertarget = ['P', 'N', 'TP', 'TN'] 225 stats_summary = ['ACC', 'ACC%'] 226 227 228 #prefixlen = Nlabelsmax + 2 + Ndigitsmax + 1 229 prefixlen = Nlabelsmax + 1 230 pref = ' '*(prefixlen) # empty prefix 231 232 if matrix.shape != (Nlabels, Nlabels): 233 raise ValueError, \ 234 "Number of labels %d doesn't correspond the size" + \ 235 " of a confusion matrix %s" % (Nlabels, matrix.shape) 236 237 # list of lists of what is printed 238 printed = [] 239 underscores = [" %s" % ("-" * L)] * Nlabels 240 if header: 241 # labels 242 printed.append(['----------. ']) 243 printed.append(['predictions\\targets'] + labels) 244 # underscores 245 printed.append([' `------'] \ 246 + underscores + stats_perpredict) 247 248 # matrix itself 249 for i, line in enumerate(matrix): 250 printed.append( 251 [labels[i]] + 252 [ str(x) for x in line ] + 253 [ '%.2g' % self.__stats[x][i] for x in stats_perpredict]) 254 255 if summary: 256 printed.append(['Per target:'] + underscores) 257 for stat in stats_pertarget: 258 printed.append([stat] + ['%.2g' \ 259 % self.__stats[stat][i] for i in xrange(Nlabels)]) 260 261 printed.append(['SUMMARY:'] + underscores) 262 263 for stat in stats_summary: 264 printed.append([stat] + ['%.2g' % self.__stats[stat]]) 265 266 # equalize number of elements in each row 267 Nelements_max = max(len(x) for x in printed) 268 for i,printed_ in enumerate(printed): 269 printed[i] += [''] * (Nelements_max - len(printed_)) 270 271 # figure out lengths within each column 272 aprinted = N.asarray(printed) 273 col_width = [ max( [len(x) for x in column] ) for column in aprinted.T ] 274 275 for i, printed_ in enumerate(printed): 276 for j, item in enumerate(printed_): 277 item = str(item) 278 NspacesL = ceil((col_width[j] - len(item))/2.0) 279 NspacesR = col_width[j] - NspacesL - len(item) 280 out.write("%%%ds%%s%%%ds " \ 281 % (NspacesL, NspacesR) % ('', item, '')) 282 out.write("\n") 283 284 285 if description: 286 out.write("\nStatistics computed in 1-vs-rest fashion per each " \ 287 "target.\n") 288 out.write("Abbreviations (for details see " \ 289 "http://en.wikipedia.org/wiki/ROC_curve):\n") 290 for d, val, eq in self._STATS_DESCRIPTION: 291 out.write(" %-3s: %s\n" % (d, val)) 292 if eq is not None: 293 out.write(" " + eq + "\n") 294 295 #out.write("%s" % printed) 296 result = out.getvalue() 297 out.close() 298 return result
299 300
301 - def __str__(self):
302 """String summary over the confusion matrix 303 304 It would print description of the summary statistics if 'CM' 305 debug target is active 306 """ 307 if __debug__: 308 description = ('CM' in debug.active) 309 else: 310 description = False 311 return self.asstring(header=True, percents=True, 312 summary=True, print_empty=False, 313 description=description)
314
315 - def __iadd__(self, other):
316 """Add the sets from `other` s `ConfusionMatrix` to current one 317 """ 318 #print "adding ", other, " to ", self 319 # need to do shallow copy, or otherwise smth like "cm += cm" 320 # would loop forever and exhaust memory eventually 321 othersets = copy.copy(other.__sets) 322 for set in othersets: 323 self.add(set[0], set[1]) 324 return self
325 326
327 - def __add__(self, other):
328 """Add two `ConfusionMatrix` 329 """ 330 result = copy.copy(self) 331 result += other 332 return result
333 334 335 @property
336 - def matrices(self):
337 """Return a list of separate confusion matrix per each stored set""" 338 return [ self.__class__(labels=self.labels, 339 targets=x[0], 340 predictions=x[1]) for x in self.__sets]
341 342 343 @property
344 - def labels(self):
345 self._compute() 346 return self.__labels
347 348 349 @property
350 - def matrix(self):
351 self._compute() 352 return self.__matrix
353 354 355 @property
356 - def percentCorrect(self):
357 self._compute() 358 return 100.0*self.__Ncorrect/sum(self.__Nsamples)
359 360 361 @property
362 - def error(self):
363 self._compute() 364 return 1.0-self.__Ncorrect*1.0/sum(self.__Nsamples)
365 366 sets = property(lambda self:self.__sets)
367
368 369 370 -class ClassifierError(Stateful):
371 """Compute (or return) some error of a (trained) classifier on a dataset. 372 """ 373 374 confusion = StateVariable(enabled=False) 375 """TODO Think that labels might be also symbolic thus can't directly 376 be indicies of the array 377 """ 378 379 training_confusion = StateVariable(enabled=False) 380 """Proxy training_confusion from underlying classifier 381 """ 382
383 - def __init__(self, clf, labels=None, train=True, **kwargs):
384 """Initialization. 385 386 :Parameters: 387 clf : Classifier 388 Either trained or untrained classifier 389 labels : list 390 if provided, should be a set of labels to add on top of the 391 ones present in testdata 392 train : bool 393 unless train=False, classifier gets trained if 394 trainingdata provided to __call__ 395 """ 396 Stateful.__init__(self, **kwargs) 397 self.__clf = clf 398 399 self._labels = labels 400 """Labels to add on top to existing in testing data""" 401 402 self.__train = train 403 """Either to train classifier if trainingdata is provided"""
404 405 406 __doc__ = enhancedDocString('ClassifierError', locals(), Stateful) 407 408
409 - def __copy__(self):
410 """TODO: think... may be we need to copy self.clf""" 411 out = ClassifierError.__new__(TransferError) 412 ClassifierError.__init__(out, self.clf) 413 return out
414 415
416 - def _precall(self, testdataset, trainingdataset=None):
417 """Generic part which trains the classifier if necessary 418 """ 419 if not trainingdataset is None: 420 if self.__train: 421 # XXX can be pretty annoying if triggered inside an algorithm 422 # where it cannot be switched of, but retraining might be 423 # intended or at least not avoidable. 424 # Additonally isTrained docs say: 425 # MUST BE USED WITH CARE IF EVER 426 # 427 # switching it off for now 428 #if self.__clf.isTrained(trainingdataset): 429 # warning('It seems that classifier %s was already trained' % 430 # self.__clf + ' on dataset %s. Please inspect' \ 431 # % trainingdataset) 432 if self.states.isEnabled('training_confusion'): 433 self.__clf.states._changeTemporarily(enable_states=['training_confusion']) 434 self.__clf.train(trainingdataset) 435 if self.states.isEnabled('training_confusion'): 436 self.training_confusion = self.__clf.training_confusion 437 self.__clf.states._resetEnabledTemporarily() 438 439 if self.__clf.states.isEnabled('trained_labels') and \ 440 not testdataset is None: 441 newlabels = Set(testdataset.uniquelabels) - self.__clf.trained_labels 442 if len(newlabels)>0: 443 warning("Classifier %s wasn't trained to classify labels %s" % 444 (`self.__clf`, `newlabels`) + 445 " present in testing dataset. Make sure that you has" + 446 " not mixed order/names of the arguments anywhere")
447 448 ### Here checking for if it was trained... might be a cause of trouble 449 # XXX disabled since it is unreliable.. just rely on explicit 450 # self.__train 451 # if not self.__clf.isTrained(trainingdataset): 452 # self.__clf.train(trainingdataset) 453 # elif __debug__: 454 # debug('CERR', 455 # 'Not training classifier %s since it was ' % `self.__clf` 456 # + ' already trained on dataset %s' % `trainingdataset`) 457 458
459 - def _call(self, testdataset, trainingdataset=None):
460 raise NotImplementedError
461 462
463 - def _postcall(self, testdataset, trainingdataset=None, error=None):
464 pass
465 466
467 - def __call__(self, testdataset, trainingdataset=None):
468 """Compute the transfer error for a certain test dataset. 469 470 If `trainingdataset` is not `None` the classifier is trained using the 471 provided dataset before computing the transfer error. Otherwise the 472 classifier is used in it's current state to make the predictions on 473 the test dataset. 474 475 Returns a scalar value of the transfer error. 476 """ 477 self._precall(testdataset, trainingdataset) 478 error = self._call(testdataset, trainingdataset) 479 self._postcall(testdataset, trainingdataset, error) 480 return error
481 482 483 @property
484 - def clf(self):
485 return self.__clf
486 487 488 @property
489 - def labels(self):
490 return self._labels
491
492 493 494 -class TransferError(ClassifierError):
495 """Compute the transfer error of a (trained) classifier on a dataset. 496 497 The actual error value is computed using a customizable error function. 498 Optionally the classifier can be trained by passing an additional 499 training dataset to the __call__() method. 500 """ 501 502 null_prob = StateVariable(enabled=True) 503 """Stores the probability of an error result under the NULL hypothesis""" 504
505 - def __init__(self, clf, errorfx=MeanMismatchErrorFx(), labels=None, 506 null_dist=None, **kwargs):
507 """Initialization. 508 509 :Parameters: 510 clf : Classifier 511 Either trained or untrained classifier 512 errorfx 513 Functor that computes a scalar error value from the vectors of 514 desired and predicted values (e.g. subclass of `ErrorFunction`) 515 labels : list 516 if provided, should be a set of labels to add on top of the 517 ones present in testdata 518 null_dist : instance of distribution estimator 519 """ 520 ClassifierError.__init__(self, clf, labels, **kwargs) 521 self.__errorfx = errorfx 522 self.__null_dist = null_dist
523 524 525 __doc__ = enhancedDocString('TransferError', locals(), ClassifierError) 526 527
528 - def __copy__(self):
529 """TODO: think... may be we need to copy self.clf""" 530 # TODO TODO -- use ClassifierError.__copy__ 531 out = TransferError.__new__(TransferError) 532 TransferError.__init__(out, self.clf, self.errorfx, self._labels) 533 534 return out
535 536
537 - def _call(self, testdataset, trainingdataset=None):
538 """Compute the transfer error for a certain test dataset. 539 540 If `trainingdataset` is not `None` the classifier is trained using the 541 provided dataset before computing the transfer error. Otherwise the 542 classifier is used in it's current state to make the predictions on 543 the test dataset. 544 545 Returns a scalar value of the transfer error. 546 """ 547 548 predictions = self.clf.predict(testdataset.samples) 549 550 # compute confusion matrix 551 # XXX should migrate into ClassifierError.__postcall? 552 # YYY probably not because other childs could estimate it 553 # not from test/train datasets explicitely, see 554 # `ConfusionBasedError`, where confusion is simply 555 # bound to classifiers confusion matrix 556 if self.states.isEnabled('confusion'): 557 self.confusion = ConfusionMatrix( 558 labels=self.labels, targets=testdataset.labels, 559 predictions=predictions) 560 561 # compute error from desired and predicted values 562 error = self.__errorfx(predictions, 563 testdataset.labels) 564 565 return error
566 567
568 - def _postcall(self, vdata, wdata=None, error=None):
569 """ 570 """ 571 # estimate the NULL distribution when functor and training data is 572 # given 573 if not self.__null_dist is None and not wdata is None: 574 # we need a matching transfer error instances (e.g. same error 575 # function), but we have to disable the estimation of the null 576 # distribution in that child to prevent infinite looping. 577 null_terr = copy.copy(self) 578 null_terr.__null_dist = None 579 self.__null_dist.fit(null_terr, wdata, vdata) 580 581 582 # get probability of error under NULL hypothesis if available 583 if not error is None and not self.__null_dist is None: 584 self.null_prob = self.__null_dist.cdf(error)
585 586 587 @property
588 - def errorfx(self): return self.__errorfx
589
590 591 592 -class ConfusionBasedError(ClassifierError):
593 """For a given classifier report an error based on internally 594 computed error measure (given by some `ConfusionMatrix` stored in 595 some state variable of `Classifier`). 596 597 This way we can perform feature selection taking as the error 598 criterion either learning error, or transfer to splits error in 599 the case of SplitClassifier 600 """ 601
602 - def __init__(self, clf, labels=None, confusion_state="training_confusion", 603 **kwargs):
604 """Initialization. 605 606 :Parameters: 607 clf : Classifier 608 Either trained or untrained classifier 609 confusion_state 610 Id of the state variable which stores `ConfusionMatrix` 611 labels : list 612 if provided, should be a set of labels to add on top of the 613 ones present in testdata 614 """ 615 ClassifierError.__init__(self, clf, labels, **kwargs) 616 617 self.__confusion_state = confusion_state 618 """What state to extract from""" 619 620 if not clf.states.isKnown(confusion_state): 621 raise ValueError, \ 622 "State variable %s is not defined for classifier %s" % \ 623 (confusion_state, `clf`) 624 if not clf.states.isEnabled(confusion_state): 625 if __debug__: 626 debug('CERR', "Forcing state %s to be enabled for %s" % 627 (confusion_state, `clf`)) 628 clf.states.enable(confusion_state)
629 630 631 __doc__ = enhancedDocString('ConfusionBasedError', locals(), 632 ClassifierError) 633 634
635 - def _call(self, testdata, trainingdata=None):
636 """Extract transfer error. Nor testdata, neither trainingdata is used 637 """ 638 confusion = self.clf.states.getvalue(self.__confusion_state) 639 self.confusion = confusion 640 return confusion.error
641