1
2
3
4
5
6
7
8
9 """Utility class to compute the transfer error of classifiers."""
10
11 __docformat__ = 'restructuredtext'
12
13 import copy
14
15 import numpy as N
16
17 from sets import Set
18 from StringIO import StringIO
19 from math import log10, ceil
20
21 from mvpa.misc.errorfx import MeanMismatchErrorFx
22 from mvpa.misc import warning
23 from mvpa.misc.state import StateVariable, Stateful
24 from mvpa.base.dochelpers import enhancedDocString
25
26 if __debug__:
27 from mvpa.misc import debug
31 """Simple class for confusion matrix computation / display.
32
33 Implementation is aimed to be simple, thus it delays actual
34 computation of confusion matrix untill all data is acquired (to
35 figure out complete set of labels. If testing data doesn't have a
36 complete set of labels, but you like to include all labels,
37 provide them as a parameter to constructor.
38 """
39
40
41 _STATS_DESCRIPTION = (
42 ('TP', 'true positive (AKA hit)', None),
43 ('TN', 'true negative (AKA correct rejection)', None),
44 ('FP', 'false positive (AKA false alarm, Type I error)', None),
45 ('FN', 'false negative (AKA miss, Type II error)', None),
46 ('TPR', 'true positive rate (AKA hit rate, recall, sensitivity)',
47 'TPR = TP / P = TP / (TP + FN)'),
48 ('FPR', 'false positive rate (AKA false alarm rate, fall-out)',
49 'FPR = FP / N = FP / (FP + TN)'),
50 ('ACC', 'accuracy', 'ACC = (TP + TN) / (P + N)'),
51 ('SPC', 'specificity', 'SPC = TN / (FP + TN) = 1 - FPR'),
52 ('PPV', 'positive predictive value (AKA precision)',
53 'PPV = TP / (TP + FP)'),
54 ('NPV', 'negative predictive value', 'NPV = TN / (TN + FN)'),
55 ('FDR', 'false discovery rate', 'FDR = FP / (FP + TP)'),
56 ('MCC', "Matthews Correlation Coefficient",
57 "MCC = (TP*TN - FP*FN)/sqrt(P N P' N')"),
58 )
59
60
61
62
63 - def __init__(self, labels=None, targets=None, predictions=None):
64 """Initialize ConfusionMatrix with optional list of `labels`
65
66 :Parameters:
67 labels : list
68 Optional set of labels to include in the matrix
69 targets
70 Optional set of targets
71 predictions
72 Optional set of predictions
73 """
74 if labels == None:
75 labels = []
76 self.__labels = labels
77 """List of known labels"""
78 self.__computed = False
79 """Flag either it was computed for a given set of data"""
80 self.__sets = []
81 """Datasets (target, prediction) to compute confusion matrix on"""
82 self.__matrix = None
83 """Resultant confusion matrix"""
84
85 if not targets is None or not predictions is None:
86 if not targets is None and not predictions is None:
87 self.add(targets=targets, predictions=predictions)
88 else:
89 raise ValueError, \
90 "Please provide none or both targets and predictions"
91
92
93 - def add(self, targets, predictions):
94 """Add new results to the set of known results"""
95 if len(targets) != len(predictions):
96 raise ValueError, \
97 "Targets[%d] and predictions[%d]" % (len(targets),
98 len(predictions)) + \
99 " have different number of samples"
100
101
102
103
104 nonetype = type(None)
105 for i in xrange(len(targets)):
106 t1, t2 = type(targets[i]), type(predictions[i])
107
108
109 if t1 != t2 and t2 != nonetype:
110
111
112 if isinstance(predictions, tuple):
113 predictions = list(predictions)
114 predictions[i] = t1(predictions[i])
115
116 self.__sets.append( (targets, predictions) )
117 self.__computed = False
118
119
121 """Actually compute the confusion matrix based on all the sets"""
122 if self.__computed:
123 return
124
125 if __debug__:
126 if not self.__matrix is None:
127 debug("LAZY", "Have to recompute ConfusionMatrix %s" % `self`)
128
129
130
131 try:
132
133 labels = \
134 list(reduce(lambda x, y: x.union(Set(y[0]).union(Set(y[1]))),
135 self.__sets,
136 Set(self.__labels)))
137 except:
138 labels = self.__labels
139
140 labels.sort()
141 self.__labels = labels
142
143 Nlabels, Nsets = len(labels), len(self.__sets)
144
145 if __debug__:
146 debug("CM", "Got labels %s" % labels)
147
148
149 mat_all = N.zeros( (Nsets, Nlabels, Nlabels), dtype=int )
150
151
152
153
154 counts_all = N.zeros( (Nsets, Nlabels) )
155
156
157 rev_map = dict([ (x[1], x[0]) for x in enumerate(labels)])
158 for iset, (targets, predictions) in enumerate(self.__sets):
159 for t,p in zip(targets, predictions):
160 mat_all[iset, rev_map[p], rev_map[t]] += 1
161
162
163
164
165
166 self.__matrix = N.sum(mat_all, axis=0)
167 self.__Nsamples = N.sum(self.__matrix, axis=0)
168 self.__Ncorrect = sum(N.diag(self.__matrix))
169
170 TP = N.diag(self.__matrix)
171 offdiag = self.__matrix - N.diag(TP)
172 stats = {'TP' : TP,
173 'FP' : N.sum(offdiag, axis=1),
174 'FN' : N.sum(offdiag, axis=0)}
175
176 stats['CORR'] = N.sum(TP)
177 stats['TN'] = stats['CORR'] - stats['TP']
178 stats['P'] = stats['TP'] + stats['FN']
179 stats['N'] = N.sum(stats['P']) - stats['P']
180 stats["P'"] = stats['TP'] + stats['FP']
181 stats["N'"] = stats['TN'] + stats['FN']
182 stats['TPR'] = stats['TP'] / (1.0*stats['P'])
183 stats['PPV'] = stats['TP'] / (1.0*stats["P'"])
184 stats['NPV'] = stats['TN'] / (1.0*stats["N'"])
185 stats['FDR'] = stats['FP'] / (1.0*stats["P'"])
186 stats['SPC'] = (stats['TN']) / (1.0*stats['FP'] + stats['TN'])
187 stats['MCC'] = \
188 (stats['TP'] * stats['TN'] - stats['FP'] * stats['FN']) \
189 / N.sqrt(1.0*stats['P']*stats['N']*stats["P'"]*stats["N'"])
190
191 stats['ACC'] = N.sum(TP)/(1.0*N.sum(stats['P']))
192 stats['ACC%'] = stats['ACC'] * 100.0
193
194 self.__stats = stats
195 self.__computed = True
196
197
198 - def asstring(self, header=True, percents=True, summary=True,
199 print_empty=False, description=False):
200 """'Pretty print' the matrix"""
201 self._compute()
202
203
204 labels = self.__labels
205 matrix = self.__matrix
206
207 out = StringIO()
208
209 Nlabels = len(labels)
210 Nsamples = self.__Nsamples.astype(int)
211
212 if len(self.__sets) == 0:
213 return "Empty confusion matrix"
214
215 Ndigitsmax = int(ceil(log10(max(Nsamples))))
216 Nlabelsmax = max( [len(str(x)) for x in labels] )
217
218
219 L = max(Ndigitsmax+2, Nlabelsmax)
220 res = ""
221
222 stats_perpredict = ["P'", "N'", 'FP', 'FN', 'PPV', 'NPV', 'TPR',
223 'SPC', 'FDR', 'MCC']
224 stats_pertarget = ['P', 'N', 'TP', 'TN']
225 stats_summary = ['ACC', 'ACC%']
226
227
228
229 prefixlen = Nlabelsmax + 1
230 pref = ' '*(prefixlen)
231
232 if matrix.shape != (Nlabels, Nlabels):
233 raise ValueError, \
234 "Number of labels %d doesn't correspond the size" + \
235 " of a confusion matrix %s" % (Nlabels, matrix.shape)
236
237
238 printed = []
239 underscores = [" %s" % ("-" * L)] * Nlabels
240 if header:
241
242 printed.append(['----------. '])
243 printed.append(['predictions\\targets'] + labels)
244
245 printed.append([' `------'] \
246 + underscores + stats_perpredict)
247
248
249 for i, line in enumerate(matrix):
250 printed.append(
251 [labels[i]] +
252 [ str(x) for x in line ] +
253 [ '%.2g' % self.__stats[x][i] for x in stats_perpredict])
254
255 if summary:
256 printed.append(['Per target:'] + underscores)
257 for stat in stats_pertarget:
258 printed.append([stat] + ['%.2g' \
259 % self.__stats[stat][i] for i in xrange(Nlabels)])
260
261 printed.append(['SUMMARY:'] + underscores)
262
263 for stat in stats_summary:
264 printed.append([stat] + ['%.2g' % self.__stats[stat]])
265
266
267 Nelements_max = max(len(x) for x in printed)
268 for i,printed_ in enumerate(printed):
269 printed[i] += [''] * (Nelements_max - len(printed_))
270
271
272 aprinted = N.asarray(printed)
273 col_width = [ max( [len(x) for x in column] ) for column in aprinted.T ]
274
275 for i, printed_ in enumerate(printed):
276 for j, item in enumerate(printed_):
277 item = str(item)
278 NspacesL = ceil((col_width[j] - len(item))/2.0)
279 NspacesR = col_width[j] - NspacesL - len(item)
280 out.write("%%%ds%%s%%%ds " \
281 % (NspacesL, NspacesR) % ('', item, ''))
282 out.write("\n")
283
284
285 if description:
286 out.write("\nStatistics computed in 1-vs-rest fashion per each " \
287 "target.\n")
288 out.write("Abbreviations (for details see " \
289 "http://en.wikipedia.org/wiki/ROC_curve):\n")
290 for d, val, eq in self._STATS_DESCRIPTION:
291 out.write(" %-3s: %s\n" % (d, val))
292 if eq is not None:
293 out.write(" " + eq + "\n")
294
295
296 result = out.getvalue()
297 out.close()
298 return result
299
300
302 """String summary over the confusion matrix
303
304 It would print description of the summary statistics if 'CM'
305 debug target is active
306 """
307 if __debug__:
308 description = ('CM' in debug.active)
309 else:
310 description = False
311 return self.asstring(header=True, percents=True,
312 summary=True, print_empty=False,
313 description=description)
314
316 """Add the sets from `other` s `ConfusionMatrix` to current one
317 """
318
319
320
321 othersets = copy.copy(other.__sets)
322 for set in othersets:
323 self.add(set[0], set[1])
324 return self
325
326
328 """Add two `ConfusionMatrix`
329 """
330 result = copy.copy(self)
331 result += other
332 return result
333
334
335 @property
337 """Return a list of separate confusion matrix per each stored set"""
338 return [ self.__class__(labels=self.labels,
339 targets=x[0],
340 predictions=x[1]) for x in self.__sets]
341
342
343 @property
345 self._compute()
346 return self.__labels
347
348
349 @property
351 self._compute()
352 return self.__matrix
353
354
355 @property
357 self._compute()
358 return 100.0*self.__Ncorrect/sum(self.__Nsamples)
359
360
361 @property
363 self._compute()
364 return 1.0-self.__Ncorrect*1.0/sum(self.__Nsamples)
365
366 sets = property(lambda self:self.__sets)
367
371 """Compute (or return) some error of a (trained) classifier on a dataset.
372 """
373
374 confusion = StateVariable(enabled=False)
375 """TODO Think that labels might be also symbolic thus can't directly
376 be indicies of the array
377 """
378
379 training_confusion = StateVariable(enabled=False)
380 """Proxy training_confusion from underlying classifier
381 """
382
383 - def __init__(self, clf, labels=None, train=True, **kwargs):
384 """Initialization.
385
386 :Parameters:
387 clf : Classifier
388 Either trained or untrained classifier
389 labels : list
390 if provided, should be a set of labels to add on top of the
391 ones present in testdata
392 train : bool
393 unless train=False, classifier gets trained if
394 trainingdata provided to __call__
395 """
396 Stateful.__init__(self, **kwargs)
397 self.__clf = clf
398
399 self._labels = labels
400 """Labels to add on top to existing in testing data"""
401
402 self.__train = train
403 """Either to train classifier if trainingdata is provided"""
404
405
406 __doc__ = enhancedDocString('ClassifierError', locals(), Stateful)
407
408
414
415
416 - def _precall(self, testdataset, trainingdataset=None):
417 """Generic part which trains the classifier if necessary
418 """
419 if not trainingdataset is None:
420 if self.__train:
421
422
423
424
425
426
427
428
429
430
431
432 if self.states.isEnabled('training_confusion'):
433 self.__clf.states._changeTemporarily(enable_states=['training_confusion'])
434 self.__clf.train(trainingdataset)
435 if self.states.isEnabled('training_confusion'):
436 self.training_confusion = self.__clf.training_confusion
437 self.__clf.states._resetEnabledTemporarily()
438
439 if self.__clf.states.isEnabled('trained_labels') and \
440 not testdataset is None:
441 newlabels = Set(testdataset.uniquelabels) - self.__clf.trained_labels
442 if len(newlabels)>0:
443 warning("Classifier %s wasn't trained to classify labels %s" %
444 (`self.__clf`, `newlabels`) +
445 " present in testing dataset. Make sure that you has" +
446 " not mixed order/names of the arguments anywhere")
447
448
449
450
451
452
453
454
455
456
457
458
459 - def _call(self, testdataset, trainingdataset=None):
460 raise NotImplementedError
461
462
463 - def _postcall(self, testdataset, trainingdataset=None, error=None):
465
466
467 - def __call__(self, testdataset, trainingdataset=None):
468 """Compute the transfer error for a certain test dataset.
469
470 If `trainingdataset` is not `None` the classifier is trained using the
471 provided dataset before computing the transfer error. Otherwise the
472 classifier is used in it's current state to make the predictions on
473 the test dataset.
474
475 Returns a scalar value of the transfer error.
476 """
477 self._precall(testdataset, trainingdataset)
478 error = self._call(testdataset, trainingdataset)
479 self._postcall(testdataset, trainingdataset, error)
480 return error
481
482
483 @property
486
487
488 @property
491
495 """Compute the transfer error of a (trained) classifier on a dataset.
496
497 The actual error value is computed using a customizable error function.
498 Optionally the classifier can be trained by passing an additional
499 training dataset to the __call__() method.
500 """
501
502 null_prob = StateVariable(enabled=True)
503 """Stores the probability of an error result under the NULL hypothesis"""
504
507 """Initialization.
508
509 :Parameters:
510 clf : Classifier
511 Either trained or untrained classifier
512 errorfx
513 Functor that computes a scalar error value from the vectors of
514 desired and predicted values (e.g. subclass of `ErrorFunction`)
515 labels : list
516 if provided, should be a set of labels to add on top of the
517 ones present in testdata
518 null_dist : instance of distribution estimator
519 """
520 ClassifierError.__init__(self, clf, labels, **kwargs)
521 self.__errorfx = errorfx
522 self.__null_dist = null_dist
523
524
525 __doc__ = enhancedDocString('TransferError', locals(), ClassifierError)
526
527
535
536
537 - def _call(self, testdataset, trainingdataset=None):
566
567
568 - def _postcall(self, vdata, wdata=None, error=None):
569 """
570 """
571
572
573 if not self.__null_dist is None and not wdata is None:
574
575
576
577 null_terr = copy.copy(self)
578 null_terr.__null_dist = None
579 self.__null_dist.fit(null_terr, wdata, vdata)
580
581
582
583 if not error is None and not self.__null_dist is None:
584 self.null_prob = self.__null_dist.cdf(error)
585
586
587 @property
588 - def errorfx(self): return self.__errorfx
589
593 """For a given classifier report an error based on internally
594 computed error measure (given by some `ConfusionMatrix` stored in
595 some state variable of `Classifier`).
596
597 This way we can perform feature selection taking as the error
598 criterion either learning error, or transfer to splits error in
599 the case of SplitClassifier
600 """
601
602 - def __init__(self, clf, labels=None, confusion_state="training_confusion",
603 **kwargs):
604 """Initialization.
605
606 :Parameters:
607 clf : Classifier
608 Either trained or untrained classifier
609 confusion_state
610 Id of the state variable which stores `ConfusionMatrix`
611 labels : list
612 if provided, should be a set of labels to add on top of the
613 ones present in testdata
614 """
615 ClassifierError.__init__(self, clf, labels, **kwargs)
616
617 self.__confusion_state = confusion_state
618 """What state to extract from"""
619
620 if not clf.states.isKnown(confusion_state):
621 raise ValueError, \
622 "State variable %s is not defined for classifier %s" % \
623 (confusion_state, `clf`)
624 if not clf.states.isEnabled(confusion_state):
625 if __debug__:
626 debug('CERR', "Forcing state %s to be enabled for %s" %
627 (confusion_state, `clf`))
628 clf.states.enable(confusion_state)
629
630
631 __doc__ = enhancedDocString('ConfusionBasedError', locals(),
632 ClassifierError)
633
634
635 - def _call(self, testdata, trainingdata=None):
641