Package mvpa :: Package misc :: Module state
[hide private]
[frames] | no frames]

Source Code for Module mvpa.misc.state

   1  #emacs: -*- mode: python-mode; py-indent-offset: 4; indent-tabs-mode: nil -*- 
   2  #ex: set sts=4 ts=4 sw=4 et: 
   3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
   4  # 
   5  #   See COPYING file distributed along with the PyMVPA package for the 
   6  #   copyright and license terms. 
   7  # 
   8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
   9  """Classes to control and store state information. 
  10   
  11  It was devised to provide conditional storage  
  12  """ 
  13   
  14  __docformat__ = 'restructuredtext' 
  15   
  16  import operator, copy 
  17  from sets import Set 
  18  from textwrap import TextWrapper 
  19   
  20  import numpy as N 
  21   
  22  from mvpa.misc.vproperty import VProperty 
  23  from mvpa.misc.exceptions import UnknownStateError 
  24  from mvpa.base.dochelpers import enhancedClassDocString 
  25   
  26  if __debug__: 
  27      from mvpa.base import debug 
  28   
  29   
  30  _object_getattribute = object.__getattribute__ 
  31  _object_setattr = object.__setattr__ 
32 33 ################################################################## 34 # Various attributes which will be collected into collections 35 # 36 -class CollectableAttribute(object):
37 """Base class for any custom behaving attribute intended to become 38 part of a collection. 39 40 Derived classes will have specific semantics: 41 42 * StateVariable: conditional storage 43 * AttributeWithUnique: easy access to a set of unique values 44 within a container 45 * Parameter: attribute with validity ranges. 46 47 - ClassifierParameter: specialization to become a part of 48 Classifier's params collection 49 - KernelParameter: --//-- to become a part of Kernel Classifier's 50 kernel_params collection 51 52 Those CollectableAttributes are to be groupped into corresponding 53 collections for each class by statecollector metaclass, ie it 54 would be done on a class creation (ie not per each object) 55 """ 56
57 - def __init__(self, name=None, doc=None):
58 self.__doc__ = doc 59 self.__name = name 60 self._value = None 61 self._isset = False 62 self.reset() 63 if __debug__: 64 debug("COL", 65 "Initialized new collectable %s " % name + `self`)
66 67 68 # Instead of going for VProperty lets make use of virtual method
69 - def _getVirtual(self):
70 return self._get()
71 72
73 - def _setVirtual(self, value):
74 return self._set(value)
75 76
77 - def _get(self):
78 return self._value
79 80
81 - def _set(self, val):
82 if __debug__: 83 # Since this call is quite often, don't convert 84 # values to strings here, rely on passing them 85 # withing msgargs 86 debug("COL", 87 "Setting %(self)s to %(val)s ", 88 msgargs={'self':self, 'val':val}) 89 self._value = val 90 self._isset = True
91 92 93 @property
94 - def isSet(self):
95 return self._isset
96 97
98 - def reset(self):
99 """Simply reset the flag""" 100 if __debug__ and self._isset: 101 debug("COL", "Reset %s to being non-modified" % self.name) 102 self._isset = False
103 104 105 # TODO XXX unify all bloody __str__
106 - def __str__(self):
107 res = "%s" % (self.name) 108 if self.isSet: 109 res += '*' # so we have the value already 110 return res
111 112
113 - def _getName(self):
114 return self.__name
115 116
117 - def _setName(self, name):
118 if name is not None: 119 if isinstance(name, basestring): 120 if name[0] == '_': 121 raise ValueError, \ 122 "Collectable attribute name must not start " \ 123 "with _. Got %s" % name 124 else: 125 raise ValueError, \ 126 "Collectable attribute name must be a string. " \ 127 "Got %s" % `name` 128 self.__name = name
129 130 131 # XXX should become vproperty? 132 # YYY yoh: not sure... someone has to do performance testing 133 # to see which is more effective. My wild guess is that 134 # _[gs]etVirtual would be faster 135 value = property(_getVirtual, _setVirtual) 136 name = property(_getName, _setName)
137
138 139 140 # XXX think that may be discard hasunique and just devise top 141 # class DatasetAttribute 142 -class AttributeWithUnique(CollectableAttribute):
143 """Container which also takes care about recomputing unique values 144 145 XXX may be we could better link original attribute to additional 146 attribute which actually stores the values (and do reverse there 147 as well). 148 149 Pros: 150 * don't need to mess with getattr since it would become just another 151 attribute 152 153 Cons: 154 * might be worse design in terms of comprehension 155 * take care about _set, since we shouldn't allow 156 change it externally 157 158 For now lets do it within a single class and tune up getattr 159 """ 160
161 - def __init__(self, name=None, hasunique=True, doc="Attribute with unique"):
162 CollectableAttribute.__init__(self, name, doc) 163 self._hasunique = hasunique 164 self._resetUnique() 165 if __debug__: 166 debug("UATTR", 167 "Initialized new AttributeWithUnique %s " % name + `self`)
168 169
170 - def reset(self):
171 super(AttributeWithUnique, self).reset() 172 self._resetUnique()
173 174
175 - def _resetUnique(self):
176 self._uniqueValues = None
177 178
179 - def _set(self, *args, **kwargs):
180 self._resetUnique() 181 CollectableAttribute._set(self, *args, **kwargs)
182 183
184 - def _getUniqueValues(self):
185 if self.value is None: 186 return None 187 if self._uniqueValues is None: 188 # XXX we might better use Set, but yoh recalls that 189 # N.unique was more efficient. May be we should check 190 # on the the class and use Set only if we are not 191 # dealing with ndarray (or lists/tuples) 192 self._uniqueValues = N.unique(N.asanyarray(self.value)) 193 return self._uniqueValues
194 195 uniqueValues = property(fget=_getUniqueValues) 196 hasunique = property(fget=lambda self:self._hasunique)
197
198 # Hooks for comprehendable semantics and automatic collection generation 199 -class SampleAttribute(AttributeWithUnique):
200 pass
201
202 -class FeatureAttribute(AttributeWithUnique):
203 pass
204
205 -class DatasetAttribute(AttributeWithUnique):
206 pass
207
208 209 -class StateVariable(CollectableAttribute):
210 """Simple container intended to conditionally store the value 211 """ 212
213 - def __init__(self, name=None, enabled=True, doc="State variable"):
214 CollectableAttribute.__init__(self, name, doc) 215 self._isenabled = enabled 216 self._defaultenabled = enabled 217 if __debug__: 218 debug("STV", 219 "Initialized new state variable %s " % name + `self`)
220 221
222 - def _get(self):
223 if not self.isSet: 224 raise UnknownStateError("Unknown yet value of %s" % (self.name)) 225 return CollectableAttribute._get(self)
226 227
228 - def _set(self, val):
229 if self.isEnabled: 230 # XXX may be should have left simple assignment 231 # self._value = val 232 CollectableAttribute._set(self, val) 233 elif __debug__: 234 debug("COL", 235 "Not setting disabled %(self)s to %(val)s ", 236 msgargs={'self':self, 'val':val})
237 238
239 - def reset(self):
240 """Simply detach the value, and reset the flag""" 241 CollectableAttribute.reset(self) 242 self._value = None
243 244 245 @property
246 - def isEnabled(self):
247 return self._isenabled
248 249
250 - def enable(self, value=False):
251 if self._isenabled == value: 252 # Do nothing since it is already in proper state 253 return 254 if __debug__: 255 debug("STV", "%s %s" % 256 ({True: 'Enabling', False: 'Disabling'}[value], str(self))) 257 self._isenabled = value
258 259
260 - def __str__(self):
261 res = CollectableAttribute.__str__(self) 262 if self.isEnabled: 263 res += '+' # it is enabled but no value is assigned yet 264 return res
265
266 267 ################################################################### 268 # Collections 269 # 270 # TODO: refactor into attributes.py and collections.py. state.py now has 271 # little in common with the main part of this file 272 # 273 -class Collection(object):
274 """Container of some CollectableAttributes. 275 276 :Groups: 277 - `Public Access Functions`: `isKnown` 278 - `Access Implementors`: `_getListing`, `_getNames` 279 - `Mutators`: `__init__` 280 - `R/O Properties`: `listing`, `names`, `items` 281 282 XXX Seems to be not used and duplicating functionality: `_getListing` 283 (thus `listing` property) 284 """ 285
286 - def __init__(self, items=None, owner=None, name=None):
287 """Initialize the Collection 288 289 :Parameters: 290 items : dict of CollectableAttribute's 291 items to initialize with 292 owner : object 293 an object to which collection belongs 294 name : basestring 295 name of the collection (as seen in the owner, e.g. 'states') 296 """ 297 298 self.__owner = owner 299 300 if items == None: 301 items = {} 302 self._items = items 303 """Dictionary to contain registered states as keys and 304 values signal either they are enabled 305 """ 306 self.__name = name
307
308 - def _setName(self, name):
309 self.__name = name
310
311 - def __str__(self):
312 num = len(self._items) 313 if __debug__ and "ST" in debug.active: 314 maxnumber = 1000 # I guess all 315 else: 316 maxnumber = 4 317 if self.__name is not None: 318 res = self.__name 319 else: 320 res = "" 321 res += "{" 322 for i in xrange(min(num, maxnumber)): 323 if i > 0: 324 res += " " 325 res += "%s" % str(self._items.values()[i]) 326 if len(self._items) > maxnumber: 327 res += "..." 328 res += "}" 329 if __debug__: 330 if "ST" in debug.active: 331 res += " owner:%s#%s" % (self.owner.__class__.__name__, 332 id(self.owner)) 333 return res
334 335
336 - def _cls_repr(self):
337 """Collection specific part of __repr__ for a class containing 338 it, ie a part of __repr__ for the owner object 339 340 :Return: 341 list of items to be appended within __repr__ after a .join() 342 """ 343 # XXX For now we do not expect any pure non-specialized 344 # collection , thus just override in derived classes 345 raise NotImplementedError, "Class %s should override _cls_repr" \ 346 % self.__class__.__name__
347
348 - def _is_initializable(self, index):
349 """Checks if index could be assigned within collection via 350 _initialize 351 352 :Return: bool value for a given `index` 353 354 It is to facilitate dynamic assignment of collections' items 355 within derived classes' __init__ depending on the present 356 collections in the class. 357 """ 358 # XXX Each collection has to provide what indexes it allows 359 # to be set within constructor. Custom handling of some 360 # arguments (like (dis|en)able_states) is to be performed 361 # in _initialize 362 # raise NotImplementedError, \ 363 # "Class %s should override _is_initializable" \ 364 # % self.__class__.__name__ 365 366 # YYY lets just check if it is in the keys 367 return index in self._items.keys()
368 369
370 - def _initialize(self, index, value):
371 """Initialize `index` (no check performed) with `value` 372 """ 373 # by default we just set corresponding value 374 self.setvalue(index, value)
375 376
377 - def __repr__(self):
378 s = "%s(" % self.__class__.__name__ 379 items_s = "" 380 sep = "" 381 for item in self._items: 382 try: 383 itemvalue = "%s" % `self._items[item].value` 384 if len(itemvalue)>50: 385 itemvalue = itemvalue[:10] + '...' + itemvalue[-10:] 386 items_s += "%s'%s':%s" % (sep, item, itemvalue) 387 sep = ', ' 388 except: 389 pass 390 if items_s != "": 391 s += "items={%s}" % items_s 392 if self.owner is not None: 393 s += "%sowner=%s" % (sep, `self.owner`) 394 s += ")" 395 return s
396 397 398 # 399 # XXX TODO: figure out if there is a way to define proper 400 # __copy__'s for a hierarchy of classes. Probably we had 401 # to define __getinitargs__, etc... read more... 402 # 403 #def __copy__(self): 404 # TODO Remove or refactor? 405 # def _copy_states_(self, fromstate, deep=False): 406 # """Copy known here states from `fromstate` object into current object 407 # 408 # Crafted to overcome a problem mentioned above in the comment 409 # and is to be called from __copy__ of derived classes 410 # 411 # Probably sooner than later will get proper __getstate__, 412 # __setstate__ 413 # """ 414 # # Bad check... doesn't generalize well... 415 # # if not issubclass(fromstate.__class__, self.__class__): 416 # # raise ValueError, \ 417 # # "Class %s is not subclass of %s, " % \ 418 # # (fromstate.__class__, self.__class__) + \ 419 # # "thus not eligible for _copy_states_" 420 # # TODO: FOR NOW NO TEST! But this beast needs to be fixed... 421 # operation = { True: copy.deepcopy, 422 # False: copy.copy }[deep] 423 # 424 # if isinstance(fromstate, Stateful): 425 # fromstate = fromstate.states 426 # 427 # self.enabled = fromstate.enabled 428 # for name in self.names: 429 # if fromstate.isKnown(name): 430 # self._items[name] = operation(fromstate._items[name]) 431 432
433 - def isKnown(self, index):
434 """Returns `True` if state `index` is known at all""" 435 return self._items.has_key(index)
436 437
438 - def __isSet1(self, index):
439 """Returns `True` if state `index` has value set""" 440 self._checkIndex(index) 441 return self._items[index].isSet
442 443
444 - def isSet(self, index=None):
445 """If item (or any in the present or listed) was set 446 447 :Parameters: 448 index : None or basestring or list of basestring 449 What items to check if they were set in the collection 450 """ 451 _items = self._items 452 if not (index is None): 453 if isinstance(index, basestring): 454 self._checkIndex(index) # process just that single index 455 return _items[index].isSet 456 else: 457 items = index # assume that we got some list 458 else: 459 items = self._items # go through all the items 460 461 for index in items: 462 self._checkIndex(index) 463 if _items[index].isSet: 464 return True 465 return False
466 467
468 - def whichSet(self):
469 """Return list of indexes which were set""" 470 result = [] 471 # go through all members and if any isSet -- return True 472 for index,v in self._items.iteritems(): 473 if v.isSet: 474 result.append(index) 475 return result
476 477
478 - def _checkIndex(self, index):
479 """Verify that given `index` is a known/registered state. 480 481 :Raise `KeyError`: if given `index` is not known 482 """ 483 # OPT: lets not reuse isKnown, to don't incure 1 more function 484 # call 485 if not self._items.has_key(index): 486 raise KeyError, \ 487 "%s of %s has no key '%s' registered" \ 488 % (self.__class__.__name__, 489 self.__owner.__class__.__name__, 490 index)
491 492
493 - def add(self, item):
494 """Add a new CollectableAttribute to the collection 495 496 :Parameters: 497 item : CollectableAttribute 498 or of derived class. Must have 'name' assigned 499 500 TODO: we should make it stricter to don't add smth of 501 wrong type into Collection since it might lead to problems 502 503 Also we might convert to __setitem__ 504 """ 505 if not isinstance(item, CollectableAttribute): 506 raise ValueError, \ 507 "Collection can add only instances of " + \ 508 "CollectableAttribute-derived classes. Got %s" % `item` 509 if item.name is None: 510 raise ValueError, \ 511 "CollectableAttribute to be added %s must have 'name' set" % \ 512 item 513 self._items[item.name] = item 514 515 if not self.owner is None: 516 self._updateOwner(item.name)
517 518
519 - def remove(self, index):
520 """Remove item from the collection 521 """ 522 self._checkIndex(index) 523 self._updateOwner(index, register=False) 524 discard = self._items.pop(index)
525 526
527 - def __getattribute__(self, index):
528 """ 529 """ 530 #return all private and protected ones first since we will not have 531 # collectable's with _ (we should not have!) 532 if index[0] == '_': 533 return _object_getattribute(self, index) 534 _items = _object_getattribute(self, '_items') 535 if index in _items: 536 return _items[index].value 537 return _object_getattribute(self, index)
538 539
540 - def __setattr__(self, index, value):
541 if index[0] == '_': 542 return _object_setattr(self, index, value) 543 _items = _object_getattribute(self, '_items') 544 if index in _items: 545 _items[index].value = value 546 else: 547 _object_setattr(self, index, value)
548 549
550 - def __getitem__(self, index):
551 _items = _object_getattribute(self, '_items') 552 if index in _items: 553 self._checkIndex(index) 554 return _items[index] 555 else: 556 raise AttributeError("State collection %s has no %s attribute" 557 % (self, index))
558 559 560 # Probably not needed -- enable if need arises 561 # 562 #def __setattr__(self, index, value): 563 # if self._items.has_key(index): 564 # self._updateOwner(index, register=False) 565 # self._items[index] = value 566 # self._updateOwner(index, register=True) 567 # 568 # _object_setattr(self, index, value) 569 570
571 - def getvalue(self, index):
572 """Returns the value by index""" 573 self._checkIndex(index) 574 return self._items[index].value
575 576
577 - def get(self, index, default):
578 """Access the value by a given index. 579 580 Mimiquing regular dictionary behavior, if value cannot be obtained 581 (i.e. if any exception is caught) return default value. 582 """ 583 try: 584 return self[index].value 585 except Exception, e: 586 #if default is not None: 587 return default
588 #else: 589 # raise e 590 591
592 - def setvalue(self, index, value):
593 """Sets the value by index""" 594 self._checkIndex(index) 595 self._items[index].value = value
596 597
598 - def _action(self, index, func, missingok=False, **kwargs):
599 """Run specific func either on a single item or on all of them 600 601 :Parameters: 602 index : basestr 603 Name of the state variable 604 func 605 Function (not bound) to call given an item, and **kwargs 606 missingok : bool 607 If True - do not complain about wrong index 608 """ 609 if isinstance(index, basestring): 610 if index.upper() == 'ALL': 611 for index_ in self._items: 612 self._action(index_, func, missingok=missingok, **kwargs) 613 else: 614 try: 615 self._checkIndex(index) 616 func(self._items[index], **kwargs) 617 except: 618 if missingok: 619 return 620 raise 621 elif operator.isSequenceType(index): 622 for item in index: 623 self._action(item, func, missingok=missingok, **kwargs) 624 else: 625 raise ValueError, \ 626 "Don't know how to handle variable given by %s" % index
627 628
629 - def reset(self, index=None):
630 """Reset the state variable defined by `index`""" 631 632 if not index is None: 633 indexes = [ index ] 634 else: 635 indexes = self.names 636 637 if len(self.items): 638 for index in indexes: 639 # XXX Check if that works as desired 640 self._action(index, self._items.values()[0].__class__.reset, 641 missingok=False)
642 643
644 - def _getListing(self):
645 """Return a list of registered states along with the documentation""" 646 647 # lets assure consistent litsting order 648 items = self._items.items() 649 items.sort() 650 return [ "%s: %s" % (str(x[1]), x[1].__doc__) for x in items ]
651 652
653 - def _getNames(self):
654 """Return ids for all registered state variables""" 655 return self._items.keys()
656 657
658 - def _getOwner(self):
659 return self.__owner
660 661
662 - def _setOwner(self, owner):
663 if not isinstance(owner, Stateful): 664 raise ValueError, \ 665 "Owner of the StateCollection must be Stateful object" 666 if __debug__: 667 try: strowner = str(owner) 668 except: strowner = "UNDEF: <%s#%s>" % (owner.__class__, id(owner)) 669 debug("ST", "Setting owner for %s to be %s" % (self, strowner)) 670 if not self.__owner is None: 671 # Remove attributes which were registered to that owner previousely 672 self._updateOwner(register=False) 673 self.__owner = owner 674 if not self.__owner is None: 675 self._updateOwner(register=True)
676 677
678 - def _updateOwner(self, index=None, register=True):
679 """Define an entry within owner's __dict__ 680 so ipython could easily complete it 681 682 :Parameters: 683 index : basestring or list of basestring 684 Name of the attribute. If None -- all known get registered 685 register : bool 686 Register if True or unregister if False 687 688 XXX Needs refactoring since we duplicate the logic of expansion of 689 index value 690 """ 691 if not index is None: 692 if not index in self._items: 693 raise ValueError, \ 694 "Attribute %s is not known to %s" % (index, self) 695 indexes = [ index ] 696 else: 697 indexes = self.names 698 699 ownerdict = self.owner.__dict__ 700 selfdict = self.__dict__ 701 owner_known = ownerdict['_known_attribs'] 702 for index_ in indexes: 703 if register: 704 if index_ in ownerdict: 705 raise RuntimeError, \ 706 "Cannot register attribute %s within %s " % \ 707 (index_, self.owner) + "since it has one already" 708 ownerdict[index_] = self._items[index_] 709 if index_ in selfdict: 710 raise RuntimeError, \ 711 "Cannot register attribute %s within %s " % \ 712 (index_, self) + "since it has one already" 713 selfdict[index_] = self._items[index_] 714 owner_known[index_] = self.__name 715 else: 716 if index_ in ownerdict: 717 # yoh doesn't think that we need to complain if False 718 ownerdict.pop(index_) 719 owner_known.pop(index_) 720 if index_ in selfdict: 721 selfdict.pop(index_)
722 723 724 # Properties 725 names = property(fget=_getNames) 726 items = property(fget=lambda x:x._items) 727 owner = property(fget=_getOwner, fset=_setOwner) 728 name = property(fget=lambda x:x.__name, fset=_setName) 729 730 # Virtual properties 731 listing = VProperty(fget=_getListing)
732
733 734 735 -class ParameterCollection(Collection):
736 """Container of Parameters for a stateful object. 737 """ 738 739 # def __init__(self, items=None, owner=None, name=None): 740 # """Initialize the state variables of a derived class 741 # 742 # :Parameters: 743 # items : dict 744 # dictionary of states 745 # """ 746 # Collection.__init__(self, items, owner, name) 747 # 748
749 - def _cls_repr(self):
750 """Part of __repr__ for the owner object 751 """ 752 prefixes = [] 753 for k in self.names: 754 # list only params with not default values 755 if self[k].isDefault: 756 continue 757 prefixes.append("%s=%s" % (k, self[k].value)) 758 return prefixes
759 760
761 - def resetvalue(self, index, missingok=False):
762 """Reset all parameters to default values""" 763 from param import Parameter 764 self._action(index, Parameter.resetvalue, missingok=False)
765
766 767 -class SampleAttributesCollection(Collection):
768 """Container for data and attributes of samples (ie data/labels/chunks/...) 769 """ 770 771 # def __init__(self, items=None, owner=None, name=None): 772 # """Initialize the state variables of a derived class 773 # 774 # :Parameters: 775 # items : dict 776 # dictionary of states 777 # """ 778 # Collection.__init__(self, items, owner, name) 779 # 780
781 - def _cls_repr(self):
782 """Part of __repr__ for the owner object 783 """ 784 return [] # TODO: return I guess samples/labels/chunks
785
786 787 788 -class StateCollection(Collection):
789 """Container of StateVariables for a stateful object. 790 791 :Groups: 792 - `Public Access Functions`: `isKnown`, `isEnabled`, `isActive` 793 - `Access Implementors`: `_getListing`, `_getNames`, `_getEnabled` 794 - `Mutators`: `__init__`, `enable`, `disable`, `_setEnabled` 795 - `R/O Properties`: `listing`, `names`, `items` 796 - `R/W Properties`: `enabled` 797 """ 798
799 - def __init__(self, items=None, owner=None):
800 """Initialize the state variables of a derived class 801 802 :Parameters: 803 items : dict 804 dictionary of states 805 owner : Stateful 806 object which owns the collection 807 name : basestring 808 literal description. Usually just attribute name for the 809 collection, e.g. 'states' 810 """ 811 Collection.__init__(self, items=items, owner=owner) 812 813 self.__storedTemporarily = [] 814 """List to contain sets of enabled states which were enabled 815 temporarily. 816 """
817 818 # 819 # XXX TODO: figure out if there is a way to define proper 820 # __copy__'s for a hierarchy of classes. Probably we had 821 # to define __getinitargs__, etc... read more... 822 # 823 #def __copy__(self): 824
825 - def _cls_repr(self):
826 """Part of __repr__ for the owner object 827 """ 828 prefixes = [] 829 for name, invert in ( ('enable', False), ('disable', True) ): 830 states = self._getEnabled(nondefault=False, 831 invert=invert) 832 if len(states): 833 prefixes.append("%s_states=%s" % (name, str(states))) 834 return prefixes
835 836
837 - def _is_initializable(self, index):
838 """Checks if index could be assigned within collection via 839 setvalue 840 """ 841 return index in ['enable_states', 'disable_states']
842 843
844 - def _initialize(self, index, value):
845 if value is None: 846 value = [] 847 if index == 'enable_states': 848 self.enable(value, missingok=True) 849 elif index == 'disable_states': 850 self.disable(value) 851 else: 852 raise ValueError, "StateCollection can accept only enable_states " \ 853 "and disable_states arguments for the initialization. " \ 854 "Got %s" % index
855 856
857 - def _copy_states_(self, fromstate, index=None, deep=False):
858 """Copy known here states from `fromstate` object into current object 859 860 :Parameters: 861 fromstate : Collection or Stateful 862 Source states to copy from 863 index : None or list of basestring 864 If not to copy all set state variables, index provides 865 selection of what to copy 866 deep : bool 867 Optional control over the way to copy 868 869 Crafted to overcome a problem mentioned above in the comment 870 and is to be called from __copy__ of derived classes 871 872 Probably sooner than later will get proper __getstate__, 873 __setstate__ 874 """ 875 # Bad check... doesn't generalize well... 876 # if not issubclass(fromstate.__class__, self.__class__): 877 # raise ValueError, \ 878 # "Class %s is not subclass of %s, " % \ 879 # (fromstate.__class__, self.__class__) + \ 880 # "thus not eligible for _copy_states_" 881 # TODO: FOR NOW NO TEST! But this beast needs to be fixed... 882 operation = { True: copy.deepcopy, 883 False: copy.copy }[deep] 884 885 if isinstance(fromstate, Stateful): 886 fromstate = fromstate.states 887 888 #self.enabled = fromstate.enabled 889 _items, from_items = self._items, fromstate._items 890 if index is None: 891 # copy all set ones 892 for name in fromstate.whichSet():#self.names: 893 #if fromstate.isKnown(name): 894 _items[name] = operation(from_items[name]) 895 else: 896 isKnown = fromstate.isKnown 897 for name in index: 898 if isKnown(name): 899 _items[name] = operation(from_items[name])
900 901
902 - def isEnabled(self, index):
903 """Returns `True` if state `index` is enabled""" 904 self._checkIndex(index) 905 return self._items[index].isEnabled
906 907
908 - def isActive(self, index):
909 """Returns `True` if state `index` is known and is enabled""" 910 return self.isKnown(index) and self.isEnabled(index)
911 912
913 - def enable(self, index, value=True, missingok=False):
914 """Enable state variable given in `index`""" 915 self._action(index, StateVariable.enable, missingok=missingok, 916 value=value)
917 918
919 - def disable(self, index):
920 """Disable state variable defined by `index` id""" 921 self._action(index, StateVariable.enable, missingok=False, value=False)
922 923 924 # TODO XXX think about some more generic way to grab temporary 925 # snapshot of CollectableAttributes to be restored later on...
926 - def _changeTemporarily(self, enable_states=None, 927 disable_states=None, other=None):
928 """Temporarily enable/disable needed states for computation 929 930 Enable or disable states which are enabled in `other` and listed in 931 `enable _states`. Use `resetEnabledTemporarily` to reset 932 to previous state of enabled. 933 934 `other` can be a Stateful object or StateCollection 935 """ 936 if enable_states == None: 937 enable_states = [] 938 if disable_states == None: 939 disable_states = [] 940 self.__storedTemporarily.append(self.enabled) 941 other_ = other 942 if isinstance(other, Stateful): 943 other = other.states 944 945 if not other is None: 946 # lets take states which are enabled in other but not in 947 # self 948 add_enable_states = list(Set(other.enabled).difference( 949 Set(enable_states)).intersection(self.names)) 950 if len(add_enable_states)>0: 951 if __debug__: 952 debug("ST", 953 "Adding states %s from %s to be enabled temporarily" % 954 (add_enable_states, other_) + 955 " since they are not enabled in %s" % 956 (self)) 957 enable_states += add_enable_states 958 959 # Lets go one by one enabling only disabled once... but could be as 960 # simple as 961 self.enable(enable_states) 962 self.disable(disable_states)
963 964
965 - def _resetEnabledTemporarily(self):
966 """Reset to previousely stored set of enabled states""" 967 if __debug__: 968 debug("ST", "Resetting to previous set of enabled states") 969 if len(self.enabled)>0: 970 self.enabled = self.__storedTemporarily.pop() 971 else: 972 raise ValueError("Trying to restore not-stored list of enabled " \ 973 "states")
974 975
976 - def _getEnabled(self, nondefault=True, invert=False):
977 """Return list of enabled states 978 979 :Parameters: 980 nondefault : bool 981 Either to return also states which are enabled simply by default 982 invert : bool 983 Would invert the meaning, ie would return disabled states 984 """ 985 if invert: 986 fmatch = lambda y: not self.isEnabled(y) 987 else: 988 fmatch = lambda y: self.isEnabled(y) 989 990 if nondefault: 991 ffunc = fmatch 992 else: 993 ffunc = lambda y: fmatch(y) and \ 994 self._items[y]._defaultenabled != self.isEnabled(y) 995 return filter(ffunc, self.names)
996 997
998 - def _setEnabled(self, indexlist):
999 """Given `indexlist` make only those in the list enabled 1000 1001 It might be handy to store set of enabled states and then to restore 1002 it later on. It can be easily accomplished now:: 1003 1004 >>> from mvpa.misc.state import Stateful, StateVariable 1005 >>> class Blah(Stateful): 1006 ... bleh = StateVariable(enabled=False, doc='Example') 1007 ... 1008 >>> blah = Blah() 1009 >>> states_enabled = blah.states.enabled 1010 >>> blah.states.enabled = ['bleh'] 1011 >>> blah.states.enabled = states_enabled 1012 """ 1013 for index in self._items.keys(): 1014 self.enable(index, index in indexlist)
1015 1016 1017 # Properties 1018 enabled = property(fget=_getEnabled, fset=_setEnabled)
1019 1020 1021 ################################################################## 1022 # Base classes (and metaclass) which use collections 1023 # 1024 1025 1026 # 1027 # Helper dictionaries for AttributesCollector 1028 # 1029 _known_collections = { 1030 # Quite a generic one but mostly in classifiers 1031 'StateVariable': ("states", StateCollection), 1032 # For classifiers only 1033 'Parameter': ("params", ParameterCollection), 1034 'KernelParameter': ("kernel_params", ParameterCollection), 1035 # For datasets 1036 # XXX custom collections needed? 1037 'SampleAttribute': ("s_attr", SampleAttributesCollection), 1038 'FeatureAttribute': ("f_attr", SampleAttributesCollection), 1039 'DatasetAttribute': ("ds_attr", SampleAttributesCollection), 1040 } 1041 1042 1043 _col2class = dict(_known_collections.values()) 1044 """Mapping from collection name into Collection class""" 1045 1046 1047 _COLLECTIONS_ORDER = ['s_attr', 'f_attr', 'ds_attr', 1048 'params', 'kernel_params', 'states']
1049 1050 1051 -class AttributesCollector(type):
1052 """Intended to collect and compose StateCollection for any child 1053 class of this metaclass 1054 """ 1055 1056
1057 - def __init__(cls, name, bases, dict):
1058 1059 if __debug__: 1060 debug( 1061 "COLR", 1062 "AttributesCollector call for %s.%s, where bases=%s, dict=%s " \ 1063 % (cls, name, bases, dict)) 1064 1065 super(AttributesCollector, cls).__init__(name, bases, dict) 1066 1067 collections = {} 1068 for name, value in dict.iteritems(): 1069 if isinstance(value, CollectableAttribute): 1070 baseclassname = value.__class__.__name__ 1071 col = _known_collections[baseclassname][0] 1072 # XXX should we allow to throw exceptions here? 1073 if not collections.has_key(col): 1074 collections[col] = {} 1075 collections[col][name] = value 1076 # and assign name if not yet was set 1077 if value.name is None: 1078 value.name = name 1079 1080 # XXX can we first collect parent's states and then populate with ours? 1081 # TODO 1082 1083 for base in bases: 1084 if hasattr(base, "__metaclass__") and \ 1085 base.__metaclass__ == AttributesCollector: 1086 # TODO take care about overriding one from super class 1087 # for state in base.states: 1088 # if state[0] = 1089 newcollections = base._collections_template 1090 if len(newcollections) == 0: 1091 continue 1092 if __debug__: 1093 debug("COLR", 1094 "Collect collections %s for %s from %s" % 1095 (newcollections, cls, base)) 1096 for col, collection in newcollections.iteritems(): 1097 newitems = collection.items 1098 if collections.has_key(col): 1099 collections[col].update(newitems) 1100 else: 1101 collections[col] = newitems 1102 1103 1104 if __debug__: 1105 debug("COLR", 1106 "Creating StateCollection template %s with collections %s" 1107 % (cls, collections.keys())) 1108 1109 # if there is an explicit 1110 if hasattr(cls, "_ATTRIBUTE_COLLECTIONS"): 1111 for col in cls._ATTRIBUTE_COLLECTIONS: 1112 if not col in _col2class: 1113 raise ValueError, \ 1114 "Requested collection %s is unknown to collector" % \ 1115 col 1116 if not col in collections: 1117 collections[col] = None 1118 1119 # TODO: check on conflict in names of Collections' items! since 1120 # otherwise even order is not definite since we use dict for 1121 # collections. 1122 # XXX should we switch to tuple? 1123 1124 for col, colitems in collections.iteritems(): 1125 collections[col] = _col2class[col](colitems) 1126 1127 setattr(cls, "_collections_template", collections) 1128 1129 # 1130 # Expand documentation for the class based on the listed 1131 # parameters an if it is stateful 1132 # 1133 # TODO -- figure nice way on how to alter __init__ doc directly... 1134 textwrapper = TextWrapper(subsequent_indent=" ", 1135 initial_indent=" ", 1136 width=70) 1137 1138 # Parameters 1139 paramsdoc = "" 1140 paramscols = [] 1141 for col in ('params', 'kernel_params'): 1142 if collections.has_key(col): 1143 paramscols.append(col) 1144 # lets at least sort the parameters for consistent output 1145 col_items = collections[col].items 1146 params = col_items.keys() 1147 params.sort() 1148 for param in params: 1149 parameter = col_items[param] 1150 paramsdoc += " %s" % param 1151 try: 1152 paramsdoc += " : %s" % parameter.allowedtype 1153 except: 1154 pass 1155 paramsdoc += "\n" 1156 try: 1157 doc = parameter.__doc__ 1158 try: 1159 doc += " (Default: %s)" % parameter.default 1160 except: 1161 pass 1162 paramsdoc += '\n'.join(textwrapper.wrap(doc))+'\n' 1163 except Exception, e: 1164 pass 1165 1166 # Parameters collection could be taked hash of to decide if 1167 # any were changed? XXX may be not needed at all? 1168 setattr(cls, "_paramscols", paramscols) 1169 1170 # States doc 1171 statesdoc = "" 1172 if collections.has_key('states'): 1173 paramsdoc += """ enable_states : None or list of basestring 1174 Names of the state variables which should be enabled additionally 1175 to default ones 1176 disable_states : None or list of basestring 1177 Names of the state variables which should be disabled 1178 """ 1179 statesdoc = "Enabled by default are listed with +\n * " 1180 statesdoc += '\n * '.join(collections['states'].listing) 1181 if __debug__: 1182 debug("COLR", "Assigning __statesdoc to be %s" % statesdoc) 1183 setattr(cls, "_statesdoc", statesdoc) 1184 1185 if paramsdoc != "": 1186 if __debug__ and 'COLR' in debug.active: 1187 debug("COLR", "Assigning __paramsdoc to be %s" % paramsdoc) 1188 setattr(cls, "_paramsdoc", paramsdoc) 1189 1190 if paramsdoc + statesdoc != "": 1191 cls.__doc__ = enhancedClassDocString(cls, *bases)
1192
1193 1194 1195 -class ClassWithCollections(object):
1196 """Base class for objects which contain any known collection 1197 1198 Classes inherited from this class gain ability to access 1199 collections and their items as simple attributes. Access to 1200 collection items "internals" is done via <collection_name> attribute 1201 and interface of a corresponding `Collection`. 1202 """ 1203 1204 _DEV__doc__ = """ 1205 TODO: rename 'descr'? -- it should simply 1206 be 'doc' -- no need to drag classes docstring imho. 1207 """ 1208 1209 __metaclass__ = AttributesCollector 1210
1211 - def __new__(cls, *args, **kwargs):
1212 """Initialize ClassWithCollections object 1213 1214 :Parameters: 1215 descr : basestring 1216 Description of the instance 1217 """ 1218 self = super(ClassWithCollections, cls).__new__(cls) 1219 1220 s__dict__ = self.__dict__ 1221 1222 # init variable 1223 # XXX: Added as pylint complained (rightfully) -- not sure if false 1224 # is the proper default 1225 self.__params_set = False 1226 1227 # need to check to avoid override of enabled states in the case 1228 # of multiple inheritance, like both Statefull and Harvestable 1229 if not s__dict__.has_key('_collections'): 1230 s__class__ = self.__class__ 1231 1232 collections = copy.deepcopy(s__class__._collections_template) 1233 s__dict__['_collections'] = collections 1234 s__dict__['_known_attribs'] = {} 1235 """Dictionary to contain 'links' to the collections from each 1236 known attribute. Is used to gain some speed up in lookup within 1237 __getattribute__ and __setattr__ 1238 """ 1239 1240 # Assign owner to all collections 1241 for col, collection in collections.iteritems(): 1242 if col in s__dict__: 1243 raise ValueError, \ 1244 "Object %s has already attribute %s" % \ 1245 (self, col) 1246 s__dict__[col] = collection 1247 collection.name = col 1248 collection.owner = self 1249 1250 self.__params_set = False 1251 1252 if __debug__: 1253 descr = kwargs.get('descr', None) 1254 debug("COL", "ClassWithCollections.__new__ was done " 1255 "for %s id %s with descr=%s" \ 1256 % (s__class__.__name__, id(self), descr)) 1257 1258 return self
1259 1260
1261 - def __init__(self, descr=None, **kwargs):
1262 1263 if not self.__params_set: 1264 self.__descr = descr 1265 """Set humane description for the object""" 1266 1267 # To avoid double initialization in case of multiple inheritance 1268 self.__params_set = True 1269 1270 collections = self._collections 1271 # Assign attributes values if they are given among 1272 # **kwargs 1273 for arg, argument in kwargs.items(): 1274 set = False 1275 for collection in collections.itervalues(): 1276 if collection._is_initializable(arg): 1277 collection._initialize(arg, argument) 1278 set = True 1279 break 1280 if set: 1281 trash = kwargs.pop(arg) 1282 else: 1283 known_params = reduce( 1284 lambda x,y:x+y, 1285 [x.items.keys() for x in collections.itervalues()], []) 1286 raise TypeError, \ 1287 "Unexpected keyword argument %s=%s for %s." \ 1288 % (arg, argument, self) \ 1289 + " Valid parameters are %s" % known_params 1290 1291 ## Initialize other base classes 1292 ## commented out since it seems to be of no use for now 1293 #if init_classes is not None: 1294 # # return back stateful arguments since they might be 1295 # # processed by underlying classes 1296 # kwargs.update(kwargs_stateful) 1297 # for cls in init_classes: 1298 # cls.__init__(self, **kwargs) 1299 #else: 1300 # if len(kwargs)>0: 1301 # known_params = reduce(lambda x, y: x + y, \ 1302 # [x.items.keys() for x in collections], 1303 # []) 1304 # raise TypeError, \ 1305 # "Unknown parameters %s for %s." % (kwargs.keys(), 1306 # self) \ 1307 # + " Valid parameters are %s" % known_params 1308 if __debug__: 1309 debug("COL", "ClassWithCollections.__init__ was done " 1310 "for %s id %s with descr=%s" \ 1311 % (self.__class__.__name__, id(self), descr))
1312 1313 1314 #__doc__ = enhancedDocString('Stateful', locals()) 1315 1316
1317 - def __getattribute__(self, index):
1318 # return all private ones first since smth like __dict__ might be 1319 # queried by copy before instance is __init__ed 1320 if index[0] == '_': 1321 return _object_getattribute(self, index) 1322 1323 s_dict = _object_getattribute(self, '__dict__') 1324 # check if it is a known collection 1325 collections = s_dict['_collections'] 1326 if index in collections: 1327 return collections[index] 1328 1329 # check if it is a part of any collection 1330 known_attribs = s_dict['_known_attribs'] 1331 if index in known_attribs: 1332 return collections[known_attribs[index]].getvalue(index) 1333 1334 # just a generic return 1335 return _object_getattribute(self, index)
1336 1337
1338 - def __setattr__(self, index, value):
1339 if index[0] == '_': 1340 return _object_setattr(self, index, value) 1341 1342 # Check if a part of a collection, and set appropriately 1343 s_dict = _object_getattribute(self, '__dict__') 1344 known_attribs = s_dict['_known_attribs'] 1345 if index in known_attribs: 1346 collections = s_dict['_collections'] 1347 return collections[known_attribs[index]].setvalue(index, value) 1348 1349 # Generic setattr 1350 return _object_setattr(self, index, value)
1351 1352 1353 # XXX not sure if we shouldn't implement anything else...
1354 - def reset(self):
1355 for collection in self._collections.values(): 1356 collection.reset()
1357 1358
1359 - def __str__(self):
1360 s = "%s:" % (self.__class__.__name__) 1361 if self.__descr is not None: 1362 s += "/%s " % self.__descr 1363 if hasattr(self, "_collections"): 1364 for col, collection in self._collections.iteritems(): 1365 s += " %d %s:%s" % (len(collection.items), col, str(collection)) 1366 return s
1367 1368
1369 - def __repr__(self, prefixes=None, fullname=False):
1370 """String definition of the object of ClassWithCollections object 1371 1372 :Parameters: 1373 fullname : bool 1374 Either to include full name of the module 1375 prefixes : list of strings 1376 What other prefixes to prepend to list of arguments 1377 """ 1378 if prefixes is None: 1379 prefixes = [] 1380 prefixes = prefixes[:] # copy list 1381 id_str = "" 1382 module_str = "" 1383 if __debug__: 1384 if 'MODULE_IN_REPR' in debug.active: 1385 fullname = True 1386 if 'ID_IN_REPR' in debug.active: 1387 id_str = '#%s' % id(self) 1388 1389 if fullname: 1390 modulename = '%s' % self.__class__.__module__ 1391 if modulename != "__main__": 1392 module_str = "%s." % modulename 1393 1394 # Collections' attributes 1395 collections = self._collections 1396 # we want them in this particular order 1397 for col in _COLLECTIONS_ORDER: 1398 collection = collections.get(col, None) 1399 if collection is None: 1400 continue 1401 prefixes += collection._cls_repr() 1402 1403 # Description if present 1404 descr = self.__descr 1405 if descr is not None: 1406 prefixes.append("descr='%s'" % (descr)) 1407 1408 return "%s%s(%s)%s" % (module_str, self.__class__.__name__, 1409 ', '.join(prefixes), id_str)
1410 1411 1412 descr = property(lambda self: self.__descr, 1413 doc="Description of the object if any")
1414 1415 1416 1417 # No actual separation is needed now between ClassWithCollections 1418 # and a specific usecase. 1419 Stateful = ClassWithCollections 1420 Parametrized = ClassWithCollections
1421 1422 1423 1424 -class Harvestable(Stateful):
1425 """Classes inherited from this class intend to collect attributes 1426 within internal processing. 1427 1428 Subclassing Harvestable we gain ability to collect any internal 1429 data from the processing which is especially important if an 1430 object performs something in loop and discards some intermidiate 1431 possibly interesting results (like in case of 1432 CrossValidatedTransferError and states of the trained classifier 1433 or TransferError). 1434 1435 """ 1436 1437 harvested = StateVariable(enabled=False, doc= 1438 """Store specified attributes of classifiers at each split""") 1439 1440 _KNOWN_COPY_METHODS = [ None, 'copy', 'deepcopy' ] 1441 1442
1443 - def __init__(self, attribs=None, copy_attribs='copy', **kwargs):
1444 """Initialize state of harvestable 1445 1446 :Parameters: 1447 attribs : list of basestr or dicts 1448 What attributes of call to store and return within 1449 harvested state variable. If an item is a dictionary, 1450 following keys are used ['name', 'copy'] 1451 copy_attribs : None or basestr 1452 Default copying. If None -- no copying, 'copy' 1453 - shallow copying, 'deepcopy' -- deepcopying 1454 1455 """ 1456 Stateful.__init__(self, **kwargs) 1457 1458 self.__atribs = attribs 1459 self.__copy_attribs = copy_attribs 1460 1461 self._setAttribs(attribs)
1462 1463
1464 - def _setAttribs(self, attribs):
1465 """Set attributes to harvest 1466 1467 Each attribute in self.__attribs must have following fields 1468 - name : functional (or arbitrary if 'obj' or 'attr' is set) 1469 description of the thing to harvest, 1470 e.g. 'transerror.clf.training_time' 1471 - obj : name of the object to harvest from (if empty, 1472 'self' is assumed), 1473 e.g 'transerror' 1474 - attr : attribute of 'obj' to harvest, 1475 e.g. 'clf.training_time' 1476 - copy : None, 'copy' or 'deepcopy' - way to copy attribute 1477 """ 1478 if attribs: 1479 # force the state 1480 self.states.enable('harvested') 1481 self.__attribs = [] 1482 for i, attrib in enumerate(attribs): 1483 if isinstance(attrib, dict): 1484 if not 'name' in attrib: 1485 raise ValueError, \ 1486 "Harvestable: attribute must be a string or " + \ 1487 "a dictionary with 'name'" 1488 else: 1489 attrib = {'name': attrib} 1490 1491 # assign default method to copy 1492 if not 'copy' in attrib: 1493 attrib['copy'] = self.__copy_attribs 1494 1495 # check copy method 1496 if not attrib['copy'] in self._KNOWN_COPY_METHODS: 1497 raise ValueError, "Unknown method %s. Known are %s" % \ 1498 (attrib['copy'], self._KNOWN_COPY_METHODS) 1499 1500 if not ('obj' in attrib or 'attr' in attrib): 1501 # Process the item to harvest 1502 # split into obj, attr. If obj is empty, then assume self 1503 split = attrib['name'].split('.', 1) 1504 if len(split)==1: 1505 obj, attr = split[0], None 1506 else: 1507 obj, attr = split 1508 attrib.update({'obj':obj, 'attr':attr}) 1509 1510 if attrib['obj'] == '': 1511 attrib['obj'] = 'self' 1512 1513 # TODO: may be enabling of the states?? 1514 1515 self.__attribs.append(attrib) # place value back 1516 else: 1517 # just to make sure it is not None or 0 1518 self.__attribs = []
1519 1520
1521 - def _harvest(self, vars):
1522 """The harvesting function: must obtain dictionary of variables 1523 from the caller. 1524 1525 :Parameters: 1526 vars : dict 1527 Dictionary of available data. Most often locals() could be 1528 passed as `vars`. Mention that desired to be harvested 1529 private attributes better be bound locally to some variable 1530 1531 :Returns: 1532 nothing 1533 """ 1534 1535 if not self.states.isEnabled('harvested') or len(self.__attribs)==0: 1536 return 1537 1538 if not self.states.isSet('harvested'): 1539 self.harvested = dict([(a['name'], []) for a in self.__attribs]) 1540 1541 for attrib in self.__attribs: 1542 attrv = vars[attrib['obj']] 1543 1544 # access particular attribute if needed 1545 if not attrib['attr'] is None: 1546 attrv = eval('attrv.%s' % attrib['attr']) 1547 1548 # copy the value if needed 1549 attrv = {'copy':copy.copy, 1550 'deepcopy':copy.deepcopy, 1551 None:lambda x:x}[attrib['copy']](attrv) 1552 1553 self.harvested[attrib['name']].append(attrv)
1554 1555 1556 harvest_attribs = property(fget=lambda self:self.__attribs, 1557 fset=_setAttribs)
1558