Package logilab-common-0 :: Package 39 :: Package 0 :: Module modutils
[frames] | no frames]

Source Code for Module logilab-common-0.39.0.modutils

  1  # -*- coding: utf-8 -*- 
  2  """Python modules manipulation utility functions. 
  3   
  4  :author: Logilab 
  5  :copyright: 2000-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  6  :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  7  :license: General Public License version 2 - http://www.gnu.org/licenses 
  8   
  9  :type PY_SOURCE_EXTS: tuple(str) 
 10  :var PY_SOURCE_EXTS: list of possible python source file extension 
 11   
 12  :type STD_LIB_DIR: str 
 13  :var STD_LIB_DIR: directory where standard modules are located 
 14   
 15  :type BUILTIN_MODULES: dict 
 16  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 17  """ 
 18  __docformat__ = "restructuredtext en" 
 19   
 20  import sys 
 21  import os 
 22  from os.path import walk, splitext, join, abspath, isdir, dirname, exists 
 23  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 24   
 25  try: 
 26      import zipimport 
 27  except ImportError: 
 28      zipimport = None 
 29       
 30  ZIPFILE = object() 
 31   
 32  from logilab.common import STD_BLACKLIST 
 33   
 34  if sys.platform.startswith('win'): 
 35      PY_SOURCE_EXTS = ('py', 'pyw') 
 36      PY_COMPILED_EXTS = ('dll', 'pyd') 
 37      STD_LIB_DIR = join(sys.prefix, 'lib') 
 38  else: 
 39      PY_SOURCE_EXTS = ('py',) 
 40      PY_COMPILED_EXTS = ('so',) 
 41      STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3]) 
 42       
 43  BUILTIN_MODULES = dict(zip(sys.builtin_module_names, 
 44                             [1]*len(sys.builtin_module_names))) 
 45   
 46   
47 -class NoSourceFile(Exception):
48 """exception raised when we are not able to get a python 49 source file for a precompiled file 50 """
51
52 -class LazyObject(object):
53 - def __init__(self, module, obj):
54 self.module = module 55 self.obj = obj 56 self._imported = None
57
58 - def __getobj(self):
59 if self._imported is None: 60 self._imported = getattr(load_module_from_name(self.module), 61 self.obj) 62 return self._imported
63
64 - def __getattribute__(self, attr):
65 try: 66 return super(LazyObject, self).__getattribute__(attr) 67 except AttributeError, ex: 68 return getattr(self.__getobj(), attr)
69
70 - def __call__(self, *args, **kwargs):
71 return self.__getobj()(*args, **kwargs)
72 73
74 -def load_module_from_name(dotted_name, path=None, use_sys=1):
75 """Load a Python module from it's name. 76 77 :type dotted_name: str 78 :param dotted_name: python name of a module or package 79 80 :type path: list or None 81 :param path: 82 optional list of path where the module or package should be 83 searched (use sys.path if nothing or None is given) 84 85 :type use_sys: bool 86 :param use_sys: 87 boolean indicating whether the sys.modules dictionary should be 88 used or not 89 90 91 :raise ImportError: if the module or package is not found 92 93 :rtype: module 94 :return: the loaded module 95 """ 96 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
97 98
99 -def load_module_from_modpath(parts, path=None, use_sys=1):
100 """Load a python module from it's splitted name. 101 102 :type parts: list(str) or tuple(str) 103 :param parts: 104 python name of a module or package splitted on '.' 105 106 :type path: list or None 107 :param path: 108 optional list of path where the module or package should be 109 searched (use sys.path if nothing or None is given) 110 111 :type use_sys: bool 112 :param use_sys: 113 boolean indicating whether the sys.modules dictionary should be used or not 114 115 :raise ImportError: if the module or package is not found 116 117 :rtype: module 118 :return: the loaded module 119 """ 120 if use_sys: 121 try: 122 return sys.modules['.'.join(parts)] 123 except KeyError: 124 pass 125 modpath = [] 126 prevmodule = None 127 for part in parts: 128 modpath.append(part) 129 curname = ".".join(modpath) 130 module = None 131 if len(modpath) != len(parts): 132 # even with use_sys=False, should try to get outer packages from sys.modules 133 module = sys.modules.get(curname) 134 if module is None: 135 mp_file, mp_filename, mp_desc = find_module(part, path) 136 module = load_module(curname, mp_file, mp_filename, mp_desc) 137 if prevmodule: 138 setattr(prevmodule, part, module) 139 _file = getattr(module, "__file__", "") 140 if not _file and len(modpath) != len(parts): 141 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) ) 142 path = [dirname( _file )] 143 prevmodule = module 144 return module
145 146
147 -def load_module_from_file(filepath, path=None, use_sys=1):
148 """Load a Python module from it's path. 149 150 :type filepath: str 151 :param filepath: path to the python module or package 152 153 :type path: list or None 154 :param path: 155 optional list of path where the module or package should be 156 searched (use sys.path if nothing or None is given) 157 158 :type use_sys: bool 159 :param use_sys: 160 boolean indicating whether the sys.modules dictionary should be 161 used or not 162 163 164 :raise ImportError: if the module or package is not found 165 166 :rtype: module 167 :return: the loaded module 168 """ 169 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
170 171
172 -def modpath_from_file(filename):
173 """given a file path return the corresponding splitted module's name 174 (i.e name of a module or package splitted on '.') 175 176 :type filename: str 177 :param filename: file's path for which we want the module's name 178 179 180 :raise ImportError: 181 if the corresponding module's name has not been found 182 183 :rtype: list(str) 184 :return: the corresponding splitted module's name 185 """ 186 base = splitext(abspath(filename))[0] 187 for path in sys.path: 188 path = abspath(path) 189 if path and base[:len(path)] == path: 190 if filename.find('site-packages') != -1 and \ 191 path.find('site-packages') == -1: 192 continue 193 mod_path = [module for module in base[len(path):].split(os.sep) 194 if module] 195 for part in mod_path[:-1]: 196 path = join(path, part) 197 if not _has_init(path): 198 break 199 else: 200 break 201 else: 202 raise ImportError('Unable to find module for %s in %s' % ( 203 filename, ', \n'.join(sys.path))) 204 return mod_path
205 206 207
208 -def file_from_modpath(modpath, path=None, context_file=None):
209 """given a mod path (ie splited module / package name), return the 210 corresponding file, giving priority to source file over precompiled 211 file if it exists 212 213 :type modpath: list or tuple 214 :param modpath: 215 splitted module's name (i.e name of a module or package splitted 216 on '.') 217 (this means explicit relative imports that start with dots have 218 empty strings in this list!) 219 220 :type path: list or None 221 :param path: 222 optional list of path where the module or package should be 223 searched (use sys.path if nothing or None is given) 224 225 :type context_file: str or None 226 :param context_file: 227 context file to consider, necessary if the identifier has been 228 introduced using a relative import unresolvable in the actual 229 context (i.e. modutils) 230 231 :raise ImportError: if there is no such module in the directory 232 233 :rtype: str or None 234 :return: 235 the path to the module's file or None if it's an integrated 236 builtin module such as 'sys' 237 """ 238 if context_file is not None: 239 context = dirname(context_file) 240 else: 241 context = context_file 242 if modpath[0] == 'xml': 243 # handle _xmlplus 244 try: 245 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 246 except ImportError: 247 return _file_from_modpath(modpath, path, context) 248 elif modpath == ['os', 'path']: 249 # FIXME: currently ignoring search_path... 250 return os.path.__file__ 251 return _file_from_modpath(modpath, path, context)
252 253 254
255 -def get_module_part(dotted_name, context_file=None):
256 """given a dotted name return the module part of the name : 257 258 >>> get_module_part('logilab.common.modutils.get_module_part') 259 'logilab.common.modutils' 260 261 262 :type dotted_name: str 263 :param dotted_name: full name of the identifier we are interested in 264 265 :type context_file: str or None 266 :param context_file: 267 context file to consider, necessary if the identifier has been 268 introduced using a relative import unresolvable in the actual 269 context (i.e. modutils) 270 271 272 :raise ImportError: if there is no such module in the directory 273 274 :rtype: str or None 275 :return: 276 the module part of the name or None if we have not been able at 277 all to import the given name 278 279 XXX: deprecated, since it doesn't handle package precedence over module 280 (see #10066) 281 """ 282 # os.path trick 283 if dotted_name.startswith('os.path'): 284 return 'os.path' 285 parts = dotted_name.split('.') 286 if context_file is not None: 287 # first check for builtin module which won't be considered latter 288 # in that case (path != None) 289 if parts[0] in BUILTIN_MODULES: 290 if len(parts) > 2: 291 raise ImportError(dotted_name) 292 return parts[0] 293 # don't use += or insert, we want a new list to be created ! 294 path = None 295 starti = 0 296 if parts[0] == '': 297 assert context_file is not None, \ 298 'explicit relative import, but no context_file?' 299 path = [] # prevent resolving the import non-relatively 300 starti = 1 301 while parts[starti] == '': # for all further dots: change context 302 starti += 1 303 context_file = dirname(context_file) 304 for i in range(starti, len(parts)): 305 try: 306 file_from_modpath(parts[starti:i+1], 307 path=path, context_file=context_file) 308 except ImportError: 309 if not i >= max(1, len(parts) - 2): 310 raise 311 return '.'.join(parts[:i]) 312 return dotted_name
313 314 315
316 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
317 """given a package directory return a list of all available python 318 modules in the package and its subpackages 319 320 :type package: str 321 :param package: the python name for the package 322 323 :type src_directory: str 324 :param src_directory: 325 path of the directory corresponding to the package 326 327 :type blacklist: list or tuple 328 :param blacklist: 329 optional list of files or directory to ignore, default to 330 the value of `logilab.common.STD_BLACKLIST` 331 332 :rtype: list 333 :return: 334 the list of all available python modules in the package and its 335 subpackages 336 """ 337 def func(modules, directory, fnames): 338 """walk handler""" 339 # remove files/directories in the black list 340 for norecurs in blacklist: 341 try: 342 fnames.remove(norecurs) 343 except ValueError: 344 continue 345 # check for __init__.py 346 if not '__init__.py' in fnames: 347 while fnames: 348 fnames.pop() 349 elif directory != src_directory: 350 #src = join(directory, file) 351 dir_package = directory[len(src_directory):].replace(os.sep, '.') 352 modules.append(package + dir_package) 353 for filename in fnames: 354 src = join(directory, filename) 355 if isdir(src): 356 continue 357 if _is_python_file(filename) and filename != '__init__.py': 358 module = package + src[len(src_directory):-3] 359 modules.append(module.replace(os.sep, '.'))
360 modules = [] 361 walk(src_directory, func, modules) 362 return modules 363 364 365
366 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
367 """given a package directory return a list of all available python 368 module's files in the package and its subpackages 369 370 :type src_directory: str 371 :param src_directory: 372 path of the directory corresponding to the package 373 374 :type blacklist: list or tuple 375 :param blacklist: 376 optional list of files or directory to ignore, default to the value of 377 `logilab.common.STD_BLACKLIST` 378 379 :rtype: list 380 :return: 381 the list of all available python module's files in the package and 382 its subpackages 383 """ 384 def func(files, directory, fnames): 385 """walk handler""" 386 # remove files/directories in the black list 387 for norecurs in blacklist: 388 try: 389 fnames.remove(norecurs) 390 except ValueError: 391 continue 392 # check for __init__.py 393 if not '__init__.py' in fnames: 394 while fnames: 395 fnames.pop() 396 for filename in fnames: 397 src = join(directory, filename) 398 if isdir(src): 399 continue 400 if _is_python_file(filename): 401 files.append(src)
402 files = [] 403 walk(src_directory, func, files) 404 return files 405 406
407 -def get_source_file(filename, include_no_ext=False):
408 """given a python module's file name return the matching source file 409 name (the filename will be returned identically if it's a already an 410 absolute path to a python source file...) 411 412 :type filename: str 413 :param filename: python module's file name 414 415 416 :raise NoSourceFile: if no source file exists on the file system 417 418 :rtype: str 419 :return: the absolute path of the source file if it exists 420 """ 421 base, orig_ext = splitext(abspath(filename)) 422 for ext in PY_SOURCE_EXTS: 423 source_path = '%s.%s' % (base, ext) 424 if exists(source_path): 425 return source_path 426 if include_no_ext and not orig_ext and exists(base): 427 return base 428 raise NoSourceFile(filename)
429 430 431
432 -def is_python_source(filename):
433 """ 434 rtype: bool 435 return: True if the filename is a python source file 436 """ 437 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
438 439 440
441 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
442 """try to guess if a module is a standard python module (by default, 443 see `std_path` parameter's description) 444 445 :type modname: str 446 :param modname: name of the module we are interested in 447 448 :type std_path: list(str) or tuple(str) 449 :param std_path: list of path considered has standard 450 451 452 :rtype: bool 453 :return: 454 true if the module: 455 - is located on the path listed in one of the directory in `std_path` 456 - is a built-in module 457 """ 458 modpath = modname.split('.') 459 modname = modpath[0] 460 try: 461 filename = file_from_modpath(modpath) 462 except ImportError: 463 # import failed, i'm probably not so wrong by supposing it's 464 # not standard... 465 return 0 466 # modules which are not living in a file are considered standard 467 # (sys and __builtin__ for instance) 468 if filename is None: 469 return 1 470 filename = abspath(filename) 471 for path in std_path: 472 path = abspath(path) 473 if filename.startswith(path): 474 pfx_len = len(path) 475 if filename[pfx_len+1:pfx_len+14] != 'site-packages': 476 return 1 477 return 0 478 return False
479 480 481
482 -def is_relative(modname, from_file):
483 """return true if the given module name is relative to the given 484 file name 485 486 :type modname: str 487 :param modname: name of the module we are interested in 488 489 :type from_file: str 490 :param from_file: 491 path of the module from which modname has been imported 492 493 :rtype: bool 494 :return: 495 true if the module has been imported relativly to `from_file` 496 """ 497 if not isdir(from_file): 498 from_file = dirname(from_file) 499 if from_file in sys.path: 500 return False 501 try: 502 find_module(modname.split('.')[0], [from_file]) 503 return True 504 except ImportError: 505 return False
506 507 508 # internal only functions ##################################################### 509
510 -def _file_from_modpath(modpath, path=None, context=None):
511 """given a mod path (ie splited module / package name), return the 512 corresponding file 513 514 this function is used internally, see `file_from_modpath`'s 515 documentation for more information 516 """ 517 assert len(modpath) > 0 518 if context is not None: 519 try: 520 mtype, mp_filename = _module_file(modpath, [context]) 521 except ImportError: 522 mtype, mp_filename = _module_file(modpath, path) 523 else: 524 mtype, mp_filename = _module_file(modpath, path) 525 if mtype == PY_COMPILED: 526 try: 527 return get_source_file(mp_filename) 528 except NoSourceFile: 529 return mp_filename 530 elif mtype == C_BUILTIN: 531 # integrated builtin module 532 return None 533 elif mtype == PKG_DIRECTORY: 534 mp_filename = _has_init(mp_filename) 535 return mp_filename
536
537 -def _search_zip(modpath, pic):
538 for filepath, importer in pic.items(): 539 if importer is not None: 540 if importer.find_module(modpath[0]): 541 if not importer.find_module('/'.join(modpath)): 542 raise ImportError('No module named %s in %s/%s' % ( 543 '.'.join(modpath[1:]), file, modpath)) 544 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 545 raise ImportError('No module named %s' % '.'.join(modpath))
546
547 -def _module_file(modpath, path=None):
548 """get a module type / file path 549 550 :type modpath: list or tuple 551 :param modpath: 552 splitted module's name (i.e name of a module or package splitted 553 on '.'), with leading empty strings for explicit relative import 554 555 :type path: list or None 556 :param path: 557 optional list of path where the module or package should be 558 searched (use sys.path if nothing or None is given) 559 560 561 :rtype: tuple(int, str) 562 :return: the module type flag and the file path for a module 563 """ 564 # egg support compat 565 try: 566 pic = sys.path_importer_cache 567 _path = (path is None and sys.path or path) 568 for __path in _path: 569 if not __path in pic: 570 try: 571 pic[__path] = zipimport.zipimporter(__path) 572 except zipimport.ZipImportError: 573 pic[__path] = None 574 checkeggs = True 575 except AttributeError: 576 checkeggs = False 577 imported = [] 578 while modpath: 579 try: 580 _, mp_filename, mp_desc = find_module(modpath[0], path) 581 except ImportError: 582 if checkeggs: 583 return _search_zip(modpath, pic)[:2] 584 raise 585 else: 586 if checkeggs: 587 fullabspath = [abspath(x) for x in _path] 588 try: 589 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 590 emtype, emp_filename, zippath = _search_zip(modpath, pic) 591 if pathindex > _path.index(zippath): 592 # an egg takes priority 593 return emtype, emp_filename 594 except ValueError: 595 # XXX not in _path 596 pass 597 except ImportError: 598 pass 599 checkeggs = False 600 imported.append(modpath.pop(0)) 601 mtype = mp_desc[2] 602 if modpath: 603 if mtype != PKG_DIRECTORY: 604 raise ImportError('No module %s in %s' % ('.'.join(modpath), 605 '.'.join(imported))) 606 path = [mp_filename] 607 return mtype, mp_filename
608
609 -def _is_python_file(filename):
610 """return true if the given filename should be considered as a python file 611 612 .pyc and .pyo are ignored 613 """ 614 for ext in ('.py', '.so', '.pyd', '.pyw'): 615 if filename.endswith(ext): 616 return True 617 return False
618 619
620 -def _has_init(directory):
621 """if the given directory has a valid __init__ file, return its path, 622 else return None 623 """ 624 mod_or_pack = join(directory, '__init__') 625 for ext in PY_SOURCE_EXTS + ('.pyc', '.pyo'): 626 if exists(mod_or_pack + '.' + ext): 627 return mod_or_pack + ext 628 return None
629