Package logilab :: Package common :: Module modutils
[frames] | no frames]

Source Code for Module logilab.common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 29  """ 
 30  __docformat__ = "restructuredtext en" 
 31   
 32  import sys 
 33  import os 
 34  from os.path import splitext, join, abspath, isdir, dirname, exists, basename 
 35  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 36  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 37   
 38  try: 
 39      import zipimport 
 40  except ImportError: 
 41      zipimport = None 
 42   
 43  ZIPFILE = object() 
 44   
 45  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 46   
 47  # Notes about STD_LIB_DIR 
 48  # Consider arch-specific installation for STD_LIB_DIR definition 
 49  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 50  # 
 51  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 52  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 53  if sys.platform.startswith('win'): 
 54      PY_SOURCE_EXTS = ('py', 'pyw') 
 55      PY_COMPILED_EXTS = ('dll', 'pyd') 
 56      STD_LIB_DIR = get_python_lib(standard_lib=1) 
 57  else: 
 58      PY_SOURCE_EXTS = ('py',) 
 59      PY_COMPILED_EXTS = ('so',) 
 60      # extend lib dir with some arch-dependant paths 
 61      STD_LIB_DIR = join(get_config_var("LIBDIR"), "python%s" % get_python_version()) 
 62   
 63  BUILTIN_MODULES = dict(zip(sys.builtin_module_names, 
 64                             [1]*len(sys.builtin_module_names))) 
 65   
 66   
67 -class NoSourceFile(Exception):
68 """exception raised when we are not able to get a python 69 source file for a precompiled file 70 """
71
72 -class LazyObject(object):
73 - def __init__(self, module, obj):
74 self.module = module 75 self.obj = obj 76 self._imported = None
77
78 - def __getobj(self):
79 if self._imported is None: 80 self._imported = getattr(load_module_from_name(self.module), 81 self.obj) 82 return self._imported
83
84 - def __getattribute__(self, attr):
85 try: 86 return super(LazyObject, self).__getattribute__(attr) 87 except AttributeError, ex: 88 return getattr(self.__getobj(), attr)
89
90 - def __call__(self, *args, **kwargs):
91 return self.__getobj()(*args, **kwargs)
92 93
94 -def load_module_from_name(dotted_name, path=None, use_sys=1):
95 """Load a Python module from it's name. 96 97 :type dotted_name: str 98 :param dotted_name: python name of a module or package 99 100 :type path: list or None 101 :param path: 102 optional list of path where the module or package should be 103 searched (use sys.path if nothing or None is given) 104 105 :type use_sys: bool 106 :param use_sys: 107 boolean indicating whether the sys.modules dictionary should be 108 used or not 109 110 111 :raise ImportError: if the module or package is not found 112 113 :rtype: module 114 :return: the loaded module 115 """ 116 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
117 118
119 -def load_module_from_modpath(parts, path=None, use_sys=1):
120 """Load a python module from it's splitted name. 121 122 :type parts: list(str) or tuple(str) 123 :param parts: 124 python name of a module or package splitted on '.' 125 126 :type path: list or None 127 :param path: 128 optional list of path where the module or package should be 129 searched (use sys.path if nothing or None is given) 130 131 :type use_sys: bool 132 :param use_sys: 133 boolean indicating whether the sys.modules dictionary should be used or not 134 135 :raise ImportError: if the module or package is not found 136 137 :rtype: module 138 :return: the loaded module 139 """ 140 if use_sys: 141 try: 142 return sys.modules['.'.join(parts)] 143 except KeyError: 144 pass 145 modpath = [] 146 prevmodule = None 147 for part in parts: 148 modpath.append(part) 149 curname = '.'.join(modpath) 150 module = None 151 if len(modpath) != len(parts): 152 # even with use_sys=False, should try to get outer packages from sys.modules 153 module = sys.modules.get(curname) 154 if module is None: 155 mp_file, mp_filename, mp_desc = find_module(part, path) 156 module = load_module(curname, mp_file, mp_filename, mp_desc) 157 if prevmodule: 158 setattr(prevmodule, part, module) 159 _file = getattr(module, '__file__', '') 160 if not _file and len(modpath) != len(parts): 161 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 162 path = [dirname( _file )] 163 prevmodule = module 164 return module
165 166
167 -def load_module_from_file(filepath, path=None, use_sys=1, extrapath=None):
168 """Load a Python module from it's path. 169 170 :type filepath: str 171 :param filepath: path to the python module or package 172 173 :type path: list or None 174 :param path: 175 optional list of path where the module or package should be 176 searched (use sys.path if nothing or None is given) 177 178 :type use_sys: bool 179 :param use_sys: 180 boolean indicating whether the sys.modules dictionary should be 181 used or not 182 183 184 :raise ImportError: if the module or package is not found 185 186 :rtype: module 187 :return: the loaded module 188 """ 189 modpath = modpath_from_file(filepath, extrapath) 190 return load_module_from_modpath(modpath, path, use_sys)
191 192
193 -def _check_init(path, mod_path):
194 """check there are some __init__.py all along the way""" 195 for part in mod_path: 196 path = join(path, part) 197 if not _has_init(path): 198 return False 199 return True
200 201
202 -def modpath_from_file(filename, extrapath=None):
203 """given a file path return the corresponding splitted module's name 204 (i.e name of a module or package splitted on '.') 205 206 :type filename: str 207 :param filename: file's path for which we want the module's name 208 209 :type extrapath: dict 210 :param extrapath: 211 optional extra search path, with path as key and package name for the path 212 as value. This is usually useful to handle package splitted in multiple 213 directories using __path__ trick. 214 215 216 :raise ImportError: 217 if the corresponding module's name has not been found 218 219 :rtype: list(str) 220 :return: the corresponding splitted module's name 221 """ 222 base = splitext(abspath(filename))[0] 223 if extrapath is not None: 224 for path_ in extrapath: 225 path = abspath(path_) 226 if path and base[:len(path)] == path: 227 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 228 if pkg] 229 if _check_init(path, submodpath[:-1]): 230 return extrapath[path_].split('.') + submodpath 231 for path in sys.path: 232 path = abspath(path) 233 if path and base[:len(path)] == path: 234 if filename.find('site-packages') != -1 and \ 235 path.find('site-packages') == -1: 236 continue 237 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 238 if _check_init(path, modpath[:-1]): 239 return modpath 240 raise ImportError('Unable to find module for %s in %s' % ( 241 filename, ', \n'.join(sys.path)))
242 243 244
245 -def file_from_modpath(modpath, path=None, context_file=None):
246 """given a mod path (i.e. splitted module / package name), return the 247 corresponding file, giving priority to source file over precompiled 248 file if it exists 249 250 :type modpath: list or tuple 251 :param modpath: 252 splitted module's name (i.e name of a module or package splitted 253 on '.') 254 (this means explicit relative imports that start with dots have 255 empty strings in this list!) 256 257 :type path: list or None 258 :param path: 259 optional list of path where the module or package should be 260 searched (use sys.path if nothing or None is given) 261 262 :type context_file: str or None 263 :param context_file: 264 context file to consider, necessary if the identifier has been 265 introduced using a relative import unresolvable in the actual 266 context (i.e. modutils) 267 268 :raise ImportError: if there is no such module in the directory 269 270 :rtype: str or None 271 :return: 272 the path to the module's file or None if it's an integrated 273 builtin module such as 'sys' 274 """ 275 if context_file is not None: 276 context = dirname(context_file) 277 else: 278 context = context_file 279 if modpath[0] == 'xml': 280 # handle _xmlplus 281 try: 282 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 283 except ImportError: 284 return _file_from_modpath(modpath, path, context) 285 elif modpath == ['os', 'path']: 286 # FIXME: currently ignoring search_path... 287 return os.path.__file__ 288 return _file_from_modpath(modpath, path, context)
289 290 291
292 -def get_module_part(dotted_name, context_file=None):
293 """given a dotted name return the module part of the name : 294 295 >>> get_module_part('logilab.common.modutils.get_module_part') 296 'logilab.common.modutils' 297 298 :type dotted_name: str 299 :param dotted_name: full name of the identifier we are interested in 300 301 :type context_file: str or None 302 :param context_file: 303 context file to consider, necessary if the identifier has been 304 introduced using a relative import unresolvable in the actual 305 context (i.e. modutils) 306 307 308 :raise ImportError: if there is no such module in the directory 309 310 :rtype: str or None 311 :return: 312 the module part of the name or None if we have not been able at 313 all to import the given name 314 315 XXX: deprecated, since it doesn't handle package precedence over module 316 (see #10066) 317 """ 318 # os.path trick 319 if dotted_name.startswith('os.path'): 320 return 'os.path' 321 parts = dotted_name.split('.') 322 if context_file is not None: 323 # first check for builtin module which won't be considered latter 324 # in that case (path != None) 325 if parts[0] in BUILTIN_MODULES: 326 if len(parts) > 2: 327 raise ImportError(dotted_name) 328 return parts[0] 329 # don't use += or insert, we want a new list to be created ! 330 path = None 331 starti = 0 332 if parts[0] == '': 333 assert context_file is not None, \ 334 'explicit relative import, but no context_file?' 335 path = [] # prevent resolving the import non-relatively 336 starti = 1 337 while parts[starti] == '': # for all further dots: change context 338 starti += 1 339 context_file = dirname(context_file) 340 for i in range(starti, len(parts)): 341 try: 342 file_from_modpath(parts[starti:i+1], 343 path=path, context_file=context_file) 344 except ImportError: 345 if not i >= max(1, len(parts) - 2): 346 raise 347 return '.'.join(parts[:i]) 348 return dotted_name
349 350
351 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
352 """given a package directory return a list of all available python 353 modules in the package and its subpackages 354 355 :type package: str 356 :param package: the python name for the package 357 358 :type src_directory: str 359 :param src_directory: 360 path of the directory corresponding to the package 361 362 :type blacklist: list or tuple 363 :param blacklist: 364 optional list of files or directory to ignore, default to 365 the value of `logilab.common.STD_BLACKLIST` 366 367 :rtype: list 368 :return: 369 the list of all available python modules in the package and its 370 subpackages 371 """ 372 modules = [] 373 for directory, dirnames, filenames in os.walk(src_directory): 374 _handle_blacklist(blacklist, dirnames, filenames) 375 # check for __init__.py 376 if not '__init__.py' in filenames: 377 dirnames[:] = () 378 continue 379 if directory != src_directory: 380 dir_package = directory[len(src_directory):].replace(os.sep, '.') 381 modules.append(package + dir_package) 382 for filename in filenames: 383 if _is_python_file(filename) and filename != '__init__.py': 384 src = join(directory, filename) 385 module = package + src[len(src_directory):-3] 386 modules.append(module.replace(os.sep, '.')) 387 return modules
388 389 390
391 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
392 """given a package directory return a list of all available python 393 module's files in the package and its subpackages 394 395 :type src_directory: str 396 :param src_directory: 397 path of the directory corresponding to the package 398 399 :type blacklist: list or tuple 400 :param blacklist: 401 optional list of files or directory to ignore, default to the value of 402 `logilab.common.STD_BLACKLIST` 403 404 :rtype: list 405 :return: 406 the list of all available python module's files in the package and 407 its subpackages 408 """ 409 files = [] 410 for directory, dirnames, filenames in os.walk(src_directory): 411 _handle_blacklist(blacklist, dirnames, filenames) 412 # check for __init__.py 413 if not '__init__.py' in filenames: 414 dirnames[:] = () 415 continue 416 for filename in filenames: 417 if _is_python_file(filename): 418 src = join(directory, filename) 419 files.append(src) 420 return files
421 422
423 -def get_source_file(filename, include_no_ext=False):
424 """given a python module's file name return the matching source file 425 name (the filename will be returned identically if it's a already an 426 absolute path to a python source file...) 427 428 :type filename: str 429 :param filename: python module's file name 430 431 432 :raise NoSourceFile: if no source file exists on the file system 433 434 :rtype: str 435 :return: the absolute path of the source file if it exists 436 """ 437 base, orig_ext = splitext(abspath(filename)) 438 for ext in PY_SOURCE_EXTS: 439 source_path = '%s.%s' % (base, ext) 440 if exists(source_path): 441 return source_path 442 if include_no_ext and not orig_ext and exists(base): 443 return base 444 raise NoSourceFile(filename)
445 446
447 -def cleanup_sys_modules(directories):
448 """remove submodules of `directories` from `sys.modules`""" 449 for modname, module in sys.modules.items(): 450 modfile = getattr(module, '__file__', None) 451 if modfile: 452 for directory in directories: 453 if modfile.startswith(directory): 454 del sys.modules[modname] 455 break
456 457
458 -def is_python_source(filename):
459 """ 460 rtype: bool 461 return: True if the filename is a python source file 462 """ 463 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
464 465 466
467 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
468 """try to guess if a module is a standard python module (by default, 469 see `std_path` parameter's description) 470 471 :type modname: str 472 :param modname: name of the module we are interested in 473 474 :type std_path: list(str) or tuple(str) 475 :param std_path: list of path considered has standard 476 477 478 :rtype: bool 479 :return: 480 true if the module: 481 - is located on the path listed in one of the directory in `std_path` 482 - is a built-in module 483 """ 484 modname = modname.split('.')[0] 485 try: 486 filename = file_from_modpath([modname]) 487 except ImportError, ex: 488 # import failed, i'm probably not so wrong by supposing it's 489 # not standard... 490 return 0 491 # modules which are not living in a file are considered standard 492 # (sys and __builtin__ for instance) 493 if filename is None: 494 return 1 495 filename = abspath(filename) 496 for path in std_path: 497 path = abspath(path) 498 if filename.startswith(path): 499 pfx_len = len(path) 500 if filename[pfx_len+1:pfx_len+14] != 'site-packages': 501 return 1 502 return 0 503 return False
504 505 506
507 -def is_relative(modname, from_file):
508 """return true if the given module name is relative to the given 509 file name 510 511 :type modname: str 512 :param modname: name of the module we are interested in 513 514 :type from_file: str 515 :param from_file: 516 path of the module from which modname has been imported 517 518 :rtype: bool 519 :return: 520 true if the module has been imported relatively to `from_file` 521 """ 522 if not isdir(from_file): 523 from_file = dirname(from_file) 524 if from_file in sys.path: 525 return False 526 try: 527 find_module(modname.split('.')[0], [from_file]) 528 return True 529 except ImportError: 530 return False
531 532 533 # internal only functions ##################################################### 534
535 -def _file_from_modpath(modpath, path=None, context=None):
536 """given a mod path (i.e. splitted module / package name), return the 537 corresponding file 538 539 this function is used internally, see `file_from_modpath`'s 540 documentation for more information 541 """ 542 assert len(modpath) > 0 543 if context is not None: 544 try: 545 mtype, mp_filename = _module_file(modpath, [context]) 546 except ImportError: 547 mtype, mp_filename = _module_file(modpath, path) 548 else: 549 mtype, mp_filename = _module_file(modpath, path) 550 if mtype == PY_COMPILED: 551 try: 552 return get_source_file(mp_filename) 553 except NoSourceFile: 554 return mp_filename 555 elif mtype == C_BUILTIN: 556 # integrated builtin module 557 return None 558 elif mtype == PKG_DIRECTORY: 559 mp_filename = _has_init(mp_filename) 560 return mp_filename
561
562 -def _search_zip(modpath, pic):
563 for filepath, importer in pic.items(): 564 if importer is not None: 565 if importer.find_module(modpath[0]): 566 if not importer.find_module('/'.join(modpath)): 567 raise ImportError('No module named %s in %s/%s' % ( 568 '.'.join(modpath[1:]), file, modpath)) 569 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 570 raise ImportError('No module named %s' % '.'.join(modpath))
571
572 -def _module_file(modpath, path=None):
573 """get a module type / file path 574 575 :type modpath: list or tuple 576 :param modpath: 577 splitted module's name (i.e name of a module or package splitted 578 on '.'), with leading empty strings for explicit relative import 579 580 :type path: list or None 581 :param path: 582 optional list of path where the module or package should be 583 searched (use sys.path if nothing or None is given) 584 585 586 :rtype: tuple(int, str) 587 :return: the module type flag and the file path for a module 588 """ 589 # egg support compat 590 try: 591 pic = sys.path_importer_cache 592 _path = (path is None and sys.path or path) 593 for __path in _path: 594 if not __path in pic: 595 try: 596 pic[__path] = zipimport.zipimporter(__path) 597 except zipimport.ZipImportError: 598 pic[__path] = None 599 checkeggs = True 600 except AttributeError: 601 checkeggs = False 602 imported = [] 603 while modpath: 604 try: 605 _, mp_filename, mp_desc = find_module(modpath[0], path) 606 except ImportError: 607 if checkeggs: 608 return _search_zip(modpath, pic)[:2] 609 raise 610 else: 611 if checkeggs: 612 fullabspath = [abspath(x) for x in _path] 613 try: 614 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 615 emtype, emp_filename, zippath = _search_zip(modpath, pic) 616 if pathindex > _path.index(zippath): 617 # an egg takes priority 618 return emtype, emp_filename 619 except ValueError: 620 # XXX not in _path 621 pass 622 except ImportError: 623 pass 624 checkeggs = False 625 imported.append(modpath.pop(0)) 626 mtype = mp_desc[2] 627 if modpath: 628 if mtype != PKG_DIRECTORY: 629 raise ImportError('No module %s in %s' % ('.'.join(modpath), 630 '.'.join(imported))) 631 path = [mp_filename] 632 return mtype, mp_filename
633
634 -def _is_python_file(filename):
635 """return true if the given filename should be considered as a python file 636 637 .pyc and .pyo are ignored 638 """ 639 for ext in ('.py', '.so', '.pyd', '.pyw'): 640 if filename.endswith(ext): 641 return True 642 return False
643 644
645 -def _has_init(directory):
646 """if the given directory has a valid __init__ file, return its path, 647 else return None 648 """ 649 mod_or_pack = join(directory, '__init__') 650 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 651 if exists(mod_or_pack + '.' + ext): 652 return mod_or_pack + '.' + ext 653 return None
654