Package logilab-common-0 :: Package 36 :: Package 1 :: Module modutils
[frames] | no frames]

Source Code for Module logilab-common-0.36.1.modutils

  1  # -*- coding: utf-8 -*- 
  2  """Python modules manipulation utility functions. 
  3   
  4  :author: Logilab 
  5  :copyright: 2000-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  6  :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  7  :license: General Public License version 2 - http://www.gnu.org/licenses 
  8   
  9  :type PY_SOURCE_EXTS: tuple(str) 
 10  :var PY_SOURCE_EXTS: list of possible python source file extension 
 11   
 12  :type STD_LIB_DIR: str 
 13  :var STD_LIB_DIR: directory where standard modules are located 
 14   
 15  :type BUILTIN_MODULES: dict 
 16  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 17  """ 
 18  __docformat__ = "restructuredtext en" 
 19   
 20  import sys 
 21  import os 
 22  from os.path import walk, splitext, join, abspath, isdir, dirname, exists 
 23  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 24   
 25  from logilab.common import STD_BLACKLIST 
 26   
 27  if sys.platform.startswith('win'): 
 28      PY_SOURCE_EXTS = ('py', 'pyw') 
 29      PY_COMPILED_EXTS = ('dll', 'pyd') 
 30      STD_LIB_DIR = join(sys.prefix, 'lib') 
 31  else: 
 32      PY_SOURCE_EXTS = ('py',) 
 33      PY_COMPILED_EXTS = ('so',) 
 34      STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3]) 
 35       
 36  BUILTIN_MODULES = dict(zip(sys.builtin_module_names, 
 37                             [1]*len(sys.builtin_module_names))) 
 38   
 39   
40 -class NoSourceFile(Exception):
41 """exception raised when we are not able to get a python 42 source file for a precompiled file 43 """
44
45 -class LazyObject(object):
46 - def __init__(self, module, obj):
47 self.module = module 48 self.obj = obj 49 self._imported = None
50
51 - def __getobj(self):
52 if self._imported is None: 53 self._imported = getattr(load_module_from_name(self.module), 54 self.obj) 55 return self._imported
56
57 - def __getattribute__(self, attr):
58 try: 59 return super(LazyObject, self).__getattribute__(attr) 60 except AttributeError, ex: 61 return getattr(self.__getobj(), attr)
62
63 - def __call__(self, *args, **kwargs):
64 return self.__getobj()(*args, **kwargs)
65 66
67 -def load_module_from_name(dotted_name, path=None, use_sys=1):
68 """Load a Python module from it's name. 69 70 :type dotted_name: str 71 :param dotted_name: python name of a module or package 72 73 :type path: list or None 74 :param path: 75 optional list of path where the module or package should be 76 searched (use sys.path if nothing or None is given) 77 78 :type use_sys: bool 79 :param use_sys: 80 boolean indicating whether the sys.modules dictionary should be 81 used or not 82 83 84 :raise ImportError: if the module or package is not found 85 86 :rtype: module 87 :return: the loaded module 88 """ 89 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
90 91
92 -def load_module_from_modpath(parts, path=None, use_sys=1):
93 """Load a python module from it's splitted name. 94 95 :type parts: list(str) or tuple(str) 96 :param parts: 97 python name of a module or package splitted on '.' 98 99 :type path: list or None 100 :param path: 101 optional list of path where the module or package should be 102 searched (use sys.path if nothing or None is given) 103 104 :type use_sys: bool 105 :param use_sys: 106 boolean indicating whether the sys.modules dictionary should be used or not 107 108 :raise ImportError: if the module or package is not found 109 110 :rtype: module 111 :return: the loaded module 112 """ 113 if use_sys: 114 try: 115 return sys.modules['.'.join(parts)] 116 except KeyError: 117 pass 118 modpath = [] 119 prevmodule = None 120 for part in parts: 121 modpath.append(part) 122 curname = ".".join(modpath) 123 module = None 124 if len(modpath) != len(parts): 125 # even with use_sys=False, should try to get outer packages from sys.modules 126 module = sys.modules.get(curname) 127 if module is None: 128 mp_file, mp_filename, mp_desc = find_module(part, path) 129 module = load_module(curname, mp_file, mp_filename, mp_desc) 130 if prevmodule: 131 setattr(prevmodule, part, module) 132 _file = getattr(module, "__file__", "") 133 if not _file and len(modpath) != len(parts): 134 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) ) 135 path = [dirname( _file )] 136 prevmodule = module 137 return module
138 139
140 -def load_module_from_file(filepath, path=None, use_sys=1):
141 """Load a Python module from it's path. 142 143 :type filepath: str 144 :param filepath: path to the python module or package 145 146 :type path: list or None 147 :param path: 148 optional list of path where the module or package should be 149 searched (use sys.path if nothing or None is given) 150 151 :type use_sys: bool 152 :param use_sys: 153 boolean indicating whether the sys.modules dictionary should be 154 used or not 155 156 157 :raise ImportError: if the module or package is not found 158 159 :rtype: module 160 :return: the loaded module 161 """ 162 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
163 164
165 -def modpath_from_file(filename):
166 """given a file path return the corresponding splitted module's name 167 (i.e name of a module or package splitted on '.') 168 169 :type filename: str 170 :param filename: file's path for which we want the module's name 171 172 173 :raise ImportError: 174 if the corresponding module's name has not been found 175 176 :rtype: list(str) 177 :return: the corresponding splitted module's name 178 """ 179 base = splitext(abspath(filename))[0] 180 for path in sys.path: 181 path = abspath(path) 182 if path and base[:len(path)] == path: 183 if filename.find('site-packages') != -1 and \ 184 path.find('site-packages') == -1: 185 continue 186 mod_path = [module for module in base[len(path):].split(os.sep) 187 if module] 188 for part in mod_path[:-1]: 189 path = join(path, part) 190 if not _has_init(path): 191 break 192 else: 193 break 194 else: 195 raise ImportError('Unable to find module for %s in %s' % ( 196 filename, ', \n'.join(sys.path))) 197 return mod_path
198 199 200
201 -def file_from_modpath(modpath, path=None, context_file=None):
202 """given a mod path (ie splited module / package name), return the 203 corresponding file, giving priority to source file over precompiled 204 file if it exists 205 206 :type modpath: list or tuple 207 :param modpath: 208 splitted module's name (i.e name of a module or package splitted 209 on '.') 210 (this means explicit relative imports that start with dots have 211 empty strings in this list!) 212 213 :type path: list or None 214 :param path: 215 optional list of path where the module or package should be 216 searched (use sys.path if nothing or None is given) 217 218 :type context_file: str or None 219 :param context_file: 220 context file to consider, necessary if the identifier has been 221 introduced using a relative import unresolvable in the actual 222 context (i.e. modutils) 223 224 :raise ImportError: if there is no such module in the directory 225 226 :rtype: str or None 227 :return: 228 the path to the module's file or None if it's an integrated 229 builtin module such as 'sys' 230 """ 231 if context_file is not None: 232 context = dirname(context_file) 233 else: 234 context = context_file 235 if modpath[0] == 'xml': 236 # handle _xmlplus 237 try: 238 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 239 except ImportError: 240 return _file_from_modpath(modpath, path, context) 241 elif modpath == ['os', 'path']: 242 # FIXME: currently ignoring search_path... 243 return os.path.__file__ 244 return _file_from_modpath(modpath, path, context)
245 246 247
248 -def get_module_part(dotted_name, context_file=None):
249 """given a dotted name return the module part of the name : 250 251 >>> get_module_part('logilab.common.modutils.get_module_part') 252 'logilab.common.modutils' 253 254 255 :type dotted_name: str 256 :param dotted_name: full name of the identifier we are interested in 257 258 :type context_file: str or None 259 :param context_file: 260 context file to consider, necessary if the identifier has been 261 introduced using a relative import unresolvable in the actual 262 context (i.e. modutils) 263 264 265 :raise ImportError: if there is no such module in the directory 266 267 :rtype: str or None 268 :return: 269 the module part of the name or None if we have not been able at 270 all to import the given name 271 272 XXX: deprecated, since it doesn't handle package precedence over module 273 (see #10066) 274 """ 275 # os.path trick 276 if dotted_name.startswith('os.path'): 277 return 'os.path' 278 parts = dotted_name.split('.') 279 if context_file is not None: 280 # first check for builtin module which won't be considered latter 281 # in that case (path != None) 282 if parts[0] in BUILTIN_MODULES: 283 if len(parts) > 2: 284 raise ImportError(dotted_name) 285 return parts[0] 286 # don't use += or insert, we want a new list to be created ! 287 path = None 288 starti = 0 289 if parts[0] == '': 290 assert context_file is not None, \ 291 'explicit relative import, but no context_file?' 292 path = [] # prevent resolving the import non-relatively 293 starti = 1 294 while parts[starti] == '': # for all further dots: change context 295 starti += 1 296 context_file = dirname(context_file) 297 for i in range(starti, len(parts)): 298 try: 299 file_from_modpath(parts[starti:i+1], 300 path=path, context_file=context_file) 301 except ImportError: 302 if not i >= max(1, len(parts) - 2): 303 raise 304 return '.'.join(parts[:i]) 305 return dotted_name
306 307 308
309 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
310 """given a package directory return a list of all available python 311 modules in the package and its subpackages 312 313 :type package: str 314 :param package: the python name for the package 315 316 :type src_directory: str 317 :param src_directory: 318 path of the directory corresponding to the package 319 320 :type blacklist: list or tuple 321 :param blacklist: 322 optional list of files or directory to ignore, default to 323 the value of `logilab.common.STD_BLACKLIST` 324 325 :rtype: list 326 :return: 327 the list of all available python modules in the package and its 328 subpackages 329 """ 330 def func(modules, directory, fnames): 331 """walk handler""" 332 # remove files/directories in the black list 333 for norecurs in blacklist: 334 try: 335 fnames.remove(norecurs) 336 except ValueError: 337 continue 338 # check for __init__.py 339 if not '__init__.py' in fnames: 340 while fnames: 341 fnames.pop() 342 elif directory != src_directory: 343 #src = join(directory, file) 344 dir_package = directory[len(src_directory):].replace(os.sep, '.') 345 modules.append(package + dir_package) 346 for filename in fnames: 347 src = join(directory, filename) 348 if isdir(src): 349 continue 350 if _is_python_file(filename) and filename != '__init__.py': 351 module = package + src[len(src_directory):-3] 352 modules.append(module.replace(os.sep, '.'))
353 modules = [] 354 walk(src_directory, func, modules) 355 return modules 356 357 358
359 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
360 """given a package directory return a list of all available python 361 module's files in the package and its subpackages 362 363 :type src_directory: str 364 :param src_directory: 365 path of the directory corresponding to the package 366 367 :type blacklist: list or tuple 368 :param blacklist: 369 optional list of files or directory to ignore, default to the value of 370 `logilab.common.STD_BLACKLIST` 371 372 :rtype: list 373 :return: 374 the list of all available python module's files in the package and 375 its subpackages 376 """ 377 def func(files, directory, fnames): 378 """walk handler""" 379 # remove files/directories in the black list 380 for norecurs in blacklist: 381 try: 382 fnames.remove(norecurs) 383 except ValueError: 384 continue 385 # check for __init__.py 386 if not '__init__.py' in fnames: 387 while fnames: 388 fnames.pop() 389 for filename in fnames: 390 src = join(directory, filename) 391 if isdir(src): 392 continue 393 if _is_python_file(filename): 394 files.append(src)
395 files = [] 396 walk(src_directory, func, files) 397 return files 398 399
400 -def get_source_file(filename, include_no_ext=False):
401 """given a python module's file name return the matching source file 402 name (the filename will be returned identically if it's a already an 403 absolute path to a python source file...) 404 405 :type filename: str 406 :param filename: python module's file name 407 408 409 :raise NoSourceFile: if no source file exists on the file system 410 411 :rtype: str 412 :return: the absolute path of the source file if it exists 413 """ 414 base, orig_ext = splitext(abspath(filename)) 415 for ext in PY_SOURCE_EXTS: 416 source_path = '%s.%s' % (base, ext) 417 if exists(source_path): 418 return source_path 419 if include_no_ext and not orig_ext and exists(base): 420 return base 421 raise NoSourceFile(filename)
422 423 424
425 -def is_python_source(filename):
426 """ 427 rtype: bool 428 return: True if the filename is a python source file 429 """ 430 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
431 432 433
434 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
435 """try to guess if a module is a standard python module (by default, 436 see `std_path` parameter's description) 437 438 :type modname: str 439 :param modname: name of the module we are interested in 440 441 :type std_path: list(str) or tuple(str) 442 :param std_path: list of path considered has standard 443 444 445 :rtype: bool 446 :return: 447 true if the module: 448 - is located on the path listed in one of the directory in `std_path` 449 - is a built-in module 450 """ 451 modpath = modname.split('.') 452 modname = modpath[0] 453 try: 454 filename = file_from_modpath(modpath) 455 except ImportError: 456 # import failed, i'm probably not so wrong by supposing it's 457 # not standard... 458 return 0 459 # modules which are not living in a file are considered standard 460 # (sys and __builtin__ for instance) 461 if filename is None: 462 return 1 463 filename = abspath(filename) 464 for path in std_path: 465 path = abspath(path) 466 if filename.startswith(path): 467 pfx_len = len(path) 468 if filename[pfx_len+1:pfx_len+14] != 'site-packages': 469 return 1 470 return 0 471 return False
472 473 474
475 -def is_relative(modname, from_file):
476 """return true if the given module name is relative to the given 477 file name 478 479 :type modname: str 480 :param modname: name of the module we are interested in 481 482 :type from_file: str 483 :param from_file: 484 path of the module from which modname has been imported 485 486 :rtype: bool 487 :return: 488 true if the module has been imported relativly to `from_file` 489 """ 490 if not isdir(from_file): 491 from_file = dirname(from_file) 492 if from_file in sys.path: 493 return False 494 try: 495 find_module(modname.split('.')[0], [from_file]) 496 return True 497 except ImportError: 498 return False
499 500 501 # internal only functions ##################################################### 502
503 -def _file_from_modpath(modpath, path=None, context=None):
504 """given a mod path (ie splited module / package name), return the 505 corresponding file 506 507 this function is used internally, see `file_from_modpath`'s 508 documentation for more information 509 """ 510 assert len(modpath) > 0 511 if context is not None: 512 try: 513 mtype, mp_filename = _module_file(modpath, [context]) 514 except ImportError: 515 mtype, mp_filename = _module_file(modpath, path) 516 else: 517 mtype, mp_filename = _module_file(modpath, path) 518 if mtype == PY_COMPILED: 519 try: 520 return get_source_file(mp_filename) 521 except NoSourceFile: 522 return mp_filename 523 elif mtype == C_BUILTIN: 524 # integrated builtin module 525 return None 526 elif mtype == PKG_DIRECTORY: 527 mp_filename = _has_init(mp_filename) 528 return mp_filename
529
530 -def _module_file(modpath, path=None):
531 """get a module type / file path 532 533 :type modpath: list or tuple 534 :param modpath: 535 splitted module's name (i.e name of a module or package splitted 536 on '.'), with leading empty strings for explicit relative import 537 538 :type path: list or None 539 :param path: 540 optional list of path where the module or package should be 541 searched (use sys.path if nothing or None is given) 542 543 544 :rtype: tuple(int, str) 545 :return: the module type flag and the file path for a module 546 """ 547 while modpath: 548 _, mp_filename, mp_desc = find_module(modpath[0], path) 549 modpath.pop(0) 550 mtype = mp_desc[2] 551 if modpath: 552 if mtype != PKG_DIRECTORY: 553 raise ImportError('No module %r' % '.'.join(modpath)) 554 path = [mp_filename] 555 return mtype, mp_filename
556
557 -def _is_python_file(filename):
558 """return true if the given filename should be considered as a python file 559 560 .pyc and .pyo are ignored 561 """ 562 for ext in ('.py', '.so', '.pyd', '.pyw'): 563 if filename.endswith(ext): 564 return True 565 return False
566 567
568 -def _has_init(directory):
569 """if the given directory has a valid __init__ file, return its path, 570 else return None 571 """ 572 mod_or_pack = join(directory, '__init__') 573 for ext in ('.py', '.pyw', '.pyc', '.pyo'): 574 if exists(mod_or_pack + ext): 575 return mod_or_pack + ext 576 return None
577