Package logilab-common-0 ::
Package 39 ::
Package 0 ::
Module modutils
|
|
1
2 """Python modules manipulation utility functions.
3
4 :author: Logilab
5 :copyright: 2000-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
6 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
7 :license: General Public License version 2 - http://www.gnu.org/licenses
8
9 :type PY_SOURCE_EXTS: tuple(str)
10 :var PY_SOURCE_EXTS: list of possible python source file extension
11
12 :type STD_LIB_DIR: str
13 :var STD_LIB_DIR: directory where standard modules are located
14
15 :type BUILTIN_MODULES: dict
16 :var BUILTIN_MODULES: dictionary with builtin module names has key
17 """
18 __docformat__ = "restructuredtext en"
19
20 import sys
21 import os
22 from os.path import walk, splitext, join, abspath, isdir, dirname, exists
23 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
24
25 try:
26 import zipimport
27 except ImportError:
28 zipimport = None
29
30 ZIPFILE = object()
31
32 from logilab.common import STD_BLACKLIST
33
34 if sys.platform.startswith('win'):
35 PY_SOURCE_EXTS = ('py', 'pyw')
36 PY_COMPILED_EXTS = ('dll', 'pyd')
37 STD_LIB_DIR = join(sys.prefix, 'lib')
38 else:
39 PY_SOURCE_EXTS = ('py',)
40 PY_COMPILED_EXTS = ('so',)
41 STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3])
42
43 BUILTIN_MODULES = dict(zip(sys.builtin_module_names,
44 [1]*len(sys.builtin_module_names)))
45
46
48 """exception raised when we are not able to get a python
49 source file for a precompiled file
50 """
51
54 self.module = module
55 self.obj = obj
56 self._imported = None
57
59 if self._imported is None:
60 self._imported = getattr(load_module_from_name(self.module),
61 self.obj)
62 return self._imported
63
65 try:
66 return super(LazyObject, self).__getattribute__(attr)
67 except AttributeError, ex:
68 return getattr(self.__getobj(), attr)
69
71 return self.__getobj()(*args, **kwargs)
72
73
75 """Load a Python module from it's name.
76
77 :type dotted_name: str
78 :param dotted_name: python name of a module or package
79
80 :type path: list or None
81 :param path:
82 optional list of path where the module or package should be
83 searched (use sys.path if nothing or None is given)
84
85 :type use_sys: bool
86 :param use_sys:
87 boolean indicating whether the sys.modules dictionary should be
88 used or not
89
90
91 :raise ImportError: if the module or package is not found
92
93 :rtype: module
94 :return: the loaded module
95 """
96 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
97
98
100 """Load a python module from it's splitted name.
101
102 :type parts: list(str) or tuple(str)
103 :param parts:
104 python name of a module or package splitted on '.'
105
106 :type path: list or None
107 :param path:
108 optional list of path where the module or package should be
109 searched (use sys.path if nothing or None is given)
110
111 :type use_sys: bool
112 :param use_sys:
113 boolean indicating whether the sys.modules dictionary should be used or not
114
115 :raise ImportError: if the module or package is not found
116
117 :rtype: module
118 :return: the loaded module
119 """
120 if use_sys:
121 try:
122 return sys.modules['.'.join(parts)]
123 except KeyError:
124 pass
125 modpath = []
126 prevmodule = None
127 for part in parts:
128 modpath.append(part)
129 curname = ".".join(modpath)
130 module = None
131 if len(modpath) != len(parts):
132
133 module = sys.modules.get(curname)
134 if module is None:
135 mp_file, mp_filename, mp_desc = find_module(part, path)
136 module = load_module(curname, mp_file, mp_filename, mp_desc)
137 if prevmodule:
138 setattr(prevmodule, part, module)
139 _file = getattr(module, "__file__", "")
140 if not _file and len(modpath) != len(parts):
141 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) )
142 path = [dirname( _file )]
143 prevmodule = module
144 return module
145
146
148 """Load a Python module from it's path.
149
150 :type filepath: str
151 :param filepath: path to the python module or package
152
153 :type path: list or None
154 :param path:
155 optional list of path where the module or package should be
156 searched (use sys.path if nothing or None is given)
157
158 :type use_sys: bool
159 :param use_sys:
160 boolean indicating whether the sys.modules dictionary should be
161 used or not
162
163
164 :raise ImportError: if the module or package is not found
165
166 :rtype: module
167 :return: the loaded module
168 """
169 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
170
171
173 """given a file path return the corresponding splitted module's name
174 (i.e name of a module or package splitted on '.')
175
176 :type filename: str
177 :param filename: file's path for which we want the module's name
178
179
180 :raise ImportError:
181 if the corresponding module's name has not been found
182
183 :rtype: list(str)
184 :return: the corresponding splitted module's name
185 """
186 base = splitext(abspath(filename))[0]
187 for path in sys.path:
188 path = abspath(path)
189 if path and base[:len(path)] == path:
190 if filename.find('site-packages') != -1 and \
191 path.find('site-packages') == -1:
192 continue
193 mod_path = [module for module in base[len(path):].split(os.sep)
194 if module]
195 for part in mod_path[:-1]:
196 path = join(path, part)
197 if not _has_init(path):
198 break
199 else:
200 break
201 else:
202 raise ImportError('Unable to find module for %s in %s' % (
203 filename, ', \n'.join(sys.path)))
204 return mod_path
205
206
207
209 """given a mod path (ie splited module / package name), return the
210 corresponding file, giving priority to source file over precompiled
211 file if it exists
212
213 :type modpath: list or tuple
214 :param modpath:
215 splitted module's name (i.e name of a module or package splitted
216 on '.')
217 (this means explicit relative imports that start with dots have
218 empty strings in this list!)
219
220 :type path: list or None
221 :param path:
222 optional list of path where the module or package should be
223 searched (use sys.path if nothing or None is given)
224
225 :type context_file: str or None
226 :param context_file:
227 context file to consider, necessary if the identifier has been
228 introduced using a relative import unresolvable in the actual
229 context (i.e. modutils)
230
231 :raise ImportError: if there is no such module in the directory
232
233 :rtype: str or None
234 :return:
235 the path to the module's file or None if it's an integrated
236 builtin module such as 'sys'
237 """
238 if context_file is not None:
239 context = dirname(context_file)
240 else:
241 context = context_file
242 if modpath[0] == 'xml':
243
244 try:
245 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
246 except ImportError:
247 return _file_from_modpath(modpath, path, context)
248 elif modpath == ['os', 'path']:
249
250 return os.path.__file__
251 return _file_from_modpath(modpath, path, context)
252
253
254
256 """given a dotted name return the module part of the name :
257
258 >>> get_module_part('logilab.common.modutils.get_module_part')
259 'logilab.common.modutils'
260
261
262 :type dotted_name: str
263 :param dotted_name: full name of the identifier we are interested in
264
265 :type context_file: str or None
266 :param context_file:
267 context file to consider, necessary if the identifier has been
268 introduced using a relative import unresolvable in the actual
269 context (i.e. modutils)
270
271
272 :raise ImportError: if there is no such module in the directory
273
274 :rtype: str or None
275 :return:
276 the module part of the name or None if we have not been able at
277 all to import the given name
278
279 XXX: deprecated, since it doesn't handle package precedence over module
280 (see #10066)
281 """
282
283 if dotted_name.startswith('os.path'):
284 return 'os.path'
285 parts = dotted_name.split('.')
286 if context_file is not None:
287
288
289 if parts[0] in BUILTIN_MODULES:
290 if len(parts) > 2:
291 raise ImportError(dotted_name)
292 return parts[0]
293
294 path = None
295 starti = 0
296 if parts[0] == '':
297 assert context_file is not None, \
298 'explicit relative import, but no context_file?'
299 path = []
300 starti = 1
301 while parts[starti] == '':
302 starti += 1
303 context_file = dirname(context_file)
304 for i in range(starti, len(parts)):
305 try:
306 file_from_modpath(parts[starti:i+1],
307 path=path, context_file=context_file)
308 except ImportError:
309 if not i >= max(1, len(parts) - 2):
310 raise
311 return '.'.join(parts[:i])
312 return dotted_name
313
314
315
317 """given a package directory return a list of all available python
318 modules in the package and its subpackages
319
320 :type package: str
321 :param package: the python name for the package
322
323 :type src_directory: str
324 :param src_directory:
325 path of the directory corresponding to the package
326
327 :type blacklist: list or tuple
328 :param blacklist:
329 optional list of files or directory to ignore, default to
330 the value of `logilab.common.STD_BLACKLIST`
331
332 :rtype: list
333 :return:
334 the list of all available python modules in the package and its
335 subpackages
336 """
337 def func(modules, directory, fnames):
338 """walk handler"""
339
340 for norecurs in blacklist:
341 try:
342 fnames.remove(norecurs)
343 except ValueError:
344 continue
345
346 if not '__init__.py' in fnames:
347 while fnames:
348 fnames.pop()
349 elif directory != src_directory:
350
351 dir_package = directory[len(src_directory):].replace(os.sep, '.')
352 modules.append(package + dir_package)
353 for filename in fnames:
354 src = join(directory, filename)
355 if isdir(src):
356 continue
357 if _is_python_file(filename) and filename != '__init__.py':
358 module = package + src[len(src_directory):-3]
359 modules.append(module.replace(os.sep, '.'))
360 modules = []
361 walk(src_directory, func, modules)
362 return modules
363
364
365
367 """given a package directory return a list of all available python
368 module's files in the package and its subpackages
369
370 :type src_directory: str
371 :param src_directory:
372 path of the directory corresponding to the package
373
374 :type blacklist: list or tuple
375 :param blacklist:
376 optional list of files or directory to ignore, default to the value of
377 `logilab.common.STD_BLACKLIST`
378
379 :rtype: list
380 :return:
381 the list of all available python module's files in the package and
382 its subpackages
383 """
384 def func(files, directory, fnames):
385 """walk handler"""
386
387 for norecurs in blacklist:
388 try:
389 fnames.remove(norecurs)
390 except ValueError:
391 continue
392
393 if not '__init__.py' in fnames:
394 while fnames:
395 fnames.pop()
396 for filename in fnames:
397 src = join(directory, filename)
398 if isdir(src):
399 continue
400 if _is_python_file(filename):
401 files.append(src)
402 files = []
403 walk(src_directory, func, files)
404 return files
405
406
408 """given a python module's file name return the matching source file
409 name (the filename will be returned identically if it's a already an
410 absolute path to a python source file...)
411
412 :type filename: str
413 :param filename: python module's file name
414
415
416 :raise NoSourceFile: if no source file exists on the file system
417
418 :rtype: str
419 :return: the absolute path of the source file if it exists
420 """
421 base, orig_ext = splitext(abspath(filename))
422 for ext in PY_SOURCE_EXTS:
423 source_path = '%s.%s' % (base, ext)
424 if exists(source_path):
425 return source_path
426 if include_no_ext and not orig_ext and exists(base):
427 return base
428 raise NoSourceFile(filename)
429
430
431
433 """
434 rtype: bool
435 return: True if the filename is a python source file
436 """
437 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
438
439
440
442 """try to guess if a module is a standard python module (by default,
443 see `std_path` parameter's description)
444
445 :type modname: str
446 :param modname: name of the module we are interested in
447
448 :type std_path: list(str) or tuple(str)
449 :param std_path: list of path considered has standard
450
451
452 :rtype: bool
453 :return:
454 true if the module:
455 - is located on the path listed in one of the directory in `std_path`
456 - is a built-in module
457 """
458 modpath = modname.split('.')
459 modname = modpath[0]
460 try:
461 filename = file_from_modpath(modpath)
462 except ImportError:
463
464
465 return 0
466
467
468 if filename is None:
469 return 1
470 filename = abspath(filename)
471 for path in std_path:
472 path = abspath(path)
473 if filename.startswith(path):
474 pfx_len = len(path)
475 if filename[pfx_len+1:pfx_len+14] != 'site-packages':
476 return 1
477 return 0
478 return False
479
480
481
483 """return true if the given module name is relative to the given
484 file name
485
486 :type modname: str
487 :param modname: name of the module we are interested in
488
489 :type from_file: str
490 :param from_file:
491 path of the module from which modname has been imported
492
493 :rtype: bool
494 :return:
495 true if the module has been imported relativly to `from_file`
496 """
497 if not isdir(from_file):
498 from_file = dirname(from_file)
499 if from_file in sys.path:
500 return False
501 try:
502 find_module(modname.split('.')[0], [from_file])
503 return True
504 except ImportError:
505 return False
506
507
508
509
511 """given a mod path (ie splited module / package name), return the
512 corresponding file
513
514 this function is used internally, see `file_from_modpath`'s
515 documentation for more information
516 """
517 assert len(modpath) > 0
518 if context is not None:
519 try:
520 mtype, mp_filename = _module_file(modpath, [context])
521 except ImportError:
522 mtype, mp_filename = _module_file(modpath, path)
523 else:
524 mtype, mp_filename = _module_file(modpath, path)
525 if mtype == PY_COMPILED:
526 try:
527 return get_source_file(mp_filename)
528 except NoSourceFile:
529 return mp_filename
530 elif mtype == C_BUILTIN:
531
532 return None
533 elif mtype == PKG_DIRECTORY:
534 mp_filename = _has_init(mp_filename)
535 return mp_filename
536
538 for filepath, importer in pic.items():
539 if importer is not None:
540 if importer.find_module(modpath[0]):
541 if not importer.find_module('/'.join(modpath)):
542 raise ImportError('No module named %s in %s/%s' % (
543 '.'.join(modpath[1:]), file, modpath))
544 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
545 raise ImportError('No module named %s' % '.'.join(modpath))
546
548 """get a module type / file path
549
550 :type modpath: list or tuple
551 :param modpath:
552 splitted module's name (i.e name of a module or package splitted
553 on '.'), with leading empty strings for explicit relative import
554
555 :type path: list or None
556 :param path:
557 optional list of path where the module or package should be
558 searched (use sys.path if nothing or None is given)
559
560
561 :rtype: tuple(int, str)
562 :return: the module type flag and the file path for a module
563 """
564
565 try:
566 pic = sys.path_importer_cache
567 _path = (path is None and sys.path or path)
568 for __path in _path:
569 if not __path in pic:
570 try:
571 pic[__path] = zipimport.zipimporter(__path)
572 except zipimport.ZipImportError:
573 pic[__path] = None
574 checkeggs = True
575 except AttributeError:
576 checkeggs = False
577 imported = []
578 while modpath:
579 try:
580 _, mp_filename, mp_desc = find_module(modpath[0], path)
581 except ImportError:
582 if checkeggs:
583 return _search_zip(modpath, pic)[:2]
584 raise
585 else:
586 if checkeggs:
587 fullabspath = [abspath(x) for x in _path]
588 try:
589 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
590 emtype, emp_filename, zippath = _search_zip(modpath, pic)
591 if pathindex > _path.index(zippath):
592
593 return emtype, emp_filename
594 except ValueError:
595
596 pass
597 except ImportError:
598 pass
599 checkeggs = False
600 imported.append(modpath.pop(0))
601 mtype = mp_desc[2]
602 if modpath:
603 if mtype != PKG_DIRECTORY:
604 raise ImportError('No module %s in %s' % ('.'.join(modpath),
605 '.'.join(imported)))
606 path = [mp_filename]
607 return mtype, mp_filename
608
610 """return true if the given filename should be considered as a python file
611
612 .pyc and .pyo are ignored
613 """
614 for ext in ('.py', '.so', '.pyd', '.pyw'):
615 if filename.endswith(ext):
616 return True
617 return False
618
619
621 """if the given directory has a valid __init__ file, return its path,
622 else return None
623 """
624 mod_or_pack = join(directory, '__init__')
625 for ext in PY_SOURCE_EXTS + ('.pyc', '.pyo'):
626 if exists(mod_or_pack + '.' + ext):
627 return mod_or_pack + ext
628 return None
629