Package logilab-common-0 ::
Package 36 ::
Package 1 ::
Module modutils
|
|
1
2 """Python modules manipulation utility functions.
3
4 :author: Logilab
5 :copyright: 2000-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
6 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
7 :license: General Public License version 2 - http://www.gnu.org/licenses
8
9 :type PY_SOURCE_EXTS: tuple(str)
10 :var PY_SOURCE_EXTS: list of possible python source file extension
11
12 :type STD_LIB_DIR: str
13 :var STD_LIB_DIR: directory where standard modules are located
14
15 :type BUILTIN_MODULES: dict
16 :var BUILTIN_MODULES: dictionary with builtin module names has key
17 """
18 __docformat__ = "restructuredtext en"
19
20 import sys
21 import os
22 from os.path import walk, splitext, join, abspath, isdir, dirname, exists
23 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
24
25 from logilab.common import STD_BLACKLIST
26
27 if sys.platform.startswith('win'):
28 PY_SOURCE_EXTS = ('py', 'pyw')
29 PY_COMPILED_EXTS = ('dll', 'pyd')
30 STD_LIB_DIR = join(sys.prefix, 'lib')
31 else:
32 PY_SOURCE_EXTS = ('py',)
33 PY_COMPILED_EXTS = ('so',)
34 STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3])
35
36 BUILTIN_MODULES = dict(zip(sys.builtin_module_names,
37 [1]*len(sys.builtin_module_names)))
38
39
41 """exception raised when we are not able to get a python
42 source file for a precompiled file
43 """
44
47 self.module = module
48 self.obj = obj
49 self._imported = None
50
52 if self._imported is None:
53 self._imported = getattr(load_module_from_name(self.module),
54 self.obj)
55 return self._imported
56
58 try:
59 return super(LazyObject, self).__getattribute__(attr)
60 except AttributeError, ex:
61 return getattr(self.__getobj(), attr)
62
64 return self.__getobj()(*args, **kwargs)
65
66
68 """Load a Python module from it's name.
69
70 :type dotted_name: str
71 :param dotted_name: python name of a module or package
72
73 :type path: list or None
74 :param path:
75 optional list of path where the module or package should be
76 searched (use sys.path if nothing or None is given)
77
78 :type use_sys: bool
79 :param use_sys:
80 boolean indicating whether the sys.modules dictionary should be
81 used or not
82
83
84 :raise ImportError: if the module or package is not found
85
86 :rtype: module
87 :return: the loaded module
88 """
89 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
90
91
93 """Load a python module from it's splitted name.
94
95 :type parts: list(str) or tuple(str)
96 :param parts:
97 python name of a module or package splitted on '.'
98
99 :type path: list or None
100 :param path:
101 optional list of path where the module or package should be
102 searched (use sys.path if nothing or None is given)
103
104 :type use_sys: bool
105 :param use_sys:
106 boolean indicating whether the sys.modules dictionary should be used or not
107
108 :raise ImportError: if the module or package is not found
109
110 :rtype: module
111 :return: the loaded module
112 """
113 if use_sys:
114 try:
115 return sys.modules['.'.join(parts)]
116 except KeyError:
117 pass
118 modpath = []
119 prevmodule = None
120 for part in parts:
121 modpath.append(part)
122 curname = ".".join(modpath)
123 module = None
124 if len(modpath) != len(parts):
125
126 module = sys.modules.get(curname)
127 if module is None:
128 mp_file, mp_filename, mp_desc = find_module(part, path)
129 module = load_module(curname, mp_file, mp_filename, mp_desc)
130 if prevmodule:
131 setattr(prevmodule, part, module)
132 _file = getattr(module, "__file__", "")
133 if not _file and len(modpath) != len(parts):
134 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) )
135 path = [dirname( _file )]
136 prevmodule = module
137 return module
138
139
141 """Load a Python module from it's path.
142
143 :type filepath: str
144 :param filepath: path to the python module or package
145
146 :type path: list or None
147 :param path:
148 optional list of path where the module or package should be
149 searched (use sys.path if nothing or None is given)
150
151 :type use_sys: bool
152 :param use_sys:
153 boolean indicating whether the sys.modules dictionary should be
154 used or not
155
156
157 :raise ImportError: if the module or package is not found
158
159 :rtype: module
160 :return: the loaded module
161 """
162 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
163
164
166 """given a file path return the corresponding splitted module's name
167 (i.e name of a module or package splitted on '.')
168
169 :type filename: str
170 :param filename: file's path for which we want the module's name
171
172
173 :raise ImportError:
174 if the corresponding module's name has not been found
175
176 :rtype: list(str)
177 :return: the corresponding splitted module's name
178 """
179 base = splitext(abspath(filename))[0]
180 for path in sys.path:
181 path = abspath(path)
182 if path and base[:len(path)] == path:
183 if filename.find('site-packages') != -1 and \
184 path.find('site-packages') == -1:
185 continue
186 mod_path = [module for module in base[len(path):].split(os.sep)
187 if module]
188 for part in mod_path[:-1]:
189 path = join(path, part)
190 if not _has_init(path):
191 break
192 else:
193 break
194 else:
195 raise ImportError('Unable to find module for %s in %s' % (
196 filename, ', \n'.join(sys.path)))
197 return mod_path
198
199
200
202 """given a mod path (ie splited module / package name), return the
203 corresponding file, giving priority to source file over precompiled
204 file if it exists
205
206 :type modpath: list or tuple
207 :param modpath:
208 splitted module's name (i.e name of a module or package splitted
209 on '.')
210 (this means explicit relative imports that start with dots have
211 empty strings in this list!)
212
213 :type path: list or None
214 :param path:
215 optional list of path where the module or package should be
216 searched (use sys.path if nothing or None is given)
217
218 :type context_file: str or None
219 :param context_file:
220 context file to consider, necessary if the identifier has been
221 introduced using a relative import unresolvable in the actual
222 context (i.e. modutils)
223
224 :raise ImportError: if there is no such module in the directory
225
226 :rtype: str or None
227 :return:
228 the path to the module's file or None if it's an integrated
229 builtin module such as 'sys'
230 """
231 if context_file is not None:
232 context = dirname(context_file)
233 else:
234 context = context_file
235 if modpath[0] == 'xml':
236
237 try:
238 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
239 except ImportError:
240 return _file_from_modpath(modpath, path, context)
241 elif modpath == ['os', 'path']:
242
243 return os.path.__file__
244 return _file_from_modpath(modpath, path, context)
245
246
247
249 """given a dotted name return the module part of the name :
250
251 >>> get_module_part('logilab.common.modutils.get_module_part')
252 'logilab.common.modutils'
253
254
255 :type dotted_name: str
256 :param dotted_name: full name of the identifier we are interested in
257
258 :type context_file: str or None
259 :param context_file:
260 context file to consider, necessary if the identifier has been
261 introduced using a relative import unresolvable in the actual
262 context (i.e. modutils)
263
264
265 :raise ImportError: if there is no such module in the directory
266
267 :rtype: str or None
268 :return:
269 the module part of the name or None if we have not been able at
270 all to import the given name
271
272 XXX: deprecated, since it doesn't handle package precedence over module
273 (see #10066)
274 """
275
276 if dotted_name.startswith('os.path'):
277 return 'os.path'
278 parts = dotted_name.split('.')
279 if context_file is not None:
280
281
282 if parts[0] in BUILTIN_MODULES:
283 if len(parts) > 2:
284 raise ImportError(dotted_name)
285 return parts[0]
286
287 path = None
288 starti = 0
289 if parts[0] == '':
290 assert context_file is not None, \
291 'explicit relative import, but no context_file?'
292 path = []
293 starti = 1
294 while parts[starti] == '':
295 starti += 1
296 context_file = dirname(context_file)
297 for i in range(starti, len(parts)):
298 try:
299 file_from_modpath(parts[starti:i+1],
300 path=path, context_file=context_file)
301 except ImportError:
302 if not i >= max(1, len(parts) - 2):
303 raise
304 return '.'.join(parts[:i])
305 return dotted_name
306
307
308
310 """given a package directory return a list of all available python
311 modules in the package and its subpackages
312
313 :type package: str
314 :param package: the python name for the package
315
316 :type src_directory: str
317 :param src_directory:
318 path of the directory corresponding to the package
319
320 :type blacklist: list or tuple
321 :param blacklist:
322 optional list of files or directory to ignore, default to
323 the value of `logilab.common.STD_BLACKLIST`
324
325 :rtype: list
326 :return:
327 the list of all available python modules in the package and its
328 subpackages
329 """
330 def func(modules, directory, fnames):
331 """walk handler"""
332
333 for norecurs in blacklist:
334 try:
335 fnames.remove(norecurs)
336 except ValueError:
337 continue
338
339 if not '__init__.py' in fnames:
340 while fnames:
341 fnames.pop()
342 elif directory != src_directory:
343
344 dir_package = directory[len(src_directory):].replace(os.sep, '.')
345 modules.append(package + dir_package)
346 for filename in fnames:
347 src = join(directory, filename)
348 if isdir(src):
349 continue
350 if _is_python_file(filename) and filename != '__init__.py':
351 module = package + src[len(src_directory):-3]
352 modules.append(module.replace(os.sep, '.'))
353 modules = []
354 walk(src_directory, func, modules)
355 return modules
356
357
358
360 """given a package directory return a list of all available python
361 module's files in the package and its subpackages
362
363 :type src_directory: str
364 :param src_directory:
365 path of the directory corresponding to the package
366
367 :type blacklist: list or tuple
368 :param blacklist:
369 optional list of files or directory to ignore, default to the value of
370 `logilab.common.STD_BLACKLIST`
371
372 :rtype: list
373 :return:
374 the list of all available python module's files in the package and
375 its subpackages
376 """
377 def func(files, directory, fnames):
378 """walk handler"""
379
380 for norecurs in blacklist:
381 try:
382 fnames.remove(norecurs)
383 except ValueError:
384 continue
385
386 if not '__init__.py' in fnames:
387 while fnames:
388 fnames.pop()
389 for filename in fnames:
390 src = join(directory, filename)
391 if isdir(src):
392 continue
393 if _is_python_file(filename):
394 files.append(src)
395 files = []
396 walk(src_directory, func, files)
397 return files
398
399
401 """given a python module's file name return the matching source file
402 name (the filename will be returned identically if it's a already an
403 absolute path to a python source file...)
404
405 :type filename: str
406 :param filename: python module's file name
407
408
409 :raise NoSourceFile: if no source file exists on the file system
410
411 :rtype: str
412 :return: the absolute path of the source file if it exists
413 """
414 base, orig_ext = splitext(abspath(filename))
415 for ext in PY_SOURCE_EXTS:
416 source_path = '%s.%s' % (base, ext)
417 if exists(source_path):
418 return source_path
419 if include_no_ext and not orig_ext and exists(base):
420 return base
421 raise NoSourceFile(filename)
422
423
424
426 """
427 rtype: bool
428 return: True if the filename is a python source file
429 """
430 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
431
432
433
435 """try to guess if a module is a standard python module (by default,
436 see `std_path` parameter's description)
437
438 :type modname: str
439 :param modname: name of the module we are interested in
440
441 :type std_path: list(str) or tuple(str)
442 :param std_path: list of path considered has standard
443
444
445 :rtype: bool
446 :return:
447 true if the module:
448 - is located on the path listed in one of the directory in `std_path`
449 - is a built-in module
450 """
451 modpath = modname.split('.')
452 modname = modpath[0]
453 try:
454 filename = file_from_modpath(modpath)
455 except ImportError:
456
457
458 return 0
459
460
461 if filename is None:
462 return 1
463 filename = abspath(filename)
464 for path in std_path:
465 path = abspath(path)
466 if filename.startswith(path):
467 pfx_len = len(path)
468 if filename[pfx_len+1:pfx_len+14] != 'site-packages':
469 return 1
470 return 0
471 return False
472
473
474
476 """return true if the given module name is relative to the given
477 file name
478
479 :type modname: str
480 :param modname: name of the module we are interested in
481
482 :type from_file: str
483 :param from_file:
484 path of the module from which modname has been imported
485
486 :rtype: bool
487 :return:
488 true if the module has been imported relativly to `from_file`
489 """
490 if not isdir(from_file):
491 from_file = dirname(from_file)
492 if from_file in sys.path:
493 return False
494 try:
495 find_module(modname.split('.')[0], [from_file])
496 return True
497 except ImportError:
498 return False
499
500
501
502
504 """given a mod path (ie splited module / package name), return the
505 corresponding file
506
507 this function is used internally, see `file_from_modpath`'s
508 documentation for more information
509 """
510 assert len(modpath) > 0
511 if context is not None:
512 try:
513 mtype, mp_filename = _module_file(modpath, [context])
514 except ImportError:
515 mtype, mp_filename = _module_file(modpath, path)
516 else:
517 mtype, mp_filename = _module_file(modpath, path)
518 if mtype == PY_COMPILED:
519 try:
520 return get_source_file(mp_filename)
521 except NoSourceFile:
522 return mp_filename
523 elif mtype == C_BUILTIN:
524
525 return None
526 elif mtype == PKG_DIRECTORY:
527 mp_filename = _has_init(mp_filename)
528 return mp_filename
529
531 """get a module type / file path
532
533 :type modpath: list or tuple
534 :param modpath:
535 splitted module's name (i.e name of a module or package splitted
536 on '.'), with leading empty strings for explicit relative import
537
538 :type path: list or None
539 :param path:
540 optional list of path where the module or package should be
541 searched (use sys.path if nothing or None is given)
542
543
544 :rtype: tuple(int, str)
545 :return: the module type flag and the file path for a module
546 """
547 while modpath:
548 _, mp_filename, mp_desc = find_module(modpath[0], path)
549 modpath.pop(0)
550 mtype = mp_desc[2]
551 if modpath:
552 if mtype != PKG_DIRECTORY:
553 raise ImportError('No module %r' % '.'.join(modpath))
554 path = [mp_filename]
555 return mtype, mp_filename
556
558 """return true if the given filename should be considered as a python file
559
560 .pyc and .pyo are ignored
561 """
562 for ext in ('.py', '.so', '.pyd', '.pyw'):
563 if filename.endswith(ext):
564 return True
565 return False
566
567
569 """if the given directory has a valid __init__ file, return its path,
570 else return None
571 """
572 mod_or_pack = join(directory, '__init__')
573 for ext in ('.py', '.pyw', '.pyc', '.pyo'):
574 if exists(mod_or_pack + ext):
575 return mod_or_pack + ext
576 return None
577