Package common :: Module modutils
[frames] | no frames]

Source Code for Module common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 29  """ 
 30   
 31   
 32  __docformat__ = "restructuredtext en" 
 33   
 34  import sys 
 35  import os 
 36  from os.path import splitext, join, abspath, isdir, dirname, exists, basename 
 37  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 38  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 39  from distutils.errors import DistutilsPlatformError 
 40   
 41  try: 
 42      import zipimport 
 43  except ImportError: 
 44      zipimport = None 
 45   
 46  ZIPFILE = object() 
 47   
 48  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 49   
 50  # Notes about STD_LIB_DIR 
 51  # Consider arch-specific installation for STD_LIB_DIR definition 
 52  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 53  # 
 54  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 55  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 56  if sys.platform.startswith('win'): 
 57      PY_SOURCE_EXTS = ('py', 'pyw') 
 58      PY_COMPILED_EXTS = ('dll', 'pyd') 
 59  else: 
 60      PY_SOURCE_EXTS = ('py',) 
 61      PY_COMPILED_EXTS = ('so',) 
 62   
 63  try: 
 64      STD_LIB_DIR = get_python_lib(standard_lib=1) 
 65  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to 
 66  # non-valid path, see https://bugs.pypy.org/issue1164 
 67  except DistutilsPlatformError: 
 68      STD_LIB_DIR = '//' 
 69   
 70  EXT_LIB_DIR = get_python_lib() 
 71   
 72  BUILTIN_MODULES = dict(list(zip(sys.builtin_module_names, 
 73                             [1]*len(sys.builtin_module_names)))) 
 74   
 75   
76 -class NoSourceFile(Exception):
77 """exception raised when we are not able to get a python 78 source file for a precompiled file 79 """
80
81 -class LazyObject(object):
82 - def __init__(self, module, obj):
83 self.module = module 84 self.obj = obj 85 self._imported = None
86
87 - def _getobj(self):
88 if self._imported is None: 89 self._imported = getattr(load_module_from_name(self.module), 90 self.obj) 91 return self._imported
92
93 - def __getattribute__(self, attr):
94 try: 95 return super(LazyObject, self).__getattribute__(attr) 96 except AttributeError as ex: 97 return getattr(self._getobj(), attr)
98
99 - def __call__(self, *args, **kwargs):
100 return self._getobj()(*args, **kwargs)
101 102
103 -def load_module_from_name(dotted_name, path=None, use_sys=1):
104 """Load a Python module from its name. 105 106 :type dotted_name: str 107 :param dotted_name: python name of a module or package 108 109 :type path: list or None 110 :param path: 111 optional list of path where the module or package should be 112 searched (use sys.path if nothing or None is given) 113 114 :type use_sys: bool 115 :param use_sys: 116 boolean indicating whether the sys.modules dictionary should be 117 used or not 118 119 120 :raise ImportError: if the module or package is not found 121 122 :rtype: module 123 :return: the loaded module 124 """ 125 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
126 127
128 -def load_module_from_modpath(parts, path=None, use_sys=1):
129 """Load a python module from its splitted name. 130 131 :type parts: list(str) or tuple(str) 132 :param parts: 133 python name of a module or package splitted on '.' 134 135 :type path: list or None 136 :param path: 137 optional list of path where the module or package should be 138 searched (use sys.path if nothing or None is given) 139 140 :type use_sys: bool 141 :param use_sys: 142 boolean indicating whether the sys.modules dictionary should be used or not 143 144 :raise ImportError: if the module or package is not found 145 146 :rtype: module 147 :return: the loaded module 148 """ 149 if use_sys: 150 try: 151 return sys.modules['.'.join(parts)] 152 except KeyError: 153 pass 154 modpath = [] 155 prevmodule = None 156 for part in parts: 157 modpath.append(part) 158 curname = '.'.join(modpath) 159 module = None 160 if len(modpath) != len(parts): 161 # even with use_sys=False, should try to get outer packages from sys.modules 162 module = sys.modules.get(curname) 163 elif use_sys: 164 # because it may have been indirectly loaded through a parent 165 module = sys.modules.get(curname) 166 if module is None: 167 mp_file, mp_filename, mp_desc = find_module(part, path) 168 module = load_module(curname, mp_file, mp_filename, mp_desc) 169 if prevmodule: 170 setattr(prevmodule, part, module) 171 _file = getattr(module, '__file__', '') 172 if not _file and len(modpath) != len(parts): 173 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 174 path = [dirname( _file )] 175 prevmodule = module 176 return module
177 178
179 -def load_module_from_file(filepath, path=None, use_sys=1, extrapath=None):
180 """Load a Python module from it's path. 181 182 :type filepath: str 183 :param filepath: path to the python module or package 184 185 :type path: list or None 186 :param path: 187 optional list of path where the module or package should be 188 searched (use sys.path if nothing or None is given) 189 190 :type use_sys: bool 191 :param use_sys: 192 boolean indicating whether the sys.modules dictionary should be 193 used or not 194 195 196 :raise ImportError: if the module or package is not found 197 198 :rtype: module 199 :return: the loaded module 200 """ 201 modpath = modpath_from_file(filepath, extrapath) 202 return load_module_from_modpath(modpath, path, use_sys)
203 204
205 -def _check_init(path, mod_path):
206 """check there are some __init__.py all along the way""" 207 for part in mod_path: 208 path = join(path, part) 209 if not _has_init(path): 210 return False 211 return True
212 213
214 -def modpath_from_file(filename, extrapath=None):
215 """given a file path return the corresponding splitted module's name 216 (i.e name of a module or package splitted on '.') 217 218 :type filename: str 219 :param filename: file's path for which we want the module's name 220 221 :type extrapath: dict 222 :param extrapath: 223 optional extra search path, with path as key and package name for the path 224 as value. This is usually useful to handle package splitted in multiple 225 directories using __path__ trick. 226 227 228 :raise ImportError: 229 if the corresponding module's name has not been found 230 231 :rtype: list(str) 232 :return: the corresponding splitted module's name 233 """ 234 base = splitext(abspath(filename))[0] 235 if extrapath is not None: 236 for path_ in extrapath: 237 path = abspath(path_) 238 if path and base[:len(path)] == path: 239 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 240 if pkg] 241 if _check_init(path, submodpath[:-1]): 242 return extrapath[path_].split('.') + submodpath 243 for path in sys.path: 244 path = abspath(path) 245 if path and base.startswith(path): 246 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 247 if _check_init(path, modpath[:-1]): 248 return modpath 249 raise ImportError('Unable to find module for %s in %s' % ( 250 filename, ', \n'.join(sys.path)))
251 252 253
254 -def file_from_modpath(modpath, path=None, context_file=None):
255 """given a mod path (i.e. splitted module / package name), return the 256 corresponding file, giving priority to source file over precompiled 257 file if it exists 258 259 :type modpath: list or tuple 260 :param modpath: 261 splitted module's name (i.e name of a module or package splitted 262 on '.') 263 (this means explicit relative imports that start with dots have 264 empty strings in this list!) 265 266 :type path: list or None 267 :param path: 268 optional list of path where the module or package should be 269 searched (use sys.path if nothing or None is given) 270 271 :type context_file: str or None 272 :param context_file: 273 context file to consider, necessary if the identifier has been 274 introduced using a relative import unresolvable in the actual 275 context (i.e. modutils) 276 277 :raise ImportError: if there is no such module in the directory 278 279 :rtype: str or None 280 :return: 281 the path to the module's file or None if it's an integrated 282 builtin module such as 'sys' 283 """ 284 if context_file is not None: 285 context = dirname(context_file) 286 else: 287 context = context_file 288 if modpath[0] == 'xml': 289 # handle _xmlplus 290 try: 291 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 292 except ImportError: 293 return _file_from_modpath(modpath, path, context) 294 elif modpath == ['os', 'path']: 295 # FIXME: currently ignoring search_path... 296 return os.path.__file__ 297 return _file_from_modpath(modpath, path, context)
298 299 300
301 -def get_module_part(dotted_name, context_file=None):
302 """given a dotted name return the module part of the name : 303 304 >>> get_module_part('logilab.common.modutils.get_module_part') 305 'logilab.common.modutils' 306 307 :type dotted_name: str 308 :param dotted_name: full name of the identifier we are interested in 309 310 :type context_file: str or None 311 :param context_file: 312 context file to consider, necessary if the identifier has been 313 introduced using a relative import unresolvable in the actual 314 context (i.e. modutils) 315 316 317 :raise ImportError: if there is no such module in the directory 318 319 :rtype: str or None 320 :return: 321 the module part of the name or None if we have not been able at 322 all to import the given name 323 324 XXX: deprecated, since it doesn't handle package precedence over module 325 (see #10066) 326 """ 327 # os.path trick 328 if dotted_name.startswith('os.path'): 329 return 'os.path' 330 parts = dotted_name.split('.') 331 if context_file is not None: 332 # first check for builtin module which won't be considered latter 333 # in that case (path != None) 334 if parts[0] in BUILTIN_MODULES: 335 if len(parts) > 2: 336 raise ImportError(dotted_name) 337 return parts[0] 338 # don't use += or insert, we want a new list to be created ! 339 path = None 340 starti = 0 341 if parts[0] == '': 342 assert context_file is not None, \ 343 'explicit relative import, but no context_file?' 344 path = [] # prevent resolving the import non-relatively 345 starti = 1 346 while parts[starti] == '': # for all further dots: change context 347 starti += 1 348 context_file = dirname(context_file) 349 for i in range(starti, len(parts)): 350 try: 351 file_from_modpath(parts[starti:i+1], 352 path=path, context_file=context_file) 353 except ImportError: 354 if not i >= max(1, len(parts) - 2): 355 raise 356 return '.'.join(parts[:i]) 357 return dotted_name
358 359
360 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
361 """given a package directory return a list of all available python 362 modules in the package and its subpackages 363 364 :type package: str 365 :param package: the python name for the package 366 367 :type src_directory: str 368 :param src_directory: 369 path of the directory corresponding to the package 370 371 :type blacklist: list or tuple 372 :param blacklist: 373 optional list of files or directory to ignore, default to 374 the value of `logilab.common.STD_BLACKLIST` 375 376 :rtype: list 377 :return: 378 the list of all available python modules in the package and its 379 subpackages 380 """ 381 modules = [] 382 for directory, dirnames, filenames in os.walk(src_directory): 383 _handle_blacklist(blacklist, dirnames, filenames) 384 # check for __init__.py 385 if not '__init__.py' in filenames: 386 dirnames[:] = () 387 continue 388 if directory != src_directory: 389 dir_package = directory[len(src_directory):].replace(os.sep, '.') 390 modules.append(package + dir_package) 391 for filename in filenames: 392 if _is_python_file(filename) and filename != '__init__.py': 393 src = join(directory, filename) 394 module = package + src[len(src_directory):-3] 395 modules.append(module.replace(os.sep, '.')) 396 return modules
397 398 399
400 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
401 """given a package directory return a list of all available python 402 module's files in the package and its subpackages 403 404 :type src_directory: str 405 :param src_directory: 406 path of the directory corresponding to the package 407 408 :type blacklist: list or tuple 409 :param blacklist: 410 optional list of files or directory to ignore, default to the value of 411 `logilab.common.STD_BLACKLIST` 412 413 :rtype: list 414 :return: 415 the list of all available python module's files in the package and 416 its subpackages 417 """ 418 files = [] 419 for directory, dirnames, filenames in os.walk(src_directory): 420 _handle_blacklist(blacklist, dirnames, filenames) 421 # check for __init__.py 422 if not '__init__.py' in filenames: 423 dirnames[:] = () 424 continue 425 for filename in filenames: 426 if _is_python_file(filename): 427 src = join(directory, filename) 428 files.append(src) 429 return files
430 431
432 -def get_source_file(filename, include_no_ext=False):
433 """given a python module's file name return the matching source file 434 name (the filename will be returned identically if it's a already an 435 absolute path to a python source file...) 436 437 :type filename: str 438 :param filename: python module's file name 439 440 441 :raise NoSourceFile: if no source file exists on the file system 442 443 :rtype: str 444 :return: the absolute path of the source file if it exists 445 """ 446 base, orig_ext = splitext(abspath(filename)) 447 for ext in PY_SOURCE_EXTS: 448 source_path = '%s.%s' % (base, ext) 449 if exists(source_path): 450 return source_path 451 if include_no_ext and not orig_ext and exists(base): 452 return base 453 raise NoSourceFile(filename)
454 455
456 -def cleanup_sys_modules(directories):
457 """remove submodules of `directories` from `sys.modules`""" 458 for modname, module in list(sys.modules.items()): 459 modfile = getattr(module, '__file__', None) 460 if modfile: 461 for directory in directories: 462 if modfile.startswith(directory): 463 del sys.modules[modname] 464 break
465 466
467 -def is_python_source(filename):
468 """ 469 rtype: bool 470 return: True if the filename is a python source file 471 """ 472 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
473 474 475
476 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
477 """try to guess if a module is a standard python module (by default, 478 see `std_path` parameter's description) 479 480 :type modname: str 481 :param modname: name of the module we are interested in 482 483 :type std_path: list(str) or tuple(str) 484 :param std_path: list of path considered has standard 485 486 487 :rtype: bool 488 :return: 489 true if the module: 490 - is located on the path listed in one of the directory in `std_path` 491 - is a built-in module 492 """ 493 modname = modname.split('.')[0] 494 try: 495 filename = file_from_modpath([modname]) 496 except ImportError as ex: 497 # import failed, i'm probably not so wrong by supposing it's 498 # not standard... 499 return 0 500 # modules which are not living in a file are considered standard 501 # (sys and __builtin__ for instance) 502 if filename is None: 503 return 1 504 filename = abspath(filename) 505 if filename.startswith(EXT_LIB_DIR): 506 return 0 507 for path in std_path: 508 if filename.startswith(abspath(path)): 509 return 1 510 return False
511 512 513
514 -def is_relative(modname, from_file):
515 """return true if the given module name is relative to the given 516 file name 517 518 :type modname: str 519 :param modname: name of the module we are interested in 520 521 :type from_file: str 522 :param from_file: 523 path of the module from which modname has been imported 524 525 :rtype: bool 526 :return: 527 true if the module has been imported relatively to `from_file` 528 """ 529 if not isdir(from_file): 530 from_file = dirname(from_file) 531 if from_file in sys.path: 532 return False 533 try: 534 find_module(modname.split('.')[0], [from_file]) 535 return True 536 except ImportError: 537 return False
538 539 540 # internal only functions ##################################################### 541
542 -def _file_from_modpath(modpath, path=None, context=None):
543 """given a mod path (i.e. splitted module / package name), return the 544 corresponding file 545 546 this function is used internally, see `file_from_modpath`'s 547 documentation for more information 548 """ 549 assert len(modpath) > 0 550 if context is not None: 551 try: 552 mtype, mp_filename = _module_file(modpath, [context]) 553 except ImportError: 554 mtype, mp_filename = _module_file(modpath, path) 555 else: 556 mtype, mp_filename = _module_file(modpath, path) 557 if mtype == PY_COMPILED: 558 try: 559 return get_source_file(mp_filename) 560 except NoSourceFile: 561 return mp_filename 562 elif mtype == C_BUILTIN: 563 # integrated builtin module 564 return None 565 elif mtype == PKG_DIRECTORY: 566 mp_filename = _has_init(mp_filename) 567 return mp_filename
568
569 -def _search_zip(modpath, pic):
570 for filepath, importer in list(pic.items()): 571 if importer is not None: 572 if importer.find_module(modpath[0]): 573 if not importer.find_module('/'.join(modpath)): 574 raise ImportError('No module named %s in %s/%s' % ( 575 '.'.join(modpath[1:]), filepath, modpath)) 576 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 577 raise ImportError('No module named %s' % '.'.join(modpath))
578 579 try: 580 import pkg_resources 581 except ImportError: 582 pkg_resources = None 583
584 -def _module_file(modpath, path=None):
585 """get a module type / file path 586 587 :type modpath: list or tuple 588 :param modpath: 589 splitted module's name (i.e name of a module or package splitted 590 on '.'), with leading empty strings for explicit relative import 591 592 :type path: list or None 593 :param path: 594 optional list of path where the module or package should be 595 searched (use sys.path if nothing or None is given) 596 597 598 :rtype: tuple(int, str) 599 :return: the module type flag and the file path for a module 600 """ 601 # egg support compat 602 try: 603 pic = sys.path_importer_cache 604 _path = (path is None and sys.path or path) 605 for __path in _path: 606 if not __path in pic: 607 try: 608 pic[__path] = zipimport.zipimporter(__path) 609 except zipimport.ZipImportError: 610 pic[__path] = None 611 checkeggs = True 612 except AttributeError: 613 checkeggs = False 614 # pkg_resources support (aka setuptools namespace packages) 615 if pkg_resources is not None and modpath[0] in pkg_resources._namespace_packages and len(modpath) > 1: 616 # setuptools has added into sys.modules a module object with proper 617 # __path__, get back information from there 618 module = sys.modules[modpath.pop(0)] 619 path = module.__path__ 620 imported = [] 621 while modpath: 622 modname = modpath[0] 623 # take care to changes in find_module implementation wrt builtin modules 624 # 625 # Python 2.6.6 (r266:84292, Sep 11 2012, 08:34:23) 626 # >>> imp.find_module('posix') 627 # (None, 'posix', ('', '', 6)) 628 # 629 # Python 3.3.1 (default, Apr 26 2013, 12:08:46) 630 # >>> imp.find_module('posix') 631 # (None, None, ('', '', 6)) 632 try: 633 _, mp_filename, mp_desc = find_module(modname, path) 634 except ImportError: 635 if checkeggs: 636 return _search_zip(modpath, pic)[:2] 637 raise 638 else: 639 if checkeggs and mp_filename: 640 fullabspath = [abspath(x) for x in _path] 641 try: 642 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 643 emtype, emp_filename, zippath = _search_zip(modpath, pic) 644 if pathindex > _path.index(zippath): 645 # an egg takes priority 646 return emtype, emp_filename 647 except ValueError: 648 # XXX not in _path 649 pass 650 except ImportError: 651 pass 652 checkeggs = False 653 imported.append(modpath.pop(0)) 654 mtype = mp_desc[2] 655 if modpath: 656 if mtype != PKG_DIRECTORY: 657 raise ImportError('No module %s in %s' % ('.'.join(modpath), 658 '.'.join(imported))) 659 # XXX guess if package is using pkgutil.extend_path by looking for 660 # those keywords in the first four Kbytes 661 try: 662 with open(join(mp_filename, '__init__.py')) as stream: 663 data = stream.read(4096) 664 except IOError: 665 path = [mp_filename] 666 else: 667 if 'pkgutil' in data and 'extend_path' in data: 668 # extend_path is called, search sys.path for module/packages 669 # of this name see pkgutil.extend_path documentation 670 path = [join(p, *imported) for p in sys.path 671 if isdir(join(p, *imported))] 672 else: 673 path = [mp_filename] 674 return mtype, mp_filename
675
676 -def _is_python_file(filename):
677 """return true if the given filename should be considered as a python file 678 679 .pyc and .pyo are ignored 680 """ 681 for ext in ('.py', '.so', '.pyd', '.pyw'): 682 if filename.endswith(ext): 683 return True 684 return False
685 686
687 -def _has_init(directory):
688 """if the given directory has a valid __init__ file, return its path, 689 else return None 690 """ 691 mod_or_pack = join(directory, '__init__') 692 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 693 if exists(mod_or_pack + '.' + ext): 694 return mod_or_pack + '.' + ext 695 return None
696