From 9d4f01972775eb6c601f5a7f55715002df2fecb1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 27 Aug 2012 20:48:15 -0400 Subject: added better common walk_packages implementation for python 2.4 and 2.5 Conflicts: src/lib/Bcfg2/Client/Tools/__init__.py --- src/lib/Bcfg2/Client/Tools/POSIX/__init__.py | 26 ++------- src/lib/Bcfg2/Client/Tools/__init__.py | 20 +------ src/lib/Bcfg2/Compat.py | 81 ++++++++++++++++++++++++++ src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 27 +++------ src/lib/Bcfg2/Server/SchemaUpdater/__init__.py | 22 +------ 5 files changed, 99 insertions(+), 77 deletions(-) (limited to 'src/lib/Bcfg2') diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py index 6316c749c..a0a8ac9f2 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/__init__.py @@ -4,9 +4,9 @@ import os import re import sys import shutil -import pkgutil from datetime import datetime import Bcfg2.Client.Tools +from Bcfg2.Compat import walk_packages try: from base import POSIXTool except ImportError: @@ -45,26 +45,12 @@ class POSIX(Bcfg2.Client.Tools.Tool): # this must be called at run-time, not at compile-time, or we # get wierd circular import issues. self._handlers = dict() - if hasattr(pkgutil, 'walk_packages'): - submodules = pkgutil.walk_packages(path=__path__) - else: - # python 2.4 - import glob - submodules = [] - for path in __path__: - for submodule in glob.glob(os.path.join(path, "*.py")): - mod = os.path.splitext(os.path.basename(submodule))[0] - if mod not in ['__init__']: - submodules.append((None, mod, True)) - - for submodule in submodules: - if submodule[1] == 'base': + for submodule in walk_packages(path=__path__, prefix=__name__ + "."): + mname = submodule[1].rsplit('.', 1)[-1] + if mname == 'base': continue - module = getattr(__import__("%s.%s" % - (__name__, - submodule[1])).Client.Tools.POSIX, - submodule[1]) - hdlr = getattr(module, "POSIX" + submodule[1]) + module = getattr(__import__(submodule[1]).Client.Tools.POSIX, mname) + hdlr = getattr(module, "POSIX" + mname) if POSIXTool in hdlr.__mro__: # figure out what entry type this handler handles etype = hdlr.__name__[5:].lower() diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index b38eca739..b058ae16c 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -3,25 +3,11 @@ import os import sys import stat import time -import pkgutil from subprocess import Popen, PIPE - import Bcfg2.Client.XML -from Bcfg2.Compat import input - -if hasattr(pkgutil, 'walk_packages'): - submodules = pkgutil.walk_packages(path=__path__) -else: - # python 2.4 - import glob - submodules = [] - for path in __path__: - for submodule in glob.glob(os.path.join(path, "*.py")): - mod = os.path.splitext(os.path.basename(submodule))[0] - if mod not in ['__init__']: - submodules.append((None, mod, True)) - -__all__ = [m[1] for m in submodules] +from Bcfg2.Compat import input, walk_packages + +__all__ = [m[1] for m in walk_packages(path=__path__)] drivers = [item for item in __all__ if item not in ['rpmtools']] default = [item for item in drivers if item not in ['RPM', 'Yum']] diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index be3954f3b..6551ffc40 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -132,3 +132,84 @@ class CmpMixin(object): def __le__(self, other): return self.__lt__(other) or self.__eq__(other) + +try: + from pkgutil import walk_packages +except ImportError: + try: + from pkgutil import iter_modules + # iter_modules was added in python 2.5; use it to get an exact + # re-implementation of walk_packages if possible + def walk_packages(path=None, prefix='', onerror=None): + def seen(p, m={}): + if p in m: + return True + m[p] = True + + for importer, name, ispkg in iter_modules(path, prefix): + yield importer, name, ispkg + + if ispkg: + try: + __import__(name) + except ImportError: + if onerror is not None: + onerror(name) + except Exception: + if onerror is not None: + onerror(name) + else: + raise + else: + path = getattr(sys.modules[name], '__path__', []) + + # don't traverse path items we've seen before + path = [p for p in path if not seen(p)] + + for item in walk_packages(path, name + '.', onerror): + yield item + except ImportError: + def walk_packages(path=None, prefix='', onerror=None): + """ imperfect, incomplete implementation of + walk_packages() for python 2.4. Differences: + + * requires a full path, not a path relative to something + in sys.path. anywhere we care about that shouldn't be + an issue + + * the first element of each tuple is None instead of an + importer object + """ + def seen(p, m={}): + if p in m: + return True + m[p] = True + + if path is None: + path = sys.path + rv = [] + for mpath in path: + for fname in os.listdir(mpath): + fpath = os.path.join(mpath, fname) + if (os.path.isfile(fpath) and fname.endswith(".py") and + fname != '__init__.py'): + yield None, prefix + fname[:-3], False + elif os.path.isdir(fpath): + mname = prefix + fname + if os.path.exists(os.path.join(fpath, "__init__.py")): + yield None, mname, True + try: + __import__(mname) + except ImportError: + if onerror is not None: + onerror(mname) + except Exception: + if onerror is not None: + onerror(mname) + else: + raise + else: + for item in walk_packages([fpath], + prefix=mname + '.', + onerror=onerror): + yield item diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 1cce36f4b..08b873948 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -4,12 +4,11 @@ import re import os import sys import stat -import pkgutil import logging import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin -from Bcfg2.Compat import u_str, unicode, b64encode +from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages import Bcfg2.Server.Lint logger = logging.getLogger(__name__) @@ -153,24 +152,12 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): global PROCESSORS if PROCESSORS is None: PROCESSORS = [] - if hasattr(pkgutil, 'walk_packages'): - submodules = pkgutil.walk_packages(path=__path__) - else: - #python 2.4 - import glob - submodules = [] - for path in __path__: - for submodule in glob.glob(os.path.join(path, "*.py")): - mod = '.'.join(submodule.split("/")[-1].split('.')[:-1]) - if mod != '__init__': - submodules.append((None, mod, True)) - - for submodule in submodules: - module = getattr(__import__("%s.%s" % - (__name__, - submodule[1])).Server.Plugins.Cfg, - submodule[1]) - proc = getattr(module, submodule[1]) + for submodule in walk_packages(path=__path__, + prefix=__name__ + "."): + mname = submodule[1].rsplit('.', 1)[-1] + module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, + mname) + proc = getattr(module, mname) if set(proc.__mro__).intersection([CfgInfo, CfgFilter, CfgGenerator, CfgVerifier]): PROCESSORS.append(proc) diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py index 79c144811..e7a3191bc 100644 --- a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py +++ b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py @@ -2,12 +2,11 @@ from django.db import connection, DatabaseError from django.core.exceptions import ImproperlyConfigured import django.core.management import logging -import pkgutil import re import sys import traceback -from Bcfg2.Compat import CmpMixin +from Bcfg2.Compat import CmpMixin, walk_packages from Bcfg2.Server.models import InternalDatabaseVersion from Bcfg2.Server.SchemaUpdater.Routines import UpdaterRoutineException, \ UpdaterRoutine @@ -23,18 +22,6 @@ class SchemaTooOldError(UpdaterError): pass -def _walk_packages(paths): - """Python 2.4 lacks this routine""" - import glob - submodules = [] - for path in paths: - for submodule in glob.glob("%s/*.py" % path): - mod = '.'.join(submodule.split("/")[-1].split('.')[:-1]) - if mod != '__init__': - submodules.append((None, mod, False)) - return submodules - - def _release_to_version(release): """ Build a release base for a version @@ -207,12 +194,7 @@ def update_database(): logger.debug("Verifying database schema") updaters = [] - if hasattr(pkgutil, 'walk_packages'): - submodules = pkgutil.walk_packages(path=Changes.__path__) - else: - #python 2.4 - submodules = _walk_packages(Changes.__path__) - for loader, submodule, ispkg in submodules: + for loader, submodule, ispkg in walk_packages(path=Changes.__path__): if ispkg: continue try: -- cgit v1.2.3-1-g7c22