diff options
31 files changed, 543 insertions, 88 deletions
diff --git a/.travis.yml b/.travis.yml index b868e83b1..af365b9d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,28 +1,82 @@ +sudo: false +dist: trusty language: python + +packages: &build_deps_optional + - swig + - libxml2-utils + - yum + - libaugeas-dev + - augeas-lenses + - libacl1-dev + - libssl-dev + matrix: include: + - env: PYTHON=2.4 NOSE_EXCLUDE_DIRS=testsuite/Testsrc/Testlib/TestServer + addons: + apt: + sources: [deadsnakes] + packages: [python2.4, python2.4-dev] + - env: PYTHON=2.5 NOSE_EXCLUDE_DIRS=testsuite/Testsrc/Testlib/TestServer + addons: + apt: + sources: [deadsnakes] + packages: [python2.5, python2.5-dev, libbluetooth-dev] + - python: "2.6" - env: WITH_OPTIONAL_DEPS=no TEST_SPHINX=no - python: "2.6" - env: WITH_OPTIONAL_DEPS=yes TEST_SPHINX=no - - python: "2.7_with_system_site_packages" - env: WITH_OPTIONAL_DEPS=no TEST_SPHINX=no - - python: "2.7_with_system_site_packages" - env: WITH_OPTIONAL_DEPS=yes TEST_SPHINX=yes + env: WITH_OPTIONAL_DEPS=yes + addons: + apt: + packages: *build_deps_optional + + - python: "2.7" + - python: "2.7" + env: WITH_OPTIONAL_DEPS=yes WITH_SYSTEM_SITE_PACKAGES=yes TEST_SPHINX=yes + addons: + apt: + packages: [*build_deps_optional, python-gamin, python-selinux] + + - python: "3.5" + env: WITH_OPTIONAL_DEPS=yes + addons: + apt: + packages: *build_deps_optional + + allow_failures: + - python: "3.5" + + fast_finish: true + +before_install: + - testsuite/prepare-python.sh + - if test -d "$HOME/custom-virtualenv/"; then source "$HOME/custom-virtualenv/bin/activate"; fi install: - testsuite/install.sh - pip install -e . script: - - nosetests testsuite + - testsuite/test.sh after_failure: - pip freeze + branches: except: - maint-1.2 - 1.1.0-stable + notifications: - email: chris.a.st.pierre@gmail.com irc: channels: - "irc.freenode.org#bcfg2" use_notice: true + +cache: + directories: + - $HOME/.cache/pip + - $HOME/.cache/wheels + - $HOME/.cache/xml + +before_cache: + - rm -f $HOME/.cache/pip/log/debug.log + - rm -f $HOME/.cache/xml/catalog.xml diff --git a/doc/installation/prerequisites.txt b/doc/installation/prerequisites.txt index d45599955..e6de2a9f6 100644 --- a/doc/installation/prerequisites.txt +++ b/doc/installation/prerequisites.txt @@ -56,7 +56,8 @@ Bcfg2 Server +-------------------------------+----------+--------------------------------+ | python-gamin or pyinotify | Any | gamin or inotify, python | +-------------------------------+----------+--------------------------------+ -| python-ssl (note | Any | python, backported ssl module | +| python-ssl (this is included | Any | | +| in Python2.6 and later) | | | +-------------------------------+----------+--------------------------------+ | python-setuptools | Any | | +-------------------------------+----------+--------------------------------+ @@ -16,11 +16,12 @@ inst_reqs = [ 'lockfile', 'lxml', 'python-daemon', + 'argparse' ] -# we only need m2crypto on < python2.6 +# Use the backported ssl module on < python2.6 if sys.version_info[:2] < (2, 6): - inst_reqs.append('M2Crypto') + inst_reqs.append('ssl') setup(name="Bcfg2", version=__version__, # Defined in src/lib/Bcfg2/version.py diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 9b3dded99..4350f6067 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -42,10 +42,10 @@ class APT(Bcfg2.Client.Tools.Tool): if reqdir not in path_entries: os.environ['PATH'] = os.environ['PATH'] + ':' + reqdir self.pkgcmd = '%s ' % self.aptget + \ - '-o DPkg::Options::=--force-confold ' + \ - '-o DPkg::Options::=--force-confmiss ' + \ - '--reinstall ' + \ - '--force-yes ' + '-o DPkg::Options::=--force-confold ' + \ + '-o DPkg::Options::=--force-confmiss ' + \ + '--reinstall ' + \ + '--force-yes ' if not Bcfg2.Options.setup.debug: self.pkgcmd += '-q=2 ' self.pkgcmd += '-y install %s' @@ -158,8 +158,8 @@ class APT(Bcfg2.Client.Tools.Tool): (entry.attrib['name'])) return False pkgname = entry.get('name') - if pkgname not in self.pkg_cache or \ - not self.pkg_cache[pkgname].is_installed: + if (pkgname not in self.pkg_cache or + not self.pkg_cache[pkgname].is_installed): self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False @@ -183,9 +183,9 @@ class APT(Bcfg2.Client.Tools.Tool): return False else: # version matches - if not Bcfg2.Options.setup.quick \ - and entry.get('verify', 'true') == 'true' \ - and checksums: + if (not Bcfg2.Options.setup.quick and + entry.get('verify', 'true') == 'true' and + checksums): pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True diff --git a/src/lib/Bcfg2/Client/Tools/BundleDeps.py b/src/lib/Bcfg2/Client/Tools/BundleDeps.py index aaa090633..c1af3f7f1 100644 --- a/src/lib/Bcfg2/Client/Tools/BundleDeps.py +++ b/src/lib/Bcfg2/Client/Tools/BundleDeps.py @@ -28,7 +28,7 @@ class BundleDeps(Bcfg2.Client.Tools.Tool): bundle_name = entry.get('name') for bundle in self.config.findall('./Bundle/Bundle'): - if bundle.get('name') == bundle_name and \ - bundle not in self.modified: + if (bundle.get('name') == bundle_name and + bundle not in self.modified): self.modified.append(bundle) return dict() diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index b535de191..b1abb376a 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -88,8 +88,8 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): if bootstatus is not None: if bootstatus == 'on': # make sure service is enabled on boot - bootcmd = '/sbin/chkconfig %s %s' % \ - (entry.get('name'), bootstatus) + bootcmd = ('/sbin/chkconfig %s %s' % + (entry.get('name'), bootstatus)) elif bootstatus == 'off': # make sure service is disabled on boot bootcmd = '/sbin/chkconfig %s %s' % (entry.get('name'), diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 53e5e7ec6..35768f0fe 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -142,8 +142,8 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): # 'disabled' means we don't attempt to modify running svcs return bootcmdrv and seqcmdrv buildmode = Bcfg2.Options.setup.service_mode == 'build' - if (entry.get('status') == 'on' and not buildmode) and \ - entry.get('current_status') == 'off': + if ((entry.get('status') == 'on' and not buildmode) and + entry.get('current_status') == 'off'): svccmdrv = self.start_service(entry) elif (entry.get('status') == 'off' or buildmode) and \ entry.get('current_status') == 'on': diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py index f4f1ee4bf..bcd695058 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/Augeas.py @@ -5,6 +5,7 @@ import Bcfg2.Client.XML from augeas import Augeas from Bcfg2.Client.Tools.POSIX.base import POSIXTool from Bcfg2.Client.Tools.POSIX.File import POSIXFile +from Bcfg2.Compat import all # pylint: disable=W0622 class AugeasCommand(object): @@ -249,8 +250,8 @@ class POSIXAugeas(POSIXTool): for cmd in self.get_commands(entry): try: if not cmd.verify(): - err = "Augeas: Command has not been applied to %s: %s" % \ - (entry.get("name"), cmd) + err = ("Augeas: Command has not been applied to %s: %s" % + (entry.get("name"), cmd)) self.logger.debug(err) entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) @@ -259,8 +260,8 @@ class POSIXAugeas(POSIXTool): else: cmd.command.set("verified", "true") except: # pylint: disable=W0702 - err = "Augeas: Unexpected error verifying %s: %s: %s" % \ - (entry.get("name"), cmd, sys.exc_info()[1]) + err = ("Augeas: Unexpected error verifying %s: %s: %s" % + (entry.get("name"), cmd, sys.exc_info()[1])) self.logger.error(err) entry.set('qtext', "\n".join([entry.get('qtext', ''), err])) rv = False diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index a482dbc00..21257f64b 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -102,8 +102,8 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): # 'disabled' means we don't attempt to modify running svcs return bootcmdrv buildmode = Bcfg2.Options.setup.service_mode == 'build' - if (entry.get('status') == 'on' and not buildmode) and \ - entry.get('current_status') == 'off': + if ((entry.get('status') == 'on' and not buildmode) and + entry.get('current_status') == 'off'): svccmdrv = self.start_service(entry) elif (entry.get('status') == 'off' or buildmode) and \ entry.get('current_status') == 'on': diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 332638de4..4eea0273f 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -119,8 +119,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if Bcfg2.Options.setup.quick or \ - entry.attrib.get('verify', 'true') == 'false': + if (Bcfg2.Options.setup.quick or + entry.attrib.get('verify', 'true') == 'false'): return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: diff --git a/src/lib/Bcfg2/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 4ba06abae..93e4facdb 100644 --- a/src/lib/Bcfg2/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -2,7 +2,7 @@ # library will use lxml, then builtin xml.etree, then ElementTree -# pylint: disable=E0611,W0611,W0613,C0103 +# pylint: disable=E0611,E1101,W0611,W0613,C0103 try: from lxml.etree import Element, SubElement, tostring, XMLParser diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index dc4dfb983..157cc7f65 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -921,8 +921,8 @@ class Client(object): """Generate XML summary of execution statistics.""" states = {} for (item, val) in list(self.states.items()): - if not Bcfg2.Options.setup.only_important or \ - item.get('important', 'false').lower() == 'true': + if (not Bcfg2.Options.setup.only_important or + item.get('important', 'false').lower() == 'true'): states[item] = val feedback = XML.Element("upload-statistics") diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 254dfa4b8..ddcf66b44 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -21,7 +21,8 @@ try: except ImportError: HAS_SOUTH = False -settings = dict( # pylint: disable=C0103 +# pylint: disable=C0103 +settings = dict( TIME_ZONE=None, TEMPLATE_DEBUG=False, DEBUG=False, @@ -106,8 +107,8 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) - if hasattr(opts, "reporting_db_engine") and \ - opts.reporting_db_engine is not None: + if (hasattr(opts, "reporting_db_engine") and + opts.reporting_db_engine is not None): settings['DATABASES']['Reporting'] = dict( ENGINE="django.db.backends.%s" % opts.reporting_db_engine, NAME=opts.reporting_db_name, @@ -180,9 +181,9 @@ def upgrade_to_django_migrations(database, logger): cursor.cursor.execute('SELECT migration FROM south_migrationhistory') applied_migrations = [name for (name,) in cursor.fetchall()] last_migration = sorted(applied_migrations).pop() + # django.db.DatabaseError is not working here, because we are + # using the low level api to interact directly with the database except: # pylint: disable=W0702 - # django.db.DatabaseError is not working here, because we are - # using the low level api to interact directly with the database logger.debug("No south migration detected for database: %s." % database) diff --git a/src/lib/Bcfg2/Logger.py b/src/lib/Bcfg2/Logger.py index e5f316a18..a26971df4 100644 --- a/src/lib/Bcfg2/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -133,10 +133,11 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): logging.WARNING), self.format(reconn))) self.socket.send(msg) + + # If we still fail then drop it. Running + # bcfg2-server as non-root can trigger permission + # denied exceptions. except: # pylint: disable=W0702 - # If we still fail then drop it. Running - # bcfg2-server as non-root can trigger permission - # denied exceptions. pass diff --git a/src/lib/Bcfg2/Options/Parser.py b/src/lib/Bcfg2/Options/Parser.py index ced61c591..51e41850c 100644 --- a/src/lib/Bcfg2/Options/Parser.py +++ b/src/lib/Bcfg2/Options/Parser.py @@ -12,12 +12,14 @@ __all__ = ["setup", "OptionParserException", "Parser", "get_parser", "new_parser"] +# pylint: disable=C0103 #: The repository option. This is specified here (and imported into #: :module:`Bcfg2.Options.Common`) rather than vice-versa due to #: circular imports. -repository = PathOption( # pylint: disable=C0103 +repository = PathOption( '-Q', '--repository', cf=('server', 'repository'), default='/var/lib/bcfg2', help="Server repository path") +# pylint: enable=C0103 #: A module-level :class:`argparse.Namespace` object that stores all diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 2fdc0c3e0..b043fd11c 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -338,7 +338,7 @@ class classproperty(object): # pylint: disable=C0103 self.getter = getter def __get__(self, instance, owner): - return self.getter(owner) + return classmethod(self.getter).__get__(None, owner)() def is_string(strng, encoding): diff --git a/src/lib/Bcfg2/manage.py b/src/lib/Bcfg2/manage.py index b156deb0f..9675a3db1 100755 --- a/src/lib/Bcfg2/manage.py +++ b/src/lib/Bcfg2/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python -import os +""" Wrapper for the django manage.py with the Bcfg2 Opitons parsing. """ + import sys import django import Bcfg2.Options @@ -10,14 +11,21 @@ try: except ImportError: pass -parser = Bcfg2.Options.get_parser() -parser.add_options([Bcfg2.Options.PositionalArgument('django_command', nargs='*')]) -parser.parse() -if __name__ == "__main__": +def main(): + parser = Bcfg2.Options.get_parser() + parser.add_options([ + Bcfg2.Options.PositionalArgument('django_command', nargs='*')]) + parser.parse() + if django.VERSION[0] == 1 and django.VERSION[1] >= 6: from django.core.management import execute_from_command_line - execute_from_command_line(sys.argv[:1] + Bcfg2.Options.setup.django_command) + execute_from_command_line( + sys.argv[:1] + Bcfg2.Options.setup.django_command) else: from django.core.management import execute_manager execute_manager(Bcfg2.DBSettings.settings) + + +if __name__ == "__main__": + main() diff --git a/testsuite/Testschema/test_schema.py b/testsuite/Testschema/test_schema.py index cd9b74cdf..138468a91 100644 --- a/testsuite/Testschema/test_schema.py +++ b/testsuite/Testschema/test_schema.py @@ -34,13 +34,18 @@ NSMAP = dict(xs=XS) class TestSchemas(Bcfg2TestCase): schema_url = "http://www.w3.org/2001/XMLSchema.xsd" + catalog_file = os.path.expanduser("~/.cache/xml/catalog.xml") @skipUnless(HAS_XMLLINT, "xmllint not installed") def test_valid(self): + env = os.environ.copy() + if os.path.exists(self.catalog_file): + print('Using cached schema files.') + env["SGML_CATALOG_FILES"] = self.catalog_file schemas = [s for s in glob.glob(os.path.join(srcpath, '*.xsd'))] - xmllint = Popen(['xmllint', '--xinclude', '--noout', '--schema', - self.schema_url] + schemas, - stdout=PIPE, stderr=STDOUT) + xmllint = Popen(['xmllint', '--xinclude', '--noout', '--catalogs', + '--schema', self.schema_url] + schemas, + stdout=PIPE, stderr=STDOUT, env=env) print(xmllint.communicate()[0].decode()) self.assertEqual(xmllint.wait(), 0) diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index c26d8c139..44c4633a7 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -79,9 +79,10 @@ no_checks = { "lib/Bcfg2/Server/migrations": ["*.py"], "lib/Bcfg2/Server/south_migrations": ["*.py"], } + if sys.version_info < (2, 6): - # multiprocessing core requires py2.6 - no_checks['lib/Bcfg2/Server'] = ['MultiprocessingCore.py'] + # Server requires python 2.6 + no_checks['lib/Bcfg2'] = ['Server'] try: any diff --git a/testsuite/Testsrc/test_doc.py b/testsuite/Testsrc/test_doc.py index 93c8d1bb4..d105254c5 100644 --- a/testsuite/Testsrc/test_doc.py +++ b/testsuite/Testsrc/test_doc.py @@ -20,7 +20,7 @@ except ImportError: HAS_SPHINX = False -TEST_SPHINX = bool(os.environ.get('TEST_SPHINX', 'yes') != 'no') +TEST_SPHINX = bool(os.environ.get('TEST_SPHINX', 'no') != 'no') class DocTest(Bcfg2TestCase): diff --git a/testsuite/common.py b/testsuite/common.py index 944471ade..e0ff3ed19 100644 --- a/testsuite/common.py +++ b/testsuite/common.py @@ -167,7 +167,7 @@ class Bcfg2TestCase(TestCase): sys.stderr = cls._stderr if hasattr(TestCase, "assertCountEqual"): - assertItemsEqual = assertCountEqual + assertItemsEqual = TestCase.assertCountEqual def assertXMLEqual(self, el1, el2, msg=None): """ Test that the two XML trees given are equal. """ diff --git a/testsuite/ext/exception_messages.py b/testsuite/ext/exception_messages.py index cd3d7112c..5c59916df 100644 --- a/testsuite/ext/exception_messages.py +++ b/testsuite/ext/exception_messages.py @@ -35,10 +35,15 @@ class ExceptionMessageChecker(BaseChecker): priority = -1 def visit_raise(self, node): - if node.exc is None: + exc = None + try: + exc = node.exc + except AttributeError: + exc = node.type + if exc is None: return - if isinstance(node.exc, ast.Name): - raised = safe_infer(node.exc) + if isinstance(exc, ast.Name): + raised = safe_infer(exc) if (isinstance(raised, ast.Class) and raised.name not in self.config.exceptions_without_args): self.add_message('R9901', node=node.exc) diff --git a/testsuite/ext/pylint_compat.py b/testsuite/ext/pylint_compat.py new file mode 100644 index 000000000..5300df8e6 --- /dev/null +++ b/testsuite/ext/pylint_compat.py @@ -0,0 +1,280 @@ +from pylint.__pkginfo__ import version as pylint_version + +try: + from logilab.astng.__pkginfo__ import version as astng_version +except ImportError: + from astroid.__pkginfo__ import version as astng_version + + +def register(linter): + if pylint_version < '0.24.0': + import pylint.utils + orig_check_message_id = pylint.utils.MessagesHandlerMixIn.check_message_id + def check_message_id(self, msgid): + # translate the new message ids back into the old ones + replacements = {'12': '65', '13': '99'} + new = msgid[1:3] + if new in replacements: + msgid = msgid[0] + replacements[new] + msgid[3:] + return orig_check_message_id(self, msgid) + pylint.utils.MessagesHandlerMixIn.check_message_id = check_message_id + + def ignore(meth, msgid, *args, **kwargs): + # ignore non-existent message ids in disable/enable comments + ignore = ['W1401', 'R0924'] + if msgid in ignore: + return + return meth(msgid, *args, **kwargs) + linter._options_methods['disable'] = lambda *args, **kwargs: ignore(linter.disable, *args, **kwargs) + linter._options_methods['enable'] = lambda *args, **kwargs: ignore(linter.enable, *args, **kwargs) + + if pylint_version < '0.22.0': + import pylint.checkers.exceptions + orig_visit_raise = pylint.checkers.exceptions.ExceptionsChecker.visit_raise + def visit_raise(self, node): + if not hasattr(node, 'type') and hasattr(node, 'exc'): + node.type = node.exc + return orig_visit_raise(self, node) + pylint.checkers.exceptions.ExceptionsChecker.visit_raise = visit_raise + + if astng_version < '0.23': + import logilab.astng.scoped_nodes + from logilab.astng.bases import InferenceContext, InferenceError + + # backport import bug fix (e642ba33ba1bdde04ac9f0c75a25dc40131c55e7) + def ancestors(self, recurs=True, context=None): + yielded = set([self]) + if context is None: + context = InferenceContext() + for stmt in self.bases: + path = set(context.path) + try: + for baseobj in stmt.infer(context): + if not isinstance(baseobj, logilab.astng.scoped_nodes.Class): + # duh ? + continue + if baseobj in yielded: + continue # cf xxx above + yielded.add(baseobj) + yield baseobj + if recurs: + for grandpa in baseobj.ancestors(True, context): + if grandpa in yielded: + continue # cf xxx above + yielded.add(grandpa) + yield grandpa + except InferenceError: + # XXX log error ? + pass + context.path = path + logilab.astng.scoped_nodes.Class.ancestors = ancestors + + # support for classpropery (d110bcf2de4b8bc48e41638cf430f17c5714ffbc) + try: + from logilab.astng.rebuilder import TreeRebuilder + except: + try: + from logilab.astng._nodes_ast import TreeRebuilder + except: + from logilab.astng._nodes_compiler import TreeRebuilder + from logilab.astng import nodes + + orig_visit_function = TreeRebuilder.visit_function + def visit_function(self, node, parent): + newnode = orig_visit_function(self, node, parent) + if newnode.decorators is not None: + for decorator_expr in newnode.decorators.nodes: + if isinstance(decorator_expr, nodes.Name): + if decorator_expr.name == 'classproperty': + newnode.type = 'classmethod' + return newnode + TreeRebuilder.visit_function = visit_function + + if astng_version < '0.22': + from logilab.astng import nodes + from logilab.astng.bases import _infer_stmts, copy_context, path_wrapper, \ + InferenceError, NotFoundError + from logilab.astng._exceptions import ASTNGBuildingException + import logilab.astng.scoped_nodes + from logilab.astng.node_classes import List, DelName + + # backport of 11886551cfdcf969f0a661f8ab63c1fa1a6dd399 with + # a bit revert of af896e299ce5e381a928a77a9c28941cad90a243 + def infer_from(self, context=None, asname=True): + name = context.lookupname + if name is None: + raise InferenceError() + if asname: + name = self.real_name(name) + module = self.do_import_module(self.modname) + try: + context = copy_context(context) + context.lookupname = name + return _infer_stmts(module.getattr(name, ignore_locals=module is self.root()), context) + except NotFoundError: + raise InferenceError(name) + nodes.From.infer = path_wrapper(infer_from) + + def getattr(self, name, context=None, ignore_locals=False): + if name in self.special_attributes: + if name == '__file__': + return [cf(self.file)] + self.locals.get(name, []) + if name == '__path__' and self.package: + return [List()] + self.locals.get(name, []) + return std_special_attributes(self, name) + if not ignore_locals and name in self.locals: + return self.locals[name] + if self.package: + try: + return [self.import_module(name, relative_only=True)] + except (KeyboardInterrupt, SystemExit): + raise + except: + pass + raise NotFoundError(name) + logilab.astng.scoped_nodes.Module.getattr = logilab.astng.scoped_nodes.remove_nodes(getattr, DelName) + + if astng_version < '0.21.1': + # backport of 3d463da455e33e7ddc53a295b6a33db7b9e4288b + from logilab.astng.scoped_nodes import Function + from logilab.astng.rebuilder import RebuildVisitor + from logilab.astng.bases import YES, Instance + + orig_init = Function.__init__ + def init(self, name, doc): + orig_init(self, name, doc) + self.instance_attrs = {} + Function.__init__ = init + + orig_getattr = Function.getattr + def getattr(self, name, context=None): + if name != '__module__' and name in self.instance_attrs: + return self.instance_attrs[name] + return orig_getattr(self, name, context) + Function.getattr = getattr + + def delayed_assattr(self, node): + """visit a AssAttr node -> add name to locals, handle members + definition + """ + try: + frame = node.frame() + for infered in node.expr.infer(): + if infered is YES: + continue + try: + if infered.__class__ is Instance: + infered = infered._proxied + iattrs = infered.instance_attrs + elif isinstance(infered, Instance): + # Const, Tuple, ... we may be wrong, may be not, but + # anyway we don't want to pollute builtin's namespace + continue + elif infered.is_function: + iattrs = infered.instance_attrs + else: + iattrs = infered.locals + except AttributeError: + # XXX log error + #import traceback + #traceback.print_exc() + continue + values = iattrs.setdefault(node.attrname, []) + if node in values: + continue + # get assign in __init__ first XXX useful ? + if frame.name == '__init__' and values and not \ + values[0].frame().name == '__init__': + values.insert(0, node) + else: + values.append(node) + except InferenceError: + pass + RebuildVisitor.delayed_assattr = delayed_assattr + + if astng_version < '0.20.4': + try: + from logilab.astng._nodes_ast import TreeRebuilder, _lineno_parent + except: + from logilab.astng._nodes_compiler import TreeRebuilder + _lineno_parent = (lambda *args: TreeRebuilder._set_infos(None, *args)) + from logilab.astng import nodes + from logilab.astng.bases import NodeNG, Instance + from logilab.astng.mixins import ParentAssignTypeMixin + + class Set(NodeNG, Instance, ParentAssignTypeMixin): + _astng_fields = ('elts',) + elts = None + + def pytype(self): + return '__builtin__.set' + + def itered(self): + return self.elts + + def visit_set(self, node, parent): + newnode = Set() + _lineno_parent(node, newnode, parent) + newnode.elts = [self.visit(child, newnode) for child in node.elts] + newnode.set_line_info(newnode.last_child()) + return newnode + TreeRebuilder.visit_set = visit_set + + def visit_setcomp(self, node, parent): + newnode = nodes.SetComp() + _lineno_parent(node, newnode, parent) + newnode.elt = self.visit(node.elt, newnode) + newnode.generators = [self.visit(child, newnode) + for child in node.generators] + newnode.set_line_info(newnode.last_child()) + return newnode + TreeRebuilder.visit_setcomp = visit_setcomp + + class DictComp(NodeNG): + _astng_fields = ('key', 'value', 'generators') + key = None + value = None + generators = None + + def visit_dictcomp(self, node, parent): + newnode = DictComp() + _lineno_parent(node, newnode, parent) + newnode.key = self.visit(node.key, newnode) + newnode.value = self.visit(node.value, newnode) + newnode.generators = [self.visit(child, newnode) + for child in node.generators] + newnode.set_line_info(newnode.last_child()) + return newnode + TreeRebuilder.visit_dictcomp = visit_dictcomp + + # backport of bfe9e5c53cfb75c3b45ebb5cb8e8902464782c7d + from logilab.astng.node_classes import From + orig_from_init = From.__init__ + def from_init(self, fromname, names, level=0): + orig_from_init(self, fromname or '', names, level) + From.__init__ = from_init + + # partial backport of 6d59ad07d722d01e458aaf8fd14fd7dfc7ebaa6e + from logilab.astng.scoped_nodes import Module + orig_absolute_modname = Module.absolute_modname + def absolute_modname(self, modname, level): + result = orig_absolute_modname(self, modname, level) + if result[-1] == '.': + return result[:-1] + return result + Module.absolute_modname = absolute_modname + + # python2.4 compatibility (no super on old-style classes) + from logilab.astng.bases import Proxy, UnboundMethod + + def unbound_igetattr(self, name, context=None): + if name == 'im_func': + return iter((self._proxied,)) + return Proxy.igetattr(self, name, context) + UnboundMethod.igetattr = unbound_igetattr + + def unbound_getattr(self, name, context=None): + if name == 'im_func': + return [self._proxied] + return Proxy.getattr(self, name, context) + UnboundMethod.getattr = unbound_getattr diff --git a/testsuite/ext/ssl_protocols.py b/testsuite/ext/ssl_protocols.py index 66068d2a9..f92e3e355 100644 --- a/testsuite/ext/ssl_protocols.py +++ b/testsuite/ext/ssl_protocols.py @@ -1,5 +1,5 @@ try: - from logilab.astng import MANAGER, scoped_nodes, node_classes + from logilab.astng import MANAGER, builder, scoped_nodes, node_classes PYLINT=0 except ImportError: from astroid import MANAGER, scoped_nodes, node_classes @@ -8,10 +8,18 @@ except ImportError: def ssl_transform(module): if module.name == 'ssl': for proto in ('SSLv23', 'TLSv1'): - module.locals['PROTOCOL_%s' % proto] = [node_classes.Const()] + module.locals['PROTOCOL_%s' % proto] = [node_classes.Const(0)] def register(linter): if PYLINT == 0: - MANAGER.register_transformer(ssl_transform) + if hasattr(MANAGER, 'register_transformer'): + MANAGER.register_transformer(ssl_transform) + else: + safe = builder.ASTNGBuilder.string_build + def _string_build(self, data, modname='', path=None): + if modname == 'ssl': + data += '\n\nPROTOCOL_SSLv23 = 0\nPROTOCOL_TLSv1 = 0' + return safe(self, data, modname, path) + builder.ASTNGBuilder.string_build = _string_build else: MANAGER.register_transform(scoped_nodes.Module, ssl_transform) diff --git a/testsuite/install.sh b/testsuite/install.sh index 4d8778ad7..af07de0a9 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -1,31 +1,77 @@ #!/bin/bash -ex # install script for Travis-CI +PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') +SITE_PACKAGES=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())') -sudo apt-get update -qq -sudo apt-get install swig libxml2-utils - -pip install -r testsuite/requirements.txt +if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" && $PYVER != "2.6" ]]; then + pip install -r testsuite/requirements-legacy.txt +else + pip install --upgrade pip -PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') + pip_wheel() { + pip wheel --find-links="$HOME/.cache/wheels/" --wheel-dir="$HOME/.cache/wheels/" "$@" + pip install --no-index --find-links="$HOME/.cache/wheels/" "$@" + } -if [[ ${PYVER:0:1} == "2" && $PYVER != "2.7" ]]; then - pip install unittest2 -fi + if [[ $PYVER == "2.6" ]]; then + pip_wheel -r testsuite/requirements-26.txt + pip_wheel unittest2 + else + pip_wheel -r testsuite/requirements.txt -if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - sudo apt-get install -y yum libaugeas0 augeas-lenses libacl1-dev libssl-dev \ - python-gamin python-selinux + if [[ ${PYVER:0:1} == "3" ]]; then + # TODO: Move to "requirements.txt" if all the new errors are fixed. + pip_wheel 'pylint>1.4' + fi + fi - pip install PyYAML pyinotify boto pylibacl Jinja2 mercurial guppy cherrypy python-augeas + if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then + pip_wheel PyYAML pyinotify boto pylibacl Jinja2 \ + cherrypy python-augeas nose-show-skipped - if [[ ${PYVER:0:1} == "2" ]]; then - pip install cheetah m2crypto + if [[ $PYVER == "2.6" ]]; then + pip install \ + --global-option='build_ext' \ + --global-option='--include-dirs=/usr/include/x86_64-linux-gnu' \ + m2crypto - if [[ $PYVER != "2.7" ]]; then - pip install 'django<1.7' 'South<0.8' + pip_wheel 'django<1.7' 'South<0.8' 'mercurial<4.3' cheetah guppy else - pip install django + if [[ $PYVER == "2.7" ]]; then + pip_wheel m2crypto guppy + fi + + pip_wheel django mercurial cheetah3 fi fi fi + +# Use system site-packages and pymodules +if [[ "$WITH_SYSTEM_SITE_PACKAGES" == "yes" ]]; then + cat <<EOF > "$SITE_PACKAGES/system-packages.pth" +/usr/lib/python$PYVER/site-packages/ +/usr/lib/python$PYVER/dist-packages/ +/usr/lib/pymodules/python$PYVER/ +EOF +fi + +# Setup the local xml schema cache +download_schema() { + if [[ ! -e "$1" ]]; then + wget -O "$1" "$2" + fi +} + +mkdir -p "$HOME/.cache/xml/" +download_schema "$HOME/.cache/xml/XMLSchema.xsd" "http://www.w3.org/2001/XMLSchema.xsd" +download_schema "$HOME/.cache/xml/xml.xsd" "http://www.w3.org/2001/xml.xsd" + +cat > "$HOME/.cache/xml/catalog.xml" <<EOF +<?xml version="1.0"?> +<catalog xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog"> + <system systemId="http://www.w3.org/2001/XMLSchema.xsd" uri="$HOME/.cache/xml/XMLSchema.xsd" /> + <system systemId="http://www.w3.org/2001/xml.xsd" uri="$HOME/.cache/xml/xml.xsd" /> + <nextCatalog catalog="/etc/xml/catalog.xml" /> +</catalog> +EOF diff --git a/testsuite/prepare-python.sh b/testsuite/prepare-python.sh new file mode 100755 index 000000000..7b72a6dc4 --- /dev/null +++ b/testsuite/prepare-python.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +if [ -n "$PYTHON" ]; then + echo "Try to use custom python version: $PYTHON" + pip install 'virtualenv<1.8' + mkdir -p "$HOME/custom-virtualenv" + virtualenv -p "python$PYTHON" --use-distribute "$HOME/custom-virtualenv" +fi diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 50ece77db..ce7e407df 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -19,7 +19,7 @@ persistent=no # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. -load-plugins=ext.exception_messages,ext.ssl_protocols +load-plugins=ext.exception_messages,ext.ssl_protocols,ext.pylint_compat [MESSAGES CONTROL] diff --git a/testsuite/requirements-26.txt b/testsuite/requirements-26.txt new file mode 100644 index 000000000..f85dc6de4 --- /dev/null +++ b/testsuite/requirements-26.txt @@ -0,0 +1,9 @@ +lxml +python-daemon<2.0.0 +argparse +genshi + +nose +mock +pylint<0.29 +pep8 diff --git a/testsuite/requirements-legacy.txt b/testsuite/requirements-legacy.txt new file mode 100644 index 000000000..7d918cb45 --- /dev/null +++ b/testsuite/requirements-legacy.txt @@ -0,0 +1,14 @@ +lxml<3.4 +lockfile<0.9 +python-daemon<1.4 +argparse +ssl + +nose +nose-exclude<0.2 +mock<1.1 +unittest2<0.6 +logilab-common==0.53.0 +logilab-astng==0.20.3 +pylint<0.22 +pep8<1.3 diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt index 0d8c297aa..665ed961d 100644 --- a/testsuite/requirements.txt +++ b/testsuite/requirements.txt @@ -1,9 +1,10 @@ lxml +python-daemon +genshi +argparse + nose mock -sphinx<1.5 pylint<0.29 pep8 -python-daemon<2.0.0 -genshi -argparse +sphinx diff --git a/testsuite/test.sh b/testsuite/test.sh new file mode 100755 index 000000000..739c3c2b0 --- /dev/null +++ b/testsuite/test.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +NOSE_OPTS="" + +if [ "$WITH_OPTIONAL_DEPS" = "yes" ]; then + NOSE_OPTS="--show-skipped" +fi + +exec nosetests $NOSE_OPTS testsuite |