From a24165b9b7dd05a104cc10176e54fa9b5bb91812 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 2 May 2012 08:28:02 -0500 Subject: doc: Fix reports installation docs (reported by calvinx on IRC) Signed-off-by: Sol Jerome --- doc/appendix/guides/web-reports-install.txt | 144 ++++------------------------ doc/reports/dynamic.txt | 2 - 2 files changed, 16 insertions(+), 130 deletions(-) diff --git a/doc/appendix/guides/web-reports-install.txt b/doc/appendix/guides/web-reports-install.txt index f6a588692..c03682974 100644 --- a/doc/appendix/guides/web-reports-install.txt +++ b/doc/appendix/guides/web-reports-install.txt @@ -11,125 +11,17 @@ Dynamic (web) Reports installation ================================== -The first step is to install the needed software components like the -Django framework and the database (SQlite2). All packages for Fedora -are in the Fedora Package Collection or in EPEL_ for CentOS/RHEL:: - - [root@system01 ~]# yum -y install Django python-simplejson python-sqlite2 - -Of course is a web server needed as well:: - - [root@system01 ~]# yum -y install httpd mod_python - -The same packages are needed for Ubuntu systems:: - - [root@system01 ~]# aptitude install python-django apache2 libapache2-mod-python - -Now we need to create the sqlite database. Use the following command on -Fedora, CentOS, or RHEL.:: - - [root@system01 ~]# python /usr/lib/python2.4/site-packages/Bcfg2/Server/Reports/manage.py syncdb - Creating table auth_permission - Creating table auth_group - Creating table auth_user - Creating table auth_message - Creating table django_content_type - Creating table django_session - Creating table django_site - Creating table django_admin_log - Creating table reports_client - Creating table reports_ping - Creating table reports_interaction - Creating table reports_reason - Creating table reports_entries - Creating table reports_entries_interactions - Creating table reports_performance - Creating table reports_internaldatabaseversion - - You just installed Django's auth system, which means you don't have any superusers defined. - Would you like to create one now? (yes/no): no - Installing index for auth.Permission model - Installing index for auth.Message model - Installing index for admin.LogEntry model - Installing index for reports.Client model - Installing index for reports.Ping model - Installing index for reports.Interaction model - Installing index for reports.Entries model - Installing index for reports.Entries_interactions model - -.. note:: There are different versions of Python available. If you are - unsure about your installed version use the following line instead of - the line above.:: - - [root@system01 ~]# PYVER=`python -c 'import sys;print(sys.version[0:3])'`; python /usr/lib/python$PYVER/site-packages/Bcfg2/site-packages/Bcfg2/Server/Reports/manage.py syncdb - -The path on Ubuntu systems is different. Please use the same path as shown -in the following command to execute the script on an Ubuntu machine in -the next steps:: - - [root@system01 ~]# python /usr/share/pyshared/Bcfg2/Server/Reports/manage.py syncdb - Creating table auth_permission - Creating table auth_group - Creating table auth_user - Creating table auth_message - Creating table django_content_type - Creating table django_session - Creating table django_site - Creating table django_admin_log - Creating table reports_client - Creating table reports_ping - Creating table reports_interaction - Creating table reports_reason - Creating table reports_entries - Creating table reports_entries_interactions - Creating table reports_performance - Creating table reports_internaldatabaseversion - - You just installed Django's auth system, which means you don't have any superusers defined. - Would you like to create one now? (yes/no): no - Installing index for auth.Permission model - Installing index for auth.Message model - Installing index for admin.LogEntry model - Installing index for reports.Client model - Installing index for reports.Ping model - Installing index for reports.Interaction model - Installing index for reports.Entries model - Installing index for reports.Entries_interactions model - -The server should be tested to make sure that there are no mistakes:: - - [root@system01 ~]# python /usr/lib/python2.6/site-packages/Bcfg2/Server/Reports/manage.py testserver - Creating test database... - Creating table auth_permission - Creating table auth_group - Creating table auth_user - Creating table auth_message - Creating table django_content_type - Creating table django_session - Creating table django_site - Creating table django_admin_log - Creating table reports_client - Creating table reports_ping - Creating table reports_interaction - Creating table reports_reason - Creating table reports_entries - Creating table reports_entries_interactions - Creating table reports_performance - Creating table reports_internaldatabaseversion - Installing index for auth.Permission model - Installing index for auth.Message model - Installing index for admin.LogEntry model - Installing index for reports.Client model - Installing index for reports.Ping model - Installing index for reports.Interaction model - Installing index for reports.Entries model - Installing index for reports.Entries_interactions model - Validating models... - 0 errors found - - Django version 1.1.1, using settings 'Reports.settings' - Development server is running at http://127.0.0.1:8000/ - Quit the server with CONTROL-C. +You need to install the bcfg2-web package that is available for your +particular distribution. All packages for Fedora are in the Fedora +Package Collection. + +You can find packages for CentOS and RHEL in EPEL_:: + + [root@system01 ~]# yum -y install bcfg2-web + +The same packages are needed for Debian/Ubuntu systems:: + + [root@system01 ~]# aptitude install bcfg2-web Add DBStats to the plugins line of ``bcfg2.conf``. The resulting **[server]** section should look something like this:: @@ -138,6 +30,10 @@ Add DBStats to the plugins line of ``bcfg2.conf``. The resulting repository = /var/lib/bcfg2 plugins = Base,Bundler,Cfg,...,DBStats +You then need to initialize the DBStats reporting database:: + + [root@system01 ~]# bcfg2-admin reports init + Start/restart the Bcfg2 server:: [root@system01 ~]# /etc/init.d/bcfg2-server restart @@ -146,16 +42,8 @@ Run the Bcfg2 client in order to populate the statistics database (this run should take a bit longer since you are uploading the client statistics to the database). -Download the static reports content:: - - [root@system01 ~]# git clone git://git.mcs.anl.gov/bcfg2 - [root@system01 ~]# cd /var/www/ - [root@system01 ~]# mv /path/to/bcfg2/checkout/reports ./ - -Configure Apache using :ref:`dynamic-http-install` as a guide - Copy server/statistics sections of ``bcfg2.conf`` to -``/etc/bcfg2-web.conf`` (make sure it is world-readable). You should +``/etc/bcfg2-web.conf`` (make sure it is world-readable). You should then have something like this:: [server] diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 07763922c..8267bffe3 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -6,8 +6,6 @@ Bcfg2 Dynamic Reporting System ============================== -.. versionadded:: 0.8.2 - Installation ============ -- cgit v1.2.3-1-g7c22 From 99680b113a6cc0e97db0ec5485d1df1d1831ca59 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 2 Apr 2012 11:11:30 -0400 Subject: made bcfg2-info work with newer IPython --- src/sbin/bcfg2-info | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 3c1a0a3da..e09b7ed87 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -167,8 +167,13 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): print("Dropping to python interpreter; press ^D to resume") try: import IPython - shell = IPython.Shell.IPShell(argv=[], user_ns=locals()) - shell.mainloop() + if hasattr(IPython, "Shell"): + shell = IPython.Shell.IPShell(argv=[], user_ns=locals()) + shell.mainloop() + elif hasattr(IPython, "embed"): + IPython.embed(user_ns=locals()) + else: + raise ImportError except ImportError: sh.interact() -- cgit v1.2.3-1-g7c22 From e3e8c73d8431c13663295b70fa73924db090a014 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 9 Apr 2012 16:16:50 -0400 Subject: fixed mode of bcfg2-test --- src/sbin/bcfg2-test | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/sbin/bcfg2-test diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test old mode 100644 new mode 100755 -- cgit v1.2.3-1-g7c22 From 592934c1f8e29fcfda53c90344c949f348ff94d4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 9 Apr 2012 16:18:03 -0400 Subject: pkg_verify works on BoundPackage tag, not just Instance --- schemas/pkgtype.xsd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/schemas/pkgtype.xsd b/schemas/pkgtype.xsd index 3655c646b..2de44b1d7 100644 --- a/schemas/pkgtype.xsd +++ b/schemas/pkgtype.xsd @@ -51,7 +51,8 @@ - + + -- cgit v1.2.3-1-g7c22 From 39350ff2e9a82aaba2d315f2a656e52f18ae0921 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 10 Apr 2012 16:54:48 -0400 Subject: add xml:base attr to all tags that can be xincluded --- schemas/bundle.xsd | 3 ++- schemas/clients.xsd | 8 ++++++-- schemas/grouppatterns.xsd | 6 +++++- schemas/packages.xsd | 18 +++++++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd index 1e67e1f3a..4e034ee3c 100644 --- a/schemas/bundle.xsd +++ b/schemas/bundle.xsd @@ -1,6 +1,6 @@ - + bundle schema for bcfg2 @@ -237,6 +237,7 @@ + diff --git a/schemas/clients.xsd b/schemas/clients.xsd index 0a9ce5202..b79be385c 100644 --- a/schemas/clients.xsd +++ b/schemas/clients.xsd @@ -1,5 +1,5 @@ - + client schema for bcfg2 @@ -8,12 +8,15 @@ + + - + @@ -35,6 +38,7 @@ + diff --git a/schemas/grouppatterns.xsd b/schemas/grouppatterns.xsd index f2bdceccd..6c63b8694 100644 --- a/schemas/grouppatterns.xsd +++ b/schemas/grouppatterns.xsd @@ -1,5 +1,5 @@ - + group patterns config schema for bcfg2 @@ -7,6 +7,9 @@ + + @@ -21,6 +24,7 @@ + diff --git a/schemas/packages.xsd b/schemas/packages.xsd index e09dcf8b5..c29a85ecf 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -1,5 +1,5 @@ - + packages config schema for bcfg2 @@ -7,6 +7,9 @@ + + @@ -45,12 +48,13 @@ - - - - - - + + + + + + + -- cgit v1.2.3-1-g7c22 From 9f31e17f1420dca584aed5b141fbc03eed284b48 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 10 Apr 2012 17:00:36 -0400 Subject: merged b82a107... removed obsolete docs about import_gpg_keys --- doc/server/plugins/generators/packages.txt | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index 276b73093..900162aaa 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -51,7 +51,7 @@ member clients. | Yum | yum | | +--------+----------+--------------+ -.. note:: +.. note:: .. versionadded:: 1.2.0 @@ -567,7 +567,7 @@ You can also view the sources applicable to a client:: Type: yum URL: http://mirror.example.com/centos-6-x86_64-updates GPG Key(s): http://mirror.example.com/centos-6-x86_64-updates/RPM-GPG-KEY-CentOS-6 - + Name: centos-6-x86_64-os Type: yum URL: http://mirror.example.com/centos-6-x86_64-os @@ -662,17 +662,8 @@ It understands the following directives: * ``gpg_keypath``: The path on the client RPM GPG keys will be copied to before they are imported on the client. Default is "/etc/pki/rpm-gpg". -* ``import_gpg_keys``: The RPM release of an RPM GPG key cannot be - reliably and automatically determined without importing the key into - the server's key chain. If ``import_gpg_keys`` is "false" (the - default), the release of automatically-generated RPM GPG key entries - in the specification will be set to "any", which disables - verification of the release. (Version will still be verified.) In - practice, this is unlikely to be an issue, as the RPM version of a - GPG key is the key's fingerprint, and collisions are rare. If you - do encounter a GPG key version collision, you will need to set this - to "true", whereupon Packages will import the keys into the server's - key chain. Python RPM libraries must be installed for this to work. +* ``version``: Set the version attribute used when binding + Packages. Default is ``auto``. [yum] section ------------- -- cgit v1.2.3-1-g7c22 From 1886e7437a445cbf60dca1c8a5d3ba2fee438573 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 13 Apr 2012 11:50:35 -0400 Subject: merged 0106905... added ability to pass options to nose, ignore tests, test specific clients to bcfg2-test --- src/lib/Options.py | 12 +++++- src/sbin/bcfg2-test | 104 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 87 insertions(+), 29 deletions(-) diff --git a/src/lib/Options.py b/src/lib/Options.py index 2f535397f..b55a8a55a 100644 --- a/src/lib/Options.py +++ b/src/lib/Options.py @@ -4,6 +4,7 @@ __revision__ = '$Revision$' import getopt import os import sys +import shlex import Bcfg2.Client.Tools # Compatibility imports from Bcfg2.Bcfg2Py3k import ConfigParser @@ -348,7 +349,16 @@ CLIENT_SERVICE_MODE = Option('Set client service mode', default='default', CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90, cmd='-t', cf=('communication', 'timeout'), odesc='') - + +# bcfg2-test options +TEST_NOSEOPTS = Option('Options to pass to nosetests', default=[], + cmd='--nose-options', cf=('bcfg2_test', 'nose_options'), + odesc='', long_arg=True, cook=shlex.split) +TEST_IGNORE = Option('Ignore these entries if they fail to build.', default=[], + cmd='--ignore', + cf=('bcfg2_test', 'ignore_entries'), long_arg=True, + odesc=':,:', cook=list_split) + # APT client tool options CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path', cf=('APT', 'install_path'), diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index af2225245..01a2a4893 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -1,41 +1,60 @@ #!/usr/bin/env python -"""This tool verifies that all clients known to the server build without failures""" +"""This tool verifies that all clients known to the server build +without failures""" import sys +import fnmatch +import logging +import Bcfg2.Logger import Bcfg2.Server.Core from nose.core import TestProgram +from nose.suite import LazySuite from unittest import TestCase class ClientTest(TestCase): """ - A test case representing the build of all of the configuration for a single host. - Checks that none of the build config entities has had a failure when it is building. - Optionally ignores some config files that we know will cause errors (because they - are private files we don't have access to, for instance) + A test case representing the build of all of the configuration for + a single host. Checks that none of the build config entities has + had a failure when it is building. Optionally ignores some config + files that we know will cause errors (because they are private + files we don't have access to, for instance) """ __test__ = False # Do not collect - def __init__(self, bcfg2_core, client): + def __init__(self, bcfg2_core, client, ignore=None): TestCase.__init__(self) self.bcfg2_core = bcfg2_core self.client = client + if ignore is None: + self.ignore = dict() + else: + self.ignore = ignore + + def ignore_entry(self, tag, name): + if tag in self.ignore: + if name in self.ignore[tag]: + return True + else: + # try wildcard matching + for pattern in self.ignore[tag]: + if fnmatch.fnmatch(name, pattern): + return True + return False def runTest(self): config = self.bcfg2_core.BuildConfiguration(self.client) - failures = config.xpath('//*[@failure]') - def format_failure(failure): - return "%s(%s): %s" % ( - failure.tag, - failure.attrib.get('name'), - failure.attrib.get('failure') - ) + failures = [] + msg = ["Failures:"] + for failure in config.xpath('//*[@failure]'): + if not self.ignore_entry(failure.tag, failure.get('name')): + failures.append(failure) + msg.append("%s:%s: %s" % (failure.tag, failure.get("name"), + failure.get("failure"))) + + assert len(failures) == 0, "\n".join(msg) - assert len(failures) == 0, "Failures:\n%s" % "\n".join( - [format_failure(failure) for failure in failures] - ) - def __str__(self): return "ClientTest(%s)" % self.client @@ -43,26 +62,55 @@ class ClientTest(TestCase): def main(): optinfo = { - 'configfile': Bcfg2.Options.CFILE, - 'help': Bcfg2.Options.HELP, - 'encoding': Bcfg2.Options.ENCODING, - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'password': Bcfg2.Options.SERVER_PASSWORD, - } + 'configfile': Bcfg2.Options.CFILE, + 'help': Bcfg2.Options.HELP, + 'encoding': Bcfg2.Options.ENCODING, + 'repo': Bcfg2.Options.SERVER_REPOSITORY, + 'plugins': Bcfg2.Options.SERVER_PLUGINS, + 'password': Bcfg2.Options.SERVER_PASSWORD, + 'verbose': Bcfg2.Options.VERBOSE, + 'noseopts': Bcfg2.Options.TEST_NOSEOPTS, + 'ignore': Bcfg2.Options.TEST_IGNORE, + } setup = Bcfg2.Options.OptionParser(optinfo) + setup.hm = \ + "bcfg2-test [options] [client] [client] [...]\nOptions:\n %s" % \ + setup.buildHelpMessage() setup.parse(sys.argv[1:]) + + if setup['verbose']: + Bcfg2.Logger.setup_logging("bcfg2-test", to_syslog=False) + core = Bcfg2.Server.Core.Core( setup['repo'], setup['plugins'], setup['password'], setup['encoding'], filemonitor='pseudo' - ) - core.fam.handle_events_in_interval(0.1) - suite = [ClientTest(core, client) for client in core.metadata.clients] + ) + + ignore = dict() + for entry in setup['ignore']: + tag, name = entry.split(":") + try: + ignore[tag].append(name) + except KeyError: + ignore[tag] = [name] + + def run_tests(): + core.fam.handle_events_in_interval(0.1) + + if setup['args']: + clients = setup['args'] + else: + clients = core.metadata.clients + + for client in clients: + logging.info("Building %s" % client) + yield ClientTest(core, client, ignore) - TestProgram(argv=sys.argv[0:1], suite = suite) + TestProgram(argv=sys.argv[0:1] + setup['noseopts'], + suite=LazySuite(run_tests)) if __name__ == "__main__": sys.exit(main()) -- cgit v1.2.3-1-g7c22 From a8421cae0d2864e8bb9010656351f37d2db941c4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 Apr 2012 11:20:47 -0400 Subject: make bcfg2-lint warn about .cat and .diff files --- src/lib/Bcfg2/Server/Lint/Deltas.py | 20 ++++++++++++++++++++ src/lib/Server/Lint/__init__.py | 7 +++++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 src/lib/Bcfg2/Server/Lint/Deltas.py diff --git a/src/lib/Bcfg2/Server/Lint/Deltas.py b/src/lib/Bcfg2/Server/Lint/Deltas.py new file mode 100644 index 000000000..7627b0cb0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Deltas.py @@ -0,0 +1,20 @@ +import Bcfg2.Server.Lint + +class Deltas(Bcfg2.Server.Lint.ServerPlugin): + """ Warn about usage of .cat and .diff files """ + + def Run(self): + """ run plugin """ + if 'Cfg' in self.core.plugins: + cfg = self.core.plugins['Cfg'] + for basename, entry in list(cfg.entries.items()): + self.check_entry(basename, entry) + + def check_entry(self, basename, entry): + for fname in list(entry.entries.keys()): + match = entry.specific.delta_reg.match(fname) + if match: + self.LintError("%s-file-used" % match.group('delta'), + "%s file used on %s: %s" % (match.group('delta'), + basename, + fname)) diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index a9d57c46c..ff5293a81 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -8,7 +8,8 @@ __all__ = ['Bundles', 'Pkgmgr', 'RequiredAttrs', 'Validate', - 'Genshi'] + 'Genshi', + 'Deltas'] import logging import os @@ -119,7 +120,9 @@ class ErrorHandler (object): "merge-probes":"warning", "input-output-error":"error", "genshi-syntax-error":"error", - "pattern-fails-to-initialize":"error"} + "pattern-fails-to-initialize":"error", + "cat-file-used":"warning", + "diff-file-used":"warning"} def __init__(self, config=None): self.errors = 0 -- cgit v1.2.3-1-g7c22 From dcca31b923a65467f9f69a44025730eece634ac1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 20 Apr 2012 11:41:05 -0400 Subject: made Deltas and Genshi lint plugins respect command-line filelist --- src/lib/Bcfg2/Server/Lint/Deltas.py | 12 ++++++------ src/lib/Server/Lint/Genshi.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/lib/Bcfg2/Server/Lint/Deltas.py b/src/lib/Bcfg2/Server/Lint/Deltas.py index 7627b0cb0..cf91d1d13 100644 --- a/src/lib/Bcfg2/Server/Lint/Deltas.py +++ b/src/lib/Bcfg2/Server/Lint/Deltas.py @@ -12,9 +12,9 @@ class Deltas(Bcfg2.Server.Lint.ServerPlugin): def check_entry(self, basename, entry): for fname in list(entry.entries.keys()): - match = entry.specific.delta_reg.match(fname) - if match: - self.LintError("%s-file-used" % match.group('delta'), - "%s file used on %s: %s" % (match.group('delta'), - basename, - fname)) + if self.HandlesFile(fname): + match = entry.specific.delta_reg.match(fname) + if match: + self.LintError("%s-file-used" % match.group('delta'), + "%s file used on %s: %s" % + (match.group('delta'), basename, fname)) diff --git a/src/lib/Server/Lint/Genshi.py b/src/lib/Server/Lint/Genshi.py index 552c495b2..56803246c 100755 --- a/src/lib/Server/Lint/Genshi.py +++ b/src/lib/Server/Lint/Genshi.py @@ -18,7 +18,8 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): for eset in entries.values(): for fname, sdata in list(eset.entries.items()): - if fname.endswith(".genshi") or fname.endswith(".newtxt"): + if (self.HandlesFile(fname) and + (fname.endswith(".genshi") or fname.endswith(".newtxt"))): try: loader.load(sdata.name, cls=genshi.template.NewTextTemplate) -- cgit v1.2.3-1-g7c22 From 77d73a68fde9f45eac91f03aba0586bc48dba6e0 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 23 Apr 2012 11:52:44 -0400 Subject: 9cd1a68... cleaner handling of line-wrapping in bcfg2-lint --- src/lib/Server/Lint/__init__.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py index ff5293a81..4d6df8c8f 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Server/Lint/__init__.py @@ -132,11 +132,12 @@ class ErrorHandler (object): termsize = get_termsize() if termsize is not None: - self._wrapper = textwrap.TextWrapper(initial_indent=" ", - subsequent_indent=" ", - width=termsize[0]) + twrap = textwrap.TextWrapper(initial_indent=" ", + subsequent_indent=" ", + width=termsize[0]) + self._wrapper = twrap.wrap else: - self._wrapper = None + self._wrapper = lambda s: [s] self._handlers = {} if config is not None: @@ -191,13 +192,10 @@ class ErrorHandler (object): rawlines = msg.splitlines() firstline = True for rawline in rawlines: - if self._wrapper: - lines = self._wrapper.wrap(rawline) - else: - lines = [rawline] + lines = self._wrapper(rawline) for line in lines: if firstline: - logfunc("%s%s" % (prefix, line.lstrip())) + logfunc(prefix + line.lstrip()) firstline = False else: logfunc(line) -- cgit v1.2.3-1-g7c22 From 16fbc634f19a03f33dcb9b07b962a5e4c62445e4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 23 Apr 2012 14:52:51 -0400 Subject: fixed confusing error message --- src/lib/Server/Core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 38f854f66..d56727e1c 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -387,9 +387,9 @@ class Core(Component): return lxml.etree.tostring(resp, encoding='UTF-8', xml_declaration=True) except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - warning = 'Client metadata resolution error for %s; check server log' % address[0] + warning = 'Client metadata resolution error for %s' % address[0] self.logger.warning(warning) - raise xmlrpclib.Fault(6, warning) + raise xmlrpclib.Fault(6, warning + "; check server log") except Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError: err_msg = 'Metadata system runtime failure' self.logger.error(err_msg) -- cgit v1.2.3-1-g7c22 From 771df2ac489131411a987f3e7e686cfc34c39bc1 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 26 Apr 2012 13:34:05 -0400 Subject: better way to divine the path to python packages --- misc/bcfg2.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index c727de948..8fbc0c7e6 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -46,7 +46,7 @@ BuildRequires: libsane1 BuildRequires: python-sphinx10 # the python-sphinx10 package doesn't set sys.path correctly, so we # have to do it for them -%define pythonpath %(find /usr/lib/python%{py_ver}/site-packages -name Sphinx*.egg) +%define pythonpath %(find %{python_sitelib} -name Sphinx*.egg) %else BuildRequires: python-sphinx >= 0.6 %endif -- cgit v1.2.3-1-g7c22 From b74454be62e19ee806a508c55030ba67b4279c92 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 27 Apr 2012 13:34:17 -0400 Subject: make %ghost work on RHEL 5, Mandriva --- misc/bcfg2.spec | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 8fbc0c7e6..24328f8de 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -264,6 +264,10 @@ mv build/dtd %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version}/ %{__mkdir_p} %{buildroot}%{_localstatedir}/cache/bcfg2 +# mandriva and RHEL 5 cannot handle %ghost without the file existing, +# so let's touch a bunch of empty config files +touch %{buildroot}%{_sysconfdir}/bcfg2.conf %{_sysconfdir}/bcfg2-web.conf + %clean [ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} || exit 2 @@ -286,10 +290,7 @@ mv build/dtd %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version}/ %{_sbindir}/rcbcfg2 %config(noreplace) /var/adm/fillup-templates/sysconfig.bcfg2 %endif -%if 0%{?mandriva_version} == 0 -# mandriva (on OBS, at least) can't handle %ghost -%ghost %attr(0600,root,root) %{_sysconfdir}/bcfg2.conf -%endif +%ghost %config(noreplace,missingok) %attr(0600,root,root) %{_sysconfdir}/bcfg2.conf %post -n bcfg2-server # enable daemon on first install only (not on update). @@ -364,9 +365,7 @@ fi %{_mandir}/man5/bcfg2-lint.conf.5* %{_mandir}/man8/*.8* %dir %{_prefix}/lib/bcfg2 -%if 0%{?mandriva_version} == 0 -%ghost %attr(0600,root,root) %{_sysconfdir}/bcfg2.conf -%endif +%ghost %config(noreplace,missingok) %attr(0600,root,root) %{_sysconfdir}/bcfg2.conf %files -n bcfg2-doc %defattr(-,root,root,-) @@ -379,9 +378,7 @@ fi %dir %{apache_conf} %dir %{apache_conf}/conf.d %config(noreplace) %{apache_conf}/conf.d/wsgi_bcfg2.conf -%if 0%{?mandriva_version} == 0 -%ghost %attr(0600,root,root) %{_sysconfdir}/bcfg2-web.conf -%endif +%ghost %config(noreplace,missingok) %attr(0640,root,apache) %{_sysconfdir}/bcfg2-web.conf %changelog * Sat Feb 18 2012 Christopher 'm4z' Holm <686f6c6d@googlemail.com> 1.2.1 -- cgit v1.2.3-1-g7c22 From 25cd0596ddc4bb4d81522f5b4e5910c368c55930 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 27 Apr 2012 14:52:27 -0400 Subject: fixed missing %{buildroot} --- misc/bcfg2.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 24328f8de..9a2f6f879 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -266,7 +266,7 @@ mv build/dtd %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version}/ # mandriva and RHEL 5 cannot handle %ghost without the file existing, # so let's touch a bunch of empty config files -touch %{buildroot}%{_sysconfdir}/bcfg2.conf %{_sysconfdir}/bcfg2-web.conf +touch %{buildroot}%{_sysconfdir}/bcfg2.conf %{buildroot}%{_sysconfdir}/bcfg2-web.conf %clean [ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} || exit 2 -- cgit v1.2.3-1-g7c22 From 49b9f002424aab1fb6a38b4c01413b6947c4dce4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 1 May 2012 10:43:46 -0400 Subject: better error message from bcfg2-info on no matching generator --- src/lib/Server/Core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index d56727e1c..82072a538 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -283,7 +283,8 @@ class Core(Component): if len(g2list) == 1: return g2list[0].HandleEntry(entry, metadata) entry.set('failure', 'no matching generator') - raise PluginExecutionError(entry.tag, entry.get('name')) + raise PluginExecutionError("No matching generator: %s:%s" % + (entry.tag, entry.get('name'))) def BuildConfiguration(self, client): """Build configuration for clients.""" -- cgit v1.2.3-1-g7c22 From 87c333f8417a967a82e3eab437a1b73c998680ea Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 2 May 2012 14:46:14 -0400 Subject: fixed total think-o in ProxyError --- src/lib/Proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Proxy.py b/src/lib/Proxy.py index 977a49228..fe2b18de8 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Proxy.py @@ -64,7 +64,7 @@ class ProxyError(Exception): err.faultCode) else: msg = str(err) - Exception(self, msg) + Exception.__init__(self, msg) class CertificateError(Exception): def __init__(self, commonName): -- cgit v1.2.3-1-g7c22 From 9e08467eba7f15af0d7b38adea014600ce34ad6b Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 13 Apr 2012 14:00:15 -0400 Subject: rhel doesn't recognize Recommends tag either --- misc/bcfg2.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 9a2f6f879..f85169eec 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -57,8 +57,8 @@ Requires: python-lxml >= 0.9 # iff we switch to the redhat one, this might not be needed anymore. Requires: redhat-lsb %endif -%if 0%{?fedora} == 0 -# fedora 15 and 16 (and possibly other distros) do not know this tag. +%if "%{_vendor}" != "redhat" +# fedora and rhel (and possibly other distros) do not know this tag. Recommends: cron %endif -- cgit v1.2.3-1-g7c22 From 98563b90b2dbaa0dae5f62f6d58c4663fb537ddf Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 18 Apr 2012 13:07:35 -0400 Subject: better error reporting --- src/lib/Server/Plugins/Cfg.py | 62 ++++++++++++++++++++++---------------- src/lib/Server/Plugins/Metadata.py | 3 +- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index ed08d454d..9ec39e108 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -120,8 +120,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): base_files = [matching.index(m) for m in matching if not m.specific.delta] if not base_files: - logger.error("No base file found for %s" % entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "No base file found for %s" % entry.get('name') + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) base = min(base_files) used = matching[:base + 1] used.reverse() @@ -133,15 +134,16 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): basefile = used.pop(0) if entry.get('perms').lower() == 'inherit': # use on-disk permissions - fname = "%s/%s" % (self.path, entry.get('name')) + fname = os.path.join(self.path, entry.get('name')) entry.set('perms', str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): if not have_genshi: - logger.error("Cfg: Genshi is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Genshi is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: template_cls = NewTextTemplate loader = TemplateLoader() @@ -159,13 +161,15 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if data == '': entry.set('empty', 'true') except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: genshi exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: genshi exception (%s): %s" % (entry.get("name"), + sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) elif basefile.name.endswith(".cheetah"): if not have_cheetah: - logger.error("Cfg: Cheetah is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Cheetah is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: fname = entry.get('realname', entry.get('name')) s = {'useStackFrames': False} @@ -178,9 +182,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if data == '': entry.set('empty', 'true') except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: cheetah exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: cheetah exception (%s): %s" % (entry.get("name"), + sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: data = basefile.data for delta in used: @@ -191,17 +196,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): try: entry.text = u_str(data, self.encoding) except UnicodeDecodeError: - e = sys.exc_info()[1] - logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) + msg = "Failed to decode %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: - e = sys.exc_info()[1] - logger.error("Error in specification for %s" % entry.get('name')) - logger.error("%s" % e) + msg = "Error in specification for %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) logger.error("You need to specify base64 encoding for %s." % entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if entry.text in ['', None]: entry.set('empty', 'true') @@ -228,16 +234,20 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if 'text' in new_entry: name = self.build_filename(specific) if os.path.exists("%s.genshi" % name): - logger.error("Cfg: Unable to pull data for genshi types") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Unable to pull data for genshi types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) elif os.path.exists("%s.cheetah" % name): - logger.error("Cfg: Unable to pull data for cheetah types") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Unable to pull data for cheetah types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: etext = new_entry['text'].encode(self.encoding) except: - logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Cannot encode content of %s as %s" % (name, + self.encoding) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) open(name, 'w').write(etext) self.debug_log("Wrote file %s" % name, flag=log) badattr = [attr for attr in ['owner', 'group', 'perms'] diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py index f837b1305..cc1f78456 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Server/Plugins/Metadata.py @@ -819,7 +819,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, try: groups_tree.xinclude() except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % dest) + self.logger.error("Failed to process XInclude for file %s: %s" % + (dest, sys.exc_info()[1])) groups = groups_tree.getroot() categories = {'default': 'grey83'} viz_str = "" -- cgit v1.2.3-1-g7c22 From ddff6db72976793db29de31f082a6e6fa3814ac5 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 12 Apr 2012 12:53:32 -0400 Subject: improved error reporting in a few places --- src/lib/Server/Core.py | 6 ++++-- src/lib/Server/Plugin.py | 2 +- src/lib/Server/Plugins/TGenshi.py | 6 ++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index 82072a538..8f02e0a40 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -7,6 +7,8 @@ import select import sys import threading import time +from traceback import format_exc + try: import lxml.etree except ImportError: @@ -240,12 +242,12 @@ class Core(Component): self.Bind(entry, metadata) except PluginExecutionError, exc: if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) + entry.set('failure', 'bind error: %s' % format_exc()) logger.error("Failed to bind entry: %s %s" % \ (entry.tag, entry.get('name'))) except Exception, exc: if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) + entry.set('failure', 'bind error: %s' % format_exc()) logger.error("Unexpected failure in BindStructure: %s %s" \ % (entry.tag, entry.get('name')), exc_info=1) diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 692aa8a81..d882c475f 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -847,7 +847,7 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): self._matches(entry, metadata, src.cache[1][entry.tag]))] if len(matching) == 0: - raise PluginExecutionError + raise PluginExecutionError('No matching source for entry when retrieving attributes for %s(%s)' % (entry.tag, entry.attrib.get('name'))) elif len(matching) == 1: index = 0 else: diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Server/Plugins/TGenshi.py index bc5e00400..072f5cd7b 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Server/Plugins/TGenshi.py @@ -115,12 +115,10 @@ class TemplateFile: if entry.text == '': entry.set('empty', 'true') except TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) + logger.exception('Genshi template error') raise Bcfg2.Server.Plugin.PluginExecutionError except AttributeError: - err = sys.exc_info()[1] - logger.error('Genshi template loading error: %s' % err) + logger.exception('Genshi template loading error') raise Bcfg2.Server.Plugin.PluginExecutionError -- cgit v1.2.3-1-g7c22 From df0261269dfb12308e56bdcd67ed664998c834dd Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 12 Apr 2012 12:49:52 -0400 Subject: e89e879... bcfg2-test requires nose --- misc/bcfg2.spec | 1 + redhat/bcfg2.spec.in | 1 + setup.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index f85169eec..9fd4c5a13 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -51,6 +51,7 @@ BuildRequires: python-sphinx10 BuildRequires: python-sphinx >= 0.6 %endif +Requires: python-nose Requires: python-lxml >= 0.9 %if 0%{?rhel_version} # the debian init script needs redhat-lsb. diff --git a/redhat/bcfg2.spec.in b/redhat/bcfg2.spec.in index 02420fe56..1b8ab5285 100644 --- a/redhat/bcfg2.spec.in +++ b/redhat/bcfg2.spec.in @@ -44,6 +44,7 @@ Requires: python-elementtree %else if "%{py_ver}" < "2.5" Requires: python-lxml %endif +Requires: python-nose Requires: initscripts Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig diff --git a/setup.py b/setup.py index 69c67c0b1..179981ca3 100755 --- a/setup.py +++ b/setup.py @@ -125,7 +125,7 @@ py3lib = 'src/lib/Bcfg2Py3Incompat.py' if sys.hexversion < 0x03000000 and os.path.exists(py3lib): os.remove(py3lib) -inst_reqs = ["lxml"] +inst_reqs = ["lxml", "nose"] if need_m2crypto: inst_reqs.append("M2Crypto") -- cgit v1.2.3-1-g7c22 From 78c498c48687feeaf19a6cdca18cc266aa17130f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 4 May 2012 09:20:34 -0400 Subject: 5ea25b9... improved some error messages; only setup pulp connection if there are pulp sources listed --- src/lib/Server/Plugins/Packages/Yum.py | 39 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py index e13b28251..1237c0e9f 100644 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ b/src/lib/Server/Plugins/Packages/Yum.py @@ -52,23 +52,23 @@ PULPCONFIG = None def _setup_pulp(config): global PULPSERVER, PULPCONFIG if not has_pulp: - logger.error("Packages: Cannot create Pulp collection: Pulp libraries not " - "found") - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Packages: Cannot create Pulp collection: Pulp libraries not found" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) if PULPSERVER is None: try: username = config.get("pulp", "username") password = config.get("pulp", "password") except ConfigParser.NoSectionError: - logger.error("Packages: No [pulp] section found in Packages/packages.conf") - raise Bcfg2.Server.Plugin.PluginInitError + msg = "Packages: No [pulp] section found in Packages/packages.conf" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) except ConfigParser.NoOptionError: - err = sys.exc_info()[1] - logger.error("Packages: Required option not found in " - "Packages/packages.conf: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError - + msg = "Packages: Required option not found in Packages/packages.conf: %s" % sys.exc_info()[1] + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginInitError(msg) + PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server @@ -113,9 +113,17 @@ class YumCollection(Collection): self.helper = self.config.get("yum", "helper", default="/usr/sbin/bcfg2-yum-helper") - if has_pulp: + if has_pulp and self.has_pulp_sources: _setup_pulp(self.config) + @property + def has_pulp_sources(self): + """ see if there are any pulp sources to handle """ + for source in self.sources: + if source.pulp_id: + return True + return False + def write_config(self): if not os.path.exists(self.cfgfile): yumconf = self.get_config(raw=True) @@ -235,14 +243,7 @@ class YumCollection(Collection): if keypkg is not None: independent.append(keypkg) - # see if there are any pulp sources to handle - has_pulp_sources = False - for source in self.sources: - if source.pulp_id: - has_pulp_sources = True - break - - if has_pulp_sources: + if self.has_pulp_sources: consumerapi = ConsumerAPI() consumer = self._get_pulp_consumer(consumerapi=consumerapi) if consumer is None: -- cgit v1.2.3-1-g7c22 From 7f64eae09fc93c37aaf90c511e7db5a56f0ba08a Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 4 May 2012 10:55:36 -0400 Subject: fixed repo name generation --- src/lib/Server/Plugins/Packages/Source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/Server/Plugins/Packages/Source.py b/src/lib/Server/Plugins/Packages/Source.py index f13267da1..5b0aa4213 100644 --- a/src/lib/Server/Plugins/Packages/Source.py +++ b/src/lib/Server/Plugins/Packages/Source.py @@ -33,7 +33,7 @@ class SourceInitError(Exception): class Source(Bcfg2.Server.Plugin.Debuggable): mrepo_re = re.compile(r'/RPMS\.([^/]+)') pulprepo_re = re.compile(r'pulp/repos/([^/]+)') - genericrepo_re = re.compile(r'https?://[^/]+/(.+?)/?$') + genericrepo_re = re.compile('https?://.*?/([^/]+)/?$') basegroups = [] def __init__(self, basepath, xsource, config): @@ -135,7 +135,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): def get_repo_name(self, url_map): # try to find a sensible name for a repo - if url_map['components']: + if 'components' in url_map and url_map['components']: # use the first component as the name rname = url_map['components'][0] else: @@ -145,6 +145,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): self.genericrepo_re): match = repo_re.search(url_map['url']) if match: + name = match.group(1) break if name is None: # couldn't figure out the name from the URL or URL map -- cgit v1.2.3-1-g7c22 From 8bc23e6ed36a435cfbab927c64487115efa33bf2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Fri, 4 May 2012 10:56:29 -0400 Subject: better support for re-reading packages.conf (and SimpleConfigs in general) --- src/lib/Server/Plugin.py | 12 ++++++++-- src/lib/Server/Plugins/Packages/PackagesSources.py | 8 ++++--- src/lib/Server/Plugins/Packages/Yum.py | 27 ++++++++++++++-------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index d882c475f..9b3c5814f 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -5,7 +5,6 @@ import copy import logging import lxml.etree import os -import os.path import pickle import posixpath import re @@ -1190,6 +1189,7 @@ class SimpleConfig(FileBacked, filename = os.path.join(plugin.data, plugin.name.lower() + ".conf") self.plugin = plugin self.fam = self.plugin.core.fam + self.read_files = set() Bcfg2.Server.Plugin.FileBacked.__init__(self, filename) ConfigParser.SafeConfigParser.__init__(self) @@ -1201,7 +1201,7 @@ class SimpleConfig(FileBacked, """ Build local data structures """ for section in self.sections(): self.remove_section(section) - self.read(self.name) + self.read_files.update(self.read(self.name)) def get(self, section, option, **kwargs): """ convenience method for getting config items """ @@ -1233,3 +1233,11 @@ class SimpleConfig(FileBacked, return default else: raise + + @property + def loaded(self): + if os.path.exists(self.name): + return self.name in self.read_files + else: + return True + diff --git a/src/lib/Server/Plugins/Packages/PackagesSources.py b/src/lib/Server/Plugins/Packages/PackagesSources.py index da79c00e9..e03303d30 100644 --- a/src/lib/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Server/Plugins/Packages/PackagesSources.py @@ -35,7 +35,6 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, (self.cachepath, err)) self.pkg_obj = packages self.parsed = set() - self.loaded = False def toggle_debug(self): Bcfg2.Server.Plugin.Debuggable.toggle_debug(self) @@ -47,10 +46,13 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, if event.filename != self.name: self.parsed.add(os.path.basename(event.filename)) - if sorted(list(self.parsed)) == sorted(self.extras): + if self.config.loaded and self.loaded: self.logger.info("Reloading Packages plugin") self.pkg_obj.Reload() - self.loaded = True + + @property + def loaded(self): + return sorted(list(self.parsed)) == sorted(self.extras) def Index(self): Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py index 1237c0e9f..be5a68aa1 100644 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ b/src/lib/Server/Plugins/Packages/Yum.py @@ -15,6 +15,7 @@ from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url +from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig logger = logging.getLogger(__name__) @@ -91,12 +92,9 @@ class YumCollection(Collection): self.keypath = os.path.join(self.basepath, "keys") if len(sources): - config = sources[0].config - self.use_yum = has_yum and config.getboolean("yum", - "use_yum_libraries", - default=False) + self.config = sources[0].config else: - self.use_yum = False + self.config = PackageConfig('Packages') if self.use_yum: self.cachefile = os.path.join(self.cachepath, @@ -110,12 +108,19 @@ class YumCollection(Collection): self.cfgfile = os.path.join(self.configdir, "%s-yum.conf" % self.cachekey) self.write_config() - - self.helper = self.config.get("yum", "helper", - default="/usr/sbin/bcfg2-yum-helper") if has_pulp and self.has_pulp_sources: _setup_pulp(self.config) + @property + def helper(self): + return self.config.get("yum", "helper", + default="/usr/sbin/bcfg2-yum-helper") + + @property + def use_yum(self): + return has_yum and self.config.getboolean("yum", "use_yum_libraries", + default=False) + @property def has_pulp_sources(self): """ see if there are any pulp sources to handle """ @@ -492,8 +497,10 @@ class YumSource(Source): self.needed_paths = set() self.file_to_arch = dict() - self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries", - default=False) + @property + def use_yum(self): + return has_yum and self.config.getboolean("yum", "use_yum_libraries", + default=False) def save_state(self): if not self.use_yum: -- cgit v1.2.3-1-g7c22