From 85d07faa0cf7a2cb264db8e88ebc4bc2fbed3db1 Mon Sep 17 00:00:00 2001 From: Matt Kemp Date: Thu, 12 Jun 2014 17:27:34 -0500 Subject: Fix email reporting bug This fixes a subtle bug by enforcing that it's a tuple of 2-tuples rather than just a single 2-tuple. --- src/lib/Bcfg2/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 834b04d36..2c5466abb 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -125,7 +125,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): # set up basic defaults. this lets manage.py work in all cases read_config(quiet=True) -ADMINS = (('Root', 'root')) +ADMINS = (('Root', 'root'),) MANAGERS = ADMINS # Language code for this installation. All choices can be found here: -- cgit v1.2.3-1-g7c22 From d225b15584db0baffc8dd05c1f437883f8d98a64 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 5 Jul 2014 02:05:12 +0800 Subject: debsums: read output from stderr instead of stdout --- src/lib/Bcfg2/Client/Tools/APT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 39816403a..0a8fe387f 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -91,7 +91,7 @@ class APT(Bcfg2.Client.Tools.Tool): def VerifyDebsums(self, entry, modlist): output = \ self.cmd.run("%s -as %s" % - (self.debsums, entry.get('name'))).stdout.splitlines() + (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: self.logger.info("Package %s has no md5sums. Cannot verify" % \ entry.get('name')) -- cgit v1.2.3-1-g7c22 From c1e30f4bc8a03a9be535bf126e993ff45e8ab382 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 10:49:44 -0400 Subject: SYSV: Implement downloading and installing SYSV packages from HTTP pkgadd has different syntax for different sources (datastream and file system format) which makes using a single pkgtool variable difficult. Also, SYSV packages in datastream format don't necessarily have uniform names. Therefore, use the existing 'simplename' attribute to specify the datastream file name. --- src/lib/Bcfg2/Client/Tools/SYSV.py | 48 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index aca7d593c..c17c3b712 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,6 +4,9 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML +import urllib +import copy + # pylint: disable=C0103 noask = ''' @@ -37,6 +40,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): # noaskfile needs to live beyond __init__ otherwise file is removed self.noaskfile = tempfile.NamedTemporaryFile() self.noaskname = self.noaskfile.name + # for any pkg files downloaded + self.tmpfiles = [] try: self.noaskfile.write(noask) # flush admin file contents to disk @@ -46,6 +51,45 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + def pkgmogrify(self, pkg): + """ Take a pkg object, check for a 'simplename' attribute. If found, + return a modified pkg object that points to to a temporary file + """ + if pkg.get('simplename'): + self.logger.debug("Pkgmogrifying %s because simplename %s found" % + (pkg.get('name'), pkg.get('simplename'))) + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + self.logger.debug("URL: %s/%s" % + (pkg.get('url'), pkg.get('simplename'))) + urllib.urlretrieve("%s/%s" % + (pkg.get('url'), pkg.get('simplename')), + tmpfile.name) + newpkg = copy.copy(pkg) + newpkg.set('url', tmpfile.name) + return newpkg + return pkg + + def Install(self, packages, states): + for pkg in packages: + pkg = self.pkgmogrify(pkg) + if self.VerifyPackage(pkg, []): + self.logger.info("Forcing state to true for pkg %s" % + (pkg.get('name'))) + states[pkg] = True + else: + self.logger.info("Installing pkg %s version %s" % + (pkg.get('name'), pkg.get('version'))) + + if self.cmd.run(self._get_package_command([pkg])): + states[pkg] = True + else: + self.logger.error("Failed to install package %s" % + pkg.get('name')) + + self.RefreshPackages() + self.modified.extend(entry for entry in packages if states[entry]) + def RefreshPackages(self): """Refresh memory hashes of packages.""" self.installed = {} @@ -80,8 +124,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Package %s not installed" % entry.get("name")) else: - if (self.setup['quick'] or - entry.attrib.get('verify', 'true') == 'false'): + if self.setup['quick'] or \ + entry.attrib.get('verify', 'true') == 'false': return True rv = self.cmd.run("/usr/sbin/pkgchk -n %s" % entry.get('name')) if rv.success: -- cgit v1.2.3-1-g7c22 From a9fdf3d7e73c4aabc0b5e5710617c0788b9311c8 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 16:05:22 -0400 Subject: Add urlretrieve to Compat and document --- src/lib/Bcfg2/Client/Tools/SYSV.py | 4 ++-- src/lib/Bcfg2/Compat.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index c17c3b712..2d835a8ae 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,8 +4,8 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML -import urllib import copy +from Bcfg2.Compat import urlretrieve # pylint: disable=C0103 @@ -62,7 +62,7 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.tmpfiles.append(tmpfile) self.logger.debug("URL: %s/%s" % (pkg.get('url'), pkg.get('simplename'))) - urllib.urlretrieve("%s/%s" % + urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), tmpfile.name) newpkg = copy.copy(pkg) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 049236e03..88956e900 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -20,6 +20,7 @@ except ImportError: # urllib imports try: from urllib import quote_plus + from urllib import urlretrieve from urlparse import urljoin, urlparse from urllib2 import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ @@ -27,7 +28,8 @@ try: except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, urlopen + HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: -- cgit v1.2.3-1-g7c22 From 570fa88a17d87d4470ff54fe7f340d03edeefd52 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 16:49:31 -0400 Subject: Fix indent. --- src/lib/Bcfg2/Compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index 88956e900..b8a75a0c5 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -29,7 +29,7 @@ except ImportError: from urllib.parse import urljoin, urlparse, quote_plus from urllib.request import HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, build_opener, install_opener, \ - urlopen, urlretrieve + urlopen, urlretrieve from urllib.error import HTTPError, URLError try: -- cgit v1.2.3-1-g7c22 From f84558cbad0d35ef557f8b767b00f2c14f8907a1 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Mon, 14 Jul 2014 16:52:37 -0400 Subject: Fix indenting --- src/lib/Bcfg2/Client/Tools/SYSV.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 2d835a8ae..811711723 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -60,11 +60,10 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): (pkg.get('name'), pkg.get('simplename'))) tmpfile = tempfile.NamedTemporaryFile() self.tmpfiles.append(tmpfile) - self.logger.debug("URL: %s/%s" % - (pkg.get('url'), pkg.get('simplename'))) - urlretrieve("%s/%s" % - (pkg.get('url'), pkg.get('simplename')), - tmpfile.name) + self.logger.debug("URL: %s/%s" % (pkg.get('url'), + pkg.get('simplename'))) + urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), + tmpfile.name) newpkg = copy.copy(pkg) newpkg.set('url', tmpfile.name) return newpkg -- cgit v1.2.3-1-g7c22 From cb06dcf5fa4d33519d84528088cdd8c7ae395f50 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 15 Jul 2014 15:37:48 -0500 Subject: Reporting: Fix prune items import and display Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 2 +- src/lib/Bcfg2/Reporting/templates/config_items/item.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 2530d2b2b..0bb3111ae 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -167,7 +167,7 @@ class DjangoORM(StorageBase): # TODO - vcs output act_dict['detail_type'] = PathEntry.DETAIL_UNUSED if path_type == 'directory' and entry.get('prune', 'false') == 'true': - unpruned_elist = [e.get('path') for e in entry.findall('Prune')] + unpruned_elist = [e.get('name') for e in entry.findall('Prune')] if unpruned_elist: act_dict['detail_type'] = PathEntry.DETAIL_PRUNED act_dict['details'] = "\n".join(unpruned_elist) diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index b03d48045..c6e6df020 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -107,7 +107,7 @@ div.entry_list h3 { {{ item.details|syntaxhilight }} {% else %} - {{ item.details }} + {{ item.details|linebreaks }} {% endif %} {% endif %} -- cgit v1.2.3-1-g7c22 From a4bf68d3c32e6654f53d8e151104850a54ad1e41 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 10:06:25 -0400 Subject: Implement _get_package_command and append _sysv_pkg_path attribute Instead of doing a partially complete Install() method for SYSV, implements a custom _get_package_command that will use the _sysv_pkg_path attribute added by the pkgmogrify call. This will allow the installs to complete. Unfortunately, the single-pass install will still fail if there are any packages with an http:// URL. The pkgadd invocation for 'device' sources doesn't take multiple packages and the 'datastream' invocation doesn't handle packages with an HTTP URL. Finally, there is no reliable standard naming convention for SYSV datastream files, so the simplename attribute is re-used. There is a known issue with this patch - if any packages specified in the PackageList have an http url, the single-pass install will produce an error like: Trying single pass package install for pkgtype sysv pkgadd: ERROR: Failure occurred with http(s) negotiation: <'Peername' doesn't match 'host' or no matching entry> pkgadd: ERROR: unable to download package datastream from . Single Pass Failed because the command that results isn't valid syntax for pkgadd. A workaround would be to add code to skip the single-pass install if any packages had the simplename attribute, or by checking the url for the presence of 'http'. I'm not sure if that should be fixed or if this is reasonable in this case. --- src/lib/Bcfg2/Client/Tools/SYSV.py | 60 +++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 811711723..4f3eea668 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -50,44 +50,38 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.pkgtool[1]) except: # pylint: disable=W0702 self.pkgtool = (self.pkgtool[0] % "", self.pkgtool[1]) + self.origpkgtool = self.pkgtool - def pkgmogrify(self, pkg): - """ Take a pkg object, check for a 'simplename' attribute. If found, - return a modified pkg object that points to to a temporary file + def pkgmogrify(self, packages): + """ Take a list of pkg objects, check for a 'simplename' attribute. + If present, insert a _sysv_pkg_path attribute to the package and + download the datastream format SYSV package to a temporary file. """ - if pkg.get('simplename'): - self.logger.debug("Pkgmogrifying %s because simplename %s found" % - (pkg.get('name'), pkg.get('simplename'))) - tmpfile = tempfile.NamedTemporaryFile() - self.tmpfiles.append(tmpfile) - self.logger.debug("URL: %s/%s" % (pkg.get('url'), - pkg.get('simplename'))) - urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), - tmpfile.name) - newpkg = copy.copy(pkg) - newpkg.set('url', tmpfile.name) - return newpkg - return pkg - - def Install(self, packages, states): for pkg in packages: - pkg = self.pkgmogrify(pkg) - if self.VerifyPackage(pkg, []): - self.logger.info("Forcing state to true for pkg %s" % - (pkg.get('name'))) - states[pkg] = True - else: - self.logger.info("Installing pkg %s version %s" % - (pkg.get('name'), pkg.get('version'))) + if pkg.get('simplename'): + tmpfile = tempfile.NamedTemporaryFile() + self.tmpfiles.append(tmpfile) + urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), + tmpfile.name) + pkg.set('_sysv_pkg_path', tmpfile.name) - if self.cmd.run(self._get_package_command([pkg])): - states[pkg] = True - else: - self.logger.error("Failed to install package %s" % - pkg.get('name')) + def _get_package_command(self, packages): + """Override the default _get_package_command, replacing the attribute + 'url' if '_sysv_pkg_path' if necessary in the returned command + string""" + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool + + pkgcmd = super(SYSV, self)._get_package_command(packages) + self.logger.debug("Calling install command: %s" % pkgcmd) + return pkgcmd - self.RefreshPackages() - self.modified.extend(entry for entry in packages if states[entry]) + def Install(self, packages, states): + self.pkgmogrify(packages) + super(SYSV, self).Install(packages, states) def RefreshPackages(self): """Refresh memory hashes of packages.""" -- cgit v1.2.3-1-g7c22 From d5be1d7f9c9456ee0c6f43b3d05902b200f1dd54 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 10:49:48 -0400 Subject: Remove unused import of copy --- src/lib/Bcfg2/Client/Tools/SYSV.py | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 4f3eea668..48d67c4f1 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -4,7 +4,6 @@ import tempfile from Bcfg2.Compat import any # pylint: disable=W0622 import Bcfg2.Client.Tools import Bcfg2.Client.XML -import copy from Bcfg2.Compat import urlretrieve -- cgit v1.2.3-1-g7c22 From b40dc8df20d8b1566da45ccea0620c60de476222 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Wed, 16 Jul 2014 15:21:39 -0400 Subject: Check for origpkgtool attribute to prevent things that subclass SYSV from breaking --- src/lib/Bcfg2/Client/Tools/SYSV.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 48d67c4f1..712dcce32 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -67,12 +67,14 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): def _get_package_command(self, packages): """Override the default _get_package_command, replacing the attribute 'url' if '_sysv_pkg_path' if necessary in the returned command - string""" - if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): - self.pkgtool = (self.pkgtool[0], ('%s %s', - ['_sysv_pkg_path', 'name'])) - else: - self.pkgtool = self.origpkgtool + string + """ + if hasattr(self, 'origpkgtool'): + if len(packages) == 1 and '_sysv_pkg_path' in packages[0].keys(): + self.pkgtool = (self.pkgtool[0], ('%s %s', + ['_sysv_pkg_path', 'name'])) + else: + self.pkgtool = self.origpkgtool pkgcmd = super(SYSV, self)._get_package_command(packages) self.logger.debug("Calling install command: %s" % pkgcmd) -- cgit v1.2.3-1-g7c22 From a04304db76ad1ce8e3658c14e8281239210da1dd Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 18 Jul 2014 08:05:52 -0500 Subject: POSIXUsers.py: Allow supplementary group = primary Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Tools/POSIXUsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 6d18cd176..bbae7abcc 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -146,7 +146,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): """ Get a list of supplmentary groups that the user in the given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() - if entry.get("name") in g[3] and g[0] != entry.get("group") + if entry.get("name") in g[3] and self._in_managed_range('POSIXGroup', g[2])] def VerifyPOSIXUser(self, entry, _): -- cgit v1.2.3-1-g7c22 From 6924046e00396763939fd2006ec13db3cbad3577 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 27 Jul 2014 00:38:27 +0200 Subject: Server/Admin: fatal errors should go to stderr If an error occurs, that leads to an termination of the process, this error should be printed to stderr. --- src/lib/Bcfg2/Server/Admin/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index 8f12a940e..ef5b2a08c 100644 --- a/src/lib/Bcfg2/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -58,7 +58,7 @@ class Mode(object): def errExit(self, emsg): """ exit with an error """ - print(emsg) + sys.stderr.write('%s\n' % emsg) raise SystemExit(1) def load_stats(self, client): -- cgit v1.2.3-1-g7c22 From d801c47773ce108de79f008f4a17b774476b0549 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 28 May 2014 03:24:35 +0200 Subject: Packages/Apt: Essential could be "no" The "Essential" field in the package control fields could be "yes" or "no". Only yes sould define the package as essential. The value "no" sould be handled same as not having the field at all. --- src/lib/Bcfg2/Server/Plugins/Packages/Apt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 27a725f23..4a78f846f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -107,7 +107,8 @@ class AptSource(Source): self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] elif words[0] == 'Essential' and self.essential: - self.essentialpkgs.add(pkgname) + if words[1].strip() == 'yes': + self.essentialpkgs.add(pkgname) elif words[0] in depfnames: vindex = 0 for dep in words[1].split(','): -- cgit v1.2.3-1-g7c22 From a516f116501737a86e2eaf99a631727d4be9ecd7 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Thu, 14 Aug 2014 08:11:46 -0500 Subject: Reporting: Remove duplicate method Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/models.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index fc9523067..71fa66086 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -715,9 +715,6 @@ class PathEntry(SuccessEntry): def has_detail(self): return self.detail_type != PathEntry.DETAIL_UNUSED - def is_sensitive(self): - return self.detail_type == PathEntry.DETAIL_SENSITIVE - def is_diff(self): return self.detail_type == PathEntry.DETAIL_DIFF -- cgit v1.2.3-1-g7c22 From d5b3c70dccd23f687c8194f6ea1807092370e586 Mon Sep 17 00:00:00 2001 From: Nathan Olla Date: Tue, 19 Aug 2014 17:58:20 -0400 Subject: Log when downloading packages via HTTP --- src/lib/Bcfg2/Client/Tools/SYSV.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index 712dcce32..a29b49efa 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -60,6 +60,8 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): if pkg.get('simplename'): tmpfile = tempfile.NamedTemporaryFile() self.tmpfiles.append(tmpfile) + self.logger.info("Downloading %s%s to %s" % (pkg.get('url'), + pkg.get('simplename'), tmpfile.name)) urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), tmpfile.name) pkg.set('_sysv_pkg_path', tmpfile.name) -- cgit v1.2.3-1-g7c22 From a42933a1ddfde4f9164e0e0818c559f8f7c9a5a1 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 5 Sep 2014 07:54:48 -0500 Subject: Version bump to 1.3.5 Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Reporting/templates/base.html | 2 +- src/lib/Bcfg2/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index ef6799c2b..a367d8ccb 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5
diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index ae82724f3..61ba7a405 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.4" +__version__ = "1.3.5" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 -- cgit v1.2.3-1-g7c22 From 43d65f4bcea9db7abc03792839ad07a51e1af6d3 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 10 Sep 2014 12:13:45 -0400 Subject: Use the older Thread.getName() interface for python 2.4 compat --- src/lib/Bcfg2/Server/Core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 44ba0fee8..f60b68f45 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -78,7 +78,7 @@ def close_db_connection(func): if self._database_available: # pylint: disable=W0212 from django import db self.logger.debug("%s: Closing database connection" % - threading.current_thread().name) + threading.current_thread().getName()) db.close_connection() return rv -- cgit v1.2.3-1-g7c22 From 788ac2be910d0cef1c888c31e9465a26f4e45090 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 10 Sep 2014 12:15:37 -0400 Subject: Use older nested try syntax for finally for python 2.4 compat --- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 0bb3111ae..98226dc4e 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -367,10 +367,11 @@ class DjangoORM(StorageBase): """Import the data into the backend""" try: - self._import_interaction(interaction) - except: - self.logger.error("Failed to import interaction: %s" % - traceback.format_exc().splitlines()[-1]) + try: + self._import_interaction(interaction) + except: + self.logger.error("Failed to import interaction: %s" % + traceback.format_exc().splitlines()[-1]) finally: self.logger.debug("%s: Closing database connection" % self.__class__.__name__) -- cgit v1.2.3-1-g7c22 From 9ffc3559ab4040fe8a02269db7ff94f8e9849af5 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 10 Sep 2014 12:38:54 -0400 Subject: use Bcfg2.Compat to get any() --- src/lib/Bcfg2/Server/Plugins/Probes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 48be1ac26..5d846b4bb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -9,7 +9,7 @@ import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin -from Bcfg2.Compat import unicode # pylint: disable=W0622 +from Bcfg2.Compat import any, unicode # pylint: disable=W0622 try: from django.db import models -- cgit v1.2.3-1-g7c22 From 43ab15a3d25667a7908ee2ef892dbf4ddff9b445 Mon Sep 17 00:00:00 2001 From: Matt Kemp Date: Wed, 1 Oct 2014 17:16:56 +0000 Subject: Attempt to break the pid lock during startup. This commit attempts to break the pidfilelock during startup in cases where the process may have exited without successfully cleaning up the lockfile. It also attempts to grab the lock before opening the context. Also applied to the Collector module, which may have been looking for the wrong exception since it does not rely on a timeout. --- src/lib/Bcfg2/Reporting/Collector.py | 34 ++++++++++++++++++++++++++-------- src/lib/Bcfg2/Server/BuiltinCore.py | 22 +++++++++++++++------- 2 files changed, 41 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 8ca145f16..50bb49f78 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -7,7 +7,7 @@ import traceback import threading # pylint: disable=E0611 -from lockfile import LockFailed, LockTimeout +from lockfile import LockFailed, LockTimeout, AlreadyLocked try: from lockfile.pidlockfile import PIDLockFile from lockfile import Error as PIDFileError @@ -118,25 +118,43 @@ class ReportingCollector(object): if self.setup['daemon']: self.logger.debug("Daemonizing") + self.context.pidfile = PIDLockFile(self.setup['daemon']) try: - self.context.pidfile = PIDLockFile(self.setup['daemon']) - self.context.open() + self.context.pidfile.acquire() except LockFailed: self.logger.error("Failed to daemonize: %s" % sys.exc_info()[1]) self.shutdown() return + except AlreadyLocked: + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except OSError: # No process with locked PID + self.context.pidfile.break_lock() + else: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + self.setup['daemon']) + self.shutdown() + return except LockTimeout: - self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) - self.shutdown() - return + try: # attempt to break the lock + os.kill(self.context.pidfile.read_pid(), 0) + except OSError: # No process with locked PID + self.context.pidfile.break_lock() + else: + self.logger.error("Failed to daemonize: " + "Failed to acquire lock on %s" % + self.setup['daemon']) + self.shutdown() + return except PIDFileError: self.logger.error("Error writing pid file: %s" % traceback.format_exc().splitlines()[-1]) self.shutdown() return + + self.context.open() self.logger.info("Starting daemon") self.transport.start_monitor(self) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 93da767c7..aeac161dd 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -1,5 +1,6 @@ """ The core of the builtin Bcfg2 server. """ +import os import sys import time import socket @@ -84,18 +85,25 @@ class Core(BaseCore): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ try: - self.context.open() - self.logger.info("%s daemonized" % self.name) - return True + self.context.pidfile.acquire() except LockFailed: err = sys.exc_info()[1] self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False except LockTimeout: - err = sys.exc_info()[1] - self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, self.setup['daemon'])) - return False + try: + os.kill(self.context.pidfile.read_pid(), 0) + except OSError: # No process with locked PID + self.context.pidfile.break_lock() + else: + err = sys.exc_info()[1] + self.logger.error("Failed to daemonize %s: Failed to acquire lock " + "on %s" % (self.name, self.setup['daemon'])) + return False + + self.context.open() + self.logger.info("%s daemonized" % self.name) + return True def _run(self): """ Create :attr:`server` to start the server listening. """ -- cgit v1.2.3-1-g7c22 From 7d11af710bbd1a6fdafcab7ad754953c1f8639b1 Mon Sep 17 00:00:00 2001 From: Matt Kemp Date: Wed, 1 Oct 2014 17:41:52 +0000 Subject: pylint fixes. --- src/lib/Bcfg2/Server/BuiltinCore.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index aeac161dd..251c39558 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -93,12 +93,13 @@ class Core(BaseCore): except LockTimeout: try: os.kill(self.context.pidfile.read_pid(), 0) - except OSError: # No process with locked PID + except OSError: # No process with locked PID self.context.pidfile.break_lock() else: err = sys.exc_info()[1] - self.logger.error("Failed to daemonize %s: Failed to acquire lock " - "on %s" % (self.name, self.setup['daemon'])) + self.logger.error("Failed to daemonize %s: Failed to acquire" + "lock on %s" % (self.name, + self.setup['daemon'])) return False self.context.open() -- cgit v1.2.3-1-g7c22 From 5d7e50dd4f55026390f9a545d0893c3ca51c7a96 Mon Sep 17 00:00:00 2001 From: Matt Kemp Date: Wed, 1 Oct 2014 21:19:20 +0000 Subject: Fixes to ensure pidfile can be opened or broken if stale. --- src/lib/Bcfg2/Reporting/Collector.py | 29 +++++++++++------------------ src/lib/Bcfg2/Server/BuiltinCore.py | 5 ++++- 2 files changed, 15 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 50bb49f78..336604daa 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -1,3 +1,4 @@ +import os import sys import atexit import daemon @@ -6,13 +7,12 @@ import time import traceback import threading +from lockfile import LockFailed, LockTimeout # pylint: disable=E0611 -from lockfile import LockFailed, LockTimeout, AlreadyLocked try: - from lockfile.pidlockfile import PIDLockFile - from lockfile import Error as PIDFileError + from daemon.pidfile import TimeoutPIDLockFile except ImportError: - from daemon.pidlockfile import PIDLockFile, PIDFileError + from daemon.pidlockfile import TimeoutPIDLockFile # pylint: enable=E0611 import Bcfg2.Logger @@ -118,7 +118,9 @@ class ReportingCollector(object): if self.setup['daemon']: self.logger.debug("Daemonizing") - self.context.pidfile = PIDLockFile(self.setup['daemon']) + self.context.pidfile = TimeoutPIDLockFile(self.setup['daemon'], + acquire_timeout=5) + # Attempt to ensure lockfile is able to be created and not stale try: self.context.pidfile.acquire() except LockFailed: @@ -126,17 +128,6 @@ class ReportingCollector(object): sys.exc_info()[1]) self.shutdown() return - except AlreadyLocked: - try: # attempt to break the lock - os.kill(self.context.pidfile.read_pid(), 0) - except OSError: # No process with locked PID - self.context.pidfile.break_lock() - else: - self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) - self.shutdown() - return except LockTimeout: try: # attempt to break the lock os.kill(self.context.pidfile.read_pid(), 0) @@ -144,8 +135,8 @@ class ReportingCollector(object): self.context.pidfile.break_lock() else: self.logger.error("Failed to daemonize: " - "Failed to acquire lock on %s" % - self.setup['daemon']) + "Failed to acquire lock on %s" % + self.setup['daemon']) self.shutdown() return except PIDFileError: @@ -153,6 +144,8 @@ class ReportingCollector(object): traceback.format_exc().splitlines()[-1]) self.shutdown() return + else: + self.context.pidfile.release() self.context.open() self.logger.info("Starting daemon") diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index 251c39558..ed959f78d 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -84,6 +84,7 @@ class Core(BaseCore): def _daemonize(self): """ Open :attr:`context` to drop privileges, write the PID file, and daemonize the server core. """ + # Attempt to ensure lockfile is able to be created and not stale try: self.context.pidfile.acquire() except LockFailed: @@ -91,7 +92,7 @@ class Core(BaseCore): self.logger.error("Failed to daemonize %s: %s" % (self.name, err)) return False except LockTimeout: - try: + try: # attempt to break the lock os.kill(self.context.pidfile.read_pid(), 0) except OSError: # No process with locked PID self.context.pidfile.break_lock() @@ -101,6 +102,8 @@ class Core(BaseCore): "lock on %s" % (self.name, self.setup['daemon'])) return False + else: + self.context.pidfile.release() self.context.open() self.logger.info("%s daemonized" % self.name) -- cgit v1.2.3-1-g7c22 From 5038a6278a1700671141ec58ea4a3cc5ce799e3b Mon Sep 17 00:00:00 2001 From: Matt Kemp Date: Thu, 2 Oct 2014 18:15:59 +0000 Subject: Catch possible typeerror resulting from None being returned when reading the pid. --- src/lib/Bcfg2/Reporting/Collector.py | 2 +- src/lib/Bcfg2/Server/BuiltinCore.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 336604daa..2ff09d483 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -131,7 +131,7 @@ class ReportingCollector(object): except LockTimeout: try: # attempt to break the lock os.kill(self.context.pidfile.read_pid(), 0) - except OSError: # No process with locked PID + except (OSError, TypeError): # No process with locked PID self.context.pidfile.break_lock() else: self.logger.error("Failed to daemonize: " diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index ed959f78d..29beb35d5 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -94,7 +94,7 @@ class Core(BaseCore): except LockTimeout: try: # attempt to break the lock os.kill(self.context.pidfile.read_pid(), 0) - except OSError: # No process with locked PID + except (OSError, TypeError): # No process with locked PID self.context.pidfile.break_lock() else: err = sys.exc_info()[1] -- cgit v1.2.3-1-g7c22 From d1b630dc6edb77f248c2c5bcaddf7526210a55da Mon Sep 17 00:00:00 2001 From: Matt Kemp Date: Thu, 2 Oct 2014 18:16:24 +0000 Subject: Remove PIDFileError as it does not always exist in the package and is rarely used. --- src/lib/Bcfg2/Reporting/Collector.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 2ff09d483..8e2fe1cb1 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -139,11 +139,6 @@ class ReportingCollector(object): self.setup['daemon']) self.shutdown() return - except PIDFileError: - self.logger.error("Error writing pid file: %s" % - traceback.format_exc().splitlines()[-1]) - self.shutdown() - return else: self.context.pidfile.release() -- cgit v1.2.3-1-g7c22 From 885e8d1ae2027fb3d62356fdc2a5cf53347df568 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 12 Oct 2014 17:38:27 +0200 Subject: Reporting: fix filter urls The regex match for the filter urls were to strict. They disallowed some charaters, that are valid in group names and so the django reverse mechanism for building urls failed. --- src/lib/Bcfg2/Reporting/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/utils.py b/src/lib/Bcfg2/Reporting/utils.py index 0d394fcd8..694f38824 100755 --- a/src/lib/Bcfg2/Reporting/utils.py +++ b/src/lib/Bcfg2/Reporting/utils.py @@ -96,12 +96,12 @@ def filteredUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] for filter in ('/state/(?P\w+)', - '/group/(?P[\w\-\.]+)', - '/group/(?P[\w\-\.]+)/(?P[A-Za-z]+)', - '/server/(?P[\w\-\.]+)', - '/server/(?P[\w\-\.]+)/(?P[A-Za-z]+)', - '/server/(?P[\w\-\.]+)/group/(?P[\w\-\.]+)', - '/server/(?P[\w\-\.]+)/group/(?P[\w\-\.]+)/(?P[A-Za-z]+)'): + '/group/(?P[^/]+)', + '/group/(?P[^/]+)/(?P[A-Za-z]+)', + '/server/(?P[^/]+)', + '/server/(?P[^/]+)/(?P[A-Za-z]+)', + '/server/(?P[^/]+)/group/(?P[^/]+)', + '/server/(?P[^/]+)/group/(?P[^/]+)/(?P[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results -- cgit v1.2.3-1-g7c22 From a2b8b3282bc07e1db362d2edd51d2bee3e425d57 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 15 Oct 2014 14:25:40 -0500 Subject: Proxy.py: Pass through SSL protocol option Previously we were not passing through the SSL protocol specified in the client's bcfg2.conf which caused it to unconditionally be set to xmlrpc/ssl. While this appears to automagically work with newer versions of openssl, the version in e.g. centos5 will fail if the server is set to use TLSv1. This commit passes through the setting from the client's bcfg2.conf so that older clients can talk to servers which are set to TLSv1 (in order to mitigate the effects of POODLE). Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Client.py | 3 ++- src/lib/Bcfg2/Options.py | 1 + src/lib/Bcfg2/Proxy.py | 11 +++++++---- 3 files changed, 10 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Client.py b/src/lib/Bcfg2/Client/Client.py index 14fe6768a..090921ab2 100644 --- a/src/lib/Bcfg2/Client/Client.py +++ b/src/lib/Bcfg2/Client/Client.py @@ -139,7 +139,8 @@ class Client(object): allowedServerCNs=self.setup['serverCN'], timeout=self.setup['timeout'], retries=int(self.setup['retries']), - delay=int(self.setup['retry_delay'])) + delay=int(self.setup['retry_delay']), + protocol=self.setup['protocol']) return self._proxy def run_probes(self, times=None): diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 206c63d4f..33b395b2e 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1292,6 +1292,7 @@ CLIENT_COMMON_OPTIONS = \ drivers=CLIENT_DRIVERS, dryrun=CLIENT_DRYRUN, paranoid=CLIENT_PARANOID, + protocol=SERVER_PROTOCOL, ppath=PARANOID_PATH, max_copies=PARANOID_MAX_COPIES, bundle=CLIENT_BUNDLE, diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index 34080da6b..736325eab 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -286,7 +286,7 @@ class SSLHTTPConnection(httplib.HTTPConnection): class XMLRPCTransport(xmlrpclib.Transport): - def __init__(self, key=None, cert=None, ca=None, + def __init__(self, key=None, cert=None, ca=None, protocol=None, scns=None, use_datetime=0, timeout=90): if hasattr(xmlrpclib.Transport, '__init__'): xmlrpclib.Transport.__init__(self, use_datetime) @@ -295,6 +295,7 @@ class XMLRPCTransport(xmlrpclib.Transport): self.ca = ca self.scns = scns self.timeout = timeout + self.protocol = protocol def make_connection(self, host): host, self._extra_headers = self.get_host_info(host)[0:2] @@ -303,7 +304,8 @@ class XMLRPCTransport(xmlrpclib.Transport): cert=self.cert, ca=self.ca, scns=self.scns, - timeout=self.timeout) + timeout=self.timeout, + protocol=self.protocol) def request(self, host, handler, request_body, verbose=0): """Send request to server and return response.""" @@ -343,7 +345,8 @@ class XMLRPCTransport(xmlrpclib.Transport): def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, - allowedServerCNs=None, timeout=90, retries=3, delay=1): + allowedServerCNs=None, timeout=90, retries=3, delay=1, + protocol=None): """Constructs proxies to components. @@ -362,6 +365,6 @@ def ComponentProxy(url, user=None, password=None, key=None, cert=None, ca=None, quote_plus(password, ''), path) else: newurl = url - ssl_trans = XMLRPCTransport(key, cert, ca, + ssl_trans = XMLRPCTransport(key, cert, ca, protocol, allowedServerCNs, timeout=float(timeout)) return xmlrpclib.ServerProxy(newurl, allow_none=True, transport=ssl_trans) -- cgit v1.2.3-1-g7c22 From 234aa9b1c5efc87d71b70e2abc5b422a487ec61e Mon Sep 17 00:00:00 2001 From: Jonathan Billings Date: Thu, 16 Oct 2014 12:46:42 -0400 Subject: YUM: Add options to enable and disable Yum plugins Adds two options you can define: * disabled_plugins: A comma-separated list of plugins to disable * enabled_plugins: A comma-separated list of plugins to enable This allows you to run bcfg2 with certain plugins enabled or disabled when they're not set that way in the yum configuration. This is useful because the Bcfg2 YUM plugin is initialized before it can read in any files that might overwrite yum plugin configuration. --- src/lib/Bcfg2/Client/Tools/YUM.py | 10 ++++++++++ src/lib/Bcfg2/Options.py | 12 ++++++++++++ 2 files changed, 22 insertions(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/YUM.py b/src/lib/Bcfg2/Client/Tools/YUM.py index 15ae5ef8b..a584fec86 100644 --- a/src/lib/Bcfg2/Client/Tools/YUM.py +++ b/src/lib/Bcfg2/Client/Tools/YUM.py @@ -191,6 +191,10 @@ class YUM(Bcfg2.Client.Tools.PkgTool): self.logger.debug("Yum: Reinstall on verify fail: %s" % self.do_reinst) self.logger.debug("Yum: installonlypkgs: %s" % self.installonlypkgs) self.logger.debug("Yum: verify_flags: %s" % self.verify_flags) + self.logger.debug("Yum: disabled_plugins: %s" % + self.setup["yum_disabled_plugins"]) + self.logger.debug("Yum: enabled_plugins: %s" % + self.setup["yum_enabled_plugins"]) def _loadYumBase(self, setup=None, logger=None): ''' this may be called before PkgTool.__init__() is called on @@ -216,6 +220,12 @@ class YUM(Bcfg2.Client.Tools.PkgTool): else: debuglevel = 0 + if setup['yum_disabled_plugins']: + rv.preconf.disabled_plugins = setup['yum_disabled_plugins'] + + if setup['yum_enabled_plugins']: + rv.preconf.enabled_plugins = setup['yum_enabled_plugins'] + # pylint: disable=E1121,W0212 try: rv.preconf.debuglevel = debuglevel diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 206c63d4f..810a726ac 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1099,6 +1099,16 @@ CLIENT_YUM_VERIFY_FLAGS = \ cf=('YUM', 'verify_flags'), deprecated_cf=('YUMng', 'verify_flags'), cook=list_split) +CLIENT_YUM_DISABLED_PLUGINS = \ + Option("YUM disabled plugins", + default=[], + cf=('YUM', 'disabled_plugins'), + cook=list_split) +CLIENT_YUM_ENABLED_PLUGINS = \ + Option("YUM enabled plugins", + default=[], + cf=('YUM', 'enabled_plugins'), + cook=list_split) CLIENT_POSIX_UID_WHITELIST = \ Option("UID ranges the POSIXUsers tool will manage", default=[], @@ -1280,6 +1290,8 @@ DRIVER_OPTIONS = \ yum_version_fail_action=CLIENT_YUM_VERSION_FAIL_ACTION, yum_verify_fail_action=CLIENT_YUM_VERIFY_FAIL_ACTION, yum_verify_flags=CLIENT_YUM_VERIFY_FLAGS, + yum_disabled_plugins=CLIENT_YUM_DISABLED_PLUGINS, + yum_enabled_plugins=CLIENT_YUM_ENABLED_PLUGINS, posix_uid_whitelist=CLIENT_POSIX_UID_WHITELIST, posix_gid_whitelist=CLIENT_POSIX_GID_WHITELIST, posix_uid_blacklist=CLIENT_POSIX_UID_BLACKLIST, -- cgit v1.2.3-1-g7c22 From 8d630569c3da497c5ccb026f175d44f02b48c4e7 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sat, 18 Oct 2014 01:35:35 +0200 Subject: Packages: add name to sources --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 22073493c..d6e3e29ca 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -209,6 +209,9 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') + #: The "name" attribute from :attr:`xsource` + self.name = xsource.get('name', None) + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -395,8 +398,10 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 doing other operations that require repository names. This function tries several approaches: - #. First, if the map contains a ``component`` key, use that as - the name. + #. First, if the source element containts a ``name`` attribute, + use that as the name. + #. If the map contains a ``component`` key, use that as the + name. #. If not, then try to match the repository URL against :attr:`Bcfg2.Server.Plugins.Packages.Source.REPO_RE`. If that succeeds, use the first matched group; additionally, @@ -426,6 +431,9 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :type url_map: dict :returns: string - the name of the repository. """ + if self.name: + return self.name + if url_map['component']: rname = url_map['component'] else: -- cgit v1.2.3-1-g7c22 From 2aea371d005e74471b6b601772e847402a9a804e Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 1 Mar 2013 04:28:47 +0100 Subject: Packages: add name to additional_data for Sources --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index d6e3e29ca..d08c7d285 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -295,6 +295,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 else: setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting + setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) @property -- cgit v1.2.3-1-g7c22 From 99f4fb302bddf6dea291ee505f90748e9c54bc71 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 12 Oct 2014 17:42:41 +0200 Subject: Reporting: better exception handling Try to keep the try-except-blocks as small as possible. --- src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py | 77 ++++++++++++---------- 1 file changed, 44 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index 489682f30..0ee5cd0d6 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -111,47 +111,58 @@ def filter_navigator(context): try: path = context['request'].META['PATH_INFO'] view, args, kwargs = resolve(path) + except (Resolver404, KeyError): + return dict() - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - # get a query string - qs = context['request'].GET.urlencode() - if qs: - qs = '?' + qs - - filters = [] - for filter in filter_list: - if filter == 'group': - continue - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + # get a query string + qs = context['request'].GET.urlencode() + if qs: + qs = '?' + qs + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + try: filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(key=lambda x: x[0]) - - myargs = kwargs.copy() - selected = True - if 'group' in myargs: - del myargs['group'] - selected = False - groups = [('---', - reverse(view, args=args, kwargs=myargs) + qs, - selected)] - for group in Group.objects.values('name'): + except NoReverseMatch: + pass + filters.sort(key=lambda x: x[0]) + + myargs = kwargs.copy() + selected = True + if 'group' in myargs: + del myargs['group'] + selected = False + + groups = [] + try: + groups.append(('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)) + except NoReverseMatch: + pass + + for group in Group.objects.values('name'): + try: myargs['group'] = group['name'] groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, group['name'] == kwargs.get('group', ''))) + except NoReverseMatch: + pass - return {'filters': filters, 'groups': groups} - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() + return {'filters': filters, 'groups': groups} def _subtract_or_na(mdict, x, y): -- cgit v1.2.3-1-g7c22 From 477841f0508b6a750f7899622729a0c9fc40f07e Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sat, 25 Oct 2014 01:30:36 +0200 Subject: Server/Core: also shutdown on exceptions during fam blocking Previously the server got stuck, if a keyboard interrupt occured during block_for_fam_events. The KeyboardInterrupt exception was only handled in the executable and it does not call shutdown for the Core. So the running fam thread does not get killed and the main thread waits for it. --- src/lib/Bcfg2/Server/Core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index f60b68f45..6dfe4df1f 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -847,15 +847,15 @@ class BaseCore(object): for plug in self.plugins_by_type(Threaded): plug.start_threads() + + if self.debug_flag: + self.set_debug(None, self.debug_flag) + self.block_for_fam_events() + self._block() except: self.shutdown() raise - if self.debug_flag: - self.set_debug(None, self.debug_flag) - self.block_for_fam_events() - self._block() - def _daemonize(self): """ Daemonize the server and write the pidfile. This must be overridden by a core implementation. """ -- cgit v1.2.3-1-g7c22 From 12c2b9cd2bec17ffe09863abed97876b10da88ed Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Sun, 26 Oct 2014 00:14:56 +0200 Subject: fix keyboard interrupt during intial event handling This removes some wildcard except handler because this drops some KeyboardInterrupt exceptions (for example previously a KeyboardInterrupt during the loading of the cache for Packages resulted in a fallback to file read). --- src/lib/Bcfg2/Server/FileMonitor/__init__.py | 2 ++ src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 52c3906fa..7a5d901fd 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -234,6 +234,8 @@ class FileMonitor(Debuggable): self.handles[event.requestID])) try: self.handles[event.requestID].HandleEvent(event) + except KeyboardInterrupt: + raise except: # pylint: disable=W0702 err = sys.exc_info()[1] LOGGER.error("Error in handling of event %s for %s: %s" % diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index d08c7d285..538215c85 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -364,7 +364,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 if os.path.exists(self.cachefile): try: self.load_state() - except: + except (OSError, cPickle.UnpicklingError): err = sys.exc_info()[1] self.logger.error("Packages: Cachefile %s load failed: %s" % (self.cachefile, err)) -- cgit v1.2.3-1-g7c22 From 835c459b9849b888fdecd045f5d64b37742eb2b3 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Wed, 29 Oct 2014 15:30:26 -0500 Subject: Tools: Fix install of non-whitelisted services Previously, a service restart would occur even when running in whitelist mode without the service being present in the whitelist. Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Client/Tools/Action.py | 22 +++------------------- src/lib/Bcfg2/Client/Tools/__init__.py | 22 ++++++++++++++++++++-- 2 files changed, 23 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index 0166e4c00..1edf4609b 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -4,7 +4,6 @@ import os import sys import select import Bcfg2.Client.Tools -from Bcfg2.Client.Frame import matches_white_list, passes_black_list from Bcfg2.Compat import input # pylint: disable=W0622 @@ -15,21 +14,6 @@ class Action(Bcfg2.Client.Tools.Tool): __req__ = {'PostInstall': ['name'], 'Action': ['name', 'timing', 'when', 'command', 'status']} - def _action_allowed(self, action): - """ Return true if the given action is allowed to be run by - the whitelist or blacklist """ - if self.setup['decision'] == 'whitelist' and \ - not matches_white_list(action, self.setup['decision_list']): - self.logger.info("In whitelist mode: suppressing Action: %s" % - action.get('name')) - return False - if self.setup['decision'] == 'blacklist' and \ - not passes_black_list(action, self.setup['decision_list']): - self.logger.info("In blacklist mode: suppressing Action: %s" % - action.get('name')) - return False - return True - def RunAction(self, entry): """This method handles command execution and status return.""" shell = False @@ -90,12 +74,12 @@ class Action(Bcfg2.Client.Tools.Tool): def BundleUpdated(self, bundle, states): """Run postinstalls when bundles have been updated.""" for postinst in bundle.findall("PostInstall"): - if not self._action_allowed(postinst): + if not self._install_allowed(postinst): continue self.cmd.run(postinst.get('name')) for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both']: - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) @@ -104,6 +88,6 @@ class Action(Bcfg2.Client.Tools.Tool): for action in bundle.findall("Action"): if action.get('timing') in ['post', 'both'] and \ action.get('when') != 'modified': - if not self._action_allowed(action): + if not self._install_allowed(action): continue states[action] = self.RunAction(action) diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 703b8ff57..0bec71e20 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -1,10 +1,12 @@ """This contains all Bcfg2 Tool modules""" import os -import sys import stat +import sys + import Bcfg2.Client import Bcfg2.Client.XML +from Bcfg2.Client.Frame import matches_white_list, passes_black_list from Bcfg2.Utils import Executor, ClassName from Bcfg2.Compat import walk_packages # pylint: disable=W0622 @@ -141,6 +143,21 @@ class Tool(object): raise ToolInstantiationError("%s: %s not executable" % (self.name, filename)) + def _install_allowed(self, entry): + """ Return true if the given entry is allowed to be installed by + the whitelist or blacklist """ + if self.setup['decision'] == 'whitelist' and \ + not matches_white_list(entry, self.setup['decision_list']): + self.logger.info("In whitelist mode: suppressing %s: %s" % + (entry.tag, entry.get('name'))) + return False + if self.setup['decision'] == 'blacklist' and \ + not passes_black_list(entry, self.setup['decision_list']): + self.logger.info("In blacklist mode: suppressing %s: %s" % + (entry.tag, entry.get('name'))) + return False + return True + def BundleUpdated(self, bundle, states): # pylint: disable=W0613 """ Callback that is invoked when a bundle has been updated. @@ -591,7 +608,8 @@ class SvcTool(Tool): return for entry in bundle: - if not self.handlesEntry(entry): + if (not self.handlesEntry(entry) + or not self._install_allowed(entry)): continue estatus = entry.get('status') -- cgit v1.2.3-1-g7c22 From 257eb0c174ef2fdaa2015975c5919979afa8da90 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 4 Nov 2014 09:57:08 -0600 Subject: Options.py: Fix database OPTIONS parsing Instead of parsing key/value pairs from bcfg2.conf, this allows the setting of the literal value which is then passed through to django as the value of the OPTIONS setting. This change allows for setting arbitrary options since some settings require nested dictionaries, etc. Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Options.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 9752ab758..652e216a5 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1,14 +1,16 @@ """Option parsing library for utilities.""" +import ast import copy import getopt +import grp import inspect import os +import pwd import re import shlex import sys -import grp -import pwd + import Bcfg2.Client.Tools from Bcfg2.Compat import ConfigParser from Bcfg2.version import __version__ @@ -329,25 +331,9 @@ def colon_split(c_string): def dict_split(c_string): - """ split an option string on commas, optionally surrounded by - whitespace and split the resulting items again on equals signs, - returning a dict """ - result = dict() - if c_string: - items = re.split(r'\s*,\s*', c_string) - for item in items: - if r'=' in item: - key, value = item.split(r'=', 1) - try: - result[key] = get_bool(value) - except ValueError: - try: - result[key] = get_int(value) - except ValueError: - result[key] = value - else: - result[item] = True - return result + """ literally evaluate the option in order to allow for arbitrarily nested + dictionaries """ + return ast.literal_eval(c_string) def get_bool(val): -- cgit v1.2.3-1-g7c22 From 14b808b5a9adf46857bdd971afe8e6717898e721 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 5 Nov 2014 15:45:42 +0100 Subject: Server/Plugins/Packages: add debsrc attribute to url_map Adding this value to the url_map makes it possible to use it in genshi templates. --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 538215c85..30cdd543f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -283,11 +283,11 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: -- cgit v1.2.3-1-g7c22 From 4e8bc5c002a54aaee1368330be6374c2e9deda59 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 5 Nov 2014 16:18:33 +0100 Subject: Client/Tools/APT: handle exceptions gracefully Exceptions during cache update (in kevlar mode) should not prevent the client tool from initializing. In the worst case the old cache would be used for the client run. --- src/lib/Bcfg2/Client/Tools/APT.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 0a8fe387f..d736b12ea 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -68,7 +68,11 @@ class APT(Bcfg2.Client.Tools.Tool): e = sys.exc_info()[1] self.logger.info("Failed to initialize APT cache: %s" % e) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except FetchFailedException: + e = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % e) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True -- cgit v1.2.3-1-g7c22 From dccd33a479c0ff14d90d2939534f91964e1a393c Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Thu, 6 Nov 2014 14:05:42 -0500 Subject: SYSV: change instances of simplename to simplefile This is to better match the schema since simplefile already exists. The previous simplename attribute would fail validation. Since pkgmgr already helpfully constructs url for you if simplefile exists, the tool no longer needs to do the concatenation itself. Given the low usage rate of SYSV.py and that the original functionality was introduced in a late 1.3 release, changing the name w/o providing backwards compatiblity seems reasonable. --- src/lib/Bcfg2/Client/Tools/SYSV.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index a29b49efa..27c3d3785 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -52,18 +52,17 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.origpkgtool = self.pkgtool def pkgmogrify(self, packages): - """ Take a list of pkg objects, check for a 'simplename' attribute. + """ Take a list of pkg objects, check for a 'simplefile' attribute. If present, insert a _sysv_pkg_path attribute to the package and download the datastream format SYSV package to a temporary file. """ for pkg in packages: - if pkg.get('simplename'): + if pkg.get('simplefile'): tmpfile = tempfile.NamedTemporaryFile() self.tmpfiles.append(tmpfile) - self.logger.info("Downloading %s%s to %s" % (pkg.get('url'), - pkg.get('simplename'), tmpfile.name)) - urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), - tmpfile.name) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) pkg.set('_sysv_pkg_path', tmpfile.name) def _get_package_command(self, packages): -- cgit v1.2.3-1-g7c22 From 26f0b52730f7fe5255a6a118b5afb564fc660423 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 10 Nov 2014 00:21:22 +0100 Subject: Client/Tools/APT: fix exception name --- src/lib/Bcfg2/Client/Tools/APT.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index d736b12ea..66ecf9610 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -70,7 +70,7 @@ class APT(Bcfg2.Client.Tools.Tool): raise Bcfg2.Client.Tools.ToolInstantiationError try: self.pkg_cache.update() - except FetchFailedException: + except apt.cache.FetchFailedException: e = sys.exc_info()[1] self.logger.info("Failed to update APT cache: %s" % e) self.pkg_cache = apt.cache.Cache() -- cgit v1.2.3-1-g7c22 From 0bfc5a946a10712a5d82daa6ae0d1cd50fbb4ba8 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 10 Nov 2014 01:26:26 +0100 Subject: Client/Tools/APT: fix pylint errors, enable check Previously pep8/pylint checks were disable for the APT tool because there were to many errors. This fix the pylint errors and enables the code style checks. --- src/lib/Bcfg2/Client/Tools/APT.py | 143 +++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 62 deletions(-) (limited to 'src') diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 66ecf9610..300c9bc51 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -6,11 +6,13 @@ warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import apt.cache import os +import sys import Bcfg2.Client.Tools + class APT(Bcfg2.Client.Tools.Tool): - """The Debian toolset implements package and service operations and inherits - the rest from Toolset.Toolset. + """The Debian toolset implements package and service operations and + inherits the rest from Toolset.Toolset. """ name = 'APT' @@ -41,21 +43,24 @@ class APT(Bcfg2.Client.Tools.Tool): if not self.setup['debug']: self.pkgcmd += '-q=2 ' self.pkgcmd += '-y install %s' - self.ignores = [entry.get('name') for struct in config \ - for entry in struct \ - if entry.tag == 'Path' and \ + self.ignores = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and entry.get('type') == 'ignore'] self.__important__ = self.__important__ + \ - ["%s/cache/debconf/config.dat" % self.var_path, - "%s/cache/debconf/templates.dat" % self.var_path, - '/etc/passwd', '/etc/group', - '%s/apt/apt.conf' % self.etc_path, - '%s/dpkg/dpkg.cfg' % self.etc_path] + \ - [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and \ - entry.get('name').startswith('%s/apt/sources.list' % self.etc_path)] - self.nonexistent = [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and entry.get('type') == 'nonexistent'] + ["%s/cache/debconf/config.dat" % self.var_path, + "%s/cache/debconf/templates.dat" % self.var_path, + '/etc/passwd', '/etc/group', + '%s/apt/apt.conf' % self.etc_path, + '%s/dpkg/dpkg.cfg' % self.etc_path] + \ + [entry.get('name') for struct in config for entry in struct + if entry.tag == 'Path' and + entry.get('name').startswith( + '%s/apt/sources.list' % self.etc_path)] + self.nonexistent = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and + entry.get('type') == 'nonexistent'] os.environ["DEBIAN_FRONTEND"] = 'noninteractive' self.actions = {} if self.setup['kevlar'] and not self.setup['dryrun']: @@ -65,14 +70,14 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError try: self.pkg_cache.update() except apt.cache.FetchFailedException: - e = sys.exc_info()[1] - self.logger.info("Failed to update APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True @@ -88,16 +93,17 @@ class APT(Bcfg2.Client.Tools.Tool): else: extras = [(p.name, p.installedVersion) for p in self.pkg_cache if p.isInstalled and p.name not in packages] - return [Bcfg2.Client.XML.Element('Package', name=name, \ - type='deb', version=version) \ - for (name, version) in extras] + return [Bcfg2.Client.XML.Element('Package', name=name, + type='deb', version=version) + for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: - self.logger.info("Package %s has no md5sums. Cannot verify" % \ + self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) entry.set('qtext', "Reinstall Package %s-%s to setup md5sums? (y/N) " % @@ -117,10 +123,10 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" \ + self.logger.error("Package %s is not fully installed" % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" \ + self.logger.error("Got Unsupported pattern %s from debsums" % item) files.append(item) files = list(set(files) - set(self.ignores)) @@ -131,31 +137,32 @@ class APT(Bcfg2.Client.Tools.Tool): modlist = [os.path.realpath(filename) for filename in modlist] bad = [filename for filename in files if filename not in modlist] if bad: - self.logger.debug("It is suggested that you either manage these " - "files, revert the changes, or ignore false " - "failures:") - self.logger.info("Package %s failed validation. Bad files are:" % \ - entry.get('name')) + self.logger.debug("It is suggested that you either manage " + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? (y/N) " % \ - (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): + if self.pkg_cache.has_key(pkgname): # noqa if self._newapi: is_installed = self.pkg_cache[pkgname].is_installed else: is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: + if not self.pkg_cache.has_key(pkgname) or not is_installed: # noqa self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False @@ -168,28 +175,33 @@ class APT(Bcfg2.Client.Tools.Tool): installed_version = pkg.installedVersion candidate_version = pkg.candidateVersion if entry.get('version') == 'auto': + # pylint: disable=W0212 if self._newapi: - is_upgradable = self.pkg_cache._depcache.is_upgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.is_upgradable( + pkg._pkg) else: - is_upgradable = self.pkg_cache._depcache.IsUpgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.IsUpgradable( + pkg._pkg) + # pylint: enable=W0212 if is_upgradable: - desiredVersion = candidate_version + desired_version = candidate_version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) - entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % \ + entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if not self.setup['quick'] and entry.get('verify', 'true') == 'true' \ - and checksums: + if not self.setup['quick'] \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -207,7 +219,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.pkg_cache[pkg].mark_delete(purge=True) else: self.pkg_cache[pkg].markDelete(purge=True) - except: + except: # pylint: disable=W0702 if self._newapi: self.pkg_cache[pkg].mark_delete() else: @@ -227,33 +239,40 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): - self.logger.error("APT has no information about package %s" % (pkg.get('name'))) + if not self.pkg_cache.has_key(pkg.get('name')): # noqa + self.logger.error("APT has no information about package %s" + % (pkg.get('name'))) continue if pkg.get('version') in ['auto', 'any']: if self._newapi: try: - ipkgs.append("%s=%s" % (pkg.get('name'), - self.pkg_cache[pkg.get('name')].candidate.version)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidate.version)) except AttributeError: - self.logger.error("Failed to find %s in apt package cache" % - pkg.get('name')) + self.logger.error("Failed to find %s in apt package " + "cache" % pkg.get('name')) continue else: - ipkgs.append("%s=%s" % (pkg.get('name'), - self.pkg_cache[pkg.get('name')].candidateVersion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidateVersion)) continue + # pylint: disable=W0212 if self._newapi: - avail_vers = [x.ver_str for x in \ - self.pkg_cache[pkg.get('name')]._pkg.version_list] + avail_vers = [ + x.ver_str for x in + self.pkg_cache[pkg.get('name')]._pkg.version_list] else: - avail_vers = [x.VerStr for x in \ - self.pkg_cache[pkg.get('name')]._pkg.VersionList] + avail_vers = [ + x.VerStr for x in + self.pkg_cache[pkg.get('name')]._pkg.VersionList] + # pylint: enable=W0212 if pkg.get('version') in avail_vers: ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" \ + self.logger.error("Package %s: desired version %s not in %s" % (pkg.get('name'), pkg.get('version'), avail_vers)) bad_pkgs.append(pkg.get('name')) @@ -271,6 +290,6 @@ class APT(Bcfg2.Client.Tools.Tool): if states[package]: self.modified.append(package) - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True -- cgit v1.2.3-1-g7c22