diff options
| -rw-r--r-- | doc/client/tools.txt | 16 | ||||
| -rw-r--r-- | schemas/packages.xsd | 2 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Client/Tools/APT.py | 145 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Client/Tools/SYSV.py | 11 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 4 | ||||
| -rw-r--r-- | testsuite/Testsrc/test_code_checks.py | 3 | ||||
| -rw-r--r-- | tools/upgrade/1.3/README | 4 | ||||
| -rwxr-xr-x | tools/upgrade/1.3/migrate_sysv_simplename.py | 51 |
8 files changed, 161 insertions, 75 deletions
diff --git a/doc/client/tools.txt b/doc/client/tools.txt index ce8732454..170b30b2e 100644 --- a/doc/client/tools.txt +++ b/doc/client/tools.txt @@ -158,14 +158,22 @@ Handles `System V Packaging <http://docs.oracle.com/cd/E19683-01/806-7008/index. .. note:: - If the Packages specified in the PackageList are datastream format packages distributed via HTTP, you must specify a simplename attribute. Such packages will be downloaded and installed from a local path. + If the Packages specified in the PackageList are datastream format + packages distributed via HTTP, you must specify a simplefile attribute. + Such packages will be downloaded and installed from a local path. - datastream format over HTTP: + Note the use of the uri attribute in the datastream format example. If + the simplefile attribute exists, the + :ref:`Pkgmgr <server-plugins-generators-pkgmgr>` plugin will + automatically construct the url attribute by concatenating the uri and + simplefile attributes (with an intervening slash). + + Datastream format over HTTP: .. code-block:: xml - <PackageList url='http://install/packages' type='sysv' priority='0'> - <Package name='SCbcfg2' version='1.3.4' simplename='bcfg-1.3.4-1' /> + <PackageList uri='http://install/packages' type='sysv' priority='0'> + <Package name='SCbcfg2' version='1.3.4' simplefile='bcfg-1.3.4-1' /> </PackageList> File system format over NFS or local path: diff --git a/schemas/packages.xsd b/schemas/packages.xsd index 8ed07baa9..46ae3c663 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -167,7 +167,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type="xsd:boolean" name="debsrc"> + <xsd:attribute type="xsd:boolean" name="debsrc" default="false"> <xsd:annotation> <xsd:documentation> Include ``deb-src`` lines in the generated APT diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 0a8fe387f..300c9bc51 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -6,11 +6,13 @@ warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import apt.cache import os +import sys import Bcfg2.Client.Tools + class APT(Bcfg2.Client.Tools.Tool): - """The Debian toolset implements package and service operations and inherits - the rest from Toolset.Toolset. + """The Debian toolset implements package and service operations and + inherits the rest from Toolset.Toolset. """ name = 'APT' @@ -41,21 +43,24 @@ class APT(Bcfg2.Client.Tools.Tool): if not self.setup['debug']: self.pkgcmd += '-q=2 ' self.pkgcmd += '-y install %s' - self.ignores = [entry.get('name') for struct in config \ - for entry in struct \ - if entry.tag == 'Path' and \ + self.ignores = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and entry.get('type') == 'ignore'] self.__important__ = self.__important__ + \ - ["%s/cache/debconf/config.dat" % self.var_path, - "%s/cache/debconf/templates.dat" % self.var_path, - '/etc/passwd', '/etc/group', - '%s/apt/apt.conf' % self.etc_path, - '%s/dpkg/dpkg.cfg' % self.etc_path] + \ - [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and \ - entry.get('name').startswith('%s/apt/sources.list' % self.etc_path)] - self.nonexistent = [entry.get('name') for struct in config for entry in struct \ - if entry.tag == 'Path' and entry.get('type') == 'nonexistent'] + ["%s/cache/debconf/config.dat" % self.var_path, + "%s/cache/debconf/templates.dat" % self.var_path, + '/etc/passwd', '/etc/group', + '%s/apt/apt.conf' % self.etc_path, + '%s/dpkg/dpkg.cfg' % self.etc_path] + \ + [entry.get('name') for struct in config for entry in struct + if entry.tag == 'Path' and + entry.get('name').startswith( + '%s/apt/sources.list' % self.etc_path)] + self.nonexistent = [entry.get('name') for struct in config + for entry in struct + if entry.tag == 'Path' and + entry.get('type') == 'nonexistent'] os.environ["DEBIAN_FRONTEND"] = 'noninteractive' self.actions = {} if self.setup['kevlar'] and not self.setup['dryrun']: @@ -65,10 +70,14 @@ class APT(Bcfg2.Client.Tools.Tool): try: self.pkg_cache = apt.cache.Cache() except SystemError: - e = sys.exc_info()[1] - self.logger.info("Failed to initialize APT cache: %s" % e) + err = sys.exc_info()[1] + self.logger.info("Failed to initialize APT cache: %s" % err) raise Bcfg2.Client.Tools.ToolInstantiationError - self.pkg_cache.update() + try: + self.pkg_cache.update() + except apt.cache.FetchFailedException: + err = sys.exc_info()[1] + self.logger.info("Failed to update APT cache: %s" % err) self.pkg_cache = apt.cache.Cache() if 'req_reinstall_pkgs' in dir(self.pkg_cache): self._newapi = True @@ -84,16 +93,17 @@ class APT(Bcfg2.Client.Tools.Tool): else: extras = [(p.name, p.installedVersion) for p in self.pkg_cache if p.isInstalled and p.name not in packages] - return [Bcfg2.Client.XML.Element('Package', name=name, \ - type='deb', version=version) \ - for (name, version) in extras] + return [Bcfg2.Client.XML.Element('Package', name=name, + type='deb', version=version) + for (name, version) in extras] def VerifyDebsums(self, entry, modlist): + """Verify the package contents with debsum information.""" output = \ self.cmd.run("%s -as %s" % (self.debsums, entry.get('name'))).stderr.splitlines() if len(output) == 1 and "no md5sums for" in output[0]: - self.logger.info("Package %s has no md5sums. Cannot verify" % \ + self.logger.info("Package %s has no md5sums. Cannot verify" % entry.get('name')) entry.set('qtext', "Reinstall Package %s-%s to setup md5sums? (y/N) " % @@ -113,10 +123,10 @@ class APT(Bcfg2.Client.Tools.Tool): # these files should not exist continue elif "is not installed" in item or "missing file" in item: - self.logger.error("Package %s is not fully installed" \ + self.logger.error("Package %s is not fully installed" % entry.get('name')) else: - self.logger.error("Got Unsupported pattern %s from debsums" \ + self.logger.error("Got Unsupported pattern %s from debsums" % item) files.append(item) files = list(set(files) - set(self.ignores)) @@ -127,31 +137,32 @@ class APT(Bcfg2.Client.Tools.Tool): modlist = [os.path.realpath(filename) for filename in modlist] bad = [filename for filename in files if filename not in modlist] if bad: - self.logger.debug("It is suggested that you either manage these " - "files, revert the changes, or ignore false " - "failures:") - self.logger.info("Package %s failed validation. Bad files are:" % \ - entry.get('name')) + self.logger.debug("It is suggested that you either manage " + "these files, revert the changes, or " + "ignore false failures:") + self.logger.info("Package %s failed validation. Bad files are:" + % entry.get('name')) self.logger.info(bad) - entry.set('qtext', - "Reinstall Package %s-%s to fix failing files? (y/N) " % \ - (entry.get('name'), entry.get('version'))) + entry.set( + 'qtext', + "Reinstall Package %s-%s to fix failing files? (y/N) " + % (entry.get('name'), entry.get('version'))) return False return True def VerifyPackage(self, entry, modlist, checksums=True): """Verify package for entry.""" - if not 'version' in entry.attrib: + if 'version' not in entry.attrib: self.logger.info("Cannot verify unversioned package %s" % (entry.attrib['name'])) return False pkgname = entry.get('name') - if self.pkg_cache.has_key(pkgname): + if self.pkg_cache.has_key(pkgname): # noqa if self._newapi: is_installed = self.pkg_cache[pkgname].is_installed else: is_installed = self.pkg_cache[pkgname].isInstalled - if not self.pkg_cache.has_key(pkgname) or not is_installed: + if not self.pkg_cache.has_key(pkgname) or not is_installed: # noqa self.logger.info("Package %s not installed" % (entry.get('name'))) entry.set('current_exists', 'false') return False @@ -164,28 +175,33 @@ class APT(Bcfg2.Client.Tools.Tool): installed_version = pkg.installedVersion candidate_version = pkg.candidateVersion if entry.get('version') == 'auto': + # pylint: disable=W0212 if self._newapi: - is_upgradable = self.pkg_cache._depcache.is_upgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.is_upgradable( + pkg._pkg) else: - is_upgradable = self.pkg_cache._depcache.IsUpgradable(pkg._pkg) + is_upgradable = self.pkg_cache._depcache.IsUpgradable( + pkg._pkg) + # pylint: enable=W0212 if is_upgradable: - desiredVersion = candidate_version + desired_version = candidate_version else: - desiredVersion = installed_version + desired_version = installed_version elif entry.get('version') == 'any': - desiredVersion = installed_version + desired_version = installed_version else: - desiredVersion = entry.get('version') - if desiredVersion != installed_version: + desired_version = entry.get('version') + if desired_version != installed_version: entry.set('current_version', installed_version) - entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % \ + entry.set('qtext', "Modify Package %s (%s -> %s)? (y/N) " % (entry.get('name'), entry.get('current_version'), - desiredVersion)) + desired_version)) return False else: # version matches - if not self.setup['quick'] and entry.get('verify', 'true') == 'true' \ - and checksums: + if not self.setup['quick'] \ + and entry.get('verify', 'true') == 'true' \ + and checksums: pkgsums = self.VerifyDebsums(entry, modlist) return pkgsums return True @@ -203,7 +219,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.pkg_cache[pkg].mark_delete(purge=True) else: self.pkg_cache[pkg].markDelete(purge=True) - except: + except: # pylint: disable=W0702 if self._newapi: self.pkg_cache[pkg].mark_delete() else: @@ -223,33 +239,40 @@ class APT(Bcfg2.Client.Tools.Tool): ipkgs = [] bad_pkgs = [] for pkg in packages: - if not self.pkg_cache.has_key(pkg.get('name')): - self.logger.error("APT has no information about package %s" % (pkg.get('name'))) + if not self.pkg_cache.has_key(pkg.get('name')): # noqa + self.logger.error("APT has no information about package %s" + % (pkg.get('name'))) continue if pkg.get('version') in ['auto', 'any']: if self._newapi: try: - ipkgs.append("%s=%s" % (pkg.get('name'), - self.pkg_cache[pkg.get('name')].candidate.version)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidate.version)) except AttributeError: - self.logger.error("Failed to find %s in apt package cache" % - pkg.get('name')) + self.logger.error("Failed to find %s in apt package " + "cache" % pkg.get('name')) continue else: - ipkgs.append("%s=%s" % (pkg.get('name'), - self.pkg_cache[pkg.get('name')].candidateVersion)) + ipkgs.append("%s=%s" % ( + pkg.get('name'), + self.pkg_cache[pkg.get('name')].candidateVersion)) continue + # pylint: disable=W0212 if self._newapi: - avail_vers = [x.ver_str for x in \ - self.pkg_cache[pkg.get('name')]._pkg.version_list] + avail_vers = [ + x.ver_str for x in + self.pkg_cache[pkg.get('name')]._pkg.version_list] else: - avail_vers = [x.VerStr for x in \ - self.pkg_cache[pkg.get('name')]._pkg.VersionList] + avail_vers = [ + x.VerStr for x in + self.pkg_cache[pkg.get('name')]._pkg.VersionList] + # pylint: enable=W0212 if pkg.get('version') in avail_vers: ipkgs.append("%s=%s" % (pkg.get('name'), pkg.get('version'))) continue else: - self.logger.error("Package %s: desired version %s not in %s" \ + self.logger.error("Package %s: desired version %s not in %s" % (pkg.get('name'), pkg.get('version'), avail_vers)) bad_pkgs.append(pkg.get('name')) @@ -267,6 +290,6 @@ class APT(Bcfg2.Client.Tools.Tool): if states[package]: self.modified.append(package) - def VerifyPath(self, entry, _): + def VerifyPath(self, entry, _): # pylint: disable=W0613 """Do nothing here since we only verify Path type=ignore.""" return True diff --git a/src/lib/Bcfg2/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index a29b49efa..27c3d3785 100644 --- a/src/lib/Bcfg2/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -52,18 +52,17 @@ class SYSV(Bcfg2.Client.Tools.PkgTool): self.origpkgtool = self.pkgtool def pkgmogrify(self, packages): - """ Take a list of pkg objects, check for a 'simplename' attribute. + """ Take a list of pkg objects, check for a 'simplefile' attribute. If present, insert a _sysv_pkg_path attribute to the package and download the datastream format SYSV package to a temporary file. """ for pkg in packages: - if pkg.get('simplename'): + if pkg.get('simplefile'): tmpfile = tempfile.NamedTemporaryFile() self.tmpfiles.append(tmpfile) - self.logger.info("Downloading %s%s to %s" % (pkg.get('url'), - pkg.get('simplename'), tmpfile.name)) - urlretrieve("%s/%s" % (pkg.get('url'), pkg.get('simplename')), - tmpfile.name) + self.logger.info("Downloading %s to %s" % (pkg.get('url'), + tmpfile.name)) + urlretrieve(pkg.get('url'), tmpfile.name) pkg.set('_sysv_pkg_path', tmpfile.name) def _get_package_command(self, packages): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 538215c85..30cdd543f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -283,11 +283,11 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch) + arch=arch, debsrc=self.debsrc) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch)] + arch=arch, debsrc=self.debsrc)] for setting in usettings: if not self.rawurl: diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index c368d40ce..d9f985104 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -35,6 +35,7 @@ contingent_checks = { "lib/Bcfg2/Server/Admin": ["Reports.py", "Syncdb.py"], "sbin": ["bcfg2-reports"]}, ("pyinotify",): {"lib/Bcfg2/Server/FileMonitor": ["Inotify.py"]}, + ("apt",): {"lib/Bcfg2/Client/Tools": ["APT.py"]}, ("yum",): {"lib/Bcfg2/Client/Tools": ["YUM.py"]}, ("genshi",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgGenshiGenerator.py"]}, ("Cheetah",): {"lib/Bcfg2/Server/Plugins/Cfg": ["CfgCheetahGenerator.py"]}, @@ -66,7 +67,7 @@ error_checks = { # perform no checks at all on the listed files no_checks = { - "lib/Bcfg2/Client/Tools": ["APT.py", "RPM.py", "rpmtools.py"], + "lib/Bcfg2/Client/Tools": ["RPM.py", "rpmtools.py"], "lib/Bcfg2/Server": ["Snapshots", "Hostbase"], "lib/Bcfg2": ["manage.py"], "lib/Bcfg2/Server/Reports": ["manage.py"], diff --git a/tools/upgrade/1.3/README b/tools/upgrade/1.3/README index 1a919f869..29fd9886b 100644 --- a/tools/upgrade/1.3/README +++ b/tools/upgrade/1.3/README @@ -24,3 +24,7 @@ migrate_probe_groups_to_db.py - Migrate Probe host and group data from XML to DB backend for Metadata and Probe plugins. Does not migrate individual probe return data. Assumes migration to BOTH Metadata and Probe to database backends. + +migrate_sysv_simplename.py + - Migrate any Pkgmgr entries which may have been using the simplename + attribute introduced in 1.3.5 to the simplefile attribute diff --git a/tools/upgrade/1.3/migrate_sysv_simplename.py b/tools/upgrade/1.3/migrate_sysv_simplename.py new file mode 100755 index 000000000..f6599756b --- /dev/null +++ b/tools/upgrade/1.3/migrate_sysv_simplename.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import lxml.etree +import Bcfg2.Options + +def main(): + opts = dict(repo=Bcfg2.Options.SERVER_REPOSITORY) + setup = Bcfg2.Options.OptionParser(opts) + setup.parse(sys.argv[1:]) + + files = [] + for plugin in ['Pkgmgr']: + files.extend(glob.glob(os.path.join(setup['repo'], plugin, "*"))) + + for bfile in files: + bdata = lxml.etree.parse(bfile) + changed = False + + if not bdata.xpath("//@type='sysv'"): + print("%s doesn't contain any sysv packages, skipping" % bfile) + continue + + pkglist = bdata.getroot() + if pkglist.tag != "PackageList": + print("%s doesn't look like a PackageList, skipping" % bfile) + continue + + for pkg in bdata.xpath("//Package"): + if "simplename" in pkg.attrib: + pkg.set("simplefile", pkg.get("simplename")) + del pkg.attrib["simplename"] + changed = True + + # if we switched to simplefile, we also need to switch to uri + if changed and "url" in pkglist.attrib: + pkglist.set("uri", pkglist.get("url")) + del pkglist.attrib["url"] + + if changed: + print("Writing %s" % bfile) + try: + open(bfile, "w").write(lxml.etree.tostring(bdata)) + except IOError: + err = sys.exc_info()[1] + print("Could not write %s: %s" % (bfile, err)) + +if __name__ == '__main__': + sys.exit(main()) |