diff options
| -rw-r--r-- | src/lib/Client/Tools/RPMng.py | 74 | ||||
| -rw-r--r-- | src/lib/Client/Tools/YUMng.py | 11 | ||||
| -rwxr-xr-x | tools/pkgmgr_gen.py | 399 |
3 files changed, 445 insertions, 39 deletions
diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Client/Tools/RPMng.py index 6ada807ee..957573fa7 100644 --- a/src/lib/Client/Tools/RPMng.py +++ b/src/lib/Client/Tools/RPMng.py @@ -125,7 +125,8 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): Constructs the text prompts for interactive mode. ''' - if len(entry) == 0: + instances = [inst for inst in entry if inst.tag == 'Instance' or inst.tag == 'Package'] + if instances == []: # We have an old style no Instance entry. Convert it to new style. version, release = entry.get('version').split('-') instance = Bcfg2.Client.XML.SubElement(entry, 'Package') @@ -134,9 +135,6 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): instance.set('version', version) instance.set('release', release) instances = [ instance ] - else: - # We have a new style entry or a previously converted old style entry. - instances = [inst for inst in entry if inst.tag == 'Instance' or inst.tag == 'Package'] self.logger.info("Verifying package instances for %s" % entry.get('name')) package_fail = False @@ -157,8 +155,6 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): for pkg in self.installed[entry.get('name')]: if self.pkg_vr_equal(inst, pkg) or self.inst_evra_equal(inst, pkg): self.logger.info(" %s" % self.str_evra(inst)) - self.logger.debug(" verify_flags = %s" % \ - (inst.get('verify_flags', []))) self.instance_status[inst]['installed'] = True flags = inst.get('verify_flags', '').split(',') @@ -170,6 +166,13 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): pkg.get('gpgkeyid', ''))) self.logger.info(' Disabling signature check.') + if self.setup.get('quick', False): + if rpmtools.prelink_exists: + flags += ['nomd5', 'nosize'] + else: + flags += ['nomd5'] + self.logger.debug(" verify_flags = %s" % flags) + vp_ts = rpmtools.rpmtransactionset() self.instance_status[inst]['verify'] = \ rpmtools.rpm_verify( vp_ts, pkg, flags) @@ -205,8 +208,6 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): for pkg in arch_match: if self.pkg_vr_equal(inst, pkg) or self.inst_evra_equal(inst, pkg): self.logger.info(" %s" % self.str_evra(inst)) - self.logger.debug(" verify_flags = %s" % \ - (inst.get('verify_flags', []))) self.instance_status[inst]['installed'] = True flags = inst.get('verify_flags', '').split(',') @@ -217,6 +218,13 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): pkg.get('gpgkeyid', ''))) self.logger.info(' Disabling signature check.') + if self.setup.get('quick', False): + if rpmtools.prelink_exists: + flags += ['nomd5', 'nosize'] + else: + flags += ['nomd5'] + self.logger.debug(" verify_flags = %s" % flags) + vp_ts = rpmtools.rpmtransactionset() self.instance_status[inst]['verify'] = \ rpmtools.rpm_verify( vp_ts, pkg, flags ) @@ -455,7 +463,15 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): # Remove extra instances. # Can not reverify because we don't have a package entry. if len(self.extra_instances) > 0: - self.RemovePackages(self.extra_instances) + if (self.setup.get('remove') == 'all' or \ + self.setup.get('remove') == 'packages') and\ + not self.setup.get('dryrun'): + self.RemovePackages(self.extra_instances) + else: + self.logger.info("The following extra package instances will be removed by the '-r' option:") + for pkg in self.extra_instances: + for inst in pkg: + self.logger.info(" %s %s" % (pkg.get('name'), self.str_evra(inst))) # Figure out which instances of the packages actually need something # doing to them and place in the appropriate work 'queue'. @@ -825,33 +841,16 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): return extra_entry - #def Inventory(self, structures=[]): - # ''' - # Wrap the Tool.Inventory() method with its own rpm.TransactionSet() - # and an explicit closeDB() as the close wasn't happening and DB4 - # locks were getting left behind on the RPM database creating a nice - # mess. - # - # ***** Do performance comparison with the transctionset/closeDB - # moved into rpmtools, which would mean a transactionset/closeDB - # per VerifyPackage() call (meaning one per RPM package) rather - # than one for the whole system. - # ''' - # self.vp_ts = rpmtools.rpmtransactionset() - # try: - # Bcfg2.Client.Tools.Tool.Inventory(self) - # # Tool is still an old style class, so super doesn't work. Change it. - # #super(RPMng, self).Inventory() - # finally: - # self.vp_ts.closeDB() - # del self.vp_ts - def str_evra(self, instance): ''' Convert evra dict entries to a string. ''' - return '%s:%s-%s.%s' % (instance.get('epoch', '*'), instance.get('version', '*'), - instance.get('release', '*'), instance.get('arch', '*')) + if instance.get('epoch', '*') == '*' or instance.get('epoch', '*') == None: + return '%s-%s.%s' % (instance.get('version', '*'), + instance.get('release', '*'), instance.get('arch', '*')) + else: + return '%s:%s-%s.%s' % (instance.get('epoch', '*'), instance.get('version', '*'), + instance.get('release', '*'), instance.get('arch', '*')) def pkg_vr_equal(self, config_entry, installed_entry): ''' @@ -875,11 +874,12 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): else: epoch = None - if (config_entry.tag == 'Instance' and\ - epoch == installed_entry.get('epoch', None) and \ - config_entry.get('version') == installed_entry.get('version') and \ - config_entry.get('release') == installed_entry.get('release') and \ - config_entry.get('arch', None) == installed_entry.get('arch', None)): + if (config_entry.tag == 'Instance' and \ + (epoch == installed_entry.get('epoch', 0) or \ + (epoch == 0 and installed_entry.get('epoch', 0) == None)) and \ + config_entry.get('version') == installed_entry.get('version') and \ + config_entry.get('release') == installed_entry.get('release') and \ + config_entry.get('arch', None) == installed_entry.get('arch', None)): return True else: return False diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Client/Tools/YUMng.py index cdc43b00b..cfc7c1192 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Client/Tools/YUMng.py @@ -1,5 +1,5 @@ '''This provides bcfg2 support for yum''' -__revision__ = '0.1' +__revision__ = '$Revision: $' import Bcfg2.Client.Tools.RPMng, ConfigParser, sys @@ -73,7 +73,14 @@ class YUMng(Bcfg2.Client.Tools.RPMng.RPMng): # Remove extra instances. # Can not reverify because we don't have a package entry. if len(self.extra_instances) > 0: - self.RemovePackages(self.extra_instances) + if (self.setup.get('remove') == 'all' or \ + self.setup.get('remove') == 'packages'): + self.RemovePackages(self.extra_instances) + else: + self.logger.info("The following extra package instances will be removed by the '-r' option:") + for pkg in self.extra_instances: + for inst in pkg: + self.logger.info(" %s %s", (pkg.get('name'), self.str_evra(inst))) # Figure out which instances of the packages actually need something # doing to them and place in the appropriate work 'queue'. diff --git a/tools/pkgmgr_gen.py b/tools/pkgmgr_gen.py new file mode 100755 index 000000000..d6d55562c --- /dev/null +++ b/tools/pkgmgr_gen.py @@ -0,0 +1,399 @@ +#!/usr/bin/python +""" Program to generate a bcfg2 Pkgmgr configuration file from a list + of directories that contain RPMS. + + All versions or only the latest may be included in the output. + rpm.labelCompare is used to compare the package versions, so that + a proper rpm version comparison is done (epoch:version-release). + + The output file may be formated for use with the RPM or Yum + bcfg2 client drivers. The output can also contain the PackageList + and nested group headers. +""" +__revision__ = '$Revision: $' +import sys +import os +import rpm +import optparse +import datetime +import glob + +def info(object, spacing=10, collapse=1): + """Print methods and doc strings. + Takes module, class, list, dictionary, or string. + """ + methodList = [method for method in dir(object) + if callable(getattr(object, method))] + processFunc = collapse and (lambda s: " ".join(s.split())) or (lambda s: s) + print "\n".join(["%s %s" % + (method.ljust(spacing), + processFunc(str(getattr(object, method).__doc__))) + for method in methodList]) + +def readRpmHeader(ts, filename): + """ + Read an rpm header from an RPM file. + """ + try: + fd = os.open(filename, os.O_RDONLY) + except: + print 'Failed to open RPM file %s' % filename + + h = ts.hdrFromFdno(fd) + os.close(fd) + return h + +def sortedDictValues(adict): + """ + Sort a dictionary by its keys and return the items in sorted key order. + """ + keys = adict.keys() + keys.sort() + return map(adict.get, keys) + +def cmpRpmHeader(a, b): + """ + cmp() implemetation suitable for use with sort. + + a and b are dictionaries as created by loadRpms(). Comparison is made + by package name and then by the full rpm version (epoch, version, release). + rpm.labelCompare is used for the version part of the comparison. + """ + n1 = str(a['name']) + e1 = str(a['epoch']) + v1 = str(a['version']) + r1 = str(a['release']) + n2 = str(b['name']) + e2 = str(b['epoch']) + v2 = str(b['version']) + r2 = str(b['release']) + + ret = cmp(n1, n2) + if ret == 0: + ret = rpm.labelCompare((e1, v1, r1),(e2, v2, r2)) + return ret + +def loadRpms(dirs): + """ + dirs is a list of directories to search for rpms. + + Builds a dictionary keyed by the package name. Dictionary item is a list, + one entry per package instance found. + + The list entries are dictionaries. Keys are 'filename', 'mtime' 'name', + 'arch', 'epoch', 'version' and 'release'. + + e.g. + + packages = { + 'bcfg2' : [ + {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'} + {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}], + 'bcfg2-server' : [ + {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'} + {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}], + } + + """ + packages = {} + ts = rpm.TransactionSet() + vsflags = 0 + vsflags |= rpm._RPMVSF_NODIGESTS + vsflags |= rpm._RPMVSF_NOSIGNATURES + ovsflags = ts.setVSFlags(vsflags) + for dir in dirs: + + if options.verbose: + print 'Scanning directory: %s' % dir + + for file in [files for files in os.listdir(dir) + if files.endswith('.rpm')]: + + filename = os.path.join( dir, file ) + + # Get the mtime of the RPM file. + file_mtime = datetime.date.fromtimestamp(os.stat(filename).st_mtime) + + # Get the RPM header + header = readRpmHeader( ts, filename ) + + # Get what we are interesting in out of the header. + name = header[rpm.RPMTAG_NAME] + epoch = header[rpm.RPMTAG_EPOCH] + version = header[rpm.RPMTAG_VERSION] + release = header[rpm.RPMTAG_RELEASE] + subarch = header[rpm.RPMTAG_ARCH] + + # Only load RPMs with subarchitectures as calculated from the --archs option. + if subarch in subarchs or 'all' in subarchs: + + # Store what we want in our structure. + if name in packages: + packages[name].append({'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \ + 'epoch':epoch, 'version':version, 'release':release}) + else: + packages[name] = [{'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \ + 'epoch':epoch, 'version':version, 'release':release}] + + # Print '.' for each package. stdio is line buffered, so have to flush it. + if options.verbose: + sys.stdout.write('.') + sys.stdout.flush() + if options.verbose: + sys.stdout.write('\n') + + return packages + +def printInstance(instance, group_count): + """ + Print the details for a package instance with the appropriate indentation and + in the specified format (rpm or yum). + """ + group_count = group_count + 1 + name = instance['name'] + epoch = instance['epoch'] + version = instance['version'] + release = instance['release'] + arch = instance['arch'] + + output_line = '' + if options.format == 'rpm': + output_line = '%s<Instance simplefile=\'%s\' ' % ( indent * group_count , instance['filename']) + else: + output_line = '%s<Instance ' % (indent * group_count) + + if epoch: + output_line += 'epoch=\'%s\' ' % (epoch) + + output_line += 'version=\'%s\' release=\'%s\' arch=\'%s\'/>\n' % ( version, release, arch) + output.write(output_line) + +def printPackage(entry, group_count): + """ + Print the details of a package with the appropriate indentation. + Only the specified (all or latest) release(s) is printed. + + entry is a single package entry as created in loadRpms(). + """ + output.write('%s<Package name=\'%s\' type=\'%s\'>\n' \ + % ( group_count * indent, entry[0]['name'], options.format) ) + + subarch_dict = {} + arch_dict = {} + # Split instances of this package into subarchitectures. + for instance in entry: + if instance['arch'] in subarch_dict: + subarch_dict[instance['arch']].append(instance) + else: + subarch_dict[instance['arch']] = [ instance ] + + # Keep track of the subarchitectures we have found in each architecture. + if subarch_mapping[instance['arch']] in arch_dict: + if instance['arch'] not in arch_dict[subarch_mapping[instance['arch']]]: + arch_dict[subarch_mapping[instance['arch']]].append(instance['arch']) + else: + arch_dict[subarch_mapping[instance['arch']]] = [ instance['arch'] ] + + # Only keep the 'highest' subarchitecture in each architecture. + for arch in arch_dict.iterkeys(): + if len(arch_dict[arch]) > 1: + arch_dict[arch].sort() + for s in arch_dict[arch][:-1]: + del subarch_dict[s] + + # Sort packages within each architecture into version order + for arch in subarch_dict: + subarch_dict[arch].sort(cmpRpmHeader) + + if options.release == 'all': + # Output all instances + for header in subarch_dict[arch]: + printInstance(header, group_count) + else: + # Output the latest + printInstance(subarch_dict[arch][-1], group_count) + + output.write('%s</Package>\n' % ( group_count * indent ) ) + +def main(): + + if options.verbose: + print 'Loading package headers' + + package_dict = loadRpms(search_dirs) + + if options.verbose: + print 'Processing package headers' + + if options.pkgmgrhdr: + if options.format == "rpm": + output.write("<PackageList uri='%s' priority='%s' type='rpm'>\n" % ( options.uri, options.priority )) + else: + output.write("<PackageList priority='%s' type='yum'>\n" %( options.priority )) + + group_count = 1 + if groups_list: + for group in groups_list: + output.write("%s<Group name='%s'>\n" % ( indent * group_count , group ) ) + group_count = group_count + 1 + + # Process packages in name order + for package_entry in sortedDictValues(package_dict): + printPackage(package_entry, group_count) + + if groups_list: + group_count = group_count - 1 + while group_count: + output.write( '%s</Group>\n' % ( indent * group_count ) ) + group_count = group_count - 1 + + if options.pkgmgrhdr: + output.write( '</PackageList>\n' ) + + if options.verbose: + print '%i package instances were processed' % len(package_dict) + + +if __name__ == "__main__": + + p = optparse.OptionParser() + + p.add_option('--archs', '-a', action='store', \ + default='all', \ + type='string', \ + help='''Comma separated list of subarchitectures to include. + The highest subarichitecture required in an + architecture group should specified. Lower + subarchitecture packages will be loaded if that + is all that is available. e.g. The higher of i386, + i486 and i586 packages will be loaded if -a i586 + is specified. (Default: all). + ''') + + p.add_option('--rpmdirs', '-d', action='store', + default='.', \ + type='string', \ + help='Comma separated list of directories to scan for RPMS. Wilcards are permitted. (Default: .)') + + p.add_option('--enddate', '-e', action='store', \ + type='string', \ + help='End date for RPM file selection.') + + p.add_option('--format', '-f', action='store', \ + default='yum', \ + type='choice', \ + choices=('yum','rpm'), \ + help='Format of the Output. Choices are yum or rpm. (Default: yum)') + + p.add_option('--groups', '-g', action='store', \ + type='string', \ + help='List of comma separated groups to nest Package entities in.') + + p.add_option('--indent', '-i', action='store', \ + default=4, \ + type='int', \ + help='Number of leading spaces to indent nested entries in the output. (Default:4)') + + p.add_option('--outfile', '-o', action='store', \ + type='string', \ + help='Output file name.') + + p.add_option('--pkgmgrhdr', '-P', action='store_true', \ + help='Include PackageList header in output.') + + p.add_option('--priority', '-p', action='store', \ + default=0, \ + type='int', \ + help='Value to set priority attribute in the PackageList Tag. (Default: 0)') + + p.add_option('--release', '-r', action='store', \ + default='latest', \ + type='choice', \ + choices=('all','latest'), \ + help='Which releases to include in the output. Choices are all or latest. (Default: latest).') + + p.add_option('--startdate', '-s', action='store', \ + type='string', \ + help='Start date for RPM file selection.') + + p.add_option('--uri', '-u', action='store', \ + type='string', \ + help='URI for PackageList header required for RPM format ouput.') + + p.add_option('--verbose', '-v', action='store_true', \ + help='Enable verbose output.') + + options, arguments = p.parse_args() + + if options.pkgmgrhdr and options.format == 'rpm' and not options.uri: + print "Option --uri must be specified to produce a PackageList Tag for rpm formatted files." + sys.exit(1) + + # Set up list of directories to search + search_dirs = [] + for d in options.rpmdirs.split(','): + search_dirs += glob.glob(d) + + # Set up list of architectures to include and some mappings + # to use later. + arch_mapping = {'x86':['i686','i586','i486','i386'], + 'x86_64':['x86_64'], + 'ia64':['ia64'], + 'ppc':['ppc'], + 'ppc64':['ppc64'], + 'sparc':['sparc'], + 'noarch':['noarch']} + subarch_mapping = {'i686':'x86', 'i586':'x86', 'i486':'x86', 'i386':'x86', + 'x86_64':'x86_64', + 'ia64':'ia64', + 'ppc':'ppc', + 'ppc64':'ppc64', + 'sparc':'sparc', + 'noarch':'noarch' } + commandline_subarchs = options.archs.split(',') + arch_list = [] + subarchs = [] + if 'all' in commandline_subarchs: + subarchs.append('all') + else: + for s in commandline_subarchs: + if s not in subarch_mapping: + print 'Error: Invalid subarchitecture specified: ', s + sys.exit(1) + # Only allow one subarchitecture per architecture to be specified. + if s not in arch_list: + arch_list.append(s) + + # Add subarchitectures lower than the one specified to the list. + # e.g. If i486 is specified this will add i386 to the list of + # subarchitectures to load. + i = arch_mapping[subarch_mapping[s]].index(s) + #if i != len(arch_mapping[subarch_mapping[s]]): + subarchs += arch_mapping[subarch_mapping[s]][i:] + else: + print 'Error: Multiple subarchitecutes of the same architecture specified.' + sys.exit(1) + + if options.verbose: + print 'The following directories will be scanned:' + for d in search_dirs: + print ' %s' % d + + indent = ' ' * options.indent + + if options.groups: + groups_list = options.groups.split(',') + else: + groups_list = None + + if options.outfile: + output = file(options.outfile, "w") + else: + output = sys.stdout + + main() + |