From c5566318388ca7a2482f061bd42a1b877bc7435b Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Thu, 19 Apr 2007 00:33:02 +0000 Subject: RPMng/YUMng driver update from mbrady git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@3055 ce84e21b-d406-0410-9b95-82705330c041 --- tools/pkgmgr_gen.py | 399 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100755 tools/pkgmgr_gen.py (limited to 'tools') diff --git a/tools/pkgmgr_gen.py b/tools/pkgmgr_gen.py new file mode 100755 index 000000000..d6d55562c --- /dev/null +++ b/tools/pkgmgr_gen.py @@ -0,0 +1,399 @@ +#!/usr/bin/python +""" Program to generate a bcfg2 Pkgmgr configuration file from a list + of directories that contain RPMS. + + All versions or only the latest may be included in the output. + rpm.labelCompare is used to compare the package versions, so that + a proper rpm version comparison is done (epoch:version-release). + + The output file may be formated for use with the RPM or Yum + bcfg2 client drivers. The output can also contain the PackageList + and nested group headers. +""" +__revision__ = '$Revision: $' +import sys +import os +import rpm +import optparse +import datetime +import glob + +def info(object, spacing=10, collapse=1): + """Print methods and doc strings. + Takes module, class, list, dictionary, or string. + """ + methodList = [method for method in dir(object) + if callable(getattr(object, method))] + processFunc = collapse and (lambda s: " ".join(s.split())) or (lambda s: s) + print "\n".join(["%s %s" % + (method.ljust(spacing), + processFunc(str(getattr(object, method).__doc__))) + for method in methodList]) + +def readRpmHeader(ts, filename): + """ + Read an rpm header from an RPM file. + """ + try: + fd = os.open(filename, os.O_RDONLY) + except: + print 'Failed to open RPM file %s' % filename + + h = ts.hdrFromFdno(fd) + os.close(fd) + return h + +def sortedDictValues(adict): + """ + Sort a dictionary by its keys and return the items in sorted key order. + """ + keys = adict.keys() + keys.sort() + return map(adict.get, keys) + +def cmpRpmHeader(a, b): + """ + cmp() implemetation suitable for use with sort. + + a and b are dictionaries as created by loadRpms(). Comparison is made + by package name and then by the full rpm version (epoch, version, release). + rpm.labelCompare is used for the version part of the comparison. + """ + n1 = str(a['name']) + e1 = str(a['epoch']) + v1 = str(a['version']) + r1 = str(a['release']) + n2 = str(b['name']) + e2 = str(b['epoch']) + v2 = str(b['version']) + r2 = str(b['release']) + + ret = cmp(n1, n2) + if ret == 0: + ret = rpm.labelCompare((e1, v1, r1),(e2, v2, r2)) + return ret + +def loadRpms(dirs): + """ + dirs is a list of directories to search for rpms. + + Builds a dictionary keyed by the package name. Dictionary item is a list, + one entry per package instance found. + + The list entries are dictionaries. Keys are 'filename', 'mtime' 'name', + 'arch', 'epoch', 'version' and 'release'. + + e.g. + + packages = { + 'bcfg2' : [ + {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'} + {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}], + 'bcfg2-server' : [ + {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'} + {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server', + ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}], + } + + """ + packages = {} + ts = rpm.TransactionSet() + vsflags = 0 + vsflags |= rpm._RPMVSF_NODIGESTS + vsflags |= rpm._RPMVSF_NOSIGNATURES + ovsflags = ts.setVSFlags(vsflags) + for dir in dirs: + + if options.verbose: + print 'Scanning directory: %s' % dir + + for file in [files for files in os.listdir(dir) + if files.endswith('.rpm')]: + + filename = os.path.join( dir, file ) + + # Get the mtime of the RPM file. + file_mtime = datetime.date.fromtimestamp(os.stat(filename).st_mtime) + + # Get the RPM header + header = readRpmHeader( ts, filename ) + + # Get what we are interesting in out of the header. + name = header[rpm.RPMTAG_NAME] + epoch = header[rpm.RPMTAG_EPOCH] + version = header[rpm.RPMTAG_VERSION] + release = header[rpm.RPMTAG_RELEASE] + subarch = header[rpm.RPMTAG_ARCH] + + # Only load RPMs with subarchitectures as calculated from the --archs option. + if subarch in subarchs or 'all' in subarchs: + + # Store what we want in our structure. + if name in packages: + packages[name].append({'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \ + 'epoch':epoch, 'version':version, 'release':release}) + else: + packages[name] = [{'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \ + 'epoch':epoch, 'version':version, 'release':release}] + + # Print '.' for each package. stdio is line buffered, so have to flush it. + if options.verbose: + sys.stdout.write('.') + sys.stdout.flush() + if options.verbose: + sys.stdout.write('\n') + + return packages + +def printInstance(instance, group_count): + """ + Print the details for a package instance with the appropriate indentation and + in the specified format (rpm or yum). + """ + group_count = group_count + 1 + name = instance['name'] + epoch = instance['epoch'] + version = instance['version'] + release = instance['release'] + arch = instance['arch'] + + output_line = '' + if options.format == 'rpm': + output_line = '%s\n' % ( version, release, arch) + output.write(output_line) + +def printPackage(entry, group_count): + """ + Print the details of a package with the appropriate indentation. + Only the specified (all or latest) release(s) is printed. + + entry is a single package entry as created in loadRpms(). + """ + output.write('%s\n' \ + % ( group_count * indent, entry[0]['name'], options.format) ) + + subarch_dict = {} + arch_dict = {} + # Split instances of this package into subarchitectures. + for instance in entry: + if instance['arch'] in subarch_dict: + subarch_dict[instance['arch']].append(instance) + else: + subarch_dict[instance['arch']] = [ instance ] + + # Keep track of the subarchitectures we have found in each architecture. + if subarch_mapping[instance['arch']] in arch_dict: + if instance['arch'] not in arch_dict[subarch_mapping[instance['arch']]]: + arch_dict[subarch_mapping[instance['arch']]].append(instance['arch']) + else: + arch_dict[subarch_mapping[instance['arch']]] = [ instance['arch'] ] + + # Only keep the 'highest' subarchitecture in each architecture. + for arch in arch_dict.iterkeys(): + if len(arch_dict[arch]) > 1: + arch_dict[arch].sort() + for s in arch_dict[arch][:-1]: + del subarch_dict[s] + + # Sort packages within each architecture into version order + for arch in subarch_dict: + subarch_dict[arch].sort(cmpRpmHeader) + + if options.release == 'all': + # Output all instances + for header in subarch_dict[arch]: + printInstance(header, group_count) + else: + # Output the latest + printInstance(subarch_dict[arch][-1], group_count) + + output.write('%s\n' % ( group_count * indent ) ) + +def main(): + + if options.verbose: + print 'Loading package headers' + + package_dict = loadRpms(search_dirs) + + if options.verbose: + print 'Processing package headers' + + if options.pkgmgrhdr: + if options.format == "rpm": + output.write("\n" % ( options.uri, options.priority )) + else: + output.write("\n" %( options.priority )) + + group_count = 1 + if groups_list: + for group in groups_list: + output.write("%s\n" % ( indent * group_count , group ) ) + group_count = group_count + 1 + + # Process packages in name order + for package_entry in sortedDictValues(package_dict): + printPackage(package_entry, group_count) + + if groups_list: + group_count = group_count - 1 + while group_count: + output.write( '%s\n' % ( indent * group_count ) ) + group_count = group_count - 1 + + if options.pkgmgrhdr: + output.write( '\n' ) + + if options.verbose: + print '%i package instances were processed' % len(package_dict) + + +if __name__ == "__main__": + + p = optparse.OptionParser() + + p.add_option('--archs', '-a', action='store', \ + default='all', \ + type='string', \ + help='''Comma separated list of subarchitectures to include. + The highest subarichitecture required in an + architecture group should specified. Lower + subarchitecture packages will be loaded if that + is all that is available. e.g. The higher of i386, + i486 and i586 packages will be loaded if -a i586 + is specified. (Default: all). + ''') + + p.add_option('--rpmdirs', '-d', action='store', + default='.', \ + type='string', \ + help='Comma separated list of directories to scan for RPMS. Wilcards are permitted. (Default: .)') + + p.add_option('--enddate', '-e', action='store', \ + type='string', \ + help='End date for RPM file selection.') + + p.add_option('--format', '-f', action='store', \ + default='yum', \ + type='choice', \ + choices=('yum','rpm'), \ + help='Format of the Output. Choices are yum or rpm. (Default: yum)') + + p.add_option('--groups', '-g', action='store', \ + type='string', \ + help='List of comma separated groups to nest Package entities in.') + + p.add_option('--indent', '-i', action='store', \ + default=4, \ + type='int', \ + help='Number of leading spaces to indent nested entries in the output. (Default:4)') + + p.add_option('--outfile', '-o', action='store', \ + type='string', \ + help='Output file name.') + + p.add_option('--pkgmgrhdr', '-P', action='store_true', \ + help='Include PackageList header in output.') + + p.add_option('--priority', '-p', action='store', \ + default=0, \ + type='int', \ + help='Value to set priority attribute in the PackageList Tag. (Default: 0)') + + p.add_option('--release', '-r', action='store', \ + default='latest', \ + type='choice', \ + choices=('all','latest'), \ + help='Which releases to include in the output. Choices are all or latest. (Default: latest).') + + p.add_option('--startdate', '-s', action='store', \ + type='string', \ + help='Start date for RPM file selection.') + + p.add_option('--uri', '-u', action='store', \ + type='string', \ + help='URI for PackageList header required for RPM format ouput.') + + p.add_option('--verbose', '-v', action='store_true', \ + help='Enable verbose output.') + + options, arguments = p.parse_args() + + if options.pkgmgrhdr and options.format == 'rpm' and not options.uri: + print "Option --uri must be specified to produce a PackageList Tag for rpm formatted files." + sys.exit(1) + + # Set up list of directories to search + search_dirs = [] + for d in options.rpmdirs.split(','): + search_dirs += glob.glob(d) + + # Set up list of architectures to include and some mappings + # to use later. + arch_mapping = {'x86':['i686','i586','i486','i386'], + 'x86_64':['x86_64'], + 'ia64':['ia64'], + 'ppc':['ppc'], + 'ppc64':['ppc64'], + 'sparc':['sparc'], + 'noarch':['noarch']} + subarch_mapping = {'i686':'x86', 'i586':'x86', 'i486':'x86', 'i386':'x86', + 'x86_64':'x86_64', + 'ia64':'ia64', + 'ppc':'ppc', + 'ppc64':'ppc64', + 'sparc':'sparc', + 'noarch':'noarch' } + commandline_subarchs = options.archs.split(',') + arch_list = [] + subarchs = [] + if 'all' in commandline_subarchs: + subarchs.append('all') + else: + for s in commandline_subarchs: + if s not in subarch_mapping: + print 'Error: Invalid subarchitecture specified: ', s + sys.exit(1) + # Only allow one subarchitecture per architecture to be specified. + if s not in arch_list: + arch_list.append(s) + + # Add subarchitectures lower than the one specified to the list. + # e.g. If i486 is specified this will add i386 to the list of + # subarchitectures to load. + i = arch_mapping[subarch_mapping[s]].index(s) + #if i != len(arch_mapping[subarch_mapping[s]]): + subarchs += arch_mapping[subarch_mapping[s]][i:] + else: + print 'Error: Multiple subarchitecutes of the same architecture specified.' + sys.exit(1) + + if options.verbose: + print 'The following directories will be scanned:' + for d in search_dirs: + print ' %s' % d + + indent = ' ' * options.indent + + if options.groups: + groups_list = options.groups.split(',') + else: + groups_list = None + + if options.outfile: + output = file(options.outfile, "w") + else: + output = sys.stdout + + main() + -- cgit v1.2.3-1-g7c22