summaryrefslogtreecommitdiffstats
path: root/tools/pkgmgr_gen.py
diff options
context:
space:
mode:
authorSol Jerome <solj@ices.utexas.edu>2009-09-08 19:47:48 +0000
committerSol Jerome <solj@ices.utexas.edu>2009-09-08 19:47:48 +0000
commit20817cb4d1a02769f64453356e04a73f2d8b5be5 (patch)
tree4a5ed5a8c72ecbe1229489fcfa1afb9a2368a38b /tools/pkgmgr_gen.py
parentd45f6d05a3c4d88177813d3df567fe219b8fdfad (diff)
downloadbcfg2-20817cb4d1a02769f64453356e04a73f2d8b5be5.tar.gz
bcfg2-20817cb4d1a02769f64453356e04a73f2d8b5be5.tar.bz2
bcfg2-20817cb4d1a02769f64453356e04a73f2d8b5be5.zip
Pkgmgr generation script patch from David Strauss (fix for ticket #712)
This patch forces the Pkgmgr generation/update scripts to parse the Yum urls in a way that is more compatible with the Yum repository spec. Signed-off-by: Sol Jerome <solj@ices.utexas.edu> git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5419 ce84e21b-d406-0410-9b95-82705330c041
Diffstat (limited to 'tools/pkgmgr_gen.py')
-rwxr-xr-xtools/pkgmgr_gen.py187
1 files changed, 104 insertions, 83 deletions
diff --git a/tools/pkgmgr_gen.py b/tools/pkgmgr_gen.py
index 8e8fd7629..c62313232 100755
--- a/tools/pkgmgr_gen.py
+++ b/tools/pkgmgr_gen.py
@@ -1,12 +1,12 @@
#!/usr/bin/python
-""" Program to generate a bcfg2 Pkgmgr configuration file from a list
- of directories that contain RPMS.
-
+""" Program to generate a bcfg2 Pkgmgr configuration file from a list
+ of directories that contain RPMS.
+
All versions or only the latest may be included in the output.
- rpm.labelCompare is used to compare the package versions, so that
+ rpm.labelCompare is used to compare the package versions, so that
a proper rpm version comparison is done (epoch:version-release).
- The output file may be formated for use with the RPM or Yum
+ The output file may be formated for use with the RPM or Yum
bcfg2 client drivers. The output can also contain the PackageList
and nested group headers.
"""
@@ -26,7 +26,7 @@ def info(object, spacing=10, collapse=1):
"""Print methods and doc strings.
Takes module, class, list, dictionary, or string.
"""
- methodList = [method for method in dir(object)
+ methodList = [method for method in dir(object)
if callable(getattr(object, method))]
processFunc = collapse and (lambda s: " ".join(s.split())) or (lambda s: s)
print "\n".join(["%s %s" %
@@ -35,8 +35,8 @@ def info(object, spacing=10, collapse=1):
for method in methodList])
def readRpmHeader(ts, filename):
- """
- Read an rpm header from an RPM file.
+ """
+ Read an rpm header from an RPM file.
"""
try:
fd = os.open(filename, os.O_RDONLY)
@@ -48,8 +48,8 @@ def readRpmHeader(ts, filename):
return h
def sortedDictValues(adict):
- """
- Sort a dictionary by its keys and return the items in sorted key order.
+ """
+ Sort a dictionary by its keys and return the items in sorted key order.
"""
keys = adict.keys()
keys.sort()
@@ -59,8 +59,8 @@ def cmpRpmHeader(a, b):
"""
cmp() implemetation suitable for use with sort.
- a and b are dictionaries as created by loadRpms(). Comparison is made
- by package name and then by the full rpm version (epoch, version, release).
+ a and b are dictionaries as created by loadRpms(). Comparison is made
+ by package name and then by the full rpm version (epoch, version, release).
rpm.labelCompare is used for the version part of the comparison.
"""
n1 = str(a['name'])
@@ -81,27 +81,27 @@ def loadRpms(dirs):
"""
dirs is a list of directories to search for rpms.
- Builds a dictionary keyed by the package name. Dictionary item is a list,
+ Builds a dictionary keyed by the package name. Dictionary item is a list,
one entry per package instance found.
-
- The list entries are dictionaries. Keys are 'filename', 'mtime' 'name',
- 'arch', 'epoch', 'version' and 'release'.
-
+
+ The list entries are dictionaries. Keys are 'filename', 'mtime' 'name',
+ 'arch', 'epoch', 'version' and 'release'.
+
e.g.
packages = {
'bcfg2' : [
- {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2',
+ {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
- {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2',
+ {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
'bcfg2-server' : [
- {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
+ {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
- {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
+ {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
}
-
+
"""
packages = {}
ts = rpm.TransactionSet()
@@ -114,16 +114,16 @@ def loadRpms(dirs):
if options.verbose:
print 'Scanning directory: %s' % dir
- for file in [files for files in os.listdir(dir)
+ for file in [files for files in os.listdir(dir)
if files.endswith('.rpm')]:
filename = os.path.join( dir, file )
-
+
# Get the mtime of the RPM file.
file_mtime = datetime.date.fromtimestamp(os.stat(filename).st_mtime)
# Get the RPM header
- header = readRpmHeader( ts, filename )
+ header = readRpmHeader( ts, filename )
# Get what we are interesting in out of the header.
name = header[rpm.RPMTAG_NAME]
@@ -134,7 +134,7 @@ def loadRpms(dirs):
# Only load RPMs with subarchitectures as calculated from the --archs option.
if subarch in subarchs or 'all' in subarchs:
-
+
# Store what we want in our structure.
packages.setdefault(name, []).append({'filename':file, 'mtime':file_mtime, 'name':name, \
'arch':subarch, 'epoch':epoch, 'version':version, \
@@ -163,38 +163,59 @@ def loadRepos(repolist):
'''
repolist is a list of urls to yum repositories.
- Builds a dictionary keyed by the package name. Dictionary item is a list,
+ Builds a dictionary keyed by the package name. Dictionary item is a list,
one entry per package instance found.
-
- The list entries are dictionaries. Keys are 'filename', 'mtime' 'name',
- 'arch', 'epoch', 'version' and 'release'.
-
+
+ The list entries are dictionaries. Keys are 'filename', 'mtime' 'name',
+ 'arch', 'epoch', 'version' and 'release'.
+
e.g.
packages = {
'bcfg2' : [
- {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2',
+ {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
- {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2',
+ {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
'bcfg2-server' : [
- {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
+ {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
- {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
+ {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
}
-
+
'''
packages = {}
for repo in repolist:
- url = urlparse.urljoin(repo, './repodata/primary.xml.gz')
+ url = urlparse.urljoin(repo, './repodata/repomd.xml')
+
+ try:
+ opener = pkgmgr_URLopener()
+ file, message = opener.retrieve(url)
+ except:
+ sys.exit()
+
+ try:
+ tree = parse(file)
+ except IOError:
+ print "ERROR: Unable to parse retrieved repomd.xml."
+ sys.exit()
+
+ repomd = tree.getroot()
+ for element in repomd:
+ if element.tag.endswith('data') and element.attrib['type'] == 'primary':
+ for property in element:
+ if property.tag.endswith('location'):
+ primaryhref = property.attrib['href']
+
+ url = urlparse.urljoin(repo, './' + primaryhref)
if options.verbose:
print 'Loading : %s' % url
try:
opener = pkgmgr_URLopener()
- file, message = opener.retrieve(url)
+ file, message = opener.retrieve(url)
except:
sys.exit()
@@ -219,7 +240,7 @@ def loadRepos(repolist):
release = property.get('rel')
elif property.tag.endswith('location'):
file = property.get('href')
-
+
# Only load RPMs with subarchitectures as calculated from the --archs option.
if subarch in subarchs or 'all' in subarchs:
packages.setdefault(name, []).append({'filename':file, 'name':name, \
@@ -237,7 +258,7 @@ def loadRepos(repolist):
def printInstance(instance, group_count):
"""
- Print the details for a package instance with the appropriate indentation and
+ Print the details for a package instance with the appropriate indentation and
in the specified format (rpm or yum).
"""
group_count = group_count + 1
@@ -277,25 +298,25 @@ def printPackage(entry, group_count):
subarch_dict[instance['arch']].append(instance)
else:
subarch_dict[instance['arch']] = [ instance ]
-
+
# Keep track of the subarchitectures we have found in each architecture.
if subarch_mapping[instance['arch']] in arch_dict:
if instance['arch'] not in arch_dict[subarch_mapping[instance['arch']]]:
arch_dict[subarch_mapping[instance['arch']]].append(instance['arch'])
else:
arch_dict[subarch_mapping[instance['arch']]] = [ instance['arch'] ]
-
+
# Only keep the 'highest' subarchitecture in each architecture.
for arch in arch_dict.iterkeys():
if len(arch_dict[arch]) > 1:
arch_dict[arch].sort()
for s in arch_dict[arch][:-1]:
del subarch_dict[s]
-
+
# Sort packages within each architecture into version order
for arch in subarch_dict:
subarch_dict[arch].sort(cmpRpmHeader)
-
+
if options.release == 'all':
# Output all instances
for header in subarch_dict[arch]:
@@ -343,7 +364,7 @@ def main():
if options.pkgmgrhdr:
output.write( '</PackageList>\n' )
-
+
if options.verbose:
print '%i package instances were processed' % len(package_dict)
@@ -355,21 +376,21 @@ if __name__ == "__main__":
p.add_option('--archs', '-a', action='store', \
default='all', \
type='string', \
- help='''Comma separated list of subarchitectures to include.
- The highest subarichitecture required in an
- architecture group should specified. Lower
- subarchitecture packages will be loaded if that
- is all that is available. e.g. The higher of i386,
- i486 and i586 packages will be loaded if -a i586
+ help='''Comma separated list of subarchitectures to include.
+ The highest subarichitecture required in an
+ architecture group should specified. Lower
+ subarchitecture packages will be loaded if that
+ is all that is available. e.g. The higher of i386,
+ i486 and i586 packages will be loaded if -a i586
is specified. (Default: all).
''')
- p.add_option('--rpmdirs', '-d', action='store',
+ p.add_option('--rpmdirs', '-d', action='store',
type='string', \
- help='''Comma separated list of directories to scan for RPMS.
+ help='''Comma separated list of directories to scan for RPMS.
Wilcards are permitted.
''')
-
+
p.add_option('--enddate', '-e', action='store', \
type='string', \
help='End date for RPM file selection.')
@@ -378,34 +399,34 @@ if __name__ == "__main__":
default='yum', \
type='choice', \
choices=('yum','rpm'), \
- help='''Format of the Output. Choices are yum or rpm.
+ help='''Format of the Output. Choices are yum or rpm.
(Default: yum)
''')
p.add_option('--groups', '-g', action='store', \
type='string', \
- help='''List of comma separated groups to nest Package
+ help='''List of comma separated groups to nest Package
entities in.
''')
-
+
p.add_option('--indent', '-i', action='store', \
default=4, \
type='int', \
- help='''Number of leading spaces to indent nested entries in the
+ help='''Number of leading spaces to indent nested entries in the
output. (Default:4)
''')
p.add_option('--outfile', '-o', action='store', \
type='string', \
help='Output file name.')
-
+
p.add_option('--pkgmgrhdr', '-P', action='store_true', \
help='Include PackageList header in output.')
-
+
p.add_option('--priority', '-p', action='store', \
default=0, \
type='int', \
- help='''Value to set priority attribute in the PackageList Tag.
+ help='''Value to set priority attribute in the PackageList Tag.
(Default: 0)
''')
@@ -413,13 +434,13 @@ if __name__ == "__main__":
default='latest', \
type='choice', \
choices=('all','latest'), \
- help='''Which releases to include in the output. Choices are
+ help='''Which releases to include in the output. Choices are
all or latest. (Default: latest).''')
-
+
p.add_option('--startdate', '-s', action='store', \
type='string', \
help='Start date for RPM file selection.')
-
+
p.add_option('--uri', '-u', action='store', \
type='string', \
help='URI for PackageList header required for RPM format ouput.')
@@ -427,7 +448,7 @@ if __name__ == "__main__":
p.add_option('--verbose', '-v', action='store_true', \
help='Enable verbose output.')
- p.add_option('--yumrepos', '-y', action='store',
+ p.add_option('--yumrepos', '-y', action='store',
type='string', \
help='''Comma separated list of YUM repository URLs to load.
NOTE: Each URL must end in a '/' character.''')
@@ -441,18 +462,18 @@ if __name__ == "__main__":
if not options.rpmdirs and not options.yumrepos:
print "One of --rpmdirs and --yumrepos must be specified"
sys.exit(1)
-
+
# Set up list of directories to search
if options.rpmdirs:
search_dirs = []
for d in options.rpmdirs.split(','):
- search_dirs += glob.glob(d)
+ search_dirs += glob.glob(d)
if options.verbose:
print 'The following directories will be scanned:'
for d in search_dirs:
print ' %s' % d
-
- # Setup list of repos
+
+ # Setup list of repos
if options.yumrepos:
repos = []
for r in options.yumrepos.split(','):
@@ -464,19 +485,19 @@ if __name__ == "__main__":
# Set up list of architectures to include and some mappings
# to use later.
- arch_mapping = {'x86':['i686', 'i586', 'i486', 'i386', 'athlon'],
- 'x86_64':['x86_64'],
- 'ia64':['ia64'],
- 'ppc':['ppc'],
- 'ppc64':['ppc64'],
- 'sparc':['sparc'],
+ arch_mapping = {'x86':['i686', 'i586', 'i486', 'i386', 'athlon'],
+ 'x86_64':['x86_64'],
+ 'ia64':['ia64'],
+ 'ppc':['ppc'],
+ 'ppc64':['ppc64'],
+ 'sparc':['sparc'],
'noarch':['noarch']}
- subarch_mapping = {'i686':'x86', 'i586':'x86', 'i486':'x86', 'i386':'x86', 'athlon':'x86',
- 'x86_64':'x86_64',
- 'ia64':'ia64',
- 'ppc':'ppc',
- 'ppc64':'ppc64',
- 'sparc':'sparc',
+ subarch_mapping = {'i686':'x86', 'i586':'x86', 'i486':'x86', 'i386':'x86', 'athlon':'x86',
+ 'x86_64':'x86_64',
+ 'ia64':'ia64',
+ 'ppc':'ppc',
+ 'ppc64':'ppc64',
+ 'sparc':'sparc',
'noarch':'noarch' }
commandline_subarchs = options.archs.split(',')
arch_list = []
@@ -491,7 +512,7 @@ if __name__ == "__main__":
# Only allow one subarchitecture per architecture to be specified.
if s not in arch_list:
arch_list.append(s)
-
+
# Add subarchitectures lower than the one specified to the list.
# e.g. If i486 is specified this will add i386 to the list of
# subarchitectures to load.
@@ -508,7 +529,7 @@ if __name__ == "__main__":
groups_list = options.groups.split(',')
else:
groups_list = None
-
+
if options.outfile:
output = file(options.outfile, "w")
else: