summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authorNarayan Desai <desai@mcs.anl.gov>2007-07-07 14:04:15 +0000
committerNarayan Desai <desai@mcs.anl.gov>2007-07-07 14:04:15 +0000
commit98b66ec8786f1d7220cc6507a847afa2ac6d16a3 (patch)
treebbc2673c5cd5dd4c283e40f9de238334ef7d767e /tools
parentce781647f7576c3cf4bf0f209f93a6e20d340e57 (diff)
downloadbcfg2-98b66ec8786f1d7220cc6507a847afa2ac6d16a3.tar.gz
bcfg2-98b66ec8786f1d7220cc6507a847afa2ac6d16a3.tar.bz2
bcfg2-98b66ec8786f1d7220cc6507a847afa2ac6d16a3.zip
pkmgr_gen/update updates from mbrady
git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@3424 ce84e21b-d406-0410-9b95-82705330c041
Diffstat (limited to 'tools')
-rwxr-xr-xtools/pkgmgr_gen.py166
-rw-r--r--tools/pkgmgr_update.py152
2 files changed, 274 insertions, 44 deletions
diff --git a/tools/pkgmgr_gen.py b/tools/pkgmgr_gen.py
index d6d55562c..5b6e23f05 100755
--- a/tools/pkgmgr_gen.py
+++ b/tools/pkgmgr_gen.py
@@ -17,6 +17,11 @@ import rpm
import optparse
import datetime
import glob
+import urllib
+import gzip
+import time
+import urlparse
+from elementtree.ElementTree import parse
def info(object, spacing=10, collapse=1):
"""Print methods and doc strings.
@@ -132,14 +137,11 @@ def loadRpms(dirs):
if subarch in subarchs or 'all' in subarchs:
# Store what we want in our structure.
- if name in packages:
- packages[name].append({'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \
- 'epoch':epoch, 'version':version, 'release':release})
- else:
- packages[name] = [{'filename':file, 'mtime':file_mtime, 'name':name, 'arch':subarch, \
- 'epoch':epoch, 'version':version, 'release':release}]
+ packages.setdefault(name, []).append({'filename':file, 'mtime':file_mtime, 'name':name, \
+ 'arch':subarch, 'epoch':epoch, 'version':version, \
+ 'release':release})
- # Print '.' for each package. stdio is line buffered, so have to flush it.
+ # Print '.' for each package. stdio is line buffered, so have to flush it.
if options.verbose:
sys.stdout.write('.')
sys.stdout.flush()
@@ -148,6 +150,92 @@ def loadRpms(dirs):
return packages
+class pkgmgr_URLopener(urllib.FancyURLopener):
+ """
+ Override default error handling so that we can see what the errors are.
+ """
+ def http_error_default(self, url, fp, errcode, errmsg, headers):
+ """
+ Override default error handling so that we can see what the errors are.
+ """
+ print "ERROR %s: Unable to retrieve %s" % (errcode, url)
+
+def loadRepos(repolist):
+ '''
+ repolist is a list of urls to yum repositories.
+
+ Builds a dictionary keyed by the package name. Dictionary item is a list,
+ one entry per package instance found.
+
+ The list entries are dictionaries. Keys are 'filename', 'mtime' 'name',
+ 'arch', 'epoch', 'version' and 'release'.
+
+ e.g.
+
+ packages = {
+ 'bcfg2' : [
+ {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2',
+ ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
+ {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2',
+ ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
+ 'bcfg2-server' : [
+ {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
+ ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
+ {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
+ ''arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}],
+ }
+
+ '''
+ packages = {}
+ for repo in repolist:
+ url = urlparse.urljoin(repo, './repodata/primary.xml.gz')
+
+ if options.verbose:
+ print 'Loading : %s' % url
+
+ try:
+ opener = pkgmgr_URLopener()
+ file, message = opener.retrieve(url)
+ except:
+ sys.exit()
+
+ try:
+ repo_file = gzip.open(file)
+ tree = parse(repo_file)
+ except IOError:
+ print "ERROR: Unable to parse retrieved file."
+ sys.exit()
+
+ repo = tree.getroot()
+ for element in repo:
+ if element.tag.endswith('package'):
+ for property in element:
+ if property.tag.endswith('name'):
+ name = property.text
+ elif property.tag.endswith('arch'):
+ subarch = property.text
+ elif property.tag.endswith('version'):
+ version = property.get('ver')
+ epoch = property.get('epoch')
+ release = property.get('rel')
+ elif property.tag.endswith('location'):
+ file = property.get('href')
+
+ # Only load RPMs with subarchitectures as calculated from the --archs option.
+ if subarch in subarchs or 'all' in subarchs:
+ packages.setdefault(name, []).append({'filename':file, 'name':name, \
+ 'arch':subarch, 'epoch':epoch, \
+ 'version':version, 'release':release})
+
+ # Print '.' for each package. stdio is line buffered, so have to flush it.
+ if options.verbose:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ if options.verbose:
+ sys.stdout.write('\n')
+ repo_file.close()
+ return packages
+
def printInstance(instance, group_count):
"""
Print the details for a package instance with the appropriate indentation and
@@ -224,7 +312,10 @@ def main():
if options.verbose:
print 'Loading package headers'
- package_dict = loadRpms(search_dirs)
+ if options.rpmdirs:
+ package_dict = loadRpms(search_dirs)
+ elif options.yumrepos:
+ package_dict = loadRepos(repos)
if options.verbose:
print 'Processing package headers'
@@ -275,9 +366,10 @@ if __name__ == "__main__":
''')
p.add_option('--rpmdirs', '-d', action='store',
- default='.', \
type='string', \
- help='Comma separated list of directories to scan for RPMS. Wilcards are permitted. (Default: .)')
+ help='''Comma separated list of directories to scan for RPMS.
+ Wilcards are permitted.
+ ''')
p.add_option('--enddate', '-e', action='store', \
type='string', \
@@ -287,16 +379,22 @@ if __name__ == "__main__":
default='yum', \
type='choice', \
choices=('yum','rpm'), \
- help='Format of the Output. Choices are yum or rpm. (Default: yum)')
+ help='''Format of the Output. Choices are yum or rpm.
+ (Default: yum)
+ ''')
p.add_option('--groups', '-g', action='store', \
type='string', \
- help='List of comma separated groups to nest Package entities in.')
+ help='''List of comma separated groups to nest Package
+ entities in.
+ ''')
p.add_option('--indent', '-i', action='store', \
default=4, \
type='int', \
- help='Number of leading spaces to indent nested entries in the output. (Default:4)')
+ help='''Number of leading spaces to indent nested entries in the
+ output. (Default:4)
+ ''')
p.add_option('--outfile', '-o', action='store', \
type='string', \
@@ -308,13 +406,16 @@ if __name__ == "__main__":
p.add_option('--priority', '-p', action='store', \
default=0, \
type='int', \
- help='Value to set priority attribute in the PackageList Tag. (Default: 0)')
+ help='''Value to set priority attribute in the PackageList Tag.
+ (Default: 0)
+ ''')
p.add_option('--release', '-r', action='store', \
default='latest', \
type='choice', \
choices=('all','latest'), \
- help='Which releases to include in the output. Choices are all or latest. (Default: latest).')
+ help='''Which releases to include in the output. Choices are
+ all or latest. (Default: latest).''')
p.add_option('--startdate', '-s', action='store', \
type='string', \
@@ -327,16 +428,40 @@ if __name__ == "__main__":
p.add_option('--verbose', '-v', action='store_true', \
help='Enable verbose output.')
+ p.add_option('--yumrepos', '-y', action='store',
+ type='string', \
+ help='''Comma separated list of YUM repository URLs to load.
+ NOTE: Each URL must end in a '/' character.''')
+
options, arguments = p.parse_args()
if options.pkgmgrhdr and options.format == 'rpm' and not options.uri:
print "Option --uri must be specified to produce a PackageList Tag for rpm formatted files."
sys.exit(1)
+ if not options.rpmdirs and not options.yumrepos:
+ print "One of --rpmdirs and --yumrepos must be specified"
+ sys.exit(1)
+
# Set up list of directories to search
- search_dirs = []
- for d in options.rpmdirs.split(','):
- search_dirs += glob.glob(d)
+ if options.rpmdirs:
+ search_dirs = []
+ for d in options.rpmdirs.split(','):
+ search_dirs += glob.glob(d)
+ if options.verbose:
+ print 'The following directories will be scanned:'
+ for d in search_dirs:
+ print ' %s' % d
+
+ # Setup list of repos
+ if options.yumrepos:
+ repos = []
+ for r in options.yumrepos.split(','):
+ repos.append(r)
+ if options.verbose:
+ print 'The following repositories will be scanned:'
+ for d in repos:
+ print ' %s' % d
# Set up list of architectures to include and some mappings
# to use later.
@@ -378,11 +503,6 @@ if __name__ == "__main__":
print 'Error: Multiple subarchitecutes of the same architecture specified.'
sys.exit(1)
- if options.verbose:
- print 'The following directories will be scanned:'
- for d in search_dirs:
- print ' %s' % d
-
indent = ' ' * options.indent
if options.groups:
diff --git a/tools/pkgmgr_update.py b/tools/pkgmgr_update.py
index a0e1fc227..e0219088e 100644
--- a/tools/pkgmgr_update.py
+++ b/tools/pkgmgr_update.py
@@ -20,6 +20,7 @@ import optparse
import datetime
import glob
from elementtree.ElementTree import parse, XML, fromstring, tostring
+import urlparse, urllib, gzip
installOnlyPkgs = ['kernel', 'kernel-bigmem', 'kernel-enterprise', 'kernel-smp',
'kernel-modules', 'kernel-debug', 'kernel-unsupported',
@@ -74,18 +75,21 @@ def loadRpms(dirs):
'arch', 'epoch', 'version' and 'release'.
e.g.
-
+
packages = {
- 'bcfg2' : { 'noarch' : [ {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':'bcfg2',
- 'arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
- {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':'bcfg2',
- 'arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}]},
- 'bcfg2-server' { 'noarch' : [ {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'' 'name':'bcfg2-server',
- 'arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc1'}
- {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'' 'name':"bcfg2-server',
- 'arch':'noarch', 'epoch':None, 'version':'0.9.2', 'release':'0.0rc5'}]},
+ 'bcfg2' : { 'noarch' : [ {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'',
+ 'name':'bcfg2', 'arch':'noarch', 'epoch':None, 'version':'0.9.2',
+ 'release':'0.0rc1'}
+ {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'',
+ 'name':'bcfg2', 'arch':'noarch', 'epoch':None, 'version':'0.9.2',
+ 'release':'0.0rc5'}]},
+ 'bcfg2-server' { 'noarch' : [ {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'',
+ 'name':'bcfg2-server', 'arch':'noarch', 'epoch':None,
+ 'version':'0.9.2', 'release':'0.0rc1'}
+ {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'',
+ 'name':"bcfg2-server', 'arch':'noarch', 'epoch':None,
+ 'version':'0.9.2', 'release':'0.0rc5'}]},
}
-
"""
packages = {}
ts = rpm.TransactionSet()
@@ -128,6 +132,91 @@ def loadRpms(dirs):
return packages
+class pkgmgr_URLopener(urllib.FancyURLopener):
+ """
+ Override default error handling so that we can see what the errors are.
+ """
+ def http_error_default(self, url, fp, errcode, errmsg, headers):
+ """
+ Override default error handling so that we can see what the errors are.
+ """
+ print "ERROR %s: Unable to retrieve %s" % (errcode, url)
+
+def loadRepos(repolist):
+ """
+ repolist is a list of urls to yum repositories.
+
+ Builds a multilevel dictionary keyed by the package name and arch.
+ Arch dictionary item is a list, one entry per package instance found.
+
+ The list entries are dictionaries. Keys are 'filename', 'mtime' 'name',
+ 'arch', 'epoch', 'version' and 'release'.
+
+ e.g.
+
+ packages = {
+ 'bcfg2' : { 'noarch' : [ {'filename':'bcfg2-0.9.2-0.0rc1.noarch.rpm', 'mtime':'',
+ 'name':'bcfg2', 'arch':'noarch', 'epoch':None, 'version':'0.9.2',
+ 'release':'0.0rc1'}
+ {'filename':'bcfg2-0.9.2-0.0rc5.noarch.rpm', 'mtime':'',
+ 'name':'bcfg2', 'arch':'noarch', 'epoch':None, 'version':'0.9.2',
+ 'release':'0.0rc5'}]},
+ 'bcfg2-server' { 'noarch' : [ {'filename':'bcfg2-server-0.9.2-0.0rc1.noarch.rpm', 'mtime':'',
+ 'name':'bcfg2-server', 'arch':'noarch', 'epoch':None,
+ 'version':'0.9.2', 'release':'0.0rc1'}
+ {'filename':'bcfg2-server-0.9.2-0.0rc5.noarch.rpm', 'mtime':'',
+ 'name':"bcfg2-server', 'arch':'noarch', 'epoch':None,
+ 'version':'0.9.2', 'release':'0.0rc5'}]},
+ }
+
+ """
+ packages = {}
+ for repo in repolist:
+ url = urlparse.urljoin(repo, './repodata/primary.xml.gz')
+
+ if options.verbose:
+ print 'Loading : %s' % url
+
+ try:
+ opener = pkgmgr_URLopener()
+ file, message = opener.retrieve(url)
+ except:
+ sys.exit()
+
+ try:
+ repo_file = gzip.open(file)
+ tree = parse(repo_file)
+ except IOError:
+ print "ERROR: Unable to parse retrieved file."
+ sys.exit()
+
+ root = tree.getroot()
+ for element in root:
+ if element.tag.endswith('package'):
+ for property in element:
+ if property.tag.endswith('name'):
+ name = property.text
+ elif property.tag.endswith('arch'):
+ subarch = property.text
+ elif property.tag.endswith('version'):
+ version = property.get('ver')
+ epoch = property.get('epoch')
+ release = property.get('rel')
+ elif property.tag.endswith('location'):
+ file = property.get('href')
+
+ if name not in installOnlyPkgs:
+ packages.setdefault(name, {}).setdefault(subarch, []).append({'filename':file, \
+ 'name':name, 'arch':subarch, \
+ 'epoch':epoch, 'version':version, 'release':release})
+ if options.verbose:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ if options.verbose:
+ sys.stdout.write('\n')
+
+ return packages
+
def str_evra(instance):
"""
Convert evra dict entries to a string.
@@ -173,7 +262,10 @@ def main():
if options.verbose:
print 'Loading package headers'
- package_dict = loadRpms(search_dirs)
+ if options.rpmdirs:
+ package_dict = loadRpms(search_dirs)
+ elif options.yumrepos:
+ package_dict = loadRepos(repos)
if options.verbose:
print 'Processing package headers'
@@ -192,9 +284,9 @@ if __name__ == "__main__":
help='Existing Pkgmgr configuration file name.')
p.add_option('--rpmdirs', '-d', action='store',
- default='.', \
type='string', \
- help='Comma separated list of directories to scan for RPMS. Wilcards are permitted. (Default: .)')
+ help='''Comma separated list of directories to scan for RPMS.
+ Wilcards are permitted.''')
p.add_option('--outfile', '-o', action='store', \
type='string', \
@@ -203,22 +295,40 @@ if __name__ == "__main__":
p.add_option('--verbose', '-v', action='store_true', \
help='Enable verbose output.')
+ p.add_option('--yumrepos', '-y', action='store',
+ type='string', \
+ help='''Comma separated list of YUM repository URLs to load.
+ NOTE: Each URL must end in a '/' character.''')
options, arguments = p.parse_args()
if not options.configfile:
print "An existing Pkgmgr configuration file must be specified with the -c option."
sys.exit()
+ if not options.rpmdirs and not options.yumrepos:
+ print "One of --rpmdirs and --yumrepos must be specified"
+ sys.exit(1)
+
# Set up list of directories to search
- search_dirs = []
- for d in options.rpmdirs.split(','):
- search_dirs += glob.glob(d)
+ if options.rpmdirs:
+ search_dirs = []
+ for d in options.rpmdirs.split(','):
+ search_dirs += glob.glob(d)
+ if options.verbose:
+ print 'The following directories will be scanned:'
+ for d in search_dirs:
+ print ' %s' % d
+
+ # Setup list of repos
+ if options.yumrepos:
+ repos = []
+ for r in options.yumrepos.split(','):
+ repos.append(r)
+ if options.verbose:
+ print 'The following repositories will be scanned:'
+ for d in repos:
+ print ' %s' % d
- if options.verbose:
- print 'The following directories will be scanned:'
- for d in search_dirs:
- print ' %s' % d
-
if options.outfile:
output = file(options.outfile, "w")
else: