From d6d2c27f891d9e796c173d07fccf86f2a67ace06 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Tue, 11 Dec 2007 18:05:47 +0000 Subject: add GP's version of the debian pkglist creator git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4077 ce84e21b-d406-0410-9b95-82705330c041 --- tools/create-debian-pkglist-gp.py | 219 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 tools/create-debian-pkglist-gp.py (limited to 'tools') diff --git a/tools/create-debian-pkglist-gp.py b/tools/create-debian-pkglist-gp.py new file mode 100644 index 000000000..d94e562bb --- /dev/null +++ b/tools/create-debian-pkglist-gp.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python + +'''Build debian/ubuntu package indexes''' +__revision__ = '$Id: create-debian-pkglist.py 11778 2007-12-11 13:46:06Z guillaume $' + +# Original code from Bcfg2 sources + +import glob, gzip, lxml.etree, os, re, urllib, cStringIO, sys, ConfigParser, commands + +def debug(msg): + '''print debug messages''' + if '-v' in sys.argv: + sys.stdout.write(msg) + +def get_as_list(somestring): + """ Input : a string like this : 'a, g, f,w' + Output : a list like this : ['a', 'g', 'f', 'w'] """ + return somestring.replace(' ', '').split(',') + +def list_contains_all_the_same_values(l): + if len(l) == 0: + return True + # The list contains all the same values if all elements in + # the list are equal to the first element. + first = l[0] + for elem in l: + if first != elem: + return False + return True + +class SourceURL: + def __init__(self, deb_url): + deb_url_tokens = deb_url.split() + # ex: deb http://somemirror.com/ubuntu dapper main restricted universe + self.url = deb_url_tokens[1] + self.distribution = deb_url_tokens[2] + self.sections = deb_url_tokens[3:] + + def __str__(self): + return "deb %s %s %s" % (self.url, self.distribution, ' '.join(self.sections)) + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, str(self)) + +class Source: + def __init__(self, confparser, section, bcfg2_repos_prefix): + self.filename = "%s/Pkgmgr/%s.xml" % (bcfg2_repos_prefix, section) + self.groups = get_as_list(confparser.get(section, "group_names")) + self.priority = confparser.getint(section, "priority") + self.architectures = get_as_list(confparser.get(section, "architectures")) + + self.source_urls = [] + self.source_urls.append(SourceURL(confparser.get(section, "deb_url"))) + # Agregate urls in the form of deb_url0, deb_url1, ... to deb_url9 + for i in range(10): # 0 to 9 + option_name = "deb_url%s" % i + if confparser.has_option(section, option_name): + self.source_urls.append(SourceURL(confparser.get(section, option_name))) + + self.file = None + self.indent_level = 0 + + def __str__(self): + return """File: %s +Groups: %s +Priority: %s +Architectures: %s +Source URLS: %s""" % (self.filename, self.groups, self.priority, self.architectures, self.source_urls) + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, str(self)) + + def _open_file(self): + self.file = open(self.filename + '~', 'w') + + def _close_file(self): + self.file.close() + + def _write_to_file(self, msg): + self.file.write("%s%s\n" % (self.indent_level * ' ', msg)) + + def _rename_file(self): + os.rename(self.filename + '~', self.filename) + + def _pkg_version_is_older(self, version1, version2): + """ Use dpkg to compare the two version + Return true if version1 < version2 """ + # Avoid forking a new process if the two strings are equals + if version1 == version2: + return False + (status, output) = commands.getstatusoutput("/usr/bin/dpkg --compare-versions %s lt %s" % (version1, version2)) + #print "%s dpkg --compare-versions %s lt %s" % (status, version1, version2) + return status == 0 + + def _update_pkgdata(self, pkgdata, source_url): + for section in source_url.sections: + for arch in self.architectures: + url = "%s/dists/%s/%s/binary-%s/Packages.gz" % (source_url.url, source_url.distribution, section, arch) + debug("Processing url %s\n" % (url)) + try: + data = urllib.urlopen(url) + buf = cStringIO.StringIO(''.join(data.readlines())) + reader = gzip.GzipFile(fileobj=buf) + for line in reader.readlines(): + if line[:8] == 'Package:': + pkgname = line.split(' ')[1].strip() + elif line[:8] == 'Version:': + version = line.split(' ')[1].strip() + if pkgdata.has_key(pkgname): + if pkgdata[pkgname].has_key(arch): + # The package is listed twice for the same architecture + # We keep the most recent version + old_version = pkgdata[pkgname][arch] + if self._pkg_version_is_older(old_version, version): + pkgdata[pkgname][arch] = version + else: + # The package data exists for another architecture, + # but not for this one. Add it. + pkgdata[pkgname][arch] = version + else: + # First entry for this package + pkgdata[pkgname] = {arch:version} + else: + continue + except: + raise Exception("Could not process URL %s\n%s\nPlease verify the URL." % (url, sys.exc_value)) + return pkgdata + + def _get_sorted_pkg_keys(self, pkgdata): + pkgs = [] + for k in pkgdata.keys(): + pkgs.append(k) + pkgs.sort() + return pkgs + + def _write_common_entries(self, pkgdata): + # Write entries for packages that have the same version + # across all architectures + #coalesced = 0 + for pkg in self._get_sorted_pkg_keys(pkgdata): + # Dictionary of archname: pkgversion + # (There is exactly one version per architecture) + archdata = pkgdata[pkg] + # List of versions for all architectures of this package + pkgversions = archdata.values() + # If the versions for all architectures are the same + if list_contains_all_the_same_values(pkgversions): + # Write the package data + ver = pkgversions[0] + self._write_to_file('' % (pkg, ver)) + #coalesced += 1 + # Remove this package entry + del pkgdata[pkg] + + def _write_perarch_entries(self, pkgdata): + # Write entries that are left, i.e. packages that have different + # versions per architecture + #perarch = 0 + if pkgdata: + for arch in self.architectures: + self._write_to_file('' % (arch)) + self.indent_level = self.indent_level + 1 + for pkg in self._get_sorted_pkg_keys(pkgdata): + if pkgdata[pkg].has_key(arch): + self._write_to_file('' % (pkg, pkgdata[pkg][arch])) + #perarch += 1 + self.indent_level = self.indent_level - 1 + self._write_to_file('') + #debug("Got %s coalesced, %s per-arch\n" % (coalesced, perarch)) + + def process(self): + '''Build package indices for source''' + + # First, build the pkgdata structure without touching the file, + # so the file does not contain incomplete informations if the + # network in not reachable. + pkgdata = {} + for source_url in self.source_urls: + pkgdata = self._update_pkgdata(pkgdata, source_url) + + # Construct the file. + self._open_file() + for source_url in self.source_urls: + self._write_to_file('' % source_url) + + self._write_to_file('' % self.priority) + + self.indent_level = self.indent_level + 1 + for group in self.groups: + self._write_to_file('' % group) + self.indent_level = self.indent_level + 1 + + self._write_common_entries(pkgdata) + self._write_perarch_entries(pkgdata) + + for group in self.groups: + self.indent_level = self.indent_level - 1 + self._write_to_file('') + self.indent_level = self.indent_level - 1 + self._write_to_file('') + self._close_file() + self._rename_file() + +if __name__ == '__main__': + # Prefix is relative to script path + complete_script_path = os.path.join(os.getcwd(), sys.argv[0]) + prefix = complete_script_path[:-len('etc/create-debian-pkglist.py')] + + confparser = ConfigParser.SafeConfigParser() + confparser.read(prefix + "etc/debian-pkglist.conf") + + # We read the whole configuration file before processing each entries + # to avoid doing work if there is a problem in the file. + sources_list = [] + for section in confparser.sections(): + sources_list.append(Source(confparser, section, prefix)) + + for source in sources_list: + source.process() -- cgit v1.2.3-1-g7c22