Modification given by Stéphane Graber to support the debian architecture

variant

include :
 * support different package version on different package list and if
   the version is different on each architecture then it's placed on
   arch specific list
 * added new possible var as dev_$arch_urlX so that architecture
   specific list are treated correctly
 * Uses python-apt for better speed, about half-time gain

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5024 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 222 insertions, 95 deletions

diff --git a/tools/create-debian-pkglist.py b/tools/create-debian-pkglist.py
index 9761d43ac..822e4922c 100755
--- a/tools/create-debian-pkglist.py
+++ b/tools/create-debian-pkglist.py
@@ -1,111 +1,238 @@
 #!/usr/bin/env python
 
 '''Build debian/ubuntu package indexes'''
-__revision__ = '$Id: $'
+__revision__ = '$Id$'
+
+# Original code from Bcfg2 sources
+
+import glob, gzip, lxml.etree, os, re, urllib, cStringIO, sys, ConfigParser, apt_pkg
+apt_pkg.init()
 
-import glob, gzip, lxml.etree, os, re, urllib, cStringIO, sys
 
 def debug(msg):
     '''print debug messages'''
     if '-v' in sys.argv:
         sys.stdout.write(msg)
 
+def get_as_list(somestring):
+    """ Input : a string like this : 'a, g, f,w'
+        Output : a list like this : ['a', 'g', 'f', 'w'] """
+    return somestring.replace(' ', '').split(',')
 
-def processSource(prefix, source, dists, archs, prio, groups, packages):
-    '''Build package indices for source'''
-    filename = "%s/%s.xml" % (prefix, source.replace('/', '_'))
-    try:
-        os.stat(filename)
-        filename += '~'
-    except:
-        pass
-    output = open(filename, 'w')
-    #depfile = open(filename + '-deps', 'w')
-    groupinfo = "".join(['<Group name="%s">' % group for group in groups])
-    output.write('<PackageList priority="%s" type="deb">%s\n' % (prio, groupinfo))
-    for dist in dists:
-        pkgdata = {}
-        pkgdeps = {}
-        for arch in archs:
-            url = "%s/%s/binary-%s/Packages.gz" % (source, dist, arch)
-            debug("Processing url %s\n" % (url))
-            data = urllib.urlopen(url)
-            buf = cStringIO.StringIO(''.join(data.readlines()))
-            reader = gzip.GzipFile(fileobj=buf)
-            for line in reader.readlines():
-                if line[:8] == 'Package:':
-                    pkgname = line.split(' ')[1].strip()
-                elif line[:8] == 'Version:':
-                    version = line.split(' ')[1].strip()
-                    if pkgname in pkgdata:
-                        pkgdata[pkgname][arch] = version
-                    else:
-                        pkgdata[pkgname] = {arch:version}
-                elif line[:8] == 'Depends:':
-                    deps = re.sub(',', '', re.sub('\(.*\)', '', line)).split()[1:]
-                    if pkgname in pkgdeps:
-                        pkgdeps[pkgname][arch] = deps
-                    else:
-                        pkgdeps[pkgname] = {arch:deps}
-                else:
+def list_contains_all_the_same_values(l):
+    if len(l) == 0:
+        return True
+    # The list contains all the same values if all elements in 
+    # the list are equal to the first element.
+    first = l[0]
+    for elem in l:
+        if first != elem:
+            return False
+    return True
+
+class SourceURL:
+    def __init__(self, deb_url, arch):
+        deb_url_tokens = deb_url.split()
+        # ex: deb http://somemirror.com/ubuntu dapper main restricted universe
+        self.url = deb_url_tokens[1]
+        self.distribution = deb_url_tokens[2]
+        self.sections = deb_url_tokens[3:]
+        self.arch = arch
+    
+    def __str__(self):
+        return "deb %s %s %s" % (self.url, self.distribution, ' '.join(self.sections))
+    
+    def __repr__(self):
+        return "<%s %s>" % (self.__class__.__name__, str(self))
+
+class Source:
+    def __init__(self, confparser, section, bcfg2_repos_prefix):
+        self.filename = "%s/Pkgmgr/%s.xml" % (bcfg2_repos_prefix, section)
+        self.groups = get_as_list(confparser.get(section, "group_names"))
+        self.priority = confparser.getint(section, "priority")
+        self.architectures = get_as_list(confparser.get(section, "architectures"))
+        self.arch_specialurl = set()
+
+        self.source_urls = []
+        self.source_urls.append(SourceURL(confparser.get(section, "deb_url"),"all"))
+        # Agregate urls in the form of deb_url0, deb_url1, ... to deb_url9
+        for i in range(10): # 0 to 9
+            option_name = "deb_url%s" % i
+            if confparser.has_option(section, option_name):
+                self.source_urls.append(SourceURL(confparser.get(section, option_name),"all"))
+
+        # Aggregate architecture specific urls (if present)
+        for arch in self.architectures:
+            if not confparser.has_option(section, "deb_"+arch+"_url"):
+                continue
+            self.source_urls.append(SourceURL(confparser.get(section, "deb_"+arch+"_url"),arch))
+            # Agregate urls in the form of deb_url0, deb_url1, ... to deb_url9
+            for i in range(10): # 0 to 9
+                option_name = "deb_"+arch+"_url%s" % i
+                if confparser.has_option(section, option_name):
+                    self.source_urls.append(SourceURL(confparser.get(section, option_name),arch))
+                    self.arch_specialurl.add(arch)
+   
+        self.file = None
+        self.indent_level = 0
+
+    def __str__(self):
+        return """File: %s
+Groups: %s
+Priority: %s
+Architectures: %s
+Source URLS: %s""" % (self.filename, self.groups, self.priority, self.architectures, self.source_urls)
+    
+    def __repr__(self):
+        return "<%s %s>" % (self.__class__.__name__, str(self))
+
+    def _open_file(self):
+        self.file = open(self.filename + '~', 'w')
+    
+    def _close_file(self):
+        self.file.close()
+
+    def _write_to_file(self, msg):
+        self.file.write("%s%s\n" % (self.indent_level * '    ', msg))
+
+    def _rename_file(self):
+        os.rename(self.filename + '~', self.filename)
+
+    def _pkg_version_is_older(self, version1, version2):
+        """ Use dpkg to compare the two version
+            Return true if version1 < version2 """
+        # Avoid forking a new process if the two strings are equals
+        if version1 == version2:
+            return False
+        status = apt_pkg.VersionCompare(version1, version2)
+        return status < 0
+
+    def _update_pkgdata(self, pkgdata, source_url):
+        for section in source_url.sections:
+            for arch in self.architectures:
+                if source_url.arch != arch and source_url.arch != "all":
                     continue
-        coalesced = 0
-        for pkg in pkgdata.keys():
-            data = pkgdata[pkg].values()
-            if data.count(data[0]) == len(data) == len(archs):
-                output.write('<Package name="%s" version="%s"/>\n' % (pkg, data[0]))
-                coalesced += 1
+                if source_url.arch == "all" and arch in self.arch_specialurl:
+                    continue
+                url = "%s/dists/%s/%s/binary-%s/Packages.gz" % (source_url.url, source_url.distribution, section, arch)
+                debug("Processing url %s\n" % (url))
+                try:
+                    data = urllib.urlopen(url)
+                    buf = cStringIO.StringIO(''.join(data.readlines()))
+                    reader = gzip.GzipFile(fileobj=buf)
+                    for line in reader.readlines():
+                        if line[:8] == 'Package:':
+                            pkgname = line.split(' ')[1].strip()
+                        elif line[:8] == 'Version:':
+                            version = line.split(' ')[1].strip()
+                            if pkgdata.has_key(pkgname):
+                                if pkgdata[pkgname].has_key(arch):
+                                    # The package is listed twice for the same architecture
+                                    # We keep the most recent version
+                                    old_version = pkgdata[pkgname][arch]
+                                    if self._pkg_version_is_older(old_version, version):
+                                        pkgdata[pkgname][arch] = version
+                                else:
+                                    # The package data exists for another architecture,
+                                    # but not for this one. Add it.
+                                    pkgdata[pkgname][arch] = version
+                            else:
+                                # First entry for this package
+                                pkgdata[pkgname] = {arch:version}
+                        else:
+                            continue
+                except:
+                    raise Exception("Could not process URL %s\n%s\nPlease verify the URL." % (url, sys.exc_value))
+        return pkgdata
+    
+    def _get_sorted_pkg_keys(self, pkgdata):
+            pkgs = []
+            for k in pkgdata.keys():
+                pkgs.append(k)
+            pkgs.sort()
+            return pkgs
+
+    def _write_common_entries(self, pkgdata):
+        # Write entries for packages that have the same version
+        # across all architectures
+        #coalesced = 0
+        for pkg in self._get_sorted_pkg_keys(pkgdata):
+            # Dictionary of archname: pkgversion
+            # (There is exactly one version per architecture)
+            archdata = pkgdata[pkg]
+            # List of versions for all architectures of this package
+            pkgversions = archdata.values()
+            # If the versions for all architectures are the same
+            if len(self.architectures) == len(pkgversions) and list_contains_all_the_same_values(pkgversions):
+                # Write the package data
+                ver = pkgversions[0]
+                self._write_to_file('<Package name="%s" version="%s"/>' % (pkg, ver))
+                #coalesced += 1
+                # Remove this package entry
                 del pkgdata[pkg]
-        for pkg in pkgdeps.keys():
-            data = pkgdeps[pkg].values()
-            if data.count(data[0]) == len(data):
-                elt = lxml.etree.Element("Package", name=pkg)
-                [lxml.etree.SubElement(elt, "Package", name=dep) for dep in data[0]]
-                #depfile.write(lxml.etree.tostring(elt) + '\n')
-                del pkgdeps[pkg]
-        # now we need to do per-arch entries
-        perarch = 0
+
+    def _write_perarch_entries(self, pkgdata):
+        # Write entries that are left, i.e. packages that have different 
+        # versions per architecture
+        #perarch = 0
         if pkgdata:
-            for arch in archs:
-                output.write('<Group name="%s">\n' % (arch))
-                for pkg in pkgdata.keys():
-                    if arch in pkgdata[pkg]:
-                        output.write('<Package name="%s" version="%s"/>\n' % (pkg, pkgdata[pkg][arch]))
-                        perarch += 1
-                output.write('</Group>\n')
-        debug("Got %s coalesced, %s per-arch\n" % (coalesced, perarch))
-    closegroup = "".join(['</Group>' for group in groups])
-    output.write('%s</PackageList>\n' % (closegroup))
-    output.close()
-    #depfile.close()
-    if filename[-1] == '~':
-        old = open(filename[:-1]).read()
-        new = open(filename).read()
-        if old != new:
-            debug("%s has changed; installing new version\n" % (filename[:-1]))
-            os.rename(filename, filename[:-1])
-        else:
-            debug("%s has not changed\n" % (filename[:-1]))
+            for arch in self.architectures:
+                self._write_to_file('<Group name="%s">' % (arch))
+                self.indent_level = self.indent_level + 1
+                for pkg in self._get_sorted_pkg_keys(pkgdata):
+                    if pkgdata[pkg].has_key(arch):
+                        self._write_to_file('<Package name="%s" version="%s"/>' % (pkg, pkgdata[pkg][arch]))
+                        #perarch += 1
+                self.indent_level = self.indent_level - 1
+                self._write_to_file('</Group>')
+        #debug("Got %s coalesced, %s per-arch\n" % (coalesced, perarch))
 
-if __name__ == '__main__':
-    hprefix = '/tmp'
-    rprefix = '/home/desai/data/bcfg2'
-    packages = []
-    for fn in glob.glob("%s/Bundler/*.xml" % rprefix) + glob.glob("%s/Base/*.xml" % rprefix):
-        doc = lxml.etree.parse(fn)
-        [packages.append(pkg.get('name')) for pkg in doc.findall('.//Package') if
-         pkg.get('name') not in packages]
+    def process(self):
+        '''Build package indices for source'''
+        
+        # First, build the pkgdata structure without touching the file,
+        # so the file does not contain incomplete informations if the
+        # network in not reachable.
+        pkgdata = {}
+        for source_url in self.source_urls:
+            pkgdata = self._update_pkgdata(pkgdata, source_url)
+        
+        # Construct the file.
+        self._open_file()
+        for source_url in self.source_urls:
+            self._write_to_file('<!-- %s -->' % source_url)
         
-    print len(packages)
-
-    SOURCES = [('http://security.debian.org/dists/stable/updates',
-                ['main', 'contrib', 'non-free'], ['i386'], '10', ['debian-sarge']),
-               ('http://volatile.debian.net/debian-volatile/dists/stable/volatile',
-                ['main', 'contrib', 'non-free'], ['i386'], '20', ['debian-sarge']),
-               ('http://us.archive.ubuntu.com/ubuntu/dists/dapper',
-                ['main', 'multiverse', 'universe'], ['i386', 'amd64'], 0, ['ubuntu-dapper']),
-               ('http://netzero.mcs.anl.gov/disks/distro/ubuntu/dists/dapper',
-                ['main', 'multiverse', 'universe'], ['i386', 'amd64'], 0, ['ubuntu-dapper'])]
-
-    for src, dsts, ars, pri, grps in SOURCES:
-        processSource(hprefix, src, dsts, ars, pri, grps, packages)
+        self._write_to_file('<PackageList priority="%s" type="deb">' % self.priority)
+        
+        self.indent_level = self.indent_level + 1
+        for group in self.groups:
+            self._write_to_file('<Group name="%s">' % group)
+            self.indent_level = self.indent_level + 1
+        
+        self._write_common_entries(pkgdata)
+        self._write_perarch_entries(pkgdata)
+        
+        for group in self.groups:
+            self.indent_level = self.indent_level - 1
+            self._write_to_file('</Group>')
+        self.indent_level = self.indent_level - 1
+        self._write_to_file('</PackageList>')
+        self._close_file()
+        self._rename_file()
+
+if __name__ == '__main__':
+    # Prefix is relative to script path
+    complete_script_path = os.path.join(os.getcwd(), sys.argv[0])
+    prefix = complete_script_path[:-len('etc/create-debian-pkglist.py')]
+    
+    confparser = ConfigParser.SafeConfigParser()
+    confparser.read(prefix + "etc/debian-pkglist.conf")
+    
+    # We read the whole configuration file before processing each entries
+    # to avoid doing work if there is a problem in the file.
+    sources_list = []
+    for section in confparser.sections():
+        sources_list.append(Source(confparser, section, prefix))
+
+    for source in sources_list:
+        source.process()