From 2a14ba6689f6104601533d0677fb9211a35c96e0 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Wed, 13 Jan 2010 02:38:44 +0100 Subject: Add support for multiple sources per overlay --- CHANGES | 10 ++++ layman/action.py | 10 ++-- layman/overlay.py | 56 +++-------------- layman/overlays/bzr.py | 9 ++- layman/overlays/cvs.py | 16 +++-- layman/overlays/darcs.py | 9 ++- layman/overlays/git.py | 9 ++- layman/overlays/mercurial.py | 5 +- layman/overlays/overlay.py | 139 ++++++++++++++++++++++++++----------------- layman/overlays/rsync.py | 7 +-- layman/overlays/source.py | 65 ++++++++++++++++++-- layman/overlays/svn.py | 9 ++- layman/overlays/tar.py | 16 +++-- 13 files changed, 205 insertions(+), 155 deletions(-) diff --git a/CHANGES b/CHANGES index dea2561..b0fa27e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,16 @@ CHANGES ------- +Version TODO +=================================== + + - TODO make test suite work again + + - Add support for several sources per overlay (also fixes #280472) + On addition all sources will be probed until a working + one is found. This should help Layman through many firewalls. + + Version 1.2.6 - Released 2010/01/12 =================================== diff --git a/layman/action.py b/layman/action.py index 6349814..323d706 100644 --- a/layman/action.py +++ b/layman/action.py @@ -113,12 +113,14 @@ class Sync: for i in self.selection: ordb = self.rdb.select(i) odb = self.db.select(i) - if ordb and odb and ordb.src != odb.src: + current_src = odb.sources[0].src + available_srcs = set(e.src for e in ordb.sources) + if ordb and odb and not current_src in available_srcs: warnings.append( 'The source of the overlay "' + i + '" seems to have c' - 'hanged. You currently sync from "' + odb.src + '" whi' - 'le the global layman list reports "' + ordb.src + '" ' - 'as correct location. Please consider removing and rea' + 'hanged. You currently sync from "' + current_src + '" whi' + 'le the global layman list reports "' + '" and "'.join(available_srcs) + '" ' + 'as correct location(s). Please consider removing and rea' 'dding the overlay!') try: diff --git a/layman/overlay.py b/layman/overlay.py index 643e652..ce0e3b1 100644 --- a/layman/overlay.py +++ b/layman/overlay.py @@ -31,34 +31,9 @@ __version__ = "$Id: overlay.py 273 2006-12-30 15:54:50Z wrobel $" import sys, codecs, os, os.path import xml.etree.ElementTree as ET # Python 2.5 -from layman.overlays.bzr import BzrOverlay -from layman.overlays.darcs import DarcsOverlay -from layman.overlays.git import GitOverlay -from layman.overlays.mercurial import MercurialOverlay -from layman.overlays.cvs import CvsOverlay -from layman.overlays.svn import SvnOverlay -from layman.overlays.rsync import RsyncOverlay -from layman.overlays.tar import TarOverlay - from layman.debug import OUT from layman.utils import indent - -#=============================================================================== -# -# Constants -# -#------------------------------------------------------------------------------- - -OVERLAY_TYPES = dict((e.type_key, e) for e in ( - GitOverlay, - CvsOverlay, - SvnOverlay, - RsyncOverlay, - TarOverlay, - BzrOverlay, - MercurialOverlay, - DarcsOverlay -)) +from layman.overlays.overlay import Overlay #=============================================================================== # @@ -124,29 +99,12 @@ class Overlays: for overlay in overlays: OUT.debug('Parsing overlay entry', 8) - - _source = overlay.find('source') - if _source != None: - element_to_scan = _source - else: - element_to_scan = overlay - - for attr_name in element_to_scan.attrib.keys(): - if attr_name == 'type': - overlay_type = element_to_scan.attrib['type'] - if overlay_type in OVERLAY_TYPES.keys(): - try: - ovl = OVERLAY_TYPES[overlay_type](overlay, - self.config, - self.ignore, - self.quiet) - self.overlays[ovl.name] = ovl - except Exception, error: - OUT.warn(str(error), 3) - else: - raise Exception('Unknown overlay type "' + - overlay_type + '"!') - break + try: + ovl = Overlay(overlay, self.config, self.ignore, self.quiet) + self.overlays[ovl.name] = ovl + except Exception, error: + OUT.warn(str(error), 3) + def write(self, path): ''' diff --git a/layman/overlays/bzr.py b/layman/overlays/bzr.py index 3f5a0c2..346f1a0 100644 --- a/layman/overlays/bzr.py +++ b/layman/overlays/bzr.py @@ -27,7 +27,6 @@ __version__ = "$Id: bzr.py 236 2006-09-05 20:39:37Z wrobel $" #------------------------------------------------------------------------------- from layman.utils import path -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -42,9 +41,9 @@ class BzrOverlay(OverlaySource): type = 'Bzr' type_key = 'bzr' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(BzrOverlay, self).__init__(xml, config, ignore) + super(BzrOverlay, self).__init__(parent, xml, config, _location, ignore, quiet) def add(self, base, quiet = False): '''Add overlay.''' @@ -52,14 +51,14 @@ class BzrOverlay(OverlaySource): self.supported() return self.cmd(self.command() + ' get "' + self.src + '/" "' +\ - path([base, self.name]) + '"') + path([base, self.parent.name]) + '"') def sync(self, base, quiet = False): '''Sync overlay.''' self.supported() - return self.cmd('cd "' + path([base, self.name]) + '" && ' + \ + return self.cmd('cd "' + path([base, self.parent.name]) + '" && ' + \ self.command() + ' pull --overwrite "' + self.src \ + '"') diff --git a/layman/overlays/cvs.py b/layman/overlays/cvs.py index 7e26461..c949f0e 100644 --- a/layman/overlays/cvs.py +++ b/layman/overlays/cvs.py @@ -27,7 +27,6 @@ __version__ = "$Id$" import xml.etree.ElementTree as ET # Python 2.5 from layman.utils import path, ensure_unicode -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -42,9 +41,9 @@ class CvsOverlay(OverlaySource): type = 'cvs' type_key = 'cvs' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(CvsOverlay, self).__init__(xml, config, ignore, quiet) + super(CvsOverlay, self).__init__(xml, parent, config, _location, ignore, quiet) _subpath = xml.find('subpath') if _subpath != None: @@ -63,13 +62,12 @@ class CvsOverlay(OverlaySource): return not self.__eq__(other) # overrider - def to_xml(self): - repo = super(CvsOverlay, self).to_xml() + def to_xml_hook(self, repo_elem): if self.subpath: _subpath = ET.Element('subpath') _subpath.text = self.subpath - repo.append(_subpath) - return repo + repo_elem.append(_subpath) + del _subpath def add(self, base, quiet = False): '''Add overlay.''' @@ -82,7 +80,7 @@ class CvsOverlay(OverlaySource): quiet_option = '' return self.cmd('cd "' + base + '" && CVSROOT="' + self.src + '" ' + - self.command() + quiet_option + ' co -d "' + self.name + self.command() + quiet_option + ' co -d "' + self.parent.name + '" "' + self.subpath + '"' ) def sync(self, base, quiet = False): @@ -95,7 +93,7 @@ class CvsOverlay(OverlaySource): else: quiet_option = '' - return self.cmd('cd "' + path([base, self.name]) + '" && ' + + return self.cmd('cd "' + path([base, self.parent.name]) + '" && ' + self.command() + quiet_option + ' update -d') def supported(self): diff --git a/layman/overlays/darcs.py b/layman/overlays/darcs.py index 7b6a462..54a06aa 100644 --- a/layman/overlays/darcs.py +++ b/layman/overlays/darcs.py @@ -26,7 +26,6 @@ __version__ = "$Id: darcs.py 236 2006-09-05 20:39:37Z wrobel $" #------------------------------------------------------------------------------- from layman.utils import path -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -41,9 +40,9 @@ class DarcsOverlay(OverlaySource): type = 'Darcs' type_key = 'darcs' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(DarcsOverlay, self).__init__(xml, config, ignore) + super(DarcsOverlay, self).__init__(parent, xml, config, _location, ignore, quiet) def add(self, base, quiet = False): '''Add overlay.''' @@ -51,14 +50,14 @@ class DarcsOverlay(OverlaySource): self.supported() return self.cmd(self.command() + ' get --partial "' + self.src + - '/" "' + path([base, self.name]) + '"') + '/" "' + path([base, self.parent.name]) + '"') def sync(self, base, quiet = False): '''Sync overlay.''' self.supported() - return self.cmd('cd "' + path([base, self.name]) + '" && ' + + return self.cmd('cd "' + path([base, self.parent.name]) + '" && ' + self.command() + ' pull --all "' + self.src + '"') def supported(self): diff --git a/layman/overlays/git.py b/layman/overlays/git.py index b3c3a2b..174de18 100644 --- a/layman/overlays/git.py +++ b/layman/overlays/git.py @@ -25,7 +25,6 @@ __version__ = "$Id: git.py 146 2006-05-27 09:52:36Z wrobel $" #------------------------------------------------------------------------------- from layman.utils import path -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -40,9 +39,9 @@ class GitOverlay(OverlaySource): type = 'Git' type_key = 'git' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(GitOverlay, self).__init__(xml, config, ignore) + super(GitOverlay, self).__init__(parent, xml, config, _location, ignore, quiet) def add(self, base, quiet = False): '''Add overlay.''' @@ -59,7 +58,7 @@ class GitOverlay(OverlaySource): if self.src.split(':')[0] == 'http': slash = '/' return self.cmd(self.command() + ' clone ' + quiet_option + '"' + self.src + slash - + '" "' + path([base, self.name]) + '"') + + '" "' + path([base, self.parent.name]) + '"') def sync(self, base, quiet = False): '''Sync overlay.''' @@ -71,7 +70,7 @@ class GitOverlay(OverlaySource): else: quiet_option = '' - return self.cmd('cd "' + path([base, self.name]) + '" && ' + return self.cmd('cd "' + path([base, self.parent.name]) + '" && ' + self.command() + ' pull' + quiet_option) def supported(self): diff --git a/layman/overlays/mercurial.py b/layman/overlays/mercurial.py index 8da1cf2..add5136 100644 --- a/layman/overlays/mercurial.py +++ b/layman/overlays/mercurial.py @@ -26,7 +26,6 @@ __version__ = "$Id: mercurial.py 236 2006-09-05 20:39:37Z wrobel $" #------------------------------------------------------------------------------- from layman.utils import path -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -41,9 +40,9 @@ class MercurialOverlay(OverlaySource): type = 'Mercurial' type_key = 'mercurial' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(MercurialOverlay, self).__init__(xml, config, ignore) + super(MercurialOverlay, self).__init__(parent, xml, config, _location, ignore, quiet) def add(self, base, quiet = False): '''Add overlay.''' diff --git a/layman/overlays/overlay.py b/layman/overlays/overlay.py index 1356d28..6ec2f38 100644 --- a/layman/overlays/overlay.py +++ b/layman/overlays/overlay.py @@ -28,7 +28,7 @@ __version__ = "$Id: overlay.py 273 2006-12-30 15:54:50Z wrobel $" # #------------------------------------------------------------------------------- -import sys, types, re, os, os.path, shutil, subprocess +import sys, re, os, os.path import codecs import locale import xml.etree.ElementTree as ET # Python 2.5 @@ -37,6 +37,32 @@ from layman.utils import path, ensure_unicode from layman.debug import OUT +from layman.overlays.bzr import BzrOverlay +from layman.overlays.darcs import DarcsOverlay +from layman.overlays.git import GitOverlay +from layman.overlays.mercurial import MercurialOverlay +from layman.overlays.cvs import CvsOverlay +from layman.overlays.svn import SvnOverlay +from layman.overlays.rsync import RsyncOverlay +from layman.overlays.tar import TarOverlay + +#=============================================================================== +# +# Constants +# +#------------------------------------------------------------------------------- + +OVERLAY_TYPES = dict((e.type_key, e) for e in ( + GitOverlay, + CvsOverlay, + SvnOverlay, + RsyncOverlay, + TarOverlay, + BzrOverlay, + MercurialOverlay, + DarcsOverlay +)) + #=============================================================================== # # Class Overlay @@ -82,14 +108,29 @@ class Overlay(object): else: raise Exception('Overlay is missing a "name" entry!') - _source = xml.find('source') - if _source == None: - if 'src' in xml.attrib: - _source = ET.Element('source') - _source.text = xml.attrib['src'] - else: - raise Exception('Overlay "' + self.name + '" is missing a "source" entry!') - self.src = ensure_unicode(_source.text.strip()) + _sources = xml.findall('source') + if _sources: + _sources = [e for e in _sources if 'type' in e.attrib] + elif ('src' in xml.attrib) and ('type' in xml.attrib): + s = ET.Element('source', type=xml.attrib['type']) + s.text = xml.attrib['src'] + _sources = [s] + del s + + if not _sources: + raise Exception('Overlay "' + self.name + '" is missing a "source" entry!') + + + def create_overlay_source(source_elem): + _type = source_elem.attrib['type'] + try: + _class = OVERLAY_TYPES[_type] + except KeyError: + raise Exception('Unknown overlay type "%s"!' % _type) + _location = ensure_unicode(source_elem.text.strip()) + return _class(self, xml, config, _location, ignore, quiet) + + self.sources = map(create_overlay_source, _sources) _owner = xml.find('owner') @@ -190,55 +231,37 @@ class Overlay(object): owner_name = ET.Element('name') owner_name.text = self.owner_name owner.append(owner_name) - source = ET.Element('source', type=self.__class__.type_key) - source.text = self.src - repo.append(source) + for i in self.sources: + source = ET.Element('source', type=i.__class__.type_key) + source.text = i.src + repo.append(source) + del source + for i in self.sources: + # NOTE: Two loops on purpose so the + # hooks are called with all sources in + i.to_xml_hook(repo) return repo def add(self, base, quiet = False): - '''Add the overlay.''' - - mdir = path([base, self.name]) - - if os.path.exists(mdir): - raise Exception('Directory ' + mdir + ' already exists. Will not ov' - 'erwrite its contents!') - - os.makedirs(mdir) + res = 1 + for s in self.sources: + try: + res = s.add(base, quiet) + if res == 0: + # Worked, throw other sources away + self.sources = [s] + break + except Exception, error: + OUT.warn(str(error), 4) + return res def sync(self, base, quiet = False): - '''Sync the overlay.''' - pass + assert len(self.sources) == 1 + return self.sources[0].sync(base, quiet) def delete(self, base): - '''Delete the overlay.''' - mdir = path([base, self.name]) - - if not os.path.exists(mdir): - OUT.warn('Directory ' + mdir + ' did not exist, no files deleted.') - return - - shutil.rmtree(mdir) - - def cmd(self, command): - '''Run a command.''' - - OUT.info('Running command "' + command + '"...', 2) - - if hasattr(sys.stdout,'encoding'): - enc = sys.stdout.encoding or sys.getfilesystemencoding() - if enc: - command = command.encode(enc) - - if not self.quiet: - return os.system(command) - else: - cmd = subprocess.Popen([command], shell = True, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - close_fds = True) - result = cmd.wait() - return result + assert len(self.sources) == 1 + return self.sources[0].delete(base) def _get_encoding(self): if hasattr(sys.stdout, 'encoding') \ @@ -273,7 +296,14 @@ class Overlay(object): result += self.name + u'\n' + (len(self.name) * u'~') - result += u'\nSource : ' + self.src + if len(self.sources) == 1: + result += u'\nSource : ' + self.sources[0].src + else: + result += u'\nSources:' + for i, v in enumerate(self.sources): + result += '\n %d. %s' % (i + 1, v.src) + result += '\n' + if self.owner_name != None: result += u'\nContact : %s <%s>' % (self.owner_name, self.owner_email) else: @@ -340,7 +370,7 @@ class Overlay(object): if not width: width = terminal_width() srclen = width - 43 - source = self.src + source = self.sources[0].src if len(source) > srclen: source = source.replace("overlays.gentoo.org", "o.g.o") source = ' (' + pad(source, srclen) + ')' @@ -352,6 +382,9 @@ class Overlay(object): return self.status == 'official' + def is_supported(self): + return any(e.is_supported() for e in self.sources) + #================================================================================ # # Testing diff --git a/layman/overlays/rsync.py b/layman/overlays/rsync.py index 857560e..9106403 100644 --- a/layman/overlays/rsync.py +++ b/layman/overlays/rsync.py @@ -25,7 +25,6 @@ __version__ = "$Id: rsync.py 236 2006-09-05 20:39:37Z wrobel $" #------------------------------------------------------------------------------- from layman.utils import path -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -41,9 +40,9 @@ class RsyncOverlay(OverlaySource): type_key = 'rsync' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(RsyncOverlay, self).__init__(xml, config, ignore) + super(RsyncOverlay, self).__init__(parent, xml, config, _location, ignore, quiet) def add(self, base, quiet = False): '''Add overlay.''' @@ -69,7 +68,7 @@ class RsyncOverlay(OverlaySource): '--exclude="packages/*" ' return self.cmd(_command + quiet_option + '"' + self.src + '/" "' + - path([base, self.name]) + '"') + path([base, self.parent.name]) + '"') def supported(self): '''Overlay type supported?''' diff --git a/layman/overlays/source.py b/layman/overlays/source.py index 1cd6b5b..aaf3881 100644 --- a/layman/overlays/source.py +++ b/layman/overlays/source.py @@ -14,11 +14,45 @@ # Sebastian Pipping import os -from layman.overlays.overlay import Overlay +import sys +import shutil +import subprocess +from layman.debug import OUT +from layman.utils import path -class OverlaySource(Overlay): - def __init__(self, xml, config, ignore = 0, quiet = False): - super(OverlaySource, self).__init__(xml, config, ignore, quiet) +class OverlaySource(object): + + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): + self.parent = parent + self.src = _location + self.config = config + self.ignore = ignore + self.quiet = quiet + + def add(self, base, quiet = False): + '''Add the overlay.''' + + mdir = path([base, self.parent.name]) + + if os.path.exists(mdir): + raise Exception('Directory ' + mdir + ' already exists. Will not ov' + 'erwrite its contents!') + + os.makedirs(mdir) + + def sync(self, base, quiet = False): + '''Sync the overlay.''' + pass + + def delete(self, base): + '''Delete the overlay.''' + mdir = path([base, self.parent.name]) + + if not os.path.exists(mdir): + OUT.warn('Directory ' + mdir + ' did not exist, no files deleted.') + return + + shutil.rmtree(mdir) def supported(self, binaries = []): '''Is the overlay type supported?''' @@ -55,3 +89,26 @@ class OverlaySource(Overlay): def command(self): return self.config['%s_command' % self.__class__.type_key] + + def cmd(self, command): + '''Run a command.''' + + OUT.info('Running command "' + command + '"...', 2) + + if hasattr(sys.stdout,'encoding'): + enc = sys.stdout.encoding or sys.getfilesystemencoding() + if enc: + command = command.encode(enc) + + if not self.quiet: + return os.system(command) + else: + cmd = subprocess.Popen([command], shell = True, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + close_fds = True) + result = cmd.wait() + return result + + def to_xml_hook(self, repo_elem): + pass diff --git a/layman/overlays/svn.py b/layman/overlays/svn.py index f031006..22b57e4 100644 --- a/layman/overlays/svn.py +++ b/layman/overlays/svn.py @@ -25,7 +25,6 @@ __version__ = "$Id: svn.py 236 2006-09-05 20:39:37Z wrobel $" #------------------------------------------------------------------------------- from layman.utils import path -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -40,9 +39,9 @@ class SvnOverlay(OverlaySource): type = 'Subversion' type_key = 'svn' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(SvnOverlay, self).__init__(xml, config, ignore) + super(SvnOverlay, self).__init__(parent, xml, config, _location, ignore, quiet) def add(self, base, quiet = False): '''Add overlay.''' @@ -57,7 +56,7 @@ class SvnOverlay(OverlaySource): quiet_option = '' return self.cmd(self.command() + ' co ' + quiet_option + - '"' + self.src + '/@" "' + path([base, self.name]) + '"') + '"' + self.src + '/@" "' + path([base, self.parent.name]) + '"') def sync(self, base, quiet = False): '''Sync overlay.''' @@ -70,7 +69,7 @@ class SvnOverlay(OverlaySource): quiet_option = '' return self.cmd(self.command() + ' up ' + quiet_option + - '"' + path([base, self.name + '@']) + '"') + '"' + path([base, self.parent.name + '@']) + '"') def supported(self): '''Overlay type supported?''' diff --git a/layman/overlays/tar.py b/layman/overlays/tar.py index 7853cb3..103c927 100644 --- a/layman/overlays/tar.py +++ b/layman/overlays/tar.py @@ -28,7 +28,6 @@ import os, os.path, sys, urllib2, shutil import xml.etree.ElementTree as ET # Python 2.5 from layman.utils import path, ensure_unicode -from layman.overlays.overlay import Overlay from layman.overlays.source import OverlaySource #=============================================================================== @@ -66,9 +65,9 @@ class TarOverlay(OverlaySource): type = 'Tar' type_key = 'tar' - def __init__(self, xml, config, ignore = 0, quiet = False): + def __init__(self, parent, xml, config, _location, ignore = 0, quiet = False): - super(TarOverlay, self).__init__(xml, config, ignore) + super(TarOverlay, self).__init__(parent, xml, config, _location, ignore, quiet) _subpath = xml.find('subpath') if _subpath != None: @@ -87,20 +86,19 @@ class TarOverlay(OverlaySource): return not self.__eq__(other) # overrider - def to_xml(self): - repo = super(TarOverlay, self).to_xml() + def to_xml_hook(self, repo_elem): if self.subpath: _subpath = ET.Element('subpath') _subpath.text = self.subpath - repo.append(_subpath) - return repo + repo_elem.append(_subpath) + del _subpath def add(self, base, quiet = False): '''Add overlay.''' self.supported() - mdir = path([base, self.name]) + mdir = path([base, self.parent.name]) if os.path.exists(mdir): raise Exception('Directory ' + mdir + ' already exists. Will not ov' @@ -122,7 +120,7 @@ class TarOverlay(OverlaySource): raise Exception('Failed to fetch the tar package from: ' + self.src + '\nError was:' + str(error)) - pkg = path([base, self.name + ext]) + pkg = path([base, self.parent.name + ext]) try: -- cgit v1.2.3-1-g7c22