From 0da4208c47162f7e48d98888b2256be378c86030 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 4 Nov 2014 11:32:19 -0600 Subject: Yum.py: Fix traceback when arch missing Sometimes repositories may not contain packages for a specific architecture group. This handles that case gracefully instead of causing a traceback and failing to bind all Package entries. Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 6139a28b5..4ee9cce52 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1287,10 +1287,13 @@ class YumSource(Source): arch = [a for a in self.arches if a in metadata.groups] if not arch: return False - return ((package in self.packages['global'] or - package in self.packages[arch[0]]) and - package not in self.blacklist and - (len(self.whitelist) == 0 or package in self.whitelist)) + try: + return ((package in self.packages['global'] or + package in self.packages[arch[0]]) and + package not in self.blacklist and + (len(self.whitelist) == 0 or package in self.whitelist)) + except KeyError: + return False is_package.__doc__ = Source.is_package.__doc__ def get_vpkgs(self, metadata): -- cgit v1.2.3-1-g7c22 From 7d6032e82ea26baf82c64435925d6991d812e768 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 14 Jan 2015 23:35:22 +0100 Subject: Plugins/Packages: fix initialization of YumSource During __init__ of the parent class get_repo_name is called. That needs fields (pump_id) that are defined later in the __init__ of YumSource. We introduce the new function _init_attributes to parse all attributes out of the tag before calling any other functions. --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 134 ++++++++++++++---------- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 7 +- 2 files changed, 81 insertions(+), 60 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 30cdd543f..c09a9988b 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -143,6 +143,83 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: source self.essentialpkgs = set() + self._init_attributes(basepath, xsource, setup) + + #: A set of all package names in this source. This will not + #: necessarily be populated, particularly by backends that + #: reimplement large portions of + #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` + self.pkgnames = set() + + #: A dict of ```` -> ````. + #: This will not necessarily be populated, particularly by + #: backends that reimplement large portions of + #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` + self.deps = dict() + + #: A dict of ```` -> ````. This will not necessarily be populated, + #: particularly by backends that reimplement large portions of + #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` + self.provides = dict() + + #: The file (or directory) used for this source's cache data + self.cachefile = os.path.join(self.basepath, + "cache-%s" % self.cachekey) + if not self.rawurl: + baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" + else: + baseurl = self.rawurl + + #: A list of dicts, each of which describes the URL to one + #: repository contained in this source. Each dict contains + #: the following keys: + #: + #: * ``version``: The version of the repo (``None`` for + #: ``rawurl`` repos) + #: * ``component``: The component use to form this URL + #: (``None`` for ``rawurl`` repos) + #: * ``arch``: The architecture of this repo + #: * ``baseurl``: Either the ``rawurl`` attribute, or the + #: format string built from the ``url`` attribute + #: * ``url``: The actual URL to the repository + self.url_map = [] + for arch in self.arches: + if self.url: + usettings = [dict(version=self.version, component=comp, + arch=arch, debsrc=self.debsrc) + for comp in self.components] + else: # rawurl given + usettings = [dict(version=self.version, component=None, + arch=arch, debsrc=self.debsrc)] + + for setting in usettings: + if not self.rawurl: + setting['baseurl'] = self.url + else: + setting['baseurl'] = self.rawurl + setting['url'] = baseurl % setting + setting['name'] = self.get_repo_name(setting) + self.url_map.extend(usettings) + + def _init_attributes(self, basepath, xsource, setup): + """ + This functions evaluates the Source tag and parses all + attributes. Override this function in a sub class to + parse specific attributes. Do not use ``__init__`` because + ``Source.__init__`` may call other functions that already + need this specific fields. This functions is called before + any other function. + + :param basepath: The base filesystem path under which cache + data for this source should be stored + :type basepath: string + :param xsource: The XML tag that describes this source + :type source: lxml.etree._Element + :param setup: A Bcfg2 options dict + :type setup: dict + """ + #: A list of the text of all 'Component' attributes of this #: source from XML self.components = [item.text for item in xsource.findall('Component')] @@ -241,63 +318,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 self.conditions.append(lambda m, el=el: el.get("name") == m.hostname) - #: A set of all package names in this source. This will not - #: necessarily be populated, particularly by backends that - #: reimplement large portions of - #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` - self.pkgnames = set() - - #: A dict of ```` -> ````. - #: This will not necessarily be populated, particularly by - #: backends that reimplement large portions of - #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` - self.deps = dict() - - #: A dict of ```` -> ````. This will not necessarily be populated, - #: particularly by backends that reimplement large portions of - #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` - self.provides = dict() - - #: The file (or directory) used for this source's cache data - self.cachefile = os.path.join(self.basepath, - "cache-%s" % self.cachekey) - if not self.rawurl: - baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" - else: - baseurl = self.rawurl - - #: A list of dicts, each of which describes the URL to one - #: repository contained in this source. Each dict contains - #: the following keys: - #: - #: * ``version``: The version of the repo (``None`` for - #: ``rawurl`` repos) - #: * ``component``: The component use to form this URL - #: (``None`` for ``rawurl`` repos) - #: * ``arch``: The architecture of this repo - #: * ``baseurl``: Either the ``rawurl`` attribute, or the - #: format string built from the ``url`` attribute - #: * ``url``: The actual URL to the repository - self.url_map = [] - for arch in self.arches: - if self.url: - usettings = [dict(version=self.version, component=comp, - arch=arch, debsrc=self.debsrc) - for comp in self.components] - else: # rawurl given - usettings = [dict(version=self.version, component=None, - arch=arch, debsrc=self.debsrc)] - - for setting in usettings: - if not self.rawurl: - setting['baseurl'] = self.url - else: - setting['baseurl'] = self.rawurl - setting['url'] = baseurl % setting - setting['name'] = self.get_repo_name(setting) - self.url_map.extend(usettings) - @property def cachekey(self): """ A unique key for this source that will be used to generate diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 6139a28b5..ae8ac5d6f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1028,8 +1028,9 @@ class YumSource(Source): #: YumSource sets the ``type`` on Package entries to "yum" ptype = 'yum' - def __init__(self, basepath, xsource, setup): - Source.__init__(self, basepath, xsource, setup) + def _init_attributes(self, basepath, xsource, setup): + Source._init_attributes(self, basepath, xsource, setup) + self.pulp_id = None if HAS_PULP and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") @@ -1071,7 +1072,7 @@ class YumSource(Source): self.needed_paths = set() self.file_to_arch = dict() self.yumgroups = dict() - __init__.__doc__ = Source.__init__.__doc__ + _init_attributes.__doc__ = Source._init_attributes.__doc__ @property def use_yum(self): -- cgit v1.2.3-1-g7c22 From d2a52e47c867f8f5864781f200bed4d0adf373b3 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 23 Jan 2015 17:44:20 +0100 Subject: Plugins/Packages: all attributes should be defined in __init__ --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 117 ++++++++++++++---------- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 17 +++- 2 files changed, 81 insertions(+), 53 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index c09a9988b..cf26c982f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -143,6 +143,75 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: source self.essentialpkgs = set() + #: A list of the text of all 'Component' attributes of this + #: source from XML + self.components = [] + + #: A list of the arches supported by this source + self.arches = [] + + #: A list of the the names of packages that are blacklisted + #: from this source + self.blacklist = [] + + #: A list of the the names of packages that are whitelisted in + #: this source + self.whitelist = [] + + #: Whether or not to include deb-src lines in the generated APT + #: configuration + self.debsrc = False + + #: A dict of repository options that will be included in the + #: configuration generated on the server side (if such is + #: applicable; most backends do not generate any sort of + #: repository configuration on the Bcfg2 server) + self.server_options = dict() + + #: A dict of repository options that will be included in the + #: configuration generated for the client (if that is + #: supported by the backend) + self.client_options = dict() + + #: A list of URLs to GPG keys that apply to this source + self.gpgkeys = [] + + #: Whether or not to include essential packages from this source + self.essential = True + + #: Whether or not to include recommended packages from this source + self.recommended = False + + #: The "rawurl" attribute from :attr:`xsource`, if applicable. + #: A trailing slash is automatically appended to this if there + #: wasn't one already present. + self.rawurl = None + + #: The "url" attribute from :attr:`xsource`, if applicable. A + #: trailing slash is automatically appended to this if there + #: wasn't one already present. + self.url = None + + #: The "version" attribute from :attr:`xsource` + self.version = None + + #: The "name" attribute from :attr:`xsource` + self.name = None + + #: A list of predicates that are used to determine if this + #: source applies to a given + #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` + #: object. + self.conditions = [] + + #: Formerly, :ref:`server-plugins-generators-packages` only + #: supported applying package sources to groups; that is, they + #: could not be assigned by more complicated logic like + #: per-client repositories and group or client negation. This + #: attribute attempts to provide for some limited backwards + #: compat with older code that relies on this. + self.groups = [] + self._init_attributes(basepath, xsource, setup) #: A set of all package names in this source. This will not @@ -220,35 +289,12 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 :type setup: dict """ - #: A list of the text of all 'Component' attributes of this - #: source from XML self.components = [item.text for item in xsource.findall('Component')] - - #: A list of the arches supported by this source self.arches = [item.text for item in xsource.findall('Arch')] - - #: A list of the the names of packages that are blacklisted - #: from this source self.blacklist = [item.text for item in xsource.findall('Blacklist')] - - #: A list of the the names of packages that are whitelisted in - #: this source self.whitelist = [item.text for item in xsource.findall('Whitelist')] - - #: Whether or not to include deb-src lines in the generated APT - #: configuration self.debsrc = xsource.get('debsrc', 'false') == 'true' - #: A dict of repository options that will be included in the - #: configuration generated on the server side (if such is - #: applicable; most backends do not generate any sort of - #: repository configuration on the Bcfg2 server) - self.server_options = dict() - - #: A dict of repository options that will be included in the - #: configuration generated for the client (if that is - #: supported by the backend) - self.client_options = dict() opts = xsource.findall("Options") for el in opts: repoopts = dict([(k, v) @@ -259,48 +305,23 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 if el.get("serveronly", "false").lower() == "false": self.client_options.update(repoopts) - #: A list of URLs to GPG keys that apply to this source self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] - #: Whether or not to include essential packages from this source self.essential = xsource.get('essential', 'true').lower() == 'true' - - #: Whether or not to include recommended packages from this source self.recommended = xsource.get('recommended', 'false').lower() == 'true' - #: The "rawurl" attribute from :attr:`xsource`, if applicable. - #: A trailing slash is automatically appended to this if there - #: wasn't one already present. self.rawurl = xsource.get('rawurl', '') if self.rawurl and not self.rawurl.endswith("/"): self.rawurl += "/" - #: The "url" attribute from :attr:`xsource`, if applicable. A - #: trailing slash is automatically appended to this if there - #: wasn't one already present. self.url = xsource.get('url', '') if self.url and not self.url.endswith("/"): self.url += "/" - #: The "version" attribute from :attr:`xsource` self.version = xsource.get('version', '') - - #: The "name" attribute from :attr:`xsource` self.name = xsource.get('name', None) - #: A list of predicates that are used to determine if this - #: source applies to a given - #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` - #: object. - self.conditions = [] - #: Formerly, :ref:`server-plugins-generators-packages` only - #: supported applying package sources to groups; that is, they - #: could not be assigned by more complicated logic like - #: per-client repositories and group or client negation. This - #: attribute attempts to provide for some limited backwards - #: compat with older code that relies on this. - self.groups = [] for el in xsource.iterancestors(): if el.tag == "Group": if el.get("negate", "false").lower() == "true": diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index ae8ac5d6f..6669d3066 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1028,10 +1028,21 @@ class YumSource(Source): #: YumSource sets the ``type`` on Package entries to "yum" ptype = 'yum' + def __init__(self, basepath, xsource, setup): + self.filemap = dict() + self.file_to_arch = dict() + self.needed_paths = set() + self.packages = dict() + self.yumgroups = dict() + self.pulp_id = None + self.repo = None + + Source.__init__(self, basepath, xsource, setup) + __init__.__doc__ = Source.__init__.__doc__ + def _init_attributes(self, basepath, xsource, setup): Source._init_attributes(self, basepath, xsource, setup) - self.pulp_id = None if HAS_PULP and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") @@ -1064,14 +1075,10 @@ class YumSource(Source): self.repo['relative_path']) self.arches = [self.repo['arch']] - self.packages = dict() self.deps = dict([('global', dict())]) self.provides = dict([('global', dict())]) self.filemap = dict([(x, dict()) for x in ['global'] + self.arches]) - self.needed_paths = set() - self.file_to_arch = dict() - self.yumgroups = dict() _init_attributes.__doc__ = Source._init_attributes.__doc__ @property -- cgit v1.2.3-1-g7c22 From eafcb0ad931c5ae2d34e564c811a8c4cc0ee6278 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 23 Jan 2015 17:57:16 +0100 Subject: Plugins/Packages/Source: Remove unused arguments of _init_attributes --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 9 ++------- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index cf26c982f..6c5b61742 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -212,7 +212,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: compat with older code that relies on this. self.groups = [] - self._init_attributes(basepath, xsource, setup) + self._init_attributes(xsource) #: A set of all package names in this source. This will not #: necessarily be populated, particularly by backends that @@ -271,7 +271,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 setting['name'] = self.get_repo_name(setting) self.url_map.extend(usettings) - def _init_attributes(self, basepath, xsource, setup): + def _init_attributes(self, xsource): """ This functions evaluates the Source tag and parses all attributes. Override this function in a sub class to @@ -280,13 +280,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 need this specific fields. This functions is called before any other function. - :param basepath: The base filesystem path under which cache - data for this source should be stored - :type basepath: string :param xsource: The XML tag that describes this source :type source: lxml.etree._Element - :param setup: A Bcfg2 options dict - :type setup: dict """ self.components = [item.text for item in xsource.findall('Component')] diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 6669d3066..20b0e103d 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1040,8 +1040,8 @@ class YumSource(Source): Source.__init__(self, basepath, xsource, setup) __init__.__doc__ = Source.__init__.__doc__ - def _init_attributes(self, basepath, xsource, setup): - Source._init_attributes(self, basepath, xsource, setup) + def _init_attributes(self, xsource): + Source._init_attributes(self, xsource) if HAS_PULP and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") -- cgit v1.2.3-1-g7c22 From 78cac1d0a6923ebc73ff221f8501885c36c112c1 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 27 Jan 2015 05:46:13 +0100 Subject: Make SSHbase also a Connector plugin. Make the public host key data avaialabe for the templates. --- src/lib/Bcfg2/Server/Plugins/SSHbase.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 2deea5f07..0152fcf70 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -93,6 +93,7 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -127,6 +128,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -444,3 +446,15 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, self.logger.error("Failed to pull %s. This file does not " "currently exist on the client" % entry.get('name')) + + def get_additional_data(self, metadata): + data = dict() + for key in self.keypatterns: + if key.endswith(".pub"): + try: + keyfile = "/etc/ssh/" + key + entry = self.entries[keyfile].best_matching(metadata) + data[key] = entry.data + except Bcfg2.Server.Plugin.PluginExecutionError: + pass + return data -- cgit v1.2.3-1-g7c22 From deb8729a09d74d2efe5bdf86e3a2d3c1274590ab Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 24 Feb 2015 18:05:13 +0100 Subject: Server/Plugins/Metadata: Reject passwd clients, if auth_type is cert If the auth type (either globally or specific for the client) is set to cert auth, we have to reject clients that does not provide a cert. --- src/lib/Bcfg2/Server/Plugins/Metadata.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 1e5544c6b..f805772a7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -1391,8 +1391,6 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # look at cert.cN client = certinfo['commonName'] self.debug_log("Got cN %s; using as client name" % client) - auth_type = self.auth.get(client, - self.core.setup['authentication']) elif user == 'root': id_method = 'address' try: @@ -1414,6 +1412,13 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # we have the client name self.debug_log("Authenticating client %s" % client) + # validate id_method + auth_type = self.auth.get(client, self.core.setup['authentication']) + if auth_type == 'cert' and id_method != 'cert': + self.logger.error("Client %s does not provide a cert, but only " + "cert auth is allowed" % client) + return False + # next we validate the address if (id_method != 'uuid' and not self.validate_client_address(client, address)): -- cgit v1.2.3-1-g7c22 From cae39b746051ff5f3257342d0659340283b2d6ef Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 24 Feb 2015 16:29:07 -0600 Subject: Fix pylint errors This also pins pylint to <= 0.28 so we don't have to keep playing whack-a-mole with it. Also removes unnecessary suppression of apt warnings. This is no longer necessary in 12.04, so should be safe to remove. If you're on Ubuntu < 12.04, upgrade for heaven's sake. --- src/lib/Bcfg2/Server/FileMonitor/__init__.py | 7 +++++-- src/lib/Bcfg2/Server/Lint/Validate.py | 25 +++++++++++-------------- src/lib/Bcfg2/Server/Lint/ValidateJSON.py | 21 ++++++++------------- src/lib/Bcfg2/Server/Lint/__init__.py | 12 ++++++++++++ src/lib/Bcfg2/Server/Plugin/__init__.py | 5 ----- src/lib/Bcfg2/Server/Plugins/GroupPatterns.py | 10 ++++------ src/lib/Bcfg2/Server/Plugins/SSLCA.py | 23 ++++++++++++----------- 7 files changed, 52 insertions(+), 51 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py index 7a5d901fd..083e50fe6 100644 --- a/src/lib/Bcfg2/Server/FileMonitor/__init__.py +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -336,8 +336,11 @@ class FileMonitor(Debuggable): available = dict() # pylint: disable=C0103 # TODO: loading the monitor drivers should be automatic -from Bcfg2.Server.FileMonitor.Pseudo import Pseudo -available['pseudo'] = Pseudo +try: + from Bcfg2.Server.FileMonitor.Pseudo import Pseudo + available['pseudo'] = Pseudo +except ImportError: + pass try: from Bcfg2.Server.FileMonitor.Fam import Fam diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 3efcc890d..1e33ec398 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -1,12 +1,16 @@ -""" Ensure that all XML files in the Bcfg2 repository validate -according to their respective schemas. """ +"""Validate XML files. +Ensure that all XML files in the Bcfg2 repository validate according +to their respective schemas. +""" + +import glob import os import sys -import glob -import fnmatch + import lxml.etree from subprocess import Popen, PIPE, STDOUT + import Bcfg2.Server.Lint @@ -204,17 +208,10 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): values are lists of the full paths to all files in the Bcfg2 repository (or given with ``bcfg2-lint --stdin``) that match the glob.""" - if self.files is not None: - listfiles = lambda p: fnmatch.filter(self.files, - os.path.join('*', p)) - else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) - for path in self.filesets.keys(): if '/**/' in path: if self.files is not None: - self.filelists[path] = listfiles(path) + self.filelists[path] = self.list_matching_files(path) else: # self.files is None fpath, fname = path.split('/**/') self.filelists[path] = [] @@ -225,9 +222,9 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): for f in files if f == fname]) else: - self.filelists[path] = listfiles(path) + self.filelists[path] = self.list_matching_files(path) - self.filelists['props'] = listfiles("Properties/*.xml") + self.filelists['props'] = self.list_matching_files("Properties/*.xml") def _load_schema(self, filename): """ Load an XML schema document, returning the Schema object diff --git a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py index 1f55962eb..bdbe6a271 100644 --- a/src/lib/Bcfg2/Server/Lint/ValidateJSON.py +++ b/src/lib/Bcfg2/Server/Lint/ValidateJSON.py @@ -1,11 +1,13 @@ -"""Ensure that all JSON files in the Bcfg2 repository are +"""Validate JSON files. + +Ensure that all JSON files in the Bcfg2 repository are valid. Currently, the only plugins that uses JSON are Ohai and -Properties.""" +Properties. +""" import os import sys -import glob -import fnmatch + import Bcfg2.Server.Lint try: @@ -48,18 +50,11 @@ class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): def get_files(self): """Return a list of all JSON files to validate, based on :attr:`Bcfg2.Server.Lint.ValidateJSON.ValidateJSON.globs`. """ - if self.files is not None: - listfiles = lambda p: fnmatch.filter(self.files, - os.path.join('*', p)) - else: - listfiles = lambda p: glob.glob(os.path.join(self.config['repo'], - p)) - rv = [] for path in self.globs: if '/**/' in path: if self.files is not None: - rv.extend(listfiles(path)) + rv.extend(self.list_matching_files(path)) else: # self.files is None fpath, fname = path.split('/**/') for root, _, files in \ @@ -68,5 +63,5 @@ class ValidateJSON(Bcfg2.Server.Lint.ServerlessPlugin): rv.extend([os.path.join(root, f) for f in files if f == fname]) else: - rv.extend(listfiles(path)) + rv.extend(self.list_matching_files(path)) return rv diff --git a/src/lib/Bcfg2/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 28644263f..ae2b81a61 100644 --- a/src/lib/Bcfg2/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -1,14 +1,19 @@ """ Base classes for Lint plugins and error handling """ import os +import fnmatch +import glob import sys import logging from copy import copy import textwrap +import time + import lxml.etree import fcntl import termios import struct + from Bcfg2.Compat import walk_packages plugins = [m[1] for m in walk_packages(path=__path__)] # pylint: disable=C0103 @@ -139,6 +144,13 @@ class Plugin(object): xml_declaration=False).decode("UTF-8").strip() return " line %s: %s" % (element.sourceline, xml) + def list_matching_files(self, path): + """list all files matching the path in self.files or the bcfg2 repo.""" + if self.files is not None: + return fnmatch.filter(self.files, os.path.join('*', path)) + else: + return glob.glob(os.path.join(self.config['repo'], path)) + class ErrorHandler(object): """ A class to handle errors for bcfg2-lint plugins """ diff --git a/src/lib/Bcfg2/Server/Plugin/__init__.py b/src/lib/Bcfg2/Server/Plugin/__init__.py index ed1282ba0..05bb92223 100644 --- a/src/lib/Bcfg2/Server/Plugin/__init__.py +++ b/src/lib/Bcfg2/Server/Plugin/__init__.py @@ -11,11 +11,6 @@ documentation it's not necessary to use the submodules. E.g., you can from Bcfg2.Server.Plugin.base import Plugin """ - -import os -import sys -sys.path.append(os.path.dirname(__file__)) - # pylint: disable=W0401 from Bcfg2.Server.Plugin.base import * from Bcfg2.Server.Plugin.interfaces import * diff --git a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 09685d972..eadd918b7 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -149,15 +149,13 @@ class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): def check(self, entry, groups, ptype="NamePattern"): """ Check a single pattern for validity """ - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - for el in entry.findall(ptype): pat = el.text try: - pmap(pat) + if ptype == "NamePattern": + PatternMap(pat, None, groups) + else: + PatternMap(None, pat, groups) except: # pylint: disable=W0702 err = sys.exc_info()[1] self.LintError("pattern-fails-to-initialize", diff --git a/src/lib/Bcfg2/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index f111ffc60..a33b2f448 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -150,15 +150,13 @@ class SSLCAEntrySet(Bcfg2.Server.Plugin.EntrySet): if passphrase: cmd.extend(["-passin", "pass:%s" % passphrase]) - def _scrub_pass(arg): - """ helper to scrub the passphrase from the - argument list """ - if arg.startswith("pass:"): - return "pass:******" - else: - return arg - else: - _scrub_pass = lambda a: a + def _scrub_pass(arg): + """ helper to scrub the passphrase from the + argument list for debugging. """ + if arg.startswith("pass:"): + return "pass:******" + else: + return arg self.debug_log("SSLCA: Generating new certificate: %s" % " ".join(_scrub_pass(a) for a in cmd)) @@ -362,10 +360,13 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): """ The SSLCA generator handles the creation and management of ssl certificates and their keys. """ __author__ = 'g.hagger@gmail.com' - # python 2.5 doesn't support mixing *magic and keyword arguments - es_cls = lambda self, *args: SSLCAEntrySet(*args, **dict(parent=self)) es_child_cls = SSLCADataFile + def es_cls(self, *args): + """Fake entry set 'class' that sets this as the parent.""" + # python 2.5 doesn't support mixing *magic and keyword arguments + return SSLCAEntrySet(*args, **dict(parent=self)) + def get_ca(self, name): """ get a dict describing a CA from the config file """ return dict(self.core.setup.cfp.items("sslca_%s" % name)) -- cgit v1.2.3-1-g7c22 From f69d2c18d1351d49f4c1ffd2a6c282df0fa3d8e3 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 16 Jan 2015 03:53:00 +0100 Subject: Server/Core: drop privileges even if not running as daemon --- src/lib/Bcfg2/Server/CherryPyCore.py | 16 ++++++++++------ src/lib/Bcfg2/Server/Core.py | 17 ++++++++++++++++- src/lib/Bcfg2/Server/MultiprocessingCore.py | 3 +++ 3 files changed, 29 insertions(+), 7 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/CherryPyCore.py b/src/lib/Bcfg2/Server/CherryPyCore.py index d097fd08f..c1581679c 100644 --- a/src/lib/Bcfg2/Server/CherryPyCore.py +++ b/src/lib/Bcfg2/Server/CherryPyCore.py @@ -103,17 +103,21 @@ class Core(BaseCore): return cherrypy.serving.response.body def _daemonize(self): - """ Drop privileges with - :class:`cherrypy.process.plugins.DropPrivileges`, daemonize - with :class:`cherrypy.process.plugins.Daemonizer`, and write a + """ Drop privileges, daemonize + with :class:`cherrypy.process.plugins.Daemonizer` and write a PID file with :class:`cherrypy.process.plugins.PIDFile`. """ + self._drop_privileges() + Daemonizer(cherrypy.engine).subscribe() + PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() + return True + + def _drop_privileges(self): + """ Drop privileges with + :class:`cherrypy.process.plugins.DropPrivileges` """ DropPrivileges(cherrypy.engine, uid=self.setup['daemon_uid'], gid=self.setup['daemon_gid'], umask=int(self.setup['umask'], 8)).subscribe() - Daemonizer(cherrypy.engine).subscribe() - PIDFile(cherrypy.engine, self.setup['daemon']).subscribe() - return True def _run(self): """ Start the server listening. """ diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 6dfe4df1f..0369da8f2 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -11,6 +11,7 @@ import threading import time import inspect import lxml.etree +import daemon import Bcfg2.settings import Bcfg2.Server import Bcfg2.Logger @@ -112,6 +113,7 @@ class BaseCore(object): :type setup: Bcfg2.Options.OptionParser .. automethod:: _daemonize + .. automethod:: _drop_privileges .. automethod:: _run .. automethod:: _block .. ----- @@ -803,7 +805,8 @@ class BaseCore(object): self.logger.debug("Slept %s seconds while handling FAM events" % slept) def run(self): - """ Run the server core. This calls :func:`_daemonize`, + """ Run the server core. This calls :func:`_daemonize` + (or :func:`_drop_privileges` if not in daemon mode), :func:`_run`, starts the :attr:`fam_thread`, and calls :func:`_block`, but note that it is the responsibility of the server core implementation to call :func:`shutdown` under @@ -830,6 +833,8 @@ class BaseCore(object): # dropped os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] else: + if os.getuid() == 0: + self._drop_privileges() os.umask(int(self.setup['umask'], 8)) if not self._run(): @@ -861,6 +866,16 @@ class BaseCore(object): overridden by a core implementation. """ raise NotImplementedError + def _drop_privileges(self): + """ This is called if not daemonized and running as root to + drop the privileges to the configured daemon_uid and daemon_gid. + """ + daemon.daemon.change_process_owner( + self.setup['daemon_uid'], + self.setup['daemon_gid']) + self.logger.debug("Dropped privileges to %s:%s." % + (os.getuid(), os.getgid())) + def _run(self): """ Start up the server; this method should return immediately. This must be overridden by a core diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 2cb3adae3..4986aac60 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -210,6 +210,9 @@ class ChildCore(BaseCore): def _daemonize(self): return True + def _drop_privileges(self): + pass + def _dispatch(self, address, data): """ Method dispatcher used for commands received from the RPC queue. """ -- cgit v1.2.3-1-g7c22 From 3829eda6e46a291875d989ecb0afa6a28ed0e3b8 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 25 Mar 2015 20:23:18 +0100 Subject: Server/Plugins/Packages: Fix _init_attributes position. _init_attributes should be called after all properties of the Source class are initialized (so that _init_attributes could overwrite some of it). The Yum class initializes self.deps with a different default entry, that should not be reset by __init__ of Source afterwards. --- src/lib/Bcfg2/Server/Plugins/Packages/Source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 6c5b61742..7aa688f12 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -212,8 +212,6 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: compat with older code that relies on this. self.groups = [] - self._init_attributes(xsource) - #: A set of all package names in this source. This will not #: necessarily be populated, particularly by backends that #: reimplement large portions of @@ -232,6 +230,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): # pylint: disable=R0902 #: :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection` self.provides = dict() + self._init_attributes(xsource) + #: The file (or directory) used for this source's cache data self.cachefile = os.path.join(self.basepath, "cache-%s" % self.cachekey) -- cgit v1.2.3-1-g7c22 From 82b8b12f6bba17201376775fe90490bdcc8642df Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 30 Mar 2015 15:03:21 -0500 Subject: Packages/Yum.py: Fix dependency resolution logic This commit helps the internal YUM resolver to choose the latest version of a package to generate dependencies. Previously, we were simply iterating through the file so that the last instance listed always won (even if that instance was an older version with differing dependencies). Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 103 ++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 20b0e103d..789df79d5 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1169,6 +1169,94 @@ class YumSource(Source): self.file_to_arch[self.escape_url(fullurl)] = arch return urls + # pylint: disable=R0911,R0912 + # disabling the pylint errors above because we are interesting in + # replicating the flow of the RPM code. + def _compare_rpm_versions(self, str1, str2): + """ Compare RPM versions. + + This is an attempt to reimplement RPM's rpmvercmp method in python. + + :param str1: package 1 version string + :param str2: package 2 version string + :return: 1 - str1 is newer than str2 + 0 - str1 and str2 are the same version + -1 - str2 is newer than str1""" + if str1 == str2: + return 0 + + front_strip_re = re.compile('^[^A-Za-z0-9~]+') + risdigit = re.compile('(^[0-9]+)') + risalpha = re.compile('(^[A-Za-z])') + lzeroes = re.compile('^0+') + + while len(str1) > 0 or len(str2) > 0: + str1 = front_strip_re.sub('', str1) + str2 = front_strip_re.sub('', str2) + + if len(str1) == 0 or len(str2) == 0: + break + + # handle the tilde separator + if str1[0] == '~' and str2[0] == '~': + str1 = str1[1:] + str2 = str2[1:] + elif str1[0] == '~': + return -1 + elif str2[0] == '~': + return 1 + + # grab continuous segments from each string + isnum = False + if risdigit.match(str1): + segment1 = risdigit.split(str1)[1] + str1 = risdigit.split(str1)[2] + if risdigit.match(str2): + segment2 = risdigit.split(str2)[1] + str2 = risdigit.split(str2)[2] + else: + segment2 = '' + isnum = True + else: + segment1 = risalpha.split(str1)[1] + str1 = risalpha.split(str1)[2] + if risalpha.match(str2): + segment2 = risalpha.split(str2)[1] + str2 = risalpha.split(str2)[2] + else: + segment2 = '' + + # numeric segments are always newer than alpha segments + if len(segment2) == 0: + if isnum: + return 1 + return -1 + + if isnum: + # discard leading zeroes + segment1 = lzeroes.sub('', segment1) + segment2 = lzeroes.sub('', segment2) + # higher number has more digits + if len(segment1) > len(segment2): + return 1 + elif len(segment2) > len(segment1): + return -1 + # do a simple string comparison + if segment1 > segment2: + return 1 + elif segment2 > segment1: + return -1 + + # if one of the strings is empty, the version of the longer + # string is higher + if len(str1) > len(str2): + return 1 + elif len(str2) > len(str1): + return -1 + else: + return 0 + # pylint: enable=R0911,R0912 + @Bcfg2.Server.Plugin.track_statistics() def read_files(self): """ When using the builtin yum parser, read and parse locally @@ -1237,13 +1325,24 @@ class YumSource(Source): if arch not in self.packages: self.packages[arch] = set() if arch not in self.deps: - self.deps[arch] = dict() + self.deps[arch] = {} if arch not in self.provides: - self.provides[arch] = dict() + self.provides[arch] = {} + versionmap = {} for pkg in data.getchildren(): if not pkg.tag.endswith('package'): continue pkgname = pkg.find(XP + 'name').text + vtag = pkg.find(XP + 'version') + version = "%s%s-%s" % (vtag.get('epoch'), vtag.get('ver'), + vtag.get('rel')) + if pkgname in self.packages[arch]: + # skip if version older than a previous version + if self._compare_rpm_versions(version, + versionmap[pkgname]) < 0: + continue + + versionmap[pkgname] = version self.packages[arch].add(pkgname) pdata = pkg.find(XP + 'format') -- cgit v1.2.3-1-g7c22 From f3e9378c975d401c061da14e8e60fb031182f289 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 31 Mar 2015 09:01:51 -0500 Subject: Yum.py: Compare epoch/version/release separately Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 789df79d5..20103820c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1334,15 +1334,24 @@ class YumSource(Source): continue pkgname = pkg.find(XP + 'name').text vtag = pkg.find(XP + 'version') - version = "%s%s-%s" % (vtag.get('epoch'), vtag.get('ver'), - vtag.get('rel')) + epoch = vtag.get('epoch') + version = vtag.get('ver') + release = vtag.get('rel') if pkgname in self.packages[arch]: # skip if version older than a previous version - if self._compare_rpm_versions(version, - versionmap[pkgname]) < 0: + if (self._compare_rpm_versions( + epoch, versionmap[pkgname]['epoch']) < 0): continue - - versionmap[pkgname] = version + elif (self._compare_rpm_versions( + version, versionmap[pkgname]['version']) < 0): + continue + elif (self._compare_rpm_versions( + release, versionmap[pkgname]['release']) < 0): + continue + versionmap[pkgname] = {} + versionmap[pkgname]['epoch'] = epoch + versionmap[pkgname]['version'] = version + versionmap[pkgname]['release'] = release self.packages[arch].add(pkgname) pdata = pkg.find(XP + 'format') -- cgit v1.2.3-1-g7c22 From 45a63e8cf0bfc5de075ee68d54b492b30f59bc77 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Tue, 28 Apr 2015 08:59:23 -0500 Subject: Yum.py: Add debug log message Log a message to debug if the package is missing from the host's primary architecture package list (as per Chris's request). Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4ee9cce52..0e4f93246 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -1293,6 +1293,8 @@ class YumSource(Source): package not in self.blacklist and (len(self.whitelist) == 0 or package in self.whitelist)) except KeyError: + self.logger.debug("Packages: Unable to find %s for arch %s" % + (package, arch[0])) return False is_package.__doc__ = Source.is_package.__doc__ -- cgit v1.2.3-1-g7c22