diff options
33 files changed, 915 insertions, 66 deletions
diff --git a/doc/releases/1.4.0pre2.txt b/doc/releases/1.4.0pre2.txt index 103af1a0a..5694fe8b4 100644 --- a/doc/releases/1.4.0pre2.txt +++ b/doc/releases/1.4.0pre2.txt @@ -46,6 +46,9 @@ backwards-incompatible user-facing changes your queries default to the class names and the Ldap plugin expires the metadata caches if the config file changes. +* Ignore directories containing a .bcfg2-ignore file in various plugins + (Bundler, Defaults, Pkgmgr, Properties, PuppetENC, TemplateHelper, Trigger). + Thanks ------ diff --git a/schemas/packages.xsd b/schemas/packages.xsd index fc5a1356c..4a9881a8f 100644 --- a/schemas/packages.xsd +++ b/schemas/packages.xsd @@ -18,6 +18,7 @@ <xsd:enumeration value="apt"/> <xsd:enumeration value="pac"/> <xsd:enumeration value="pkgng"/> + <xsd:enumeration value="dummy"/> </xsd:restriction> </xsd:simpleType> @@ -225,6 +226,25 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute type="xsd:integer" name="priority"> + <xsd:annotation> + <xsd:documentation> + The priority of the source. This is used to order the + sources. After sorting, the first source, that could + deliver the package, is used. If not supplied the default + priority is 500. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type="xsd:string" name="pin"> + <xsd:annotation> + <xsd:documentation> + Extra information for pinning. This information is used + to differ between the sources. Should be used in the + supported format of apt. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attributeGroup ref="py:genshiAttrs"/> </xsd:complexType> diff --git a/schemas/pkgvars.xsd b/schemas/pkgvars.xsd new file mode 100644 index 000000000..dbd02726d --- /dev/null +++ b/schemas/pkgvars.xsd @@ -0,0 +1,43 @@ +<xsd:schema xmlns:xsd='http://www.w3.org/2001/XMLSchema' + xmlns:py="http://genshi.edgewall.org/"> + + <xsd:annotation> + <xsd:documentation> + XML-Schema-Definition für PkgVars/*.xml + Alexander Sulfrian + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://genshi.edgewall.org/" + schemaLocation="genshi.xsd"/> + + <xsd:complexType name='pkgVarType'> + <xsd:attribute type='xsd:string' name='name'/> + + <xsd:attribute type='xsd:string' name='pin'/> + <xsd:attribute type='xsd:string' name='use'/> + <xsd:attribute type='xsd:string' name='keywords'/> + + <xsd:attributeGroup ref="py:genshiAttrs"/> + </xsd:complexType> + + <xsd:complexType name='containerType'> + <xsd:choice maxOccurs='unbounded'> + <xsd:element name='Package' type='pkgVarType'/> + <xsd:element name='Client' type='containerType'/> + <xsd:element name='Group' type='containerType'/> + </xsd:choice> + <xsd:attribute name='name' type='xsd:string' use='required'/> + <xsd:attribute name='negate' type='xsd:boolean'/> + </xsd:complexType> + + <xsd:complexType name='pkgVarsType'> + <xsd:choice minOccurs='0' maxOccurs='unbounded'> + <xsd:element name='Package' type='pkgVarType'/> + <xsd:element name='Client' type='containerType'/> + <xsd:element name='Group' type='containerType'/> + </xsd:choice> + </xsd:complexType> + + <xsd:element name='PkgVars' type='pkgVarsType'/> +</xsd:schema> diff --git a/schemas/types.xsd b/schemas/types.xsd index dc57c5cc8..59e9149e2 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -28,6 +28,7 @@ <xsd:enumeration value='ebuild' /> <xsd:enumeration value='yum' /> <xsd:enumeration value='freebsdpkg' /> + <xsd:enumeration value='dummy' /> </xsd:restriction> </xsd:simpleType> @@ -47,6 +47,7 @@ setup(name="Bcfg2", "Bcfg2.Server.Plugin", "Bcfg2.Server.Plugins", "Bcfg2.Server.Plugins.Packages", + "Bcfg2.Server.Plugins.Packages.Readers", "Bcfg2.Server.Plugins.Cfg", "Bcfg2.Server.Reports", "Bcfg2.Server.Reports.reports", diff --git a/src/lib/Bcfg2/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index 4350f6067..77610d9bc 100644 --- a/src/lib/Bcfg2/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -35,6 +35,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.debsums = '%s/bin/debsums' % Bcfg2.Options.setup.apt_install_path self.aptget = '%s/bin/apt-get' % Bcfg2.Options.setup.apt_install_path self.dpkg = '%s/bin/dpkg' % Bcfg2.Options.setup.apt_install_path + self.aptmark = '%s/bin/apt-mark' % Bcfg2.Options.setup.apt_install_path self.__execs__ = [self.debsums, self.aptget, self.dpkg] path_entries = os.environ['PATH'].split(':') @@ -45,6 +46,7 @@ class APT(Bcfg2.Client.Tools.Tool): '-o DPkg::Options::=--force-confold ' + \ '-o DPkg::Options::=--force-confmiss ' + \ '--reinstall ' + \ + '--no-install-recommends ' + \ '--force-yes ' if not Bcfg2.Options.setup.debug: self.pkgcmd += '-q=2 ' @@ -87,6 +89,23 @@ class APT(Bcfg2.Client.Tools.Tool): except apt.cache.FetchFailedException: err = sys.exc_info()[1] self.logger.info("Failed to update APT cache: %s" % err) + # mark dependencies as being automatically installed and vice versa + mark = [] + unmark = [] + try: + installed_pkgs = [p.name for p in self.pkg_cache if p.is_installed] + except AttributeError: + installed_pkgs = [p.name for p in self.pkg_cache if p.isInstalled] + for pkg in self.getSupportedEntries(): + if pkg.get('name') in installed_pkgs: + if pkg.get('origin') == 'Packages': + mark.append(pkg.get('name')) + else: + unmark.append(pkg.get('name')) + if mark: + self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark)))) + if unmark: + self.cmd.run("%s unmarkauto %s" % (self.aptmark, (" ".join(unmark)))) self.pkg_cache = apt.cache.Cache() def FindExtra(self): @@ -166,13 +185,15 @@ class APT(Bcfg2.Client.Tools.Tool): pkg = self.pkg_cache[pkgname] installed_version = pkg.installed.version - if entry.get('version') == 'auto': + if entry.get('version').startswith('auto'): if pkg.is_upgradable: desired_version = pkg.candidate.version else: desired_version = installed_version - elif entry.get('version') == 'any': + entry.set('version', "auto: %s" % desired_version) + elif entry.get('version').startswith('any'): desired_version = installed_version + entry.set('version', "any: %s" % desired_version) else: desired_version = entry.get('version') if desired_version != installed_version: @@ -215,7 +236,7 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.error("APT has no information about package %s" % pkgname) continue - if pkg.get('version') in ['auto', 'any']: + if any([pkg.get('version').startswith(v) for v in ['auto', 'any']]): try: ipkgs.append("%s=%s" % ( pkgname, @@ -241,11 +262,16 @@ class APT(Bcfg2.Client.Tools.Tool): self.logger.error("APT command failed") self.pkg_cache = apt.cache.Cache() self.extra = self.FindExtra() + mark = [] states = dict() for package in packages: states[package] = self.VerifyPackage(package, [], checksums=False) if states[package]: self.modified.append(package) + if package.get('origin') == 'Packages': + mark.append(package.get('name')) + if mark: + self.cmd.run("%s markauto %s" % (self.aptmark, (" ".join(mark)))) return states def VerifyPath(self, entry, _): # pylint: disable=W0613 diff --git a/src/lib/Bcfg2/Client/Tools/Dummy.py b/src/lib/Bcfg2/Client/Tools/Dummy.py new file mode 100644 index 000000000..9a96eb904 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/Dummy.py @@ -0,0 +1,16 @@ +"""This is the Bcfg2 tool for the Dummy package system.""" + +import re +import Bcfg2.Client.Tools + + +class Dummy(Bcfg2.Client.Tools.PkgTool): + __handles__ = [('Package', 'dummy')] + __req__ = {'Package': []} + pkgtype = 'dummy' + + def RefreshPackages(self): + pass + + def VerifyPackage(self, _entry, _): + return True diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 40598541e..224119a79 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -27,13 +27,23 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): type=uid_range_type, help="GID ranges the POSIXUsers tool will manage"), Bcfg2.Options.Option( + cf=('POSIXUsers', 'supgid_whitelist'), default=[], + type=uid_range_type, + help="GID ranges for supplementary groups the POSIXUsers" + "tool will manage"), + Bcfg2.Options.Option( cf=('POSIXUsers', 'uid_blacklist'), default=[], type=uid_range_type, help="UID ranges the POSIXUsers tool will not manage"), Bcfg2.Options.Option( cf=('POSIXUsers', 'gid_blacklist'), default=[], type=uid_range_type, - help="GID ranges the POSIXUsers tool will not manage")] + help="GID ranges the POSIXUsers tool will not manage"), + Bcfg2.Options.Option( + cf=('POSIXUsers', 'supgid_blacklist'), default=[], + type=uid_range_type, + help="GID ranges for supplementary groups the POSIXUsers" + "tool will not manage")] __execs__ = ['/usr/sbin/useradd', '/usr/sbin/usermod', '/usr/sbin/userdel', '/usr/sbin/groupadd', '/usr/sbin/groupmod', @@ -58,10 +68,19 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): self.set_defaults = dict(POSIXUser=self.populate_user_entry, POSIXGroup=lambda g: g) self._existing = None + + supgid_whitelist = Bcfg2.Options.setup.supgid_whitelist + supgid_blacklist = Bcfg2.Options.setup.supgid_blacklist + if supgid_whitelist is None and supgid_blacklist is None: + supgid_whitelist = Bcfg2.Options.setup.gid_whitelist + supgid_blacklist = Bcfg2.Options.setup.gid_blacklist + self._whitelist = dict(POSIXUser=Bcfg2.Options.setup.uid_whitelist, - POSIXGroup=Bcfg2.Options.setup.gid_whitelist) + POSIXGroup=Bcfg2.Options.setup.gid_whitelist, + POSIXSupGroup=supgid_whitelist) self._blacklist = dict(POSIXUser=Bcfg2.Options.setup.uid_blacklist, - POSIXGroup=Bcfg2.Options.setup.gid_blacklist) + POSIXGroup=Bcfg2.Options.setup.gid_blacklist, + POSIXSupGroup=supgid_blacklist) @property def existing(self): @@ -161,7 +180,7 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): given entry is a member of """ return [g for g in self.existing['POSIXGroup'].values() if entry.get("name") in g[3] and - self._in_managed_range('POSIXGroup', g[2])] + self._in_managed_range('POSIXSupGroup', g[2])] def VerifyPOSIXUser(self, entry, _): """ Verify a POSIXUser entry """ diff --git a/src/lib/Bcfg2/Reporting/Reports.py b/src/lib/Bcfg2/Reporting/Reports.py index e60b2e82e..7e1661c5a 100755 --- a/src/lib/Bcfg2/Reporting/Reports.py +++ b/src/lib/Bcfg2/Reporting/Reports.py @@ -17,7 +17,7 @@ def print_entries(interaction, etype): class _FlagsFilterMixin(object): """ Mixin that allows to filter the interactions based on the - only_important and/or the dry_run flag """ + only_important, the dry_run and/or the ready flag """ options = [ Bcfg2.Options.BooleanOption( @@ -27,10 +27,15 @@ class _FlagsFilterMixin(object): Bcfg2.Options.BooleanOption( "-i", "--no-only-important", help="Do not consider interactions created with the " - "--only-important flag")] + "--only-important flag"), + Bcfg2.Options.BooleanOption( + "-r", "--ready", + help="Only consider interactions fully imported into the " + "database")] def get_interaction(self, client, setup): - if not setup.no_dry_run and not setup.no_only_important: + if not setup.no_dry_run and not setup.no_only_important \ + and not setup.ready: return client.current_interaction filter = {} @@ -38,6 +43,8 @@ class _FlagsFilterMixin(object): filter['dry_run'] = False if setup.no_only_important: filter['only_important'] = False + if setup.ready: + filter['ready'] = True from Bcfg2.Reporting.models import Interaction try: diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index 493cbdfdf..e0566a51b 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -406,6 +406,9 @@ class DjangoORM(StorageBase): metric=metric, value=value).save() + inter.ready = True + inter.save() + def import_interaction(self, interaction): """Import the data into the backend""" try: diff --git a/src/lib/Bcfg2/Reporting/migrations/0008_add_ready_flag_interaction.py b/src/lib/Bcfg2/Reporting/migrations/0008_add_ready_flag_interaction.py new file mode 100644 index 000000000..18ea6e8ba --- /dev/null +++ b/src/lib/Bcfg2/Reporting/migrations/0008_add_ready_flag_interaction.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('Reporting', '0007_add_flag_fields_interaction'), + ] + + operations = [ + migrations.AddField( + model_name='interaction', + name='ready', + field=models.BooleanField(default=False), + preserve_default=True, + ), + ] diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index fe465672d..7bff0ab3b 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -182,6 +182,9 @@ class Interaction(models.Model): groups = models.ManyToManyField("Group") bundles = models.ManyToManyField("Bundle") + # Is the import ready? + ready = models.BooleanField(default=False) + objects = InteractionManager() def __str__(self): diff --git a/src/lib/Bcfg2/Reporting/south_migrations/0008_add_ready_flag_interaction.py b/src/lib/Bcfg2/Reporting/south_migrations/0008_add_ready_flag_interaction.py new file mode 100644 index 000000000..cfe42f35b --- /dev/null +++ b/src/lib/Bcfg2/Reporting/south_migrations/0008_add_ready_flag_interaction.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'Interaction.ready' + db.add_column('Reporting_interaction', 'ready', + self.gf('django.db.models.fields.BooleanField')(default=False), + keep_default=False) + + + def backwards(self, orm): + # Deleting field 'Interaction.ready' + db.delete_column('Reporting_interaction', 'ready') + + + models = { + 'Reporting.actionentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ActionEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'output': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'check'", 'max_length': '128'}) + }, + 'Reporting.bundle': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Bundle'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'Reporting.client': { + 'Meta': {'object_name': 'Client'}, + 'creation': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'current_interaction': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'parent_client'", 'null': 'True', 'to': "orm['Reporting.Interaction']"}), + 'expiration': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.deviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'DeviceEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_major': ('django.db.models.fields.IntegerField', [], {}), + 'current_minor': ('django.db.models.fields.IntegerField', [], {}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_major': ('django.db.models.fields.IntegerField', [], {}), + 'target_minor': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.failureentry': { + 'Meta': {'object_name': 'FailureEntry'}, + 'entry_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileacl': { + 'Meta': {'object_name': 'FileAcl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}) + }, + 'Reporting.fileperms': { + 'Meta': {'unique_together': "(('owner', 'group', 'mode'),)", 'object_name': 'FilePerms'}, + 'group': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mode': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'owner': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'Reporting.group': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Group'}, + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'category': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'comment': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'profile': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'Reporting.interaction': { + 'Meta': {'ordering': "['-timestamp']", 'unique_together': "(('client', 'timestamp'),)", 'object_name': 'Interaction'}, + 'actions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ActionEntry']", 'symmetrical': 'False'}), + 'bad_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bundles': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Bundle']", 'symmetrical': 'False'}), + 'client': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'interactions'", 'to': "orm['Reporting.Client']"}), + 'dry_run': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'extra_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'failures': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FailureEntry']", 'symmetrical': 'False'}), + 'good_count': ('django.db.models.fields.IntegerField', [], {}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.Group']", 'symmetrical': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'modified_count': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'only_important': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PackageEntry']", 'symmetrical': 'False'}), + 'paths': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.PathEntry']", 'symmetrical': 'False'}), + 'posixgroups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.POSIXGroupEntry']", 'symmetrical': 'False'}), + 'posixusers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.POSIXUserEntry']", 'symmetrical': 'False'}), + 'profile': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'to': "orm['Reporting.Group']"}), + 'ready': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'repo_rev_code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'sebooleans': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEBooleanEntry']", 'symmetrical': 'False'}), + 'sefcontexts': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEFcontextEntry']", 'symmetrical': 'False'}), + 'seinterfaces': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEInterfaceEntry']", 'symmetrical': 'False'}), + 'selogins': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SELoginEntry']", 'symmetrical': 'False'}), + 'semodules': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEModuleEntry']", 'symmetrical': 'False'}), + 'senodes': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SENodeEntry']", 'symmetrical': 'False'}), + 'sepermissives': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPermissiveEntry']", 'symmetrical': 'False'}), + 'seports': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEPortEntry']", 'symmetrical': 'False'}), + 'server': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'services': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.ServiceEntry']", 'symmetrical': 'False'}), + 'seusers': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.SEUserEntry']", 'symmetrical': 'False'}), + 'state': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'total_count': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.linkentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'LinkEntry', '_ormbases': ['Reporting.PathEntry']}, + 'current_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}), + 'pathentry_ptr': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['Reporting.PathEntry']", 'unique': 'True', 'primary_key': 'True'}), + 'target_path': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'blank': 'True'}) + }, + 'Reporting.packageentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PackageEntry'}, + 'current_version': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_version': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}), + 'verification_details': ('django.db.models.fields.TextField', [], {'default': "''"}) + }, + 'Reporting.pathentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'PathEntry'}, + 'acls': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['Reporting.FileAcl']", 'symmetrical': 'False'}), + 'current_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}), + 'detail_type': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'details': ('django.db.models.fields.TextField', [], {'default': "''"}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'path_type': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_perms': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['Reporting.FilePerms']"}) + }, + 'Reporting.performance': { + 'Meta': {'object_name': 'Performance'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interaction': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'performance_items'", 'to': "orm['Reporting.Interaction']"}), + 'metric': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'value': ('django.db.models.fields.DecimalField', [], {'max_digits': '32', 'decimal_places': '16'}) + }, + 'Reporting.posixgroupentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'POSIXGroupEntry'}, + 'current_gid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'gid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.posixuserentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'POSIXUserEntry'}, + 'current_gecos': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_group': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True'}), + 'current_home': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_shell': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True'}), + 'current_uid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'gecos': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'group': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'home': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'shell': ('django.db.models.fields.CharField', [], {'default': "'/bin/bash'", 'max_length': '1024'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'uid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}) + }, + 'Reporting.sebooleanentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEBooleanEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'value': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'Reporting.sefcontextentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEFcontextEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'filetype': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seinterfaceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEInterfaceEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seloginentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SELoginEntry'}, + 'current_selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxuser': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.semoduleentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEModuleEntry'}, + 'current_disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'disabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.senodeentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SENodeEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'proto': ('django.db.models.fields.CharField', [], {'max_length': '4'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.sepermissiveentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPermissiveEntry'}, + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.seportentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEPortEntry'}, + 'current_selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'selinuxtype': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + }, + 'Reporting.serviceentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'ServiceEntry'}, + 'current_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'state': ('django.db.models.fields.IntegerField', [], {}), + 'target_status': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '128'}) + }, + 'Reporting.seuserentry': { + 'Meta': {'ordering': "('state', 'name')", 'object_name': 'SEUserEntry'}, + 'current_prefix': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'current_roles': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True'}), + 'exists': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'hash_key': ('django.db.models.fields.BigIntegerField', [], {'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128', 'db_index': 'True'}), + 'prefix': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'state': ('django.db.models.fields.IntegerField', [], {}) + } + } + + complete_apps = ['Reporting'] diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index d6f18afbb..146f18b0c 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -61,7 +61,8 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", - "GroupLogic/groups.xml": "grouplogic.xsd" + "GroupLogic/groups.xml": "grouplogic.xsd", + "PkgVars/*.xml": "pkgvars.xsd" } self.filelists = {} diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 762d018eb..ca0fe8188 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -456,7 +456,9 @@ class DirectoryBacked(Debuggable): # again without having to add a new monitor. elif os.path.isdir(abspath): # Deal with events for directories - if action in ['exists', 'created']: + if os.path.exists(os.path.join(abspath, '.bcfg2-ignore')): + self.logger.debug("Ignoring directory %s" % abspath) + elif action in ['exists', 'created']: self.add_directory_monitor(relpath) elif action == 'changed': if relpath in self.entries: diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index f5bcbe797..4f5a79465 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -133,7 +133,7 @@ class Bundler(Plugin, if child.get('inherit_modification', 'false') == 'true': if metadata.version_info >= \ Bcfg2VersionInfo('1.4.0pre2'): - lxml.etree.SubElement(data, 'Bundle', + lxml.etree.SubElement(data, 'BoundBundle', name=child.get('name')) else: self.logger.warning( diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py index 92fcc4cd8..b9ced6682 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgSSLCACertCreator.py @@ -216,15 +216,12 @@ class CfgSSLCACertCreator(XMLCfgCreator, CfgVerifier): chaincert = ca.get('chaincert') cmd = ["openssl", "verify"] is_root = ca.get('root_ca', "false").lower() == 'true' - if is_root: - cmd.append("-CAfile") - else: - # verifying based on an intermediate cert - cmd.extend(["-purpose", "sslserver", "-untrusted"]) - cmd.extend([chaincert, filename]) + if not is_root: + cmd.append("-partial_chain") + cmd.extend(["-trusted", chaincert, filename]) self.debug_log("Cfg: Verifying %s against CA" % entry.get("name")) result = self.cmd.run(cmd) - if result.stdout == cert + ": OK\n": + if result.stdout == filename + ": OK\n": self.debug_log("Cfg: %s verified successfully against CA" % entry.get("name")) else: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index 956cb9f51..c3a3dc6ad 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -1,7 +1,6 @@ """ APT backend for :mod:`Bcfg2.Server.Plugins.Packages` """ import re -import gzip from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source @@ -68,19 +67,34 @@ class AptSource(Source): #: AptSource sets the ``type`` on Package entries to "deb" ptype = 'deb' + #: Most (3rd-party) debian repositories still only support "gzip". + default_compression = 'gzip' + @property def urls(self): """ A list of URLs to the base metadata file for each repository described by this source. """ + fname = self.build_filename('Packages') + if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: - rv.append("%sdists/%s/%s/binary-%s/Packages.gz" % - (self.url, self.version, part, arch)) + rv.append("%sdists/%s/%s/binary-%s/%s" % + (self.url, self.version, part, arch, fname)) return rv else: - return ["%sPackages.gz" % self.rawurl] + return ["%s%s" % (self.rawurl, fname)] + + def _get_arch(self, fname): + if not self.rawurl: + return [x + for x in fname.split('@') + if x.startswith('binary-')][0][7:] + + # RawURL entries assume that they only have one <Arch></Arch> + # element and that it is the architecture of the source. + return self.arches[0] def read_files(self): # pylint: disable=R0912 bdeps = dict() @@ -89,23 +103,13 @@ class AptSource(Source): self.pkgnames = set() self.essentialpkgs = set() for fname in self.files: - if not self.rawurl: - barch = [x - for x in fname.split('@') - if x.startswith('binary-')][0][7:] - else: - # RawURL entries assume that they only have one <Arch></Arch> - # element and that it is the architecture of the source. - barch = self.arches[0] + barch = self._get_arch(fname) if barch not in bdeps: bdeps[barch] = dict() brecs[barch] = dict() bprov[barch] = dict() - try: - reader = gzip.GzipFile(fname) - except IOError: - self.logger.error("Packages: Failed to read file %s" % fname) - raise + + reader = self.open_file(fname) for line in reader.readlines(): if not isinstance(line, str): line = line.decode('utf-8') @@ -150,5 +154,6 @@ class AptSource(Source): if dname not in bprov[barch]: bprov[barch][dname] = set() bprov[barch][dname].add(pkgname) + reader.close() self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 004e27874..e0d6e1fc3 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -289,7 +289,7 @@ class Collection(list, Debuggable): return any(source.is_virtual_package(self.metadata, package) for source in self) - def get_deps(self, package, recs=None): + def get_deps(self, package, recs=None, pinnings=None): """ Get a list of the dependencies of the given package. The base implementation simply aggregates the results of @@ -297,16 +297,35 @@ class Collection(list, Debuggable): :param package: The name of the symbol, but see :ref:`pkg-objects` :type package: string + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: list of strings, but see :ref:`pkg-objects` """ recommended = None if recs and package in recs: recommended = recs[package] + pin_found = False + pin_source = None + if pinnings and package in pinnings: + pin_source = pinnings[package] + for source in self: + if pin_source and source.name not in pin_source: + continue + pin_found = True + if source.is_package(self.metadata, package): return source.get_deps(self.metadata, package, recommended) + if not pin_found: + if pin_source: + self.logger.error("Packages: Source '%s' for package '%s' not found" % + (' or '.join(pin_source), package)) + else: + self.logger.error("Packages: No source found for package '%s'" % + package); + return [] def get_essential(self): @@ -471,12 +490,14 @@ class Collection(list, Debuggable): @track_statistics() def complete(self, packagelist, # pylint: disable=R0912,R0914 - recommended=None): + recommended=None, pinnings=None): """ Build a complete list of all packages and their dependencies. :param packagelist: Set of initial packages computed from the specification. :type packagelist: set of strings, but see :ref:`pkg-objects` + :param pinnings: Mapping from package names to source names. + :type pinnings: dict :returns: tuple of sets - The first element contains a set of strings (but see :ref:`pkg-objects`) describing the complete package list, and the second element is a @@ -535,7 +556,7 @@ class Collection(list, Debuggable): self.debug_log("Packages: handling package requirement %s" % (current,)) packages.add(current) - deps = self.get_deps(current, recommended) + deps = self.get_deps(current, recommended, pinnings) newdeps = set(deps).difference(examined) if newdeps: self.debug_log("Packages: Package %s added requirements %s" diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Dummy.py b/src/lib/Bcfg2/Server/Plugins/Packages/Dummy.py new file mode 100644 index 000000000..f47b8f22c --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Dummy.py @@ -0,0 +1,35 @@ +""" Dummy backend for :mod:`Bcfg2.Server.Plugins.Packages` """ + +from Bcfg2.Server.Plugins.Packages.Collection import Collection +from Bcfg2.Server.Plugins.Packages.Source import Source + + +class DummyCollection(Collection): + """ Handle collections of Dummy sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Collection.Collection`, + overrides nothing, and defers all operations to :class:`PacSource` + """ + + def __init__(self, metadata, sources, cachepath, basepath, debug=False): + # we define an __init__ that just calls the parent __init__, + # so that we can set the docstring on __init__ to something + # different from the parent __init__ -- namely, the parent + # __init__ docstring, minus everything after ``.. -----``, + # which we use to delineate the actual docs from the + # .. autoattribute hacks we have to do to get private + # attributes included in sphinx 1.0 """ + Collection.__init__(self, metadata, sources, cachepath, basepath, + debug=debug) + __init__.__doc__ = Collection.__init__.__doc__.split(".. -----")[0] + + +class DummySource(Source): + """ Handle Dummy sources """ + + #: DummySource sets the ``type`` on Package entries to "dummy" + ptype = 'dummy' + + def __init__(self, basepath, xsource): + xsource.set('rawurl', 'http://example.com/') + Source.__init__(self, basepath, xsource) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 6fc084cc4..e3432c934 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -87,6 +87,9 @@ class PacSource(Source): #: PacSource sets the ``type`` on Package entries to "pacman" ptype = 'pacman' + #: The database of pacman repositories is compressed with "gzip" + default_compression = 'gzip' + def __init__(self, basepath, xsource): self.pacgroups = {} @@ -113,9 +116,10 @@ class PacSource(Source): if not self.rawurl: rv = [] for part in self.components: + filename = self.build_filename("%s.db.tar" % part) for arch in self.arches: - rv.append("%s%s/os/%s/%s.db.tar.gz" % - (self.url, part, arch, part)) + rv.append("%s%s/os/%s/%s" % + (self.url, part, arch, filename)) return rv else: raise Exception("PacSource : RAWUrl not supported (yet)") @@ -140,7 +144,8 @@ class PacSource(Source): bprov[barch] = {} try: self.debug_log("Packages: try to read %s" % fname) - tar = tarfile.open(fname, "r") + reader = self.open_file(fname) + tar = tarfile.open(fileobj=reader) except (IOError, tarfile.TarError): self.logger.error("Packages: Failed to read file %s" % fname) raise @@ -185,6 +190,7 @@ class PacSource(Source): self.pacgroups[group].append(pkgname) tar.close() + reader.close() self.process_files(bdeps, bprov, brecs) read_files.__doc__ = Source.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 1af046ec0..d982626d0 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -106,6 +106,8 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile): source = self.source_from_xml(xsource) if source is not None: self.entries.append(source) + self.entries.sort(key=(lambda source: source.priority), + reverse=True) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py index 4938efb94..5248ad896 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pkgng.py @@ -1,6 +1,5 @@ """ pkgng backend for :mod:`Bcfg2.Server.Plugins.Packages` """ -import lzma import tarfile try: @@ -40,19 +39,30 @@ class PkgngSource(Source): #: PkgngSource sets the ``type`` on Package entries to "pkgng" ptype = 'pkgng' + #: The "packagesite" files of pkgng repositories are compressed + #: with "xz" + default_compression = 'xz' + + def _get_extension(self): + extension = super(PkgngSource, self)._get_extension() + if extension == '': + return 'tar' + return 't%s' % extension + @property def urls(self): """ A list of URLs to the base metadata file for each repository described by this source. """ + fname = self.build_filename("packagesite") if not self.rawurl: rv = [] for part in self.components: for arch in self.arches: - rv.append("%s/freebsd:%s:%s/%s/packagesite.txz" % - (self.url, self.version, arch, part)) + rv.append("%s/freebsd:%s:%s/%s/%s" % + (self.url, self.version, arch, part, fname)) return rv else: - return ["%s/packagesite.txz" % self.rawurl] + return ["%s/%s" % (self.rawurl, fname)] def read_files(self): bdeps = dict() @@ -70,15 +80,14 @@ class PkgngSource(Source): if barch not in bdeps: bdeps[barch] = dict() try: - tar = tarfile.open(fileobj=lzma.LZMAFile(fname)) - reader = tar.extractfile('packagesite.yaml') + reader = self.open_file(fname) + tar = tarfile.open(fileobj=reader) + packagesite = tar.extractfile('packagesite.yaml') except (IOError, tarfile.TarError): self.logger.error("Packages: Failed to read file %s" % fname) raise - for line in reader.readlines(): - if not isinstance(line, str): - line = line.decode('utf-8') - pkg = json.loads(line) + for line in packagesite.readlines(): + pkg = json.loads(unicode(line, errors='ignore')) pkgname = pkg['name'] self.pkgnames.add(pkgname) if 'deps' in pkg: diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py new file mode 100644 index 000000000..5e0eb55be --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pyapt.py @@ -0,0 +1,93 @@ +""" +APT backend for :mod:`Bcfg2.Server.Plugins.Packages` using +apt_pkg python bindings. +""" + +import apt_pkg +from Bcfg2.Server.Plugins.Packages.Apt import AptCollection, AptSource + + +class PyaptCollection(AptCollection): + """ Handle collections of PyAPT sources. This is a no-op object + that simply inherits from + :class:`Bcfg2.Server.Plugins.Packages.Apt.AptCollection` and + overrides nothing. + """ + pass + + +class PyaptSource(AptSource): + """ Handle PyAPT sources """ + + def read_files(self): # pylint: disable=R0912 + bdeps = dict() + brecs = dict() + bprov = dict() + bvers = dict() + self.pkgnames = set() + self.essentialpkgs = set() + for fname in self.files: + barch = self._get_arch(fname) + if barch not in bdeps: + bdeps[barch] = dict() + brecs[barch] = dict() + bprov[barch] = dict() + + apt_pkg.init_system() + with apt_pkg.TagFile(fname) as tagfile: + for section in tagfile: + pkgname = section['Package'] + + if pkgname in bvers: + new = section['Version'] + old = bvers[pkgname] + if apt_pkg.version_compare(new, old) <= 0: + continue + + self.pkgnames.add(pkgname) + bvers[pkgname] = section['Version'] + bdeps[barch][pkgname] = [] + brecs[barch][pkgname] = [] + + if section.find_flag('Essential'): + self.essentialpkgs.add(pkgname) + + for dep_type in ['Depends', 'Pre-Depends', 'Recommends']: + dep_str = section.find(dep_type) + if dep_str is None: + continue + + vindex = 0 + for dep in apt_pkg.parse_depends(dep_str): + if len(dep) > 1: + cdeps = [cdep for (cdep, _, _) in dep] + dyn_dname = "choice-%s-%s-%s" % (pkgname, + barch, + vindex) + vindex += 1 + + if dep_type == 'Recommends': + brecs[barch][pkgname].append(dyn_dname) + else: + bdeps[barch][pkgname].append(dyn_dname) + bprov[barch][dyn_dname] = set(cdeps) + else: + (raw_dep, _, _) = dep[0] + if dep_type == 'Recommends': + brecs[barch][pkgname].append(raw_dep) + else: + bdeps[barch][pkgname].append(raw_dep) + + provides = section.find('Provides') + if provides is not None: + provided_packages = [ + pkg + for group in apt_pkg.parse_depends(provides) + for (pkg, _, _) in group] + for dname in provided_packages: + if dname not in bprov[barch]: + bprov[barch][dname] = set() + bprov[barch][dname].add(pkgname) + + self.process_files(bdeps, bprov, brecs) + read_files.__doc__ = AptSource.read_files.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py new file mode 100644 index 000000000..2267197ca --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Bzip2.py @@ -0,0 +1,14 @@ +""" Reader for bzip2 compressed package sources. """ + +import bz2 +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class Bzip2Reader(Reader): + extension = 'bz' + + def _open(self, filename): + return bz2.BZ2File(filename) + + def readable(self): + return True diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py new file mode 100644 index 000000000..8e67e2b33 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Gzip.py @@ -0,0 +1,11 @@ +""" Reader for gzip compressed package sources. """ + +import gzip +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class GzipReader(Reader): + extension = 'gz' + + def _open(self, filename): + return gzip.GzipFile(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py new file mode 100644 index 000000000..2f7a18d84 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/None.py @@ -0,0 +1,10 @@ +""" Reader for uncompressed package sources. """ + +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class NoneReader(Reader): + extension = '' + + def _open(self, filename): + return open(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py new file mode 100644 index 000000000..39a058767 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/Xz.py @@ -0,0 +1,11 @@ +""" Reader for lzma compressed package sources. """ + +import lzma +from Bcfg2.Server.Plugins.Packages.Readers import Reader + + +class XzReader(Reader): + extension = 'xz' + + def _open(self, filename): + return lzma.LZMAFile(filename) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py new file mode 100644 index 000000000..51b4fb79c --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Readers/__init__.py @@ -0,0 +1,36 @@ +"""This module implements different readers for package files.""" + +from io import IOBase +from Bcfg2.Compat import walk_packages + + +def get_readers(): + """ Return all available packages readers. """ + return [m[1] # pylint: disable=C0103 + for m in walk_packages(path=__path__)] + + +class Reader(IOBase): + extension = None + + def __init__(self, name): + self.name = name + self._file = self._open(name) + + def _open(self, filename): + raise NotImplementedError + + def read(self, size=-1): + return self._file.read(size) + + def readable(self): + return self._file.readable() + + def readline(self, size=-1): + return self._file.readline(size) + + def readlines(self, hint=None): + return self._file.readlines(size) + + def writelines(self, lines): + self._unsupported("writelines") diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 86f7698f7..574dbd851 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -49,6 +49,7 @@ in your ``Source`` subclass. For an example of this kind of import os import re import sys +import Bcfg2.Options from Bcfg2.Logger import Debuggable from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, urlopen, \ @@ -56,7 +57,7 @@ from Bcfg2.Compat import HTTPError, HTTPBasicAuthHandler, \ from Bcfg2.Server.Statistics import track_statistics -def fetch_url(url): +def fetch_url(url, opts): """ Return the content of the given URL. :param url: The URL to fetch content from. @@ -73,8 +74,14 @@ def fetch_url(url): url = mobj.group(1) + mobj.group(4) auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) auth.add_password(None, url, user, passwd) - install_opener(build_opener(auth)) - return urlopen(url).read() + req = build_opener(auth) + else: + req = build_opener() + + if 'user-agent' in opts: + req.addheaders = [('User-Agent', opts['user-agent'])] + + return req.open(url).read() class SourceInitError(Exception): @@ -111,6 +118,12 @@ class Source(Debuggable): # pylint: disable=R0902 #: when they are handled by :mod:`Bcfg2.Server.Plugins.Packages`. ptype = None + #: The default compression format used by this Source class. This + #: is the file the package metadata files should be loaded. It is + #: used if a source has no custom compression format specified + #: in the :attr:`server_options`. + default_compression = 'None' + def __init__(self, basepath, xsource): # pylint: disable=R0912 """ :param basepath: The base filesystem path under which cache @@ -188,6 +201,12 @@ class Source(Debuggable): # pylint: disable=R0902 #: The "name" attribute from :attr:`xsource` self.name = None + #: The "priority" attribute from :attr:`xsource` + self.priority = xsource.get('priority', 500) + + #: The "pin" attribute from :attr:`xsource` + self.pin = xsource.get('pin', '') + #: A list of predicates that are used to determine if this #: source applies to a given #: :class:`Bcfg2.Server.Plugins.Metadata.ClientMetadata` @@ -250,11 +269,13 @@ class Source(Debuggable): # pylint: disable=R0902 for arch in self.arches: if self.url: usettings = [dict(version=self.version, component=comp, - arch=arch, debsrc=self.debsrc) + arch=arch, debsrc=self.debsrc, + priority=self.priority, pin=self.pin) for comp in self.components] else: # rawurl given usettings = [dict(version=self.version, component=None, - arch=arch, debsrc=self.debsrc)] + arch=arch, debsrc=self.debsrc, + priority=self.priority, pin=self.pin)] for setting in usettings: if not self.rawurl: @@ -263,6 +284,8 @@ class Source(Debuggable): # pylint: disable=R0902 setting['baseurl'] = self.rawurl setting['url'] = baseurl % setting setting['name'] = self.get_repo_name(setting) + setting['options'] = dict(server=self.server_options, + client=self.client_options) self.url_map.extend(usettings) def _init_attributes(self, xsource): @@ -328,6 +351,38 @@ class Source(Debuggable): # pylint: disable=R0902 self.conditions.append(lambda m, el=el: el.get("name") == m.hostname) + def _get_reader(self): + ctype = self.default_compression + if 'compression' in self.server_options: + ctype = self.server_options['compression'] + + for mod in Bcfg2.Options.setup.packages_readers: + if mod.__name__.endswith(".%s" % ctype.title()): + return getattr(mod, "%sReader" % ctype.title()) + + raise ValueError("Packages: Unknown compression type %s" % ctype) + + def _get_extension(self): + cls = self._get_reader() + if cls.extension is None: + raise ValueError("%s does not define an extension" % + cls.__name__) + return cls.extension + + def build_filename(self, basename): + extension = self._get_extension() + if extension == '': + return basename + return "%s.%s" % (basename, extension) + + def open_file(self, fname): + try: + cls = self._get_reader() + return cls(fname) + except IOError: + self.logger.error("Packages: Failed to read file %s" % fname) + raise + @property def cachekey(self): """ A unique key for this source that will be used to generate @@ -534,6 +589,9 @@ class Source(Debuggable): # pylint: disable=R0902 self.logger.warning("%s provides no packages for %s" % (self, agrp)) continue + if (agrp in self.blacklist or + (len(self.whitelist) != 0 and agrp not in self.whitelist)): + continue for key, value in list(self.provides[agrp].items()): if key not in vdict: vdict[key] = set(value) @@ -686,7 +744,7 @@ class Source(Debuggable): # pylint: disable=R0902 self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: - open(fname, 'wb').write(fetch_url(url)) + open(fname, 'wb').write(fetch_url(url, self.server_options)) except ValueError: self.logger.error("Packages: Bad url string %s" % url) raise @@ -760,7 +818,9 @@ class Source(Debuggable): # pylint: disable=R0902 :returns: list of strings """ for arch in self.get_arches(metadata): - if package in self.provides[arch]: + if (package in self.provides[arch] and + package not in self.blacklist and + (len(self.whitelist) == 0 or package in self.whitelist)): return self.provides[arch][package] return [] diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 846fb89cd..acb11f1ab 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -850,7 +850,7 @@ class YumCollection(Collection): return new @track_statistics() - def complete(self, packagelist, recommended=None): + def complete(self, packagelist, recommended=None, pinnings=None): """ Build a complete list of all packages and their dependencies. When using the Python yum libraries, this defers to the @@ -868,7 +868,8 @@ class YumCollection(Collection): resolved. """ if not self.use_yum: - return Collection.complete(self, packagelist, recommended) + return Collection.complete(self, packagelist, recommended, + pinnings) lock = FileLock(os.path.join(self.cachefile, "lock")) slept = 0 diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 23ccd7b8e..1a9673891 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -14,6 +14,7 @@ from Bcfg2.Compat import urlopen, HTTPError, URLError from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources +from Bcfg2.Server.Plugins.Packages.Readers import get_readers from Bcfg2.Server.Statistics import track_statistics @@ -36,6 +37,12 @@ class PackagesBackendAction(Bcfg2.Options.ComponentAction): fail_silently = True +class PackagesReadersAction(Bcfg2.Options.ComponentAction): + """ ComponentAction to load Packages readers """ + bases = ['Bcfg2.Server.Plugins.Packages.Readers'] + module = True + + class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -57,7 +64,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, help="Packages backends to load", type=Bcfg2.Options.Types.comma_list, action=PackagesBackendAction, - default=['Yum', 'Apt', 'Pac', 'Pkgng']), + default=['Yum', 'Apt', 'Pac', 'Pkgng', 'Dummy', 'Pyapt']), Bcfg2.Options.PathOption( cf=("packages", "cache"), dest="packages_cache", help="Path to the Packages cache", @@ -82,7 +89,13 @@ class Packages(Bcfg2.Server.Plugin.Plugin, cf=("packages", "apt_config"), help="The default path for generated apt configs", default="/etc/apt/sources.list.d/" - "bcfg2-packages-generated-sources.list")] + "bcfg2-packages-generated-sources.list"), + Bcfg2.Options.Option( + cf=("packages", "readers"), dest="packages_readers", + help="Packages readers to load", + type=Bcfg2.Options.Types.comma_list, + action=PackagesReadersAction, + default=get_readers())] #: Packages is an alternative to #: :mod:`Bcfg2.Server.Plugins.Pkgmgr` and conflicts with it. @@ -315,6 +328,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin, groups = [] recommended = dict() + pinned_src = dict() + if hasattr(metadata, 'PkgVars'): + pinned_src = metadata.PkgVars['pin'] + for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): if pkg.get("name"): @@ -356,11 +373,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, base.update(collection.get_essential()) # check for this set of packages in the package cache - pkey = hash(tuple(base)) + pkey = hash((tuple(base), tuple(recommended), tuple(pinned_src))) pcache = Bcfg2.Server.Cache.Cache("Packages", "pkg_sets", collection.cachekey) if pkey not in pcache: - pcache[pkey] = collection.complete(base, recommended) + pcache[pkey] = collection.complete(base, recommended, pinned_src) packages, unknown = pcache[pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) diff --git a/src/lib/Bcfg2/Server/Plugins/PkgVars.py b/src/lib/Bcfg2/Server/Plugins/PkgVars.py new file mode 100644 index 000000000..9a2649d02 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/PkgVars.py @@ -0,0 +1,65 @@ +import os +import re +import sys +import copy +import logging +import lxml.etree +import Bcfg2.Server.Plugin + +logger = logging.getLogger('Bcfg2.Plugins.PkgVars') +vars = ['pin', 'use', 'keywords'] + +class PkgVarsFile(Bcfg2.Server.Plugin.StructFile): + def get_additional_data(self, meta): + data = self.Match(meta) + results = {} + for d in data: + name = d.get('name', '') + + for v in vars: + value = d.get(v, None) + if value: + if v not in results: + results[v] = {} + if name not in results[v]: + results[v][name] = set() + + results[v][name].add(value) + + return results + +class PkgVarsDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): + __child__ = PkgVarsFile + patterns = re.compile(r'.*\.xml$') + + def get_additional_data(self, meta): + results = {} + for v in vars: + results[v] = {} + + for files in self.entries: + new = self.entries[files].get_additional_data(meta) + for x in vars: + if x in new: + results[x].update(new[x]) + + return results + +class PkgVars(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + name = 'PkgVars' + version = '$Revision$' + + def __init__(self, core): + Bcfg2.Server.Plugin.Plugin.__init__(self, core) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + self.store = PkgVarsDirectoryBacked(self.data) + except OSError: + e = sys.exc_info()[1] + self.logger.error("Error while creating PkgVars store: %s %s" % + (e.strerror, e.filename)) + raise Bcfg2.Server.Plugin.PluginInitError + + def get_additional_data(self, meta): + return self.store.get_additional_data(meta) |