Reporting: Merge new reporting data

Move reporting data to a new schema
Use south for django migrations
Add bcfg2-report-collector daemon

Conflicts:
	doc/development/index.txt
	doc/server/plugins/connectors/properties.txt
	doc/server/plugins/generators/packages.txt
	setup.py
	src/lib/Bcfg2/Client/Tools/SELinux.py
	src/lib/Bcfg2/Compat.py
	src/lib/Bcfg2/Encryption.py
	src/lib/Bcfg2/Options.py
	src/lib/Bcfg2/Server/Admin/Init.py
	src/lib/Bcfg2/Server/Admin/Reports.py
	src/lib/Bcfg2/Server/BuiltinCore.py
	src/lib/Bcfg2/Server/Core.py
	src/lib/Bcfg2/Server/FileMonitor/Inotify.py
	src/lib/Bcfg2/Server/Plugin/base.py
	src/lib/Bcfg2/Server/Plugin/interfaces.py
	src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py
	src/lib/Bcfg2/Server/Plugins/FileProbes.py
	src/lib/Bcfg2/Server/Plugins/Ohai.py
	src/lib/Bcfg2/Server/Plugins/Packages/Collection.py
	src/lib/Bcfg2/Server/Plugins/Packages/Source.py
	src/lib/Bcfg2/Server/Plugins/Packages/Yum.py
	src/lib/Bcfg2/Server/Plugins/Packages/__init__.py
	src/lib/Bcfg2/Server/Plugins/Probes.py
	src/lib/Bcfg2/Server/Plugins/Properties.py
	src/lib/Bcfg2/Server/Reports/backends.py
	src/lib/Bcfg2/Server/Reports/manage.py
	src/lib/Bcfg2/Server/Reports/nisauth.py
	src/lib/Bcfg2/settings.py
	src/sbin/bcfg2-crypt
	src/sbin/bcfg2-yum-helper
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py
	testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestSEModules.py

1 files changed, 582 insertions, 0 deletions

diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py
new file mode 100644
index 000000000..3a540587a
--- /dev/null
+++ b/src/lib/Bcfg2/Reporting/models.py
@@ -0,0 +1,582 @@
+"""Django models for Bcfg2 reports."""
+import sys
+
+from django.core.exceptions import ImproperlyConfigured
+try:
+    from django.db import models
+except ImproperlyConfigured:
+    e = sys.exc_info()[1]
+    print("Reports: unable to import django models: %s" % e)
+    sys.exit(1)
+
+from django.core.cache import cache
+from datetime import datetime, timedelta
+
+try:
+    import cPickle as pickle
+except:
+    import pickle
+
+KIND_CHOICES = (
+    #These are the kinds of config elements
+    ('Package', 'Package'),
+    ('Path', 'directory'),
+    ('Path', 'file'),
+    ('Path', 'permissions'),
+    ('Path', 'symlink'),
+    ('Service', 'Service'),
+)
+TYPE_GOOD = 0
+TYPE_BAD = 1
+TYPE_MODIFIED = 2
+TYPE_EXTRA = 3
+
+TYPE_CHOICES = (
+    (TYPE_GOOD, 'Good'),
+    (TYPE_BAD, 'Bad'),
+    (TYPE_MODIFIED, 'Modified'),
+    (TYPE_EXTRA, 'Extra'),
+)
+
+
+def convert_entry_type_to_id(type_name):
+    """Convert a entry type to its entry id"""
+    for e_id, e_name in TYPE_CHOICES:
+        if e_name.lower() == type_name.lower():
+            return e_id
+    return -1
+
+
+def hash_entry(entry_dict):
+    """
+    Build a key for this based on its data
+
+    entry_dict = a dict of all the data identifying this
+    """
+    dataset = []
+    for key in sorted(entry_dict.keys()):
+        if key in ('id', 'hash_key') or key.startswith('_'):
+            continue
+        dataset.append( (key, entry_dict[key]) )
+    return hash(pickle.dumps(dataset))
+
+
+class Client(models.Model):
+    """Object representing every client we have seen stats for."""
+    creation = models.DateTimeField(auto_now_add=True)
+    name = models.CharField(max_length=128,)
+    current_interaction = models.ForeignKey('Interaction',
+                                            null=True, blank=True,
+                                            related_name="parent_client")
+    expiration = models.DateTimeField(blank=True, null=True)
+
+    def __str__(self):
+        return self.name
+
+
+class InteractionManager(models.Manager):
+    """Manages interactions objects."""
+
+    def recent_ids(self, maxdate=None):
+        """
+        Returns the ids of most recent interactions for clients as of a date.
+
+        Arguments:
+        maxdate -- datetime object.  Most recent date to pull. (dafault None)
+
+        """
+        from django.db import connection
+        cursor = connection.cursor()
+        cfilter = "expiration is null"
+
+        sql = 'select ri.id, x.client_id from (select client_id, MAX(timestamp) ' + \
+                    'as timer from Reporting_interaction'
+        if maxdate:
+            if not isinstance(maxdate, datetime):
+                raise ValueError('Expected a datetime object')
+            sql = sql + " where timestamp <= '%s' " % maxdate
+            cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate)
+        sql = sql + ' GROUP BY client_id) x, Reporting_interaction ri where ' + \
+                    'ri.client_id = x.client_id AND ri.timestamp = x.timer'
+        sql = sql + " and x.client_id in (select id from Reporting_client where %s)" % cfilter
+        try:
+            cursor.execute(sql)
+            return [item[0] for item in cursor.fetchall()]
+        except:
+            '''FIXME - really need some error handling'''
+            pass
+        return []
+
+
+    def recent(self, maxdate=None):
+        """
+        Returns the most recent interactions for clients as of a date
+        Arguments:
+        maxdate -- datetime object.  Most recent date to pull. (dafault None)
+
+        """
+        if maxdate and not isinstance(maxdate, datetime):
+            raise ValueError('Expected a datetime object')
+        return self.filter(id__in=self.recent_ids(maxdate))
+
+
+class Interaction(models.Model):
+    """Models each reconfiguration operation interaction between client and server."""
+    client = models.ForeignKey(Client, related_name="interactions")
+    timestamp = models.DateTimeField(db_index=True)  # Timestamp for this record
+    state = models.CharField(max_length=32)  # good/bad/modified/etc
+    repo_rev_code = models.CharField(max_length=64)  # repo revision at time of interaction
+    server = models.CharField(max_length=256)  # Name of the server used for the interaction
+    good_count = models.IntegerField()  # of good config-items
+    total_count = models.IntegerField()  # of total config-items
+    bad_count = models.IntegerField(default=0)
+    modified_count = models.IntegerField(default=0)
+    extra_count = models.IntegerField(default=0)
+
+    actions = models.ManyToManyField("ActionEntry")
+    packages = models.ManyToManyField("PackageEntry")
+    paths = models.ManyToManyField("PathEntry")
+    services = models.ManyToManyField("ServiceEntry")
+    failures = models.ManyToManyField("FailureEntry")
+
+    # Formerly InteractionMetadata
+    profile = models.ForeignKey("Group", related_name="+")
+    groups = models.ManyToManyField("Group")
+    bundles = models.ManyToManyField("Bundle")
+
+    objects = InteractionManager()
+
+    def __str__(self):
+        return "With " + self.client.name + " @ " + self.timestamp.isoformat()
+
+    def percentgood(self):
+        if not self.total_count == 0:
+            return (self.good_count / float(self.total_count)) * 100
+        else:
+            return 0
+
+    def percentbad(self):
+        if not self.total_count == 0:
+            return ((self.total_count - self.good_count) / (float(self.total_count))) * 100
+        else:
+            return 0
+
+    def isclean(self):
+        if (self.bad_count == 0 and self.good_count == self.total_count):
+            return True
+        else:
+            return False
+
+    def isstale(self):
+        if (self == self.client.current_interaction):  # Is Mostrecent
+            if(datetime.now() - self.timestamp > timedelta(hours=25)):
+                return True
+            else:
+                return False
+        else:
+            #Search for subsequent Interaction for this client
+            #Check if it happened more than 25 hrs ago.
+            if (self.client.interactions.filter(timestamp__gt=self.timestamp)
+                    .order_by('timestamp')[0].timestamp -
+                    self.timestamp > timedelta(hours=25)):
+                return True
+            else:
+                return False
+
+    def save(self):
+        super(Interaction, self).save()  # call the real save...
+        self.client.current_interaction = self.client.interactions.latest()
+        self.client.save()  # save again post update
+
+    def delete(self):
+        '''Override the default delete.  Allows us to remove Performance items'''
+        pitems = list(self.performance_items.all())
+        super(Interaction, self).delete()
+        for perf in pitems:
+            if perf.interaction.count() == 0:
+                perf.delete()
+
+    def badcount(self):
+        return self.total_count - self.good_count
+
+    def bad(self):
+        rv = []
+        for entry in ('actions', 'packages', 'paths', 'services'):
+            rv.extend(getattr(self, entry).filter(state=TYPE_BAD))
+        return rv
+
+    def modified(self):
+        rv = []
+        for entry in ('actions', 'packages', 'paths', 'services'):
+            rv.extend(getattr(self, entry).filter(state=TYPE_MODIFIED))
+        return rv
+
+    def extra(self):
+        rv = []
+        for entry in ('actions', 'packages', 'paths', 'services'):
+            rv.extend(getattr(self, entry).filter(state=TYPE_EXTRA))
+        return rv
+
+    class Meta:
+        get_latest_by = 'timestamp'
+        ordering = ['-timestamp']
+        unique_together = ("client", "timestamp")
+
+
+class Performance(models.Model):
+    """Object representing performance data for any interaction."""
+    interaction = models.ForeignKey(Interaction, related_name="performance_items")
+    metric = models.CharField(max_length=128)
+    value = models.DecimalField(max_digits=32, decimal_places=16)
+
+    def __str__(self):
+        return self.metric
+
+
+class Group(models.Model):
+    """
+    Groups extracted from interactions
+
+    name - The group name
+
+    TODO - Most of this is for future use
+    TODO - set a default group
+    """
+
+    name = models.CharField(max_length=255, unique=True)
+    profile = models.BooleanField(default=False)
+    public = models.BooleanField(default=False)
+    category = models.CharField(max_length=1024, blank=True)
+    comment = models.TextField(blank=True)
+
+    groups = models.ManyToManyField("self", symmetrical=False)
+    bundles = models.ManyToManyField("Bundle")
+
+    def __unicode__(self):
+        return self.name
+
+    class Meta:
+        ordering = ('name',)
+
+
+class Bundle(models.Model):
+    """
+    Bundles extracted from interactions
+
+    name - The bundle name
+    """
+
+    name = models.CharField(max_length=255, unique=True)
+
+    def __unicode__(self):
+        return self.name
+
+    class Meta:
+        ordering = ('name',)
+
+
+# new interaction models
+class FilePerms(models.Model):
+    owner = models.CharField(max_length=128)
+    group = models.CharField(max_length=128)
+    perms = models.CharField(max_length=128)
+
+    class Meta:
+        unique_together = ('owner', 'group', 'perms')
+
+    def empty(self):
+        """Return true if we have no real data"""
+        if self.owner or self.group or self.perms:
+            return False
+        else:
+            return True
+
+
+class FileAcl(models.Model):
+    """Placeholder"""
+    name = models.CharField(max_length=128, db_index=True)
+
+
+class BaseEntry(models.Model):
+    """ Abstract base for all entry types """
+    name = models.CharField(max_length=128, db_index=True)
+    hash_key = models.IntegerField(editable=False, db_index=True)
+
+    class Meta:
+        abstract = True
+
+    def save(self, *args, **kwargs):
+        if 'hash_key' in kwargs:
+            self.hash_key = kwargs['hash_key']
+            del kwargs['hash_key']
+        else:
+            self.hash_key = hash_entry(self.__dict__)
+        super(BaseEntry, self).save(*args, **kwargs)
+
+
+    def class_name(self):
+        return self.__class__.__name__
+
+    def short_list(self):
+        """todo"""
+        return []
+
+
+    @classmethod
+    def entry_from_name(cls, name):
+        try:
+            newcls = globals()[name]
+            if not isinstance(newcls(), cls):
+                raise ValueError("%s is not an instance of %s" % (name, cls))
+            return newcls
+        except KeyError:
+            raise ValueError("Invalid type %s" % name)
+
+
+    @classmethod
+    def entry_get_or_create(cls, act_dict):
+        """Helper to quickly lookup an object"""
+        cls_name = cls().__class__.__name__
+        act_hash = hash_entry(act_dict)
+    
+        # TODO - get form cache and validate
+        act_key = "%s_%s" % (cls_name, act_hash)
+        newact = cache.get(act_key)
+        if newact:
+            return newact
+    
+        acts = cls.objects.filter(hash_key=act_hash)
+        if len(acts) > 0:
+            for act in acts:
+                for key in act_dict:
+                    if act_dict[key] != getattr(act, key):
+                        continue
+                    #match found
+                    newact = act
+                    break
+    
+        # worst case, its new
+        if not newact:
+            newact = cls(**act_dict)
+            newact.save(hash_key=act_hash)
+    
+        cache.set(act_key, newact)
+        return newact
+
+
+    def is_failure(self):
+        return isinstance(self, FailureEntry)
+
+
+class SuccessEntry(BaseEntry):
+    """Base for successful entries"""
+    state = models.IntegerField(choices=TYPE_CHOICES)
+    exists = models.BooleanField(default=True)
+
+    ENTRY_TYPE = r"Success"
+
+    @property
+    def entry_type(self):
+        return self.ENTRY_TYPE
+
+    def is_extra(self):
+        return self.state == TYPE_EXTRA
+
+    class Meta:
+        abstract = True
+        ordering = ('state', 'name')
+
+    def short_list(self):
+        """Return a list of problems"""
+        rv = []
+        if self.is_extra():
+            rv.append("Extra")
+        elif not self.exists:
+            rv.append("Missing")
+        return rv
+
+
+class FailureEntry(BaseEntry):
+    """Represents objects that failed to bind"""
+    entry_type = models.CharField(max_length=128)
+    message = models.TextField()
+
+    def is_failure(self):
+        return True
+
+
+class ActionEntry(SuccessEntry):
+    """ The new model for package information """
+    status = models.CharField(max_length=128, default="check")
+    output = models.IntegerField(default=0)
+
+    ENTRY_TYPE = r"Action"
+    #TODO - prune
+
+
+class PackageEntry(SuccessEntry):
+    """ The new model for package information """
+
+    # if this is an extra entry trget_version will be empty
+    target_version = models.CharField(max_length=1024, default='')
+    current_version = models.CharField(max_length=1024)
+    verification_details = models.TextField(default="")
+
+    ENTRY_TYPE = r"Package"
+    #TODO - prune
+
+    def version_problem(self):
+        """Check for a version problem."""
+        if not self.current_version:
+            return True
+        if self.target_version != self.current_version:
+            return True
+        elif self.target_version == 'auto':
+            return True
+        else:
+            return False
+
+    def short_list(self):
+        """Return a list of problems"""
+        rv = super(PackageEntry, self).short_list()
+        if self.is_extra():
+            return rv
+        if not self.version_problem() or not self.exists:
+            return rv
+        if not self.current_version:
+            rv.append("Missing")
+        else:
+            rv.append("Wrong version")
+        return rv
+
+
+class PathEntry(SuccessEntry):
+    """reason why modified or bad entry did not verify, or changed."""
+
+    PATH_TYPES = (
+        ("device", "Device"),
+        ("directory", "Directory"),
+        ("hardlink", "Hard Link"),
+        ("nonexistent", "Non Existent"),
+        ("permissions", "Permissions"),
+        ("symlink", "Symlink"),
+    )
+
+    DETAIL_UNUSED = 0
+    DETAIL_DIFF = 1
+    DETAIL_BINARY = 2
+    DETAIL_SENSITIVE = 3
+    DETAIL_SIZE_LIMIT = 4
+    DETAIL_VCS = 5
+    DETAIL_PRUNED = 6
+
+    DETAIL_CHOICES = (
+        (DETAIL_UNUSED, 'Unused'),
+        (DETAIL_DIFF, 'Diff'),
+        (DETAIL_BINARY, 'Binary'),
+        (DETAIL_SENSITIVE, 'Sensitive'),
+        (DETAIL_SIZE_LIMIT, 'Size limit exceeded'),
+        (DETAIL_VCS, 'VCS output'),
+        (DETAIL_PRUNED, 'Pruned paths'),
+    )
+
+    path_type = models.CharField(max_length=128, choices=PATH_TYPES)
+
+    target_perms = models.ForeignKey(FilePerms, related_name="+")
+    current_perms = models.ForeignKey(FilePerms, related_name="+")
+
+    acls = models.ManyToManyField(FileAcl)
+
+    detail_type = models.IntegerField(default=0,
+        choices=DETAIL_CHOICES)
+    details = models.TextField(default='')
+
+    ENTRY_TYPE = r"Path"
+
+    def perms_problem(self):
+        if self.current_perms.empty():
+            return False
+        elif self.target_perms.perms != self.current_perms.perms:
+            return True
+        else:
+            return False
+
+    def has_detail(self):
+        return self.detail_type != PathEntry.DETAIL_UNUSED
+
+    def is_sensitive(self):
+        return self.detail_type == PathEntry.DETAIL_SENSITIVE
+
+    def is_diff(self):
+        return self.detail_type == PathEntry.DETAIL_DIFF
+
+    def is_sensitive(self):
+        return self.detail_type == PathEntry.DETAIL_SENSITIVE
+
+    def is_binary(self):
+        return self.detail_type == PathEntry.DETAIL_BINARY
+
+    def is_too_large(self):
+        return self.detail_type == PathEntry.DETAIL_SIZE_LIMIT
+
+    def short_list(self):
+        """Return a list of problems"""
+        rv = super(PathEntry, self).short_list()
+        if self.is_extra():
+            return rv
+        if self.perms_problem():
+            rv.append("File permissions")
+        if self.detail_type == PathEntry.DETAIL_PRUNED:
+            rv.append("Directory has extra files")
+        elif self.detail_type != PathEntry.DETAIL_UNUSED:
+            rv.append("Incorrect data")
+        if hasattr(self, 'linkentry') and \
+                self.linkentry.target_path != self.linkentry.current_path:
+            rv.append("Incorrect target")
+        return rv
+
+
+class LinkEntry(PathEntry):
+    """Sym/Hard Link types"""
+    target_path = models.CharField(max_length=1024, blank=True)
+    current_path = models.CharField(max_length=1024, blank=True)
+
+    def link_problem(self):
+        return self.target_path != self.current_path
+
+
+class DeviceEntry(PathEntry):
+    """Device types.  Best I can tell the client driver needs work here"""
+    DEVICE_TYPES = (
+        ("block", "Block"),
+        ("char", "Char"),
+        ("fifo", "Fifo"),
+    )
+
+    device_type = models.CharField(max_length=16, choices=DEVICE_TYPES)
+
+    target_major = models.IntegerField()
+    target_minor = models.IntegerField()
+    current_major = models.IntegerField()
+    current_minor = models.IntegerField()
+
+
+class ServiceEntry(SuccessEntry):
+    """ The new model for package information """
+    target_status = models.CharField(max_length=128, default='')
+    current_status = models.CharField(max_length=128, default='')
+
+    ENTRY_TYPE = r"Service"
+    #TODO - prune
+
+    def status_problem(self):
+        return self.target_status != self.current_status
+
+    def short_list(self):
+        """Return a list of problems"""
+        rv = super(ServiceEntry, self).short_list()
+        if self.status_problem():
+            rv.append("Incorrect status")
+        return rv
+
+