DBStats: New db update routines

Replace updatefix.py with the Updater class.  This streamlines some
of the common tasks and groups database updates by release.  Upgrades
from pre 1.1.x are no longer supported.

12 files changed, 631 insertions, 332 deletions

diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py
index 041eacccc..97db140b7 100644
--- a/src/lib/Bcfg2/Server/Admin/Reports.py
+++ b/src/lib/Bcfg2/Server/Admin/Reports.py
@@ -26,7 +26,7 @@ import Bcfg2.Server.Reports.settings
 # Load django and reports stuff _after_ we know we can load settings
 import django.core.management
 from Bcfg2.Server.Reports.importscript import load_stats
-from Bcfg2.Server.Reports.updatefix import update_database
+from Bcfg2.Server.Reports.Updater import update_database, UpdaterError
 from Bcfg2.Server.Reports.utils import *
 
 project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__)
@@ -111,10 +111,14 @@ class Reports(Bcfg2.Server.Admin.Mode):
             self.django_command_proxy(args[0])
         elif args[0] == 'scrub':
             self.scrub()
-        elif args[0] == 'init':
-            update_database()
-        elif args[0] == 'update':
-            update_database()
+        elif args[0] in ['init', 'update']:
+            try:
+                update_database()
+            #except SchemaTooOldError:
+            #    logger.error("Sc
+            except UpdaterError:
+                print "Update failed"
+                raise SystemExit(-1)
         elif args[0] == 'load_stats':
             quick = '-O3' in args
             stats_file = None
@@ -266,6 +270,8 @@ class Reports(Bcfg2.Server.Admin.Mode):
                        self.log,
                        quick=quick,
                        location=platform.node())
+        except UpdaterError:
+            self.errExit("StatReports: Database updater failed")
         except:
             pass
 
diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py
index 131b6b059..b28484039 100644
--- a/src/lib/Bcfg2/Server/Plugins/DBStats.py
+++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py
@@ -14,7 +14,7 @@ import Bcfg2.Server.Plugin
 import Bcfg2.Server.Reports.importscript
 from Bcfg2.Server.Reports.reports.models import Client
 import Bcfg2.Server.Reports.settings
-from Bcfg2.Server.Reports.updatefix import update_database
+from Bcfg2.Server.Reports.Updater import update_database, UpdaterError
 # for debugging output only
 logger = logging.getLogger('Bcfg2.Plugins.DBStats')
 
@@ -34,6 +34,8 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
                      "add to the statistics database")
         try:
             update_database()
+        except UpdaterError:
+            raise Bcfg2.Server.Plugin.PluginInitError
         except Exception:
             inst = sys.exc_info()[1]
             logger.debug(str(inst))
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py
new file mode 100644
index 000000000..54ba07554
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py
@@ -0,0 +1,11 @@
+"""
+1_0_x.py
+
+This file should contain updates relevant to the 1.0.x branches ONLY.
+The updates() method must be defined and it should return an Updater object
+"""
+from Bcfg2.Server.Reports.Updater import UnsupportedUpdate
+
+def updates():
+    return UnsupportedUpdate("1.0", 10)
+
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py
new file mode 100644
index 000000000..26194cb67
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py
@@ -0,0 +1,59 @@
+"""
+1_1_x.py
+
+This file should contain updates relevant to the 1.1.x branches ONLY.
+The updates() method must be defined and it should return an Updater object
+"""
+from Bcfg2.Server.Reports.Updater import Updater
+from Bcfg2.Server.Reports.Updater.Routines import updatercallable
+
+from django.db import connection
+import sys
+import Bcfg2.Server.Reports.settings
+from Bcfg2.Server.Reports.reports.models import \
+                TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA
+
+@updatercallable
+def _interactions_constraint_or_idx():
+    """sqlite doesn't support alter tables.. or constraints"""
+    cursor = connection.cursor()
+    try:
+        cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)')
+    except:
+        cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)')
+
+
+@updatercallable
+def _populate_interaction_entry_counts():
+    '''Populate up the type totals for the interaction table'''
+    cursor = connection.cursor()
+    count_field = {TYPE_BAD: 'bad_entries',
+                   TYPE_MODIFIED: 'modified_entries',
+                   TYPE_EXTRA: 'extra_entries'}
+
+    for type in list(count_field.keys()):
+        cursor.execute("select count(type), interaction_id " +
+                "from reports_entries_interactions where type = %s group by interaction_id" % type)
+        updates = []
+        for row in cursor.fetchall():
+            updates.append(row)
+        try:
+            cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates)
+        except Exception:
+            e = sys.exc_info()[1]
+            print(e)
+    cursor.close()
+
+
+def updates():
+    fixes = Updater("1.1")
+    fixes.override_base_version(12) # Do not do this in new code
+
+    fixes.add('alter table reports_interaction add column bad_entries integer not null default -1;')
+    fixes.add('alter table reports_interaction add column modified_entries integer not null default -1;')
+    fixes.add('alter table reports_interaction add column extra_entries integer not null default -1;')
+    fixes.add(_populate_interaction_entry_counts())
+    fixes.add(_interactions_constraint_or_idx())
+    fixes.add('alter table reports_reason add is_binary bool NOT NULL default False;')
+    return fixes
+
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py
new file mode 100644
index 000000000..22bd937c2
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py
@@ -0,0 +1,15 @@
+"""
+1_2_x.py
+
+This file should contain updates relevant to the 1.2.x branches ONLY.
+The updates() method must be defined and it should return an Updater object
+"""
+from Bcfg2.Server.Reports.Updater import Updater
+from Bcfg2.Server.Reports.Updater.Routines import updatercallable
+
+def updates():
+    fixes = Updater("1.2")
+    fixes.override_base_version(18) # Do not do this in new code
+    fixes.add('alter table reports_reason add is_sensitive bool NOT NULL default False;')
+    return fixes
+
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py
new file mode 100644
index 000000000..b09b06302
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py
@@ -0,0 +1,26 @@
+"""
+1_3_0.py
+
+This file should contain updates relevant to the 1.3.x branches ONLY.
+The updates() method must be defined and it should return an Updater object
+"""
+from Bcfg2.Server.Reports.Updater import Updater, UpdaterError
+from Bcfg2.Server.Reports.Updater.Routines import AddColumns, \
+        RemoveColumns, RebuildTable
+
+from Bcfg2.Server.Reports.reports.models import Reason, Interaction
+
+
+def updates():
+    fixes = Updater("1.3")
+    fixes.add(RemoveColumns(Interaction, 'client_version'))
+    fixes.add(AddColumns(Reason))
+    fixes.add(RebuildTable(Reason, [
+               'owner', 'current_owner',
+               'group', 'current_group',
+               'perms', 'current_perms',
+               'status', 'current_status',
+               'to', 'current_to']))
+
+    return fixes
+
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Routines.py b/src/lib/Bcfg2/Server/Reports/Updater/Routines.py
new file mode 100644
index 000000000..1d41848e4
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Reports/Updater/Routines.py
@@ -0,0 +1,258 @@
+import logging
+from django.db.models.fields import NOT_PROVIDED
+from django.db import connection, DatabaseError, backend, models
+from django.core.management.color import no_style
+from django.core.management.sql import sql_create
+import django.core.management
+
+import Bcfg2.Server.Reports.settings
+
+logger = logging.getLogger(__name__)
+
+def _quote(value):
+    """
+    Quote a string to use as a table name or column
+    """
+    return backend.DatabaseOperations().quote_name(value)
+
+
+def _rebuild_sqlite_table(model):
+    """Sqlite doesn't support most alter table statments.  This streamlines the
+       rebuild process"""
+    try:
+        cursor = connection.cursor()
+        table_name = model._meta.db_table
+
+        # Build create staement from django
+        model._meta.db_table = "%s_temp" % table_name
+        sql, references = connection.creation.sql_create_model(model, no_style())
+        columns = ",".join([_quote(f.column) \
+                               for f in model._meta.fields])
+
+        # Create a temp table
+        [cursor.execute(s) for s in sql]
+
+        # Fill the table
+        tbl_name = _quote(table_name)
+        tmp_tbl_name = _quote(model._meta.db_table)
+        # Reset this
+        model._meta.db_table = table_name
+        cursor.execute("insert into %s(%s) select %s from %s;" % (
+            tmp_tbl_name,
+            columns,
+            columns,
+            tbl_name))
+        cursor.execute("drop table %s" % tbl_name)
+
+        # Call syncdb to create the table again
+        django.core.management.call_command("syncdb", interactive=False, verbosity=0)
+        # syncdb closes our cursor
+        cursor = connection.cursor()
+        # Repopulate
+        cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name,
+                                                                  columns,
+                                                                  columns,
+                                                                  tmp_tbl_name))
+        cursor.execute('DROP TABLE %s;' % tmp_tbl_name)
+    except DatabaseError:
+        logger.error("Failed to rebuild sqlite table %s" % table_name, exc_info=1)
+        raise UpdaterError
+
+
+class UpdaterRoutineException(Exception):
+    pass
+
+
+class UpdaterRoutine(object):
+    """Base for routines."""
+    def __init__(self):
+        pass
+
+    def __str__(self):
+        return __name__
+
+    def run(self):
+        """Called to execute the action"""
+        raise UpdaterRoutineException
+
+
+
+class AddColumns(UpdaterRoutine):
+    """
+    Routine to add new columns to an existing model
+    """
+    def __init__(self, model):
+        self.model = model
+        self.model_name = model.__name__
+
+    def __str__(self):
+        return "Add new columns for model %s" % self.model_name
+
+    def run(self):
+        try:
+            cursor = connection.cursor()
+        except DatabaseError:
+            logger.error("Failed to connect to the db")
+            raise UpdaterRoutineException
+
+        try:
+            desc = {}
+            for d in connection.introspection.get_table_description(cursor, 
+                    self.model._meta.db_table):
+                desc[d[0]] = d
+        except DatabaseError:
+            logger.error("Failed to get table description", exc_info=1)
+            raise UpdaterRoutineException
+
+        for field in self.model._meta.fields:
+            if field.column in desc:
+                continue
+            logger.debug("Column %s does not exist yet" % field.column)
+            if field.default == NOT_PROVIDED:
+                logger.error("Cannot add a column with out a default value")
+                raise UpdaterRoutineException
+
+            sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % (
+                    _quote(self.model._meta.db_table),
+                    _quote(field.column), field.db_type(), )
+            db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
+            if db_engine == 'django.db.backends.sqlite3':
+                sql += _quote(field.default)
+                sql_values = ()
+            else:
+                sql += '%s'
+                sql_values = (field.default, )
+            try:
+                cursor.execute(sql, sql_values)
+                logger.debug("Added column %s to %s" % 
+                        (field.column, self.model._meta.db_table))
+            except DatabaseError:
+                logger.error("Unable to add column %s" % field.column)
+                raise UpdaterRoutineException
+
+
+class RebuildTable(UpdaterRoutine):
+    """
+    Rebuild the table for an existing model.  Use this if field types have changed.
+    """
+    def __init__(self, model, columns):
+        self.model = model
+        self.model_name = model.__name__
+
+        if type(columns) == str:
+            self.columns = [columns]
+        elif type(columns) in (tuple, list):
+            self.columns = columns
+        else:
+            logger.error("Columns must be a str, tuple, or list")
+            raise UpdaterRoutineException
+
+
+    def __str__(self):
+        return "Rebuild columns for model %s" % self.model_name
+
+    def run(self):
+        try:
+            cursor = connection.cursor()
+        except DatabaseError:
+            logger.error("Failed to connect to the db")
+            raise UpdaterRoutineException
+
+        db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
+        if db_engine == 'django.db.backends.sqlite3':
+            """ Sqlite is a special case.  Altering columns is not supported. """
+            _rebuild_sqlite_table(self.model)
+            return
+
+        if db_engine == 'django.db.backends.mysql':
+            modify_cmd = 'MODIFY '
+        else:
+            modify_cmd = 'ALTER COLUMN '
+
+        col_strings = []
+        for column in self.columns:
+            col_strings.append("%s %s %s" % ( \
+                modify_cmd,
+                _quote(column),
+                self.model._meta.get_field(column).db_type()
+            ))
+
+        try:
+            cursor.execute('ALTER TABLE %s %s' %
+                (_quote(self.model._meta.db_table), ", ".join(col_strings)))
+        except DatabaseError:
+            logger.debug("Failed modify table %s" % self.model._meta.db_table)
+            raise UpdaterRoutineException
+
+
+
+class RemoveColumns(RebuildTable):
+    """
+    Routine to remove columns from an existing model
+    """
+    def __init__(self, model, columns):
+        super(RemoveColumns, self).__init__(model, columns)
+
+
+    def __str__(self):
+        return "Remove columns from model %s" % self.model_name
+
+    def run(self):
+        try:
+            cursor = connection.cursor()
+        except DatabaseError:
+            logger.error("Failed to connect to the db")
+            raise UpdaterRoutineException
+
+        try:
+            columns = [d[0] for d in connection.introspection.get_table_description(cursor, 
+                    self.model._meta.db_table)]
+        except DatabaseError:
+            logger.error("Failed to get table description", exc_info=1)
+            raise UpdaterRoutineException
+
+        for column in self.columns:
+            if column not in columns:
+                logger.warning("Cannot drop column %s: does not exist" % column)
+                continue
+
+            logger.debug("Dropping column %s" % column)
+
+            db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
+            if db_engine == 'django.db.backends.sqlite3':
+                _rebuild_sqlite_table(self.model)
+            else:
+                sql = "alter table %s drop column %s" % \
+                    (_quote(self.model._meta.db_table), _quote(column), )
+                try:
+                    cursor.execute(sql)
+                except DatabaseError:
+                    logger.debug("Failed to drop column %s from %s" % 
+                        (column, self.model._meta.db_table))
+                    raise UpdaterRoutineException
+
+
+class UpdaterCallable(UpdaterRoutine):
+    """Helper for routines.  Basically delays execution"""
+    def __init__(self, fn):
+        self.fn = fn
+        self.args = []
+        self.kwargs = {}
+
+    def __call__(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+        return self
+
+    def __str__(self):
+        return self.fn.__name__
+
+    def run(self):
+        self.fn(*self.args, **self.kwargs)
+
+def updatercallable(fn):
+    """Decorator for UpdaterCallable.  Use for any function passed
+       into the fixes list"""
+    return UpdaterCallable(fn)
+
+
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/__init__.py b/src/lib/Bcfg2/Server/Reports/Updater/__init__.py
new file mode 100644
index 000000000..3038e9691
--- /dev/null
+++ b/src/lib/Bcfg2/Server/Reports/Updater/__init__.py
@@ -0,0 +1,239 @@
+from django.db import connection, DatabaseError
+import django.core.management
+import logging
+import pkgutil
+import re
+import sys
+import traceback
+
+from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion
+from Bcfg2.Server.Reports.Updater.Routines import UpdaterRoutineException, \
+                UpdaterRoutine
+from Bcfg2.Server.Reports.Updater import Changes
+
+logger = logging.getLogger(__name__)
+
+class UpdaterError(Exception):
+    pass
+
+
+class SchemaTooOldError(UpdaterError):
+    pass
+
+
+def _walk_packages(path):
+    """Python 2.4 lacks this routine"""
+    import glob
+    submodules = []
+    for path in __path__:
+        for submodule in glob.glob("%s/*.py" % path):
+            mod = '.'.join(submodule.split("/")[-1].split('.')[:-1])
+            if mod != '__init__':
+                submodules.append((None, mod, False))
+    return submodules
+
+
+def _release_to_version(release):
+    """
+    Build a release base for a version
+
+    Expects a string of the form 00.00
+
+    returns an integer of the form MMmm00
+    """
+    regex = re.compile("^(\d+)\.(\d+)$")
+    m = regex.match(release)
+    if not m:
+        logger.error("Invalid release string: %s" % release)
+        raise TypeError
+    return int("%02d%02d00" % (int(m.group(1)), int(m.group(2))))
+
+
+class Updater(object):
+    """Database updater to standardize updates"""
+
+    def __init__(self, release):
+        self._cursor = None
+        self._release = release
+        try:
+            self._base_version = _release_to_version(release)
+        except:
+            raise UpdaterError
+
+        self._fixes = []
+        self._version = -1
+
+
+    def __cmp__(self, other):
+        return self._base_version - other._base_version
+
+    @property
+    def release(self):
+        return self._release
+
+    @property
+    def version(self):
+        if self._version < 0:
+            try:
+                iv = InternalDatabaseVersion.objects.latest()
+                self._version = iv.version
+            except InternalDatabaseVersion.DoesNotExist:
+                raise UpdaterError
+        return self._version
+
+    @property
+    def cursor(self):
+        if not self._cursor:
+            self._cursor = connection.cursor()
+        return self._cursor
+
+    @property
+    def target_version(self):
+        if(len(self._fixes) == 0):
+            return self._base_version
+        else:
+            return self._base_version + len(self._fixes) - 1
+
+
+    def add(self, update):
+        if type(update) == str or isinstance(update, UpdaterRoutine):
+            self._fixes.append(update)
+        else:
+            raise TypeError
+
+
+    def override_base_version(self, version):
+        """Override our starting point for old releases.  New code should
+           not use this method"""
+        self._base_version = int(version)
+
+
+    @staticmethod
+    def get_current_version():
+        """Queries the db for the latest version.  Returns 0 for a fresh install"""
+
+        if "call_command" in dir(django.core.management):
+            django.core.management.call_command("syncdb", interactive=False, verbosity=0)
+        else:
+            logger.warning("Unable to call syndb routine")
+            raise UpdaterError
+
+        try:
+            iv = InternalDatabaseVersion.objects.latest()
+            version = iv.version
+        except InternalDatabaseVersion.DoesNotExist:
+            version = 0
+
+        return version
+
+
+    def syncdb(self):
+        """Function to do the syncronisation for the models"""
+
+        self._version = Updater.get_current_version()
+        self._cursor = None
+
+
+    def increment(self):
+        """Increment schema version in the database"""
+        if self._version < self._base_version:
+            self._version = self._base_version
+        else:
+            self._version += 1
+        InternalDatabaseVersion.objects.create(version=self._version)
+
+    def apply(self):
+        """Apply pending schema changes"""
+
+        if self.version >= self.target_version:
+            logger.debug("No updates for release %s" % self._release)
+            return
+
+        logger.debug("Applying updates for release %s" % self._release)
+
+        if self.version < self._base_version:
+            start = 0
+        else:
+            start = self.version - self._base_version + 1
+
+        try:
+            for fix in self._fixes[start:]:
+                if type(fix) == str:
+                    self.cursor.execute(fix)
+                elif isinstance(fix, UpdaterRoutine):
+                    fix.run()
+                else:
+                    logger.error("Invalid schema change at %s" % \
+                        self._version + 1)
+                self.increment()
+                logger.debug("Applied schema change number %s: %s" % \
+                    (self.version, fix))
+            logger.info("Applied schema changes for release %s" % self._release)
+        except:
+            logger.error("Failed to perform db update %s (%s): %s" % \
+                (self._version + 1, fix, traceback.format_exc().splitlines()[-1]))
+            raise UpdaterError
+
+
+class UnsupportedUpdate(Updater):
+    """Handle an unsupported update"""
+
+    def __init__(self, release, version):
+        super(UnsupportedUpdate, self).__init__(release)
+        self._base_version = version
+
+    def apply(self):
+        """Raise an exception if we're too old"""
+
+        if self.version < self.target_version:
+            logger.error("Upgrade from release %s unsupported" % self._release)
+            raise SchemaTooOldError
+
+
+def update_database():
+    """method to search where we are in the revision
+    of the database models and update them"""
+    try:
+        logger.debug("Verifying database schema")
+
+        updaters = []
+        if hasattr(pkgutil, 'walk_packages'):
+            submodules = pkgutil.walk_packages(path=Changes.__path__)
+        else:
+            #python 2.4
+            submodules = _walk_packages(Changes.__path__)
+        for loader, submodule, ispkg in submodules:
+            if ispkg:
+                continue
+            try:
+                updates = getattr(
+                    __import__("%s.%s" % (Changes.__name__, submodule), 
+                        globals(), locals(), ['*']),
+                    "updates")
+                updaters.append(updates())
+            except ImportError:
+                logger.error("Failed to import %s" % submodule)
+            except AttributeError:
+                logger.warning("Module %s does not have an updates function" % submodule)
+            except:
+                logger.error("Failed to build updater for %s" % submodule, exc_info=1)
+                raise UpdaterError
+
+        current_version = Updater.get_current_version()
+        logger.debug("Database version at %s" % current_version)
+
+        if current_version > 0:
+            [u.apply() for u in sorted(updaters)]
+            logger.debug("Database version at %s" % Updater.get_current_version())
+        else:
+            target = updaters[-1].target_version
+            InternalDatabaseVersion.objects.create(version=target)
+            logger.info("A new database was created")
+
+    except UpdaterError:
+        raise
+    except:
+        logger.error("Error while updating the database")
+        for x in traceback.format_exc().splitlines():
+            logger.error(x)
+        raise UpdaterError
diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py
index 16df86a9b..11603197b 100755
--- a/src/lib/Bcfg2/Server/Reports/importscript.py
+++ b/src/lib/Bcfg2/Server/Reports/importscript.py
@@ -28,7 +28,7 @@ from getopt import getopt, GetoptError
 from datetime import datetime
 from time import strptime
 from django.db import connection
-from Bcfg2.Server.Reports.updatefix import update_database
+from Bcfg2.Server.Reports.Updater import update_database, UpdaterError
 import logging
 import Bcfg2.Logger
 import platform
@@ -304,7 +304,10 @@ if __name__ == '__main__':
 
     q = '-O3' in sys.argv
     # Be sure the database is ready for new schema
-    update_database()
+    try:
+        update_database()
+    except UpdaterError:
+        raise SystemExit(1)
     load_stats(clientsdata,
                statsdata,
                encoding,
diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py
index 84bdc5291..b58633c38 100644
--- a/src/lib/Bcfg2/Server/Reports/reports/models.py
+++ b/src/lib/Bcfg2/Server/Reports/reports/models.py
@@ -286,7 +286,7 @@ class Reason(models.Model):
     current_diff = models.TextField(max_length=1024*1024, blank=True)
     is_binary = models.BooleanField(default=False)
     is_sensitive = models.BooleanField(default=False)
-    unpruned = models.TextField(max_length=4096, blank=True)
+    unpruned = models.TextField(max_length=4096, blank=True, default='')
 
     def _str_(self):
         return "Reason"
@@ -350,3 +350,6 @@ class InternalDatabaseVersion(models.Model):
 
     def __str__(self):
         return "version %d updated the %s" % (self.version, self.updated.isoformat())
+
+    class Meta:
+        get_latest_by = "version"
diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py
deleted file mode 100644
index b93ae0eb9..000000000
--- a/src/lib/Bcfg2/Server/Reports/updatefix.py
+++ /dev/null
@@ -1,323 +0,0 @@
-import Bcfg2.Server.Reports.settings
-
-from django.db import connection, DatabaseError, backend
-import django.core.management
-import logging
-import sys
-import traceback
-from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion, \
-                Reason, TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA
-logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix')
-
-
-# all update function should go here
-def _merge_database_table_entries():
-    cursor = connection.cursor()
-    insert_cursor = connection.cursor()
-    find_cursor = connection.cursor()
-    cursor.execute("""
-    Select name, kind from reports_bad
-    union
-    select name, kind from reports_modified
-    union
-    select name, kind from reports_extra
-    """)
-    # this fetch could be better done
-    entries_map = {}
-    for row in cursor.fetchall():
-        insert_cursor.execute("insert into reports_entries (name, kind) \
-            values (%s, %s)", (row[0], row[1]))
-        entries_map[(row[0], row[1])] = insert_cursor.lastrowid
-
-    cursor.execute("""
-        Select name, kind, reason_id, interaction_id, 1 from reports_bad
-        inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id
-        union
-        Select name, kind, reason_id, interaction_id, 2 from reports_modified
-        inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id
-        union
-        Select name, kind, reason_id, interaction_id, 3 from reports_extra
-        inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id
-    """)
-    for row in cursor.fetchall():
-        key = (row[0], row[1])
-        if entries_map.get(key, None):
-            entry_id = entries_map[key]
-        else:
-            find_cursor.execute("Select id from reports_entries where name=%s and kind=%s", key)
-            rowe = find_cursor.fetchone()
-            entry_id = rowe[0]
-        insert_cursor.execute("insert into reports_entries_interactions \
-            (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4]))
-
-
-def _interactions_constraint_or_idx():
-    """sqlite doesn't support alter tables.. or constraints"""
-    cursor = connection.cursor()
-    try:
-        cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)')
-    except:
-        cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)')
-
-
-def _rebuild_reports_reason():
-    """Rebuild the reports_reason table with better data types"""
-    cursor = connection.cursor()
-    columns = ['owner', 'current_owner',
-               'group', 'current_group',
-               'perms', 'current_perms',
-               'status', 'current_status',
-               'to', 'current_to']
-
-    tbl_name = backend.DatabaseOperations().quote_name('reports_reason')
-
-    db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
-    if db_engine == 'django.db.backends.mysql':
-        modify_cmd = 'MODIFY '
-    elif db_engine == 'django.db.backends.sqlite3':
-        """ Sqlite is a special case.  Altering columns is not supported. """
-        tmp_tbl_name = backend.DatabaseOperations().quote_name('reports_reason_temp')
-        cursor.execute('ALTER TABLE %s RENAME TO %s' % (tbl_name, tmp_tbl_name))
-        django.core.management.call_command("syncdb", interactive=False, verbosity=0)
-        columns = ",".join([backend.DatabaseOperations().quote_name(f.name) \
-                            for f in Reason._meta.fields])
-        cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name,
-                                                                  columns,
-                                                                  columns,
-                                                                  tmp_tbl_name))
-        cursor.execute('DROP TABLE %s;' % tmp_tbl_name)
-        return
-    else:
-        modify_cmd = 'ALTER COLUMN '
-
-    col_strings = []
-    for column in columns:
-        col_strings.append("%s %s %s" % ( \
-            modify_cmd,
-            backend.DatabaseOperations().quote_name(column),
-            Reason._meta.get_field(column).db_type()
-        ))
-    cursor.execute('ALTER TABLE %s %s' % (tbl_name, ", ".join(col_strings)))
-
-def _remove_table_column(tbl, col):
-    """sqlite doesn't support deleting a column via alter table"""
-    cursor = connection.cursor()
-    db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
-    if db_engine == 'django.db.backends.mysql':
-        db_name = Bcfg2.Server.Reports.settings.DATABASES['default']['NAME']
-        column_exists = cursor.execute('select * from information_schema.columns '
-                                       'where table_schema="%s" and '
-                                       'table_name="%s" '
-                                       'and column_name="%s";' % (db_name, tbl, col))
-        if not column_exists:
-            # column doesn't exist
-            return
-        # if column exists from previous database, remove it
-        cursor.execute('alter table %s '
-                       'drop column %s;' % (tbl, col))
-    elif db_engine == 'django.db.backends.sqlite3':
-        # check if table exists
-        try:
-            cursor.execute('select * from sqlite_master where name=%s and type="table";' % tbl)
-        except DatabaseError:
-            # table doesn't exist
-            return
-
-        # sqlite wants us to create a new table containing the columns we want
-        # and copy into it http://www.sqlite.org/faq.html#q11
-        tmptbl_name = "t_backup"
-        _tmptbl_create = \
-"""create temporary table "%s" (
-    "id" integer NOT NULL PRIMARY KEY,
-    "client_id" integer NOT NULL REFERENCES "reports_client" ("id"),
-    "timestamp" datetime NOT NULL,
-    "state" varchar(32) NOT NULL,
-    "repo_rev_code" varchar(64) NOT NULL,
-    "goodcount" integer NOT NULL,
-    "totalcount" integer NOT NULL,
-    "server" varchar(256) NOT NULL,
-    "bad_entries" integer NOT NULL,
-    "modified_entries" integer NOT NULL,
-    "extra_entries" integer NOT NULL,
-    UNIQUE ("client_id", "timestamp")
-);""" % tmptbl_name
-        _newtbl_create = \
-"""create table "%s" (
-    "id" integer NOT NULL PRIMARY KEY,
-    "client_id" integer NOT NULL REFERENCES "reports_client" ("id"),
-    "timestamp" datetime NOT NULL,
-    "state" varchar(32) NOT NULL,
-    "repo_rev_code" varchar(64) NOT NULL,
-    "goodcount" integer NOT NULL,
-    "totalcount" integer NOT NULL,
-    "server" varchar(256) NOT NULL,
-    "bad_entries" integer NOT NULL,
-    "modified_entries" integer NOT NULL,
-    "extra_entries" integer NOT NULL,
-    UNIQUE ("client_id", "timestamp")
-);""" % tbl
-        new_cols = "id,\
-                    client_id,\
-                    timestamp,\
-                    state,\
-                    repo_rev_code,\
-                    goodcount,\
-                    totalcount,\
-                    server,\
-                    bad_entries,\
-                    modified_entries,\
-                    extra_entries"
-
-        delete_col = [_tmptbl_create,
-                      "insert into %s select %s from %s;" % (tmptbl_name, new_cols, tbl),
-                      "drop table %s" % tbl,
-                      _newtbl_create,
-                      "create index reports_interaction_client_id on %s (client_id);" % tbl,
-                      "insert into %s select %s from %s;" % (tbl, new_cols,
-                                                             tmptbl_name),
-                      "drop table %s;" % tmptbl_name]
-
-        for sql in delete_col:
-            cursor.execute(sql)
-
-
-def _populate_interaction_entry_counts():
-    '''Populate up the type totals for the interaction table'''
-    cursor = connection.cursor()
-    count_field = {TYPE_BAD: 'bad_entries',
-                   TYPE_MODIFIED: 'modified_entries',
-                   TYPE_EXTRA: 'extra_entries'}
-
-    for type in list(count_field.keys()):
-        cursor.execute("select count(type), interaction_id " +
-                "from reports_entries_interactions where type = %s group by interaction_id" % type)
-        updates = []
-        for row in cursor.fetchall():
-            updates.append(row)
-        try:
-            cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates)
-        except Exception:
-            e = sys.exc_info()[1]
-            print(e)
-    cursor.close()
-
-
-# be sure to test your upgrade query before reflecting the change in the models
-# the list of function and sql command to do should go here
-_fixes = [_merge_database_table_entries,
-          # this will remove unused tables
-          "drop table reports_bad;",
-          "drop table reports_bad_interactions;",
-          "drop table reports_extra;",
-          "drop table reports_extra_interactions;",
-          "drop table reports_modified;",
-          "drop table reports_modified_interactions;",
-          "drop table reports_repository;",
-          "drop table reports_metadata;",
-          "alter table reports_interaction add server varchar(256) not null default 'N/A';",
-          # fix revision data type to support $VCS hashes
-          "alter table reports_interaction add repo_rev_code varchar(64) default '';",
-          # Performance enhancements for large sites
-          'alter table reports_interaction add column bad_entries integer not null default -1;',
-          'alter table reports_interaction add column modified_entries integer not null default -1;',
-          'alter table reports_interaction add column extra_entries integer not null default -1;',
-          _populate_interaction_entry_counts,
-          _interactions_constraint_or_idx,
-          'alter table reports_reason add is_binary bool NOT NULL default False;',
-          'alter table reports_reason add is_sensitive bool NOT NULL default False;',
-          _remove_table_column('reports_interaction', 'client_version'),
-          "alter table reports_reason add unpruned varchar(1280) not null default '';",
-          _rebuild_reports_reason,
-]
-
-# this will calculate the last possible version of the database
-lastversion = len(_fixes)
-
-
-def rollupdate(current_version):
-    """function responsible to coordinates all the updates
-    need current_version as integer
-    """
-    ret = None
-    if current_version < lastversion:
-        for i in range(current_version, lastversion):
-            try:
-                if type(_fixes[i]) == str:
-                    connection.cursor().execute(_fixes[i])
-                else:
-                    _fixes[i]()
-            except:
-                logger.error("Failed to perform db update %s" % (_fixes[i]),
-                                                                 exc_info=1)
-            # since the array starts at 0 but version
-            # starts at 1 we add 1 to the normal count
-            ret = InternalDatabaseVersion.objects.create(version=i + 1)
-        return ret
-    else:
-        return None
-
-
-def dosync():
-    """Function to do the syncronisation for the models"""
-    # try to detect if it's a fresh new database
-    try:
-        cursor = connection.cursor()
-        # If this table goes missing,
-        # don't forget to change it to the new one
-        cursor.execute("Select * from reports_client")
-        # if we get here with no error then the database has existing tables
-        fresh = False
-    except:
-        logger.debug("there was an error while detecting "
-                     "the freshness of the database")
-        #we should get here if the database is new
-        fresh = True
-
-    # ensure database connections are closed
-    # so that the management can do its job right
-    try:
-        cursor.close()
-        connection.close()
-    except:
-        # ignore any errors from missing/invalid dbs
-        pass
-    # Do the syncdb according to the django version
-    if "call_command" in dir(django.core.management):
-        # this is available since django 1.0 alpha.
-        # not yet tested for full functionnality
-        django.core.management.call_command("syncdb", interactive=False, verbosity=0)
-        if fresh:
-            iv = InternalDatabaseVersion.objects.create(version=len(_fixes))
-            logger.debug("loading the initial version at %s" % iv.version)
-    elif "syncdb" in dir(django.core.management):
-        # this exist only for django 0.96.*
-        django.core.management.syncdb(interactive=False, verbosity=0)
-        if fresh:
-            iv = InternalDatabaseVersion.objects.create(version=len(_fixes))
-            logger.debug("loading the initial version at %s" % iv.version)
-    else:
-        logger.warning("Don't forget to run syncdb")
-
-
-def update_database():
-    """method to search where we are in the revision
-    of the database models and update them"""
-    try:
-        logger.debug("Running upgrade of models to the new one")
-        dosync()
-        know_version = InternalDatabaseVersion.objects.order_by('-version')
-        if not know_version:
-            logger.debug("No version, creating initial version")
-            know_version = InternalDatabaseVersion.objects.create(version=0)
-        else:
-            know_version = know_version[0]
-        logger.debug("Presently at %s" % know_version)
-        if know_version.version < lastversion:
-            logger.info("upgrading database")
-            new_version = rollupdate(know_version.version)
-            if new_version:
-                logger.info("upgraded to %s" % new_version)
-    except:
-        logger.error("Error while updating the database")
-        for x in traceback.format_exc().splitlines():
-            logger.error(x)