diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/lib/Bcfg2/Server/Admin/Reports.py | 16 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Plugins/DBStats.py | 4 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py | 11 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py | 59 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py | 15 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py | 26 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py | 0 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Routines.py | 258 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/__init__.py | 239 | ||||
| -rwxr-xr-x | src/lib/Bcfg2/Server/Reports/importscript.py | 7 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/models.py | 5 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/Reports/updatefix.py | 323 |
12 files changed, 631 insertions, 332 deletions
diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 041eacccc..97db140b7 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -26,7 +26,7 @@ import Bcfg2.Server.Reports.settings # Load django and reports stuff _after_ we know we can load settings import django.core.management from Bcfg2.Server.Reports.importscript import load_stats -from Bcfg2.Server.Reports.updatefix import update_database +from Bcfg2.Server.Reports.Updater import update_database, UpdaterError from Bcfg2.Server.Reports.utils import * project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) @@ -111,10 +111,14 @@ class Reports(Bcfg2.Server.Admin.Mode): self.django_command_proxy(args[0]) elif args[0] == 'scrub': self.scrub() - elif args[0] == 'init': - update_database() - elif args[0] == 'update': - update_database() + elif args[0] in ['init', 'update']: + try: + update_database() + #except SchemaTooOldError: + # logger.error("Sc + except UpdaterError: + print "Update failed" + raise SystemExit(-1) elif args[0] == 'load_stats': quick = '-O3' in args stats_file = None @@ -266,6 +270,8 @@ class Reports(Bcfg2.Server.Admin.Mode): self.log, quick=quick, location=platform.node()) + except UpdaterError: + self.errExit("StatReports: Database updater failed") except: pass diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index 131b6b059..b28484039 100644 --- a/src/lib/Bcfg2/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -14,7 +14,7 @@ import Bcfg2.Server.Plugin import Bcfg2.Server.Reports.importscript from Bcfg2.Server.Reports.reports.models import Client import Bcfg2.Server.Reports.settings -from Bcfg2.Server.Reports.updatefix import update_database +from Bcfg2.Server.Reports.Updater import update_database, UpdaterError # for debugging output only logger = logging.getLogger('Bcfg2.Plugins.DBStats') @@ -34,6 +34,8 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, "add to the statistics database") try: update_database() + except UpdaterError: + raise Bcfg2.Server.Plugin.PluginInitError except Exception: inst = sys.exc_info()[1] logger.debug(str(inst)) diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py new file mode 100644 index 000000000..54ba07554 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py @@ -0,0 +1,11 @@ +""" +1_0_x.py + +This file should contain updates relevant to the 1.0.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import UnsupportedUpdate + +def updates(): + return UnsupportedUpdate("1.0", 10) + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py new file mode 100644 index 000000000..26194cb67 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py @@ -0,0 +1,59 @@ +""" +1_1_x.py + +This file should contain updates relevant to the 1.1.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import Updater +from Bcfg2.Server.Reports.Updater.Routines import updatercallable + +from django.db import connection +import sys +import Bcfg2.Server.Reports.settings +from Bcfg2.Server.Reports.reports.models import \ + TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA + +@updatercallable +def _interactions_constraint_or_idx(): + """sqlite doesn't support alter tables.. or constraints""" + cursor = connection.cursor() + try: + cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') + except: + cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') + + +@updatercallable +def _populate_interaction_entry_counts(): + '''Populate up the type totals for the interaction table''' + cursor = connection.cursor() + count_field = {TYPE_BAD: 'bad_entries', + TYPE_MODIFIED: 'modified_entries', + TYPE_EXTRA: 'extra_entries'} + + for type in list(count_field.keys()): + cursor.execute("select count(type), interaction_id " + + "from reports_entries_interactions where type = %s group by interaction_id" % type) + updates = [] + for row in cursor.fetchall(): + updates.append(row) + try: + cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) + except Exception: + e = sys.exc_info()[1] + print(e) + cursor.close() + + +def updates(): + fixes = Updater("1.1") + fixes.override_base_version(12) # Do not do this in new code + + fixes.add('alter table reports_interaction add column bad_entries integer not null default -1;') + fixes.add('alter table reports_interaction add column modified_entries integer not null default -1;') + fixes.add('alter table reports_interaction add column extra_entries integer not null default -1;') + fixes.add(_populate_interaction_entry_counts()) + fixes.add(_interactions_constraint_or_idx()) + fixes.add('alter table reports_reason add is_binary bool NOT NULL default False;') + return fixes + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py new file mode 100644 index 000000000..22bd937c2 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py @@ -0,0 +1,15 @@ +""" +1_2_x.py + +This file should contain updates relevant to the 1.2.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import Updater +from Bcfg2.Server.Reports.Updater.Routines import updatercallable + +def updates(): + fixes = Updater("1.2") + fixes.override_base_version(18) # Do not do this in new code + fixes.add('alter table reports_reason add is_sensitive bool NOT NULL default False;') + return fixes + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py new file mode 100644 index 000000000..b09b06302 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py @@ -0,0 +1,26 @@ +""" +1_3_0.py + +This file should contain updates relevant to the 1.3.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import Updater, UpdaterError +from Bcfg2.Server.Reports.Updater.Routines import AddColumns, \ + RemoveColumns, RebuildTable + +from Bcfg2.Server.Reports.reports.models import Reason, Interaction + + +def updates(): + fixes = Updater("1.3") + fixes.add(RemoveColumns(Interaction, 'client_version')) + fixes.add(AddColumns(Reason)) + fixes.add(RebuildTable(Reason, [ + 'owner', 'current_owner', + 'group', 'current_group', + 'perms', 'current_perms', + 'status', 'current_status', + 'to', 'current_to'])) + + return fixes + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Routines.py b/src/lib/Bcfg2/Server/Reports/Updater/Routines.py new file mode 100644 index 000000000..1d41848e4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Routines.py @@ -0,0 +1,258 @@ +import logging +from django.db.models.fields import NOT_PROVIDED +from django.db import connection, DatabaseError, backend, models +from django.core.management.color import no_style +from django.core.management.sql import sql_create +import django.core.management + +import Bcfg2.Server.Reports.settings + +logger = logging.getLogger(__name__) + +def _quote(value): + """ + Quote a string to use as a table name or column + """ + return backend.DatabaseOperations().quote_name(value) + + +def _rebuild_sqlite_table(model): + """Sqlite doesn't support most alter table statments. This streamlines the + rebuild process""" + try: + cursor = connection.cursor() + table_name = model._meta.db_table + + # Build create staement from django + model._meta.db_table = "%s_temp" % table_name + sql, references = connection.creation.sql_create_model(model, no_style()) + columns = ",".join([_quote(f.column) \ + for f in model._meta.fields]) + + # Create a temp table + [cursor.execute(s) for s in sql] + + # Fill the table + tbl_name = _quote(table_name) + tmp_tbl_name = _quote(model._meta.db_table) + # Reset this + model._meta.db_table = table_name + cursor.execute("insert into %s(%s) select %s from %s;" % ( + tmp_tbl_name, + columns, + columns, + tbl_name)) + cursor.execute("drop table %s" % tbl_name) + + # Call syncdb to create the table again + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + # syncdb closes our cursor + cursor = connection.cursor() + # Repopulate + cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name, + columns, + columns, + tmp_tbl_name)) + cursor.execute('DROP TABLE %s;' % tmp_tbl_name) + except DatabaseError: + logger.error("Failed to rebuild sqlite table %s" % table_name, exc_info=1) + raise UpdaterError + + +class UpdaterRoutineException(Exception): + pass + + +class UpdaterRoutine(object): + """Base for routines.""" + def __init__(self): + pass + + def __str__(self): + return __name__ + + def run(self): + """Called to execute the action""" + raise UpdaterRoutineException + + + +class AddColumns(UpdaterRoutine): + """ + Routine to add new columns to an existing model + """ + def __init__(self, model): + self.model = model + self.model_name = model.__name__ + + def __str__(self): + return "Add new columns for model %s" % self.model_name + + def run(self): + try: + cursor = connection.cursor() + except DatabaseError: + logger.error("Failed to connect to the db") + raise UpdaterRoutineException + + try: + desc = {} + for d in connection.introspection.get_table_description(cursor, + self.model._meta.db_table): + desc[d[0]] = d + except DatabaseError: + logger.error("Failed to get table description", exc_info=1) + raise UpdaterRoutineException + + for field in self.model._meta.fields: + if field.column in desc: + continue + logger.debug("Column %s does not exist yet" % field.column) + if field.default == NOT_PROVIDED: + logger.error("Cannot add a column with out a default value") + raise UpdaterRoutineException + + sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % ( + _quote(self.model._meta.db_table), + _quote(field.column), field.db_type(), ) + db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] + if db_engine == 'django.db.backends.sqlite3': + sql += _quote(field.default) + sql_values = () + else: + sql += '%s' + sql_values = (field.default, ) + try: + cursor.execute(sql, sql_values) + logger.debug("Added column %s to %s" % + (field.column, self.model._meta.db_table)) + except DatabaseError: + logger.error("Unable to add column %s" % field.column) + raise UpdaterRoutineException + + +class RebuildTable(UpdaterRoutine): + """ + Rebuild the table for an existing model. Use this if field types have changed. + """ + def __init__(self, model, columns): + self.model = model + self.model_name = model.__name__ + + if type(columns) == str: + self.columns = [columns] + elif type(columns) in (tuple, list): + self.columns = columns + else: + logger.error("Columns must be a str, tuple, or list") + raise UpdaterRoutineException + + + def __str__(self): + return "Rebuild columns for model %s" % self.model_name + + def run(self): + try: + cursor = connection.cursor() + except DatabaseError: + logger.error("Failed to connect to the db") + raise UpdaterRoutineException + + db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] + if db_engine == 'django.db.backends.sqlite3': + """ Sqlite is a special case. Altering columns is not supported. """ + _rebuild_sqlite_table(self.model) + return + + if db_engine == 'django.db.backends.mysql': + modify_cmd = 'MODIFY ' + else: + modify_cmd = 'ALTER COLUMN ' + + col_strings = [] + for column in self.columns: + col_strings.append("%s %s %s" % ( \ + modify_cmd, + _quote(column), + self.model._meta.get_field(column).db_type() + )) + + try: + cursor.execute('ALTER TABLE %s %s' % + (_quote(self.model._meta.db_table), ", ".join(col_strings))) + except DatabaseError: + logger.debug("Failed modify table %s" % self.model._meta.db_table) + raise UpdaterRoutineException + + + +class RemoveColumns(RebuildTable): + """ + Routine to remove columns from an existing model + """ + def __init__(self, model, columns): + super(RemoveColumns, self).__init__(model, columns) + + + def __str__(self): + return "Remove columns from model %s" % self.model_name + + def run(self): + try: + cursor = connection.cursor() + except DatabaseError: + logger.error("Failed to connect to the db") + raise UpdaterRoutineException + + try: + columns = [d[0] for d in connection.introspection.get_table_description(cursor, + self.model._meta.db_table)] + except DatabaseError: + logger.error("Failed to get table description", exc_info=1) + raise UpdaterRoutineException + + for column in self.columns: + if column not in columns: + logger.warning("Cannot drop column %s: does not exist" % column) + continue + + logger.debug("Dropping column %s" % column) + + db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] + if db_engine == 'django.db.backends.sqlite3': + _rebuild_sqlite_table(self.model) + else: + sql = "alter table %s drop column %s" % \ + (_quote(self.model._meta.db_table), _quote(column), ) + try: + cursor.execute(sql) + except DatabaseError: + logger.debug("Failed to drop column %s from %s" % + (column, self.model._meta.db_table)) + raise UpdaterRoutineException + + +class UpdaterCallable(UpdaterRoutine): + """Helper for routines. Basically delays execution""" + def __init__(self, fn): + self.fn = fn + self.args = [] + self.kwargs = {} + + def __call__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + return self + + def __str__(self): + return self.fn.__name__ + + def run(self): + self.fn(*self.args, **self.kwargs) + +def updatercallable(fn): + """Decorator for UpdaterCallable. Use for any function passed + into the fixes list""" + return UpdaterCallable(fn) + + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/__init__.py b/src/lib/Bcfg2/Server/Reports/Updater/__init__.py new file mode 100644 index 000000000..3038e9691 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/__init__.py @@ -0,0 +1,239 @@ +from django.db import connection, DatabaseError +import django.core.management +import logging +import pkgutil +import re +import sys +import traceback + +from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion +from Bcfg2.Server.Reports.Updater.Routines import UpdaterRoutineException, \ + UpdaterRoutine +from Bcfg2.Server.Reports.Updater import Changes + +logger = logging.getLogger(__name__) + +class UpdaterError(Exception): + pass + + +class SchemaTooOldError(UpdaterError): + pass + + +def _walk_packages(path): + """Python 2.4 lacks this routine""" + import glob + submodules = [] + for path in __path__: + for submodule in glob.glob("%s/*.py" % path): + mod = '.'.join(submodule.split("/")[-1].split('.')[:-1]) + if mod != '__init__': + submodules.append((None, mod, False)) + return submodules + + +def _release_to_version(release): + """ + Build a release base for a version + + Expects a string of the form 00.00 + + returns an integer of the form MMmm00 + """ + regex = re.compile("^(\d+)\.(\d+)$") + m = regex.match(release) + if not m: + logger.error("Invalid release string: %s" % release) + raise TypeError + return int("%02d%02d00" % (int(m.group(1)), int(m.group(2)))) + + +class Updater(object): + """Database updater to standardize updates""" + + def __init__(self, release): + self._cursor = None + self._release = release + try: + self._base_version = _release_to_version(release) + except: + raise UpdaterError + + self._fixes = [] + self._version = -1 + + + def __cmp__(self, other): + return self._base_version - other._base_version + + @property + def release(self): + return self._release + + @property + def version(self): + if self._version < 0: + try: + iv = InternalDatabaseVersion.objects.latest() + self._version = iv.version + except InternalDatabaseVersion.DoesNotExist: + raise UpdaterError + return self._version + + @property + def cursor(self): + if not self._cursor: + self._cursor = connection.cursor() + return self._cursor + + @property + def target_version(self): + if(len(self._fixes) == 0): + return self._base_version + else: + return self._base_version + len(self._fixes) - 1 + + + def add(self, update): + if type(update) == str or isinstance(update, UpdaterRoutine): + self._fixes.append(update) + else: + raise TypeError + + + def override_base_version(self, version): + """Override our starting point for old releases. New code should + not use this method""" + self._base_version = int(version) + + + @staticmethod + def get_current_version(): + """Queries the db for the latest version. Returns 0 for a fresh install""" + + if "call_command" in dir(django.core.management): + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + else: + logger.warning("Unable to call syndb routine") + raise UpdaterError + + try: + iv = InternalDatabaseVersion.objects.latest() + version = iv.version + except InternalDatabaseVersion.DoesNotExist: + version = 0 + + return version + + + def syncdb(self): + """Function to do the syncronisation for the models""" + + self._version = Updater.get_current_version() + self._cursor = None + + + def increment(self): + """Increment schema version in the database""" + if self._version < self._base_version: + self._version = self._base_version + else: + self._version += 1 + InternalDatabaseVersion.objects.create(version=self._version) + + def apply(self): + """Apply pending schema changes""" + + if self.version >= self.target_version: + logger.debug("No updates for release %s" % self._release) + return + + logger.debug("Applying updates for release %s" % self._release) + + if self.version < self._base_version: + start = 0 + else: + start = self.version - self._base_version + 1 + + try: + for fix in self._fixes[start:]: + if type(fix) == str: + self.cursor.execute(fix) + elif isinstance(fix, UpdaterRoutine): + fix.run() + else: + logger.error("Invalid schema change at %s" % \ + self._version + 1) + self.increment() + logger.debug("Applied schema change number %s: %s" % \ + (self.version, fix)) + logger.info("Applied schema changes for release %s" % self._release) + except: + logger.error("Failed to perform db update %s (%s): %s" % \ + (self._version + 1, fix, traceback.format_exc().splitlines()[-1])) + raise UpdaterError + + +class UnsupportedUpdate(Updater): + """Handle an unsupported update""" + + def __init__(self, release, version): + super(UnsupportedUpdate, self).__init__(release) + self._base_version = version + + def apply(self): + """Raise an exception if we're too old""" + + if self.version < self.target_version: + logger.error("Upgrade from release %s unsupported" % self._release) + raise SchemaTooOldError + + +def update_database(): + """method to search where we are in the revision + of the database models and update them""" + try: + logger.debug("Verifying database schema") + + updaters = [] + if hasattr(pkgutil, 'walk_packages'): + submodules = pkgutil.walk_packages(path=Changes.__path__) + else: + #python 2.4 + submodules = _walk_packages(Changes.__path__) + for loader, submodule, ispkg in submodules: + if ispkg: + continue + try: + updates = getattr( + __import__("%s.%s" % (Changes.__name__, submodule), + globals(), locals(), ['*']), + "updates") + updaters.append(updates()) + except ImportError: + logger.error("Failed to import %s" % submodule) + except AttributeError: + logger.warning("Module %s does not have an updates function" % submodule) + except: + logger.error("Failed to build updater for %s" % submodule, exc_info=1) + raise UpdaterError + + current_version = Updater.get_current_version() + logger.debug("Database version at %s" % current_version) + + if current_version > 0: + [u.apply() for u in sorted(updaters)] + logger.debug("Database version at %s" % Updater.get_current_version()) + else: + target = updaters[-1].target_version + InternalDatabaseVersion.objects.create(version=target) + logger.info("A new database was created") + + except UpdaterError: + raise + except: + logger.error("Error while updating the database") + for x in traceback.format_exc().splitlines(): + logger.error(x) + raise UpdaterError diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py index 16df86a9b..11603197b 100755 --- a/src/lib/Bcfg2/Server/Reports/importscript.py +++ b/src/lib/Bcfg2/Server/Reports/importscript.py @@ -28,7 +28,7 @@ from getopt import getopt, GetoptError from datetime import datetime from time import strptime from django.db import connection -from Bcfg2.Server.Reports.updatefix import update_database +from Bcfg2.Server.Reports.Updater import update_database, UpdaterError import logging import Bcfg2.Logger import platform @@ -304,7 +304,10 @@ if __name__ == '__main__': q = '-O3' in sys.argv # Be sure the database is ready for new schema - update_database() + try: + update_database() + except UpdaterError: + raise SystemExit(1) load_stats(clientsdata, statsdata, encoding, diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py index 84bdc5291..b58633c38 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/models.py +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -286,7 +286,7 @@ class Reason(models.Model): current_diff = models.TextField(max_length=1024*1024, blank=True) is_binary = models.BooleanField(default=False) is_sensitive = models.BooleanField(default=False) - unpruned = models.TextField(max_length=4096, blank=True) + unpruned = models.TextField(max_length=4096, blank=True, default='') def _str_(self): return "Reason" @@ -350,3 +350,6 @@ class InternalDatabaseVersion(models.Model): def __str__(self): return "version %d updated the %s" % (self.version, self.updated.isoformat()) + + class Meta: + get_latest_by = "version" diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py deleted file mode 100644 index b93ae0eb9..000000000 --- a/src/lib/Bcfg2/Server/Reports/updatefix.py +++ /dev/null @@ -1,323 +0,0 @@ -import Bcfg2.Server.Reports.settings - -from django.db import connection, DatabaseError, backend -import django.core.management -import logging -import sys -import traceback -from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion, \ - Reason, TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA -logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') - - -# all update function should go here -def _merge_database_table_entries(): - cursor = connection.cursor() - insert_cursor = connection.cursor() - find_cursor = connection.cursor() - cursor.execute(""" - Select name, kind from reports_bad - union - select name, kind from reports_modified - union - select name, kind from reports_extra - """) - # this fetch could be better done - entries_map = {} - for row in cursor.fetchall(): - insert_cursor.execute("insert into reports_entries (name, kind) \ - values (%s, %s)", (row[0], row[1])) - entries_map[(row[0], row[1])] = insert_cursor.lastrowid - - cursor.execute(""" - Select name, kind, reason_id, interaction_id, 1 from reports_bad - inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id - union - Select name, kind, reason_id, interaction_id, 2 from reports_modified - inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id - union - Select name, kind, reason_id, interaction_id, 3 from reports_extra - inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id - """) - for row in cursor.fetchall(): - key = (row[0], row[1]) - if entries_map.get(key, None): - entry_id = entries_map[key] - else: - find_cursor.execute("Select id from reports_entries where name=%s and kind=%s", key) - rowe = find_cursor.fetchone() - entry_id = rowe[0] - insert_cursor.execute("insert into reports_entries_interactions \ - (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) - - -def _interactions_constraint_or_idx(): - """sqlite doesn't support alter tables.. or constraints""" - cursor = connection.cursor() - try: - cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') - except: - cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') - - -def _rebuild_reports_reason(): - """Rebuild the reports_reason table with better data types""" - cursor = connection.cursor() - columns = ['owner', 'current_owner', - 'group', 'current_group', - 'perms', 'current_perms', - 'status', 'current_status', - 'to', 'current_to'] - - tbl_name = backend.DatabaseOperations().quote_name('reports_reason') - - db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.mysql': - modify_cmd = 'MODIFY ' - elif db_engine == 'django.db.backends.sqlite3': - """ Sqlite is a special case. Altering columns is not supported. """ - tmp_tbl_name = backend.DatabaseOperations().quote_name('reports_reason_temp') - cursor.execute('ALTER TABLE %s RENAME TO %s' % (tbl_name, tmp_tbl_name)) - django.core.management.call_command("syncdb", interactive=False, verbosity=0) - columns = ",".join([backend.DatabaseOperations().quote_name(f.name) \ - for f in Reason._meta.fields]) - cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name, - columns, - columns, - tmp_tbl_name)) - cursor.execute('DROP TABLE %s;' % tmp_tbl_name) - return - else: - modify_cmd = 'ALTER COLUMN ' - - col_strings = [] - for column in columns: - col_strings.append("%s %s %s" % ( \ - modify_cmd, - backend.DatabaseOperations().quote_name(column), - Reason._meta.get_field(column).db_type() - )) - cursor.execute('ALTER TABLE %s %s' % (tbl_name, ", ".join(col_strings))) - -def _remove_table_column(tbl, col): - """sqlite doesn't support deleting a column via alter table""" - cursor = connection.cursor() - db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.mysql': - db_name = Bcfg2.Server.Reports.settings.DATABASES['default']['NAME'] - column_exists = cursor.execute('select * from information_schema.columns ' - 'where table_schema="%s" and ' - 'table_name="%s" ' - 'and column_name="%s";' % (db_name, tbl, col)) - if not column_exists: - # column doesn't exist - return - # if column exists from previous database, remove it - cursor.execute('alter table %s ' - 'drop column %s;' % (tbl, col)) - elif db_engine == 'django.db.backends.sqlite3': - # check if table exists - try: - cursor.execute('select * from sqlite_master where name=%s and type="table";' % tbl) - except DatabaseError: - # table doesn't exist - return - - # sqlite wants us to create a new table containing the columns we want - # and copy into it http://www.sqlite.org/faq.html#q11 - tmptbl_name = "t_backup" - _tmptbl_create = \ -"""create temporary table "%s" ( - "id" integer NOT NULL PRIMARY KEY, - "client_id" integer NOT NULL REFERENCES "reports_client" ("id"), - "timestamp" datetime NOT NULL, - "state" varchar(32) NOT NULL, - "repo_rev_code" varchar(64) NOT NULL, - "goodcount" integer NOT NULL, - "totalcount" integer NOT NULL, - "server" varchar(256) NOT NULL, - "bad_entries" integer NOT NULL, - "modified_entries" integer NOT NULL, - "extra_entries" integer NOT NULL, - UNIQUE ("client_id", "timestamp") -);""" % tmptbl_name - _newtbl_create = \ -"""create table "%s" ( - "id" integer NOT NULL PRIMARY KEY, - "client_id" integer NOT NULL REFERENCES "reports_client" ("id"), - "timestamp" datetime NOT NULL, - "state" varchar(32) NOT NULL, - "repo_rev_code" varchar(64) NOT NULL, - "goodcount" integer NOT NULL, - "totalcount" integer NOT NULL, - "server" varchar(256) NOT NULL, - "bad_entries" integer NOT NULL, - "modified_entries" integer NOT NULL, - "extra_entries" integer NOT NULL, - UNIQUE ("client_id", "timestamp") -);""" % tbl - new_cols = "id,\ - client_id,\ - timestamp,\ - state,\ - repo_rev_code,\ - goodcount,\ - totalcount,\ - server,\ - bad_entries,\ - modified_entries,\ - extra_entries" - - delete_col = [_tmptbl_create, - "insert into %s select %s from %s;" % (tmptbl_name, new_cols, tbl), - "drop table %s" % tbl, - _newtbl_create, - "create index reports_interaction_client_id on %s (client_id);" % tbl, - "insert into %s select %s from %s;" % (tbl, new_cols, - tmptbl_name), - "drop table %s;" % tmptbl_name] - - for sql in delete_col: - cursor.execute(sql) - - -def _populate_interaction_entry_counts(): - '''Populate up the type totals for the interaction table''' - cursor = connection.cursor() - count_field = {TYPE_BAD: 'bad_entries', - TYPE_MODIFIED: 'modified_entries', - TYPE_EXTRA: 'extra_entries'} - - for type in list(count_field.keys()): - cursor.execute("select count(type), interaction_id " + - "from reports_entries_interactions where type = %s group by interaction_id" % type) - updates = [] - for row in cursor.fetchall(): - updates.append(row) - try: - cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) - except Exception: - e = sys.exc_info()[1] - print(e) - cursor.close() - - -# be sure to test your upgrade query before reflecting the change in the models -# the list of function and sql command to do should go here -_fixes = [_merge_database_table_entries, - # this will remove unused tables - "drop table reports_bad;", - "drop table reports_bad_interactions;", - "drop table reports_extra;", - "drop table reports_extra_interactions;", - "drop table reports_modified;", - "drop table reports_modified_interactions;", - "drop table reports_repository;", - "drop table reports_metadata;", - "alter table reports_interaction add server varchar(256) not null default 'N/A';", - # fix revision data type to support $VCS hashes - "alter table reports_interaction add repo_rev_code varchar(64) default '';", - # Performance enhancements for large sites - 'alter table reports_interaction add column bad_entries integer not null default -1;', - 'alter table reports_interaction add column modified_entries integer not null default -1;', - 'alter table reports_interaction add column extra_entries integer not null default -1;', - _populate_interaction_entry_counts, - _interactions_constraint_or_idx, - 'alter table reports_reason add is_binary bool NOT NULL default False;', - 'alter table reports_reason add is_sensitive bool NOT NULL default False;', - _remove_table_column('reports_interaction', 'client_version'), - "alter table reports_reason add unpruned varchar(1280) not null default '';", - _rebuild_reports_reason, -] - -# this will calculate the last possible version of the database -lastversion = len(_fixes) - - -def rollupdate(current_version): - """function responsible to coordinates all the updates - need current_version as integer - """ - ret = None - if current_version < lastversion: - for i in range(current_version, lastversion): - try: - if type(_fixes[i]) == str: - connection.cursor().execute(_fixes[i]) - else: - _fixes[i]() - except: - logger.error("Failed to perform db update %s" % (_fixes[i]), - exc_info=1) - # since the array starts at 0 but version - # starts at 1 we add 1 to the normal count - ret = InternalDatabaseVersion.objects.create(version=i + 1) - return ret - else: - return None - - -def dosync(): - """Function to do the syncronisation for the models""" - # try to detect if it's a fresh new database - try: - cursor = connection.cursor() - # If this table goes missing, - # don't forget to change it to the new one - cursor.execute("Select * from reports_client") - # if we get here with no error then the database has existing tables - fresh = False - except: - logger.debug("there was an error while detecting " - "the freshness of the database") - #we should get here if the database is new - fresh = True - - # ensure database connections are closed - # so that the management can do its job right - try: - cursor.close() - connection.close() - except: - # ignore any errors from missing/invalid dbs - pass - # Do the syncdb according to the django version - if "call_command" in dir(django.core.management): - # this is available since django 1.0 alpha. - # not yet tested for full functionnality - django.core.management.call_command("syncdb", interactive=False, verbosity=0) - if fresh: - iv = InternalDatabaseVersion.objects.create(version=len(_fixes)) - logger.debug("loading the initial version at %s" % iv.version) - elif "syncdb" in dir(django.core.management): - # this exist only for django 0.96.* - django.core.management.syncdb(interactive=False, verbosity=0) - if fresh: - iv = InternalDatabaseVersion.objects.create(version=len(_fixes)) - logger.debug("loading the initial version at %s" % iv.version) - else: - logger.warning("Don't forget to run syncdb") - - -def update_database(): - """method to search where we are in the revision - of the database models and update them""" - try: - logger.debug("Running upgrade of models to the new one") - dosync() - know_version = InternalDatabaseVersion.objects.order_by('-version') - if not know_version: - logger.debug("No version, creating initial version") - know_version = InternalDatabaseVersion.objects.create(version=0) - else: - know_version = know_version[0] - logger.debug("Presently at %s" % know_version) - if know_version.version < lastversion: - logger.info("upgrading database") - new_version = rollupdate(know_version.version) - if new_version: - logger.info("upgraded to %s" % new_version) - except: - logger.error("Error while updating the database") - for x in traceback.format_exc().splitlines(): - logger.error(x) |