diff options
Diffstat (limited to 'src/lib/Bcfg2/Server/SchemaUpdater')
| -rw-r--r-- | src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py | 11 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py | 59 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py | 15 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py | 27 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py | 0 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/SchemaUpdater/Routines.py | 279 | ||||
| -rw-r--r-- | src/lib/Bcfg2/Server/SchemaUpdater/__init__.py | 239 |
7 files changed, 0 insertions, 630 deletions
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py deleted file mode 100644 index ff4c24328..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -1_0_x.py - -This file should contain updates relevant to the 1.0.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import UnsupportedUpdate - -def updates(): - return UnsupportedUpdate("1.0", 10) - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py deleted file mode 100644 index 0d28786fd..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -1_1_x.py - -This file should contain updates relevant to the 1.1.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater -from Bcfg2.Server.SchemaUpdater.Routines import updatercallable - -from django.db import connection -import sys -import Bcfg2.settings -from Bcfg2.Server.Reports.reports.models import \ - TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA - -@updatercallable -def _interactions_constraint_or_idx(): - """sqlite doesn't support alter tables.. or constraints""" - cursor = connection.cursor() - try: - cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') - except: - cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') - - -@updatercallable -def _populate_interaction_entry_counts(): - '''Populate up the type totals for the interaction table''' - cursor = connection.cursor() - count_field = {TYPE_BAD: 'bad_entries', - TYPE_MODIFIED: 'modified_entries', - TYPE_EXTRA: 'extra_entries'} - - for type in list(count_field.keys()): - cursor.execute("select count(type), interaction_id " + - "from reports_entries_interactions where type = %s group by interaction_id" % type) - updates = [] - for row in cursor.fetchall(): - updates.append(row) - try: - cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) - except Exception: - e = sys.exc_info()[1] - print(e) - cursor.close() - - -def updates(): - fixes = Updater("1.1") - fixes.override_base_version(12) # Do not do this in new code - - fixes.add('alter table reports_interaction add column bad_entries integer not null default -1;') - fixes.add('alter table reports_interaction add column modified_entries integer not null default -1;') - fixes.add('alter table reports_interaction add column extra_entries integer not null default -1;') - fixes.add(_populate_interaction_entry_counts()) - fixes.add(_interactions_constraint_or_idx()) - fixes.add('alter table reports_reason add is_binary bool NOT NULL default False;') - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py deleted file mode 100644 index 024965bd5..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -1_2_x.py - -This file should contain updates relevant to the 1.2.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater -from Bcfg2.Server.SchemaUpdater.Routines import updatercallable - -def updates(): - fixes = Updater("1.2") - fixes.override_base_version(18) # Do not do this in new code - fixes.add('alter table reports_reason add is_sensitive bool NOT NULL default False;') - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py deleted file mode 100644 index 4fc57c653..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -1_3_0.py - -This file should contain updates relevant to the 1.3.x branches ONLY. -The updates() method must be defined and it should return an Updater object -""" -from Bcfg2.Server.SchemaUpdater import Updater, UpdaterError -from Bcfg2.Server.SchemaUpdater.Routines import AddColumns, \ - RemoveColumns, RebuildTable, DropTable - -from Bcfg2.Server.Reports.reports.models import Reason, Interaction - - -def updates(): - fixes = Updater("1.3") - fixes.add(RemoveColumns(Interaction, 'client_version')) - fixes.add(AddColumns(Reason)) - fixes.add(RebuildTable(Reason, [ - 'owner', 'current_owner', - 'group', 'current_group', - 'perms', 'current_perms', - 'status', 'current_status', - 'to', 'current_to'])) - fixes.add(DropTable('reports_ping')) - - return fixes - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py +++ /dev/null diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py b/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py deleted file mode 100644 index 4fcf0e6bf..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py +++ /dev/null @@ -1,279 +0,0 @@ -import logging -import traceback -from django.db.models.fields import NOT_PROVIDED -from django.db import connection, DatabaseError, backend, models -from django.core.management.color import no_style -from django.core.management.sql import sql_create -import django.core.management - -import Bcfg2.settings - -logger = logging.getLogger(__name__) - -def _quote(value): - """ - Quote a string to use as a table name or column - """ - return backend.DatabaseOperations().quote_name(value) - - -def _rebuild_sqlite_table(model): - """Sqlite doesn't support most alter table statments. This streamlines the - rebuild process""" - try: - cursor = connection.cursor() - table_name = model._meta.db_table - - # Build create staement from django - model._meta.db_table = "%s_temp" % table_name - sql, references = connection.creation.sql_create_model(model, no_style()) - columns = ",".join([_quote(f.column) \ - for f in model._meta.fields]) - - # Create a temp table - [cursor.execute(s) for s in sql] - - # Fill the table - tbl_name = _quote(table_name) - tmp_tbl_name = _quote(model._meta.db_table) - # Reset this - model._meta.db_table = table_name - cursor.execute("insert into %s(%s) select %s from %s;" % ( - tmp_tbl_name, - columns, - columns, - tbl_name)) - cursor.execute("drop table %s" % tbl_name) - - # Call syncdb to create the table again - django.core.management.call_command("syncdb", interactive=False, verbosity=0) - # syncdb closes our cursor - cursor = connection.cursor() - # Repopulate - cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name, - columns, - columns, - tmp_tbl_name)) - cursor.execute('DROP TABLE %s;' % tmp_tbl_name) - except DatabaseError: - logger.error("Failed to rebuild sqlite table %s" % table_name, exc_info=1) - raise UpdaterRoutineException - - -class UpdaterRoutineException(Exception): - pass - - -class UpdaterRoutine(object): - """Base for routines.""" - def __init__(self): - pass - - def __str__(self): - return __name__ - - def run(self): - """Called to execute the action""" - raise UpdaterRoutineException - - - -class AddColumns(UpdaterRoutine): - """ - Routine to add new columns to an existing model - """ - def __init__(self, model): - self.model = model - self.model_name = model.__name__ - - def __str__(self): - return "Add new columns for model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - try: - desc = {} - for d in connection.introspection.get_table_description(cursor, - self.model._meta.db_table): - desc[d[0]] = d - except DatabaseError: - logger.error("Failed to get table description", exc_info=1) - raise UpdaterRoutineException - - for field in self.model._meta.fields: - if field.column in desc: - continue - logger.debug("Column %s does not exist yet" % field.column) - if field.default == NOT_PROVIDED: - logger.error("Cannot add a column with out a default value") - raise UpdaterRoutineException - - sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % ( - _quote(self.model._meta.db_table), - _quote(field.column), field.db_type(), ) - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - sql += _quote(field.default) - sql_values = () - else: - sql += '%s' - sql_values = (field.default, ) - try: - cursor.execute(sql, sql_values) - logger.debug("Added column %s to %s" % - (field.column, self.model._meta.db_table)) - except DatabaseError: - logger.error("Unable to add column %s" % field.column) - raise UpdaterRoutineException - - -class RebuildTable(UpdaterRoutine): - """ - Rebuild the table for an existing model. Use this if field types have changed. - """ - def __init__(self, model, columns): - self.model = model - self.model_name = model.__name__ - - if type(columns) == str: - self.columns = [columns] - elif type(columns) in (tuple, list): - self.columns = columns - else: - logger.error("Columns must be a str, tuple, or list") - raise UpdaterRoutineException - - - def __str__(self): - return "Rebuild columns for model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - """ Sqlite is a special case. Altering columns is not supported. """ - _rebuild_sqlite_table(self.model) - return - - if db_engine == 'django.db.backends.mysql': - modify_cmd = 'MODIFY ' - else: - modify_cmd = 'ALTER COLUMN ' - - col_strings = [] - for column in self.columns: - col_strings.append("%s %s %s" % ( \ - modify_cmd, - _quote(column), - self.model._meta.get_field(column).db_type() - )) - - try: - cursor.execute('ALTER TABLE %s %s' % - (_quote(self.model._meta.db_table), ", ".join(col_strings))) - except DatabaseError: - logger.debug("Failed modify table %s" % self.model._meta.db_table) - raise UpdaterRoutineException - - - -class RemoveColumns(RebuildTable): - """ - Routine to remove columns from an existing model - """ - def __init__(self, model, columns): - super(RemoveColumns, self).__init__(model, columns) - - - def __str__(self): - return "Remove columns from model %s" % self.model_name - - def run(self): - try: - cursor = connection.cursor() - except DatabaseError: - logger.error("Failed to connect to the db") - raise UpdaterRoutineException - - try: - columns = [d[0] for d in connection.introspection.get_table_description(cursor, - self.model._meta.db_table)] - except DatabaseError: - logger.error("Failed to get table description", exc_info=1) - raise UpdaterRoutineException - - for column in self.columns: - if column not in columns: - logger.warning("Cannot drop column %s: does not exist" % column) - continue - - logger.debug("Dropping column %s" % column) - - db_engine = Bcfg2.settings.DATABASES['default']['ENGINE'] - if db_engine == 'django.db.backends.sqlite3': - _rebuild_sqlite_table(self.model) - else: - sql = "alter table %s drop column %s" % \ - (_quote(self.model._meta.db_table), _quote(column), ) - try: - cursor.execute(sql) - except DatabaseError: - logger.debug("Failed to drop column %s from %s" % - (column, self.model._meta.db_table)) - raise UpdaterRoutineException - - -class DropTable(UpdaterRoutine): - """ - Drop a table - """ - def __init__(self, table_name): - self.table_name = table_name - - def __str__(self): - return "Drop table %s" % self.table_name - - def run(self): - try: - cursor = connection.cursor() - cursor.execute('DROP TABLE %s' % _quote(self.table_name)) - except DatabaseError: - logger.error("Failed to drop table: %s" % - traceback.format_exc().splitlines()[-1]) - raise UpdaterRoutineException - - -class UpdaterCallable(UpdaterRoutine): - """Helper for routines. Basically delays execution""" - def __init__(self, fn): - self.fn = fn - self.args = [] - self.kwargs = {} - - def __call__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - return self - - def __str__(self): - return self.fn.__name__ - - def run(self): - self.fn(*self.args, **self.kwargs) - -def updatercallable(fn): - """Decorator for UpdaterCallable. Use for any function passed - into the fixes list""" - return UpdaterCallable(fn) - - diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py deleted file mode 100644 index e7a3191bc..000000000 --- a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py +++ /dev/null @@ -1,239 +0,0 @@ -from django.db import connection, DatabaseError -from django.core.exceptions import ImproperlyConfigured -import django.core.management -import logging -import re -import sys -import traceback - -from Bcfg2.Compat import CmpMixin, walk_packages -from Bcfg2.Server.models import InternalDatabaseVersion -from Bcfg2.Server.SchemaUpdater.Routines import UpdaterRoutineException, \ - UpdaterRoutine -from Bcfg2.Server.SchemaUpdater import Changes - -logger = logging.getLogger(__name__) - -class UpdaterError(Exception): - pass - - -class SchemaTooOldError(UpdaterError): - pass - - -def _release_to_version(release): - """ - Build a release base for a version - - Expects a string of the form 00.00 - - returns an integer of the form MMmm00 - """ - regex = re.compile("^(\d+)\.(\d+)$") - m = regex.match(release) - if not m: - logger.error("Invalid release string: %s" % release) - raise TypeError - return int("%02d%02d00" % (int(m.group(1)), int(m.group(2)))) - - -class Updater(CmpMixin): - """Database updater to standardize updates""" - - def __init__(self, release): - CmpMixin.__init__(self) - - self._cursor = None - self._release = release - try: - self._base_version = _release_to_version(release) - except: - err = "Invalid release string: %s" % release - logger.error(err) - raise UpdaterError(err) - - self._fixes = [] - self._version = -1 - - def __cmp__(self, other): - return self._base_version - other._base_version - - @property - def release(self): - return self._release - - @property - def version(self): - if self._version < 0: - try: - iv = InternalDatabaseVersion.objects.latest() - self._version = iv.version - except InternalDatabaseVersion.DoesNotExist: - raise UpdaterError("No database version stored internally") - return self._version - - @property - def cursor(self): - if not self._cursor: - self._cursor = connection.cursor() - return self._cursor - - @property - def target_version(self): - if(len(self._fixes) == 0): - return self._base_version - else: - return self._base_version + len(self._fixes) - 1 - - - def add(self, update): - if type(update) == str or isinstance(update, UpdaterRoutine): - self._fixes.append(update) - else: - raise TypeError - - - def override_base_version(self, version): - """Override our starting point for old releases. New code should - not use this method""" - self._base_version = int(version) - - - @staticmethod - def get_current_version(): - """Queries the db for the latest version. Returns 0 for a - fresh install""" - - if "call_command" in dir(django.core.management): - django.core.management.call_command("syncdb", interactive=False, - verbosity=0) - else: - msg = "Unable to call syndb routine" - logger.warning(msg) - raise UpdaterError(msg) - - try: - iv = InternalDatabaseVersion.objects.latest() - version = iv.version - except InternalDatabaseVersion.DoesNotExist: - version = 0 - - return version - - - def syncdb(self): - """Function to do the syncronisation for the models""" - - self._version = Updater.get_current_version() - self._cursor = None - - - def increment(self): - """Increment schema version in the database""" - if self._version < self._base_version: - self._version = self._base_version - else: - self._version += 1 - InternalDatabaseVersion.objects.create(version=self._version) - - def apply(self): - """Apply pending schema changes""" - - if self.version >= self.target_version: - logger.debug("No updates for release %s" % self._release) - return - - logger.debug("Applying updates for release %s" % self._release) - - if self.version < self._base_version: - start = 0 - else: - start = self.version - self._base_version + 1 - - try: - for fix in self._fixes[start:]: - if type(fix) == str: - self.cursor.execute(fix) - elif isinstance(fix, UpdaterRoutine): - fix.run() - else: - logger.error("Invalid schema change at %s" % \ - self._version + 1) - self.increment() - logger.debug("Applied schema change number %s: %s" % \ - (self.version, fix)) - logger.info("Applied schema changes for release %s" % self._release) - except: - msg = "Failed to perform db update %s (%s): %s" % \ - (self._version + 1, fix, - traceback.format_exc().splitlines()[-1]) - logger.error(msg) - raise UpdaterError(msg) - - -class UnsupportedUpdate(Updater): - """Handle an unsupported update""" - - def __init__(self, release, version): - super(UnsupportedUpdate, self).__init__(release) - self._base_version = version - - def apply(self): - """Raise an exception if we're too old""" - - if self.version < self.target_version: - logger.error("Upgrade from release %s unsupported" % self._release) - raise SchemaTooOldError - - -def update_database(): - """method to search where we are in the revision - of the database models and update them""" - try: - logger.debug("Verifying database schema") - - updaters = [] - for loader, submodule, ispkg in walk_packages(path=Changes.__path__): - if ispkg: - continue - try: - updates = getattr( - __import__("%s.%s" % (Changes.__name__, submodule), - globals(), locals(), ['*']), - "updates") - updaters.append(updates()) - except ImportError: - logger.error("Failed to import %s" % submodule) - except AttributeError: - logger.warning("Module %s does not have an updates function" % - submodule) - except: - msg = "Failed to build updater for %s" % submodule - logger.error(msg, exc_info=1) - raise UpdaterError(msg) - - current_version = Updater.get_current_version() - logger.debug("Database version at %s" % current_version) - - updaters.sort() - if current_version > 0: - [u.apply() for u in updaters] - logger.debug("Database version at %s" % - Updater.get_current_version()) - else: - target = updaters[-1].target_version - InternalDatabaseVersion.objects.create(version=target) - logger.info("A new database was created") - - except UpdaterError: - raise - except ImproperlyConfigured: - logger.error("Django is not properly configured: %s" % - traceback.format_exc().splitlines()[-1]) - raise UpdaterError - except: - logger.error("Error while updating the database") - for x in traceback.format_exc().splitlines(): - logger.error(x) - raise UpdaterError |