summaryrefslogtreecommitdiffstats
path: root/src/lib/Bcfg2/Server/SchemaUpdater
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib/Bcfg2/Server/SchemaUpdater')
-rw-r--r--src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py11
-rw-r--r--src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py59
-rw-r--r--src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py15
-rw-r--r--src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py27
-rw-r--r--src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py0
-rw-r--r--src/lib/Bcfg2/Server/SchemaUpdater/Routines.py279
-rw-r--r--src/lib/Bcfg2/Server/SchemaUpdater/__init__.py239
7 files changed, 0 insertions, 630 deletions
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py
deleted file mode 100644
index ff4c24328..000000000
--- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py
+++ /dev/null
@@ -1,11 +0,0 @@
-"""
-1_0_x.py
-
-This file should contain updates relevant to the 1.0.x branches ONLY.
-The updates() method must be defined and it should return an Updater object
-"""
-from Bcfg2.Server.SchemaUpdater import UnsupportedUpdate
-
-def updates():
- return UnsupportedUpdate("1.0", 10)
-
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py
deleted file mode 100644
index 0d28786fd..000000000
--- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""
-1_1_x.py
-
-This file should contain updates relevant to the 1.1.x branches ONLY.
-The updates() method must be defined and it should return an Updater object
-"""
-from Bcfg2.Server.SchemaUpdater import Updater
-from Bcfg2.Server.SchemaUpdater.Routines import updatercallable
-
-from django.db import connection
-import sys
-import Bcfg2.settings
-from Bcfg2.Server.Reports.reports.models import \
- TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA
-
-@updatercallable
-def _interactions_constraint_or_idx():
- """sqlite doesn't support alter tables.. or constraints"""
- cursor = connection.cursor()
- try:
- cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)')
- except:
- cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)')
-
-
-@updatercallable
-def _populate_interaction_entry_counts():
- '''Populate up the type totals for the interaction table'''
- cursor = connection.cursor()
- count_field = {TYPE_BAD: 'bad_entries',
- TYPE_MODIFIED: 'modified_entries',
- TYPE_EXTRA: 'extra_entries'}
-
- for type in list(count_field.keys()):
- cursor.execute("select count(type), interaction_id " +
- "from reports_entries_interactions where type = %s group by interaction_id" % type)
- updates = []
- for row in cursor.fetchall():
- updates.append(row)
- try:
- cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates)
- except Exception:
- e = sys.exc_info()[1]
- print(e)
- cursor.close()
-
-
-def updates():
- fixes = Updater("1.1")
- fixes.override_base_version(12) # Do not do this in new code
-
- fixes.add('alter table reports_interaction add column bad_entries integer not null default -1;')
- fixes.add('alter table reports_interaction add column modified_entries integer not null default -1;')
- fixes.add('alter table reports_interaction add column extra_entries integer not null default -1;')
- fixes.add(_populate_interaction_entry_counts())
- fixes.add(_interactions_constraint_or_idx())
- fixes.add('alter table reports_reason add is_binary bool NOT NULL default False;')
- return fixes
-
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py
deleted file mode 100644
index 024965bd5..000000000
--- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-1_2_x.py
-
-This file should contain updates relevant to the 1.2.x branches ONLY.
-The updates() method must be defined and it should return an Updater object
-"""
-from Bcfg2.Server.SchemaUpdater import Updater
-from Bcfg2.Server.SchemaUpdater.Routines import updatercallable
-
-def updates():
- fixes = Updater("1.2")
- fixes.override_base_version(18) # Do not do this in new code
- fixes.add('alter table reports_reason add is_sensitive bool NOT NULL default False;')
- return fixes
-
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py
deleted file mode 100644
index 4fc57c653..000000000
--- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-1_3_0.py
-
-This file should contain updates relevant to the 1.3.x branches ONLY.
-The updates() method must be defined and it should return an Updater object
-"""
-from Bcfg2.Server.SchemaUpdater import Updater, UpdaterError
-from Bcfg2.Server.SchemaUpdater.Routines import AddColumns, \
- RemoveColumns, RebuildTable, DropTable
-
-from Bcfg2.Server.Reports.reports.models import Reason, Interaction
-
-
-def updates():
- fixes = Updater("1.3")
- fixes.add(RemoveColumns(Interaction, 'client_version'))
- fixes.add(AddColumns(Reason))
- fixes.add(RebuildTable(Reason, [
- 'owner', 'current_owner',
- 'group', 'current_group',
- 'perms', 'current_perms',
- 'status', 'current_status',
- 'to', 'current_to']))
- fixes.add(DropTable('reports_ping'))
-
- return fixes
-
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py
+++ /dev/null
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py b/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py
deleted file mode 100644
index 4fcf0e6bf..000000000
--- a/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py
+++ /dev/null
@@ -1,279 +0,0 @@
-import logging
-import traceback
-from django.db.models.fields import NOT_PROVIDED
-from django.db import connection, DatabaseError, backend, models
-from django.core.management.color import no_style
-from django.core.management.sql import sql_create
-import django.core.management
-
-import Bcfg2.settings
-
-logger = logging.getLogger(__name__)
-
-def _quote(value):
- """
- Quote a string to use as a table name or column
- """
- return backend.DatabaseOperations().quote_name(value)
-
-
-def _rebuild_sqlite_table(model):
- """Sqlite doesn't support most alter table statments. This streamlines the
- rebuild process"""
- try:
- cursor = connection.cursor()
- table_name = model._meta.db_table
-
- # Build create staement from django
- model._meta.db_table = "%s_temp" % table_name
- sql, references = connection.creation.sql_create_model(model, no_style())
- columns = ",".join([_quote(f.column) \
- for f in model._meta.fields])
-
- # Create a temp table
- [cursor.execute(s) for s in sql]
-
- # Fill the table
- tbl_name = _quote(table_name)
- tmp_tbl_name = _quote(model._meta.db_table)
- # Reset this
- model._meta.db_table = table_name
- cursor.execute("insert into %s(%s) select %s from %s;" % (
- tmp_tbl_name,
- columns,
- columns,
- tbl_name))
- cursor.execute("drop table %s" % tbl_name)
-
- # Call syncdb to create the table again
- django.core.management.call_command("syncdb", interactive=False, verbosity=0)
- # syncdb closes our cursor
- cursor = connection.cursor()
- # Repopulate
- cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name,
- columns,
- columns,
- tmp_tbl_name))
- cursor.execute('DROP TABLE %s;' % tmp_tbl_name)
- except DatabaseError:
- logger.error("Failed to rebuild sqlite table %s" % table_name, exc_info=1)
- raise UpdaterRoutineException
-
-
-class UpdaterRoutineException(Exception):
- pass
-
-
-class UpdaterRoutine(object):
- """Base for routines."""
- def __init__(self):
- pass
-
- def __str__(self):
- return __name__
-
- def run(self):
- """Called to execute the action"""
- raise UpdaterRoutineException
-
-
-
-class AddColumns(UpdaterRoutine):
- """
- Routine to add new columns to an existing model
- """
- def __init__(self, model):
- self.model = model
- self.model_name = model.__name__
-
- def __str__(self):
- return "Add new columns for model %s" % self.model_name
-
- def run(self):
- try:
- cursor = connection.cursor()
- except DatabaseError:
- logger.error("Failed to connect to the db")
- raise UpdaterRoutineException
-
- try:
- desc = {}
- for d in connection.introspection.get_table_description(cursor,
- self.model._meta.db_table):
- desc[d[0]] = d
- except DatabaseError:
- logger.error("Failed to get table description", exc_info=1)
- raise UpdaterRoutineException
-
- for field in self.model._meta.fields:
- if field.column in desc:
- continue
- logger.debug("Column %s does not exist yet" % field.column)
- if field.default == NOT_PROVIDED:
- logger.error("Cannot add a column with out a default value")
- raise UpdaterRoutineException
-
- sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % (
- _quote(self.model._meta.db_table),
- _quote(field.column), field.db_type(), )
- db_engine = Bcfg2.settings.DATABASES['default']['ENGINE']
- if db_engine == 'django.db.backends.sqlite3':
- sql += _quote(field.default)
- sql_values = ()
- else:
- sql += '%s'
- sql_values = (field.default, )
- try:
- cursor.execute(sql, sql_values)
- logger.debug("Added column %s to %s" %
- (field.column, self.model._meta.db_table))
- except DatabaseError:
- logger.error("Unable to add column %s" % field.column)
- raise UpdaterRoutineException
-
-
-class RebuildTable(UpdaterRoutine):
- """
- Rebuild the table for an existing model. Use this if field types have changed.
- """
- def __init__(self, model, columns):
- self.model = model
- self.model_name = model.__name__
-
- if type(columns) == str:
- self.columns = [columns]
- elif type(columns) in (tuple, list):
- self.columns = columns
- else:
- logger.error("Columns must be a str, tuple, or list")
- raise UpdaterRoutineException
-
-
- def __str__(self):
- return "Rebuild columns for model %s" % self.model_name
-
- def run(self):
- try:
- cursor = connection.cursor()
- except DatabaseError:
- logger.error("Failed to connect to the db")
- raise UpdaterRoutineException
-
- db_engine = Bcfg2.settings.DATABASES['default']['ENGINE']
- if db_engine == 'django.db.backends.sqlite3':
- """ Sqlite is a special case. Altering columns is not supported. """
- _rebuild_sqlite_table(self.model)
- return
-
- if db_engine == 'django.db.backends.mysql':
- modify_cmd = 'MODIFY '
- else:
- modify_cmd = 'ALTER COLUMN '
-
- col_strings = []
- for column in self.columns:
- col_strings.append("%s %s %s" % ( \
- modify_cmd,
- _quote(column),
- self.model._meta.get_field(column).db_type()
- ))
-
- try:
- cursor.execute('ALTER TABLE %s %s' %
- (_quote(self.model._meta.db_table), ", ".join(col_strings)))
- except DatabaseError:
- logger.debug("Failed modify table %s" % self.model._meta.db_table)
- raise UpdaterRoutineException
-
-
-
-class RemoveColumns(RebuildTable):
- """
- Routine to remove columns from an existing model
- """
- def __init__(self, model, columns):
- super(RemoveColumns, self).__init__(model, columns)
-
-
- def __str__(self):
- return "Remove columns from model %s" % self.model_name
-
- def run(self):
- try:
- cursor = connection.cursor()
- except DatabaseError:
- logger.error("Failed to connect to the db")
- raise UpdaterRoutineException
-
- try:
- columns = [d[0] for d in connection.introspection.get_table_description(cursor,
- self.model._meta.db_table)]
- except DatabaseError:
- logger.error("Failed to get table description", exc_info=1)
- raise UpdaterRoutineException
-
- for column in self.columns:
- if column not in columns:
- logger.warning("Cannot drop column %s: does not exist" % column)
- continue
-
- logger.debug("Dropping column %s" % column)
-
- db_engine = Bcfg2.settings.DATABASES['default']['ENGINE']
- if db_engine == 'django.db.backends.sqlite3':
- _rebuild_sqlite_table(self.model)
- else:
- sql = "alter table %s drop column %s" % \
- (_quote(self.model._meta.db_table), _quote(column), )
- try:
- cursor.execute(sql)
- except DatabaseError:
- logger.debug("Failed to drop column %s from %s" %
- (column, self.model._meta.db_table))
- raise UpdaterRoutineException
-
-
-class DropTable(UpdaterRoutine):
- """
- Drop a table
- """
- def __init__(self, table_name):
- self.table_name = table_name
-
- def __str__(self):
- return "Drop table %s" % self.table_name
-
- def run(self):
- try:
- cursor = connection.cursor()
- cursor.execute('DROP TABLE %s' % _quote(self.table_name))
- except DatabaseError:
- logger.error("Failed to drop table: %s" %
- traceback.format_exc().splitlines()[-1])
- raise UpdaterRoutineException
-
-
-class UpdaterCallable(UpdaterRoutine):
- """Helper for routines. Basically delays execution"""
- def __init__(self, fn):
- self.fn = fn
- self.args = []
- self.kwargs = {}
-
- def __call__(self, *args, **kwargs):
- self.args = args
- self.kwargs = kwargs
- return self
-
- def __str__(self):
- return self.fn.__name__
-
- def run(self):
- self.fn(*self.args, **self.kwargs)
-
-def updatercallable(fn):
- """Decorator for UpdaterCallable. Use for any function passed
- into the fixes list"""
- return UpdaterCallable(fn)
-
-
diff --git a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py
deleted file mode 100644
index e7a3191bc..000000000
--- a/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py
+++ /dev/null
@@ -1,239 +0,0 @@
-from django.db import connection, DatabaseError
-from django.core.exceptions import ImproperlyConfigured
-import django.core.management
-import logging
-import re
-import sys
-import traceback
-
-from Bcfg2.Compat import CmpMixin, walk_packages
-from Bcfg2.Server.models import InternalDatabaseVersion
-from Bcfg2.Server.SchemaUpdater.Routines import UpdaterRoutineException, \
- UpdaterRoutine
-from Bcfg2.Server.SchemaUpdater import Changes
-
-logger = logging.getLogger(__name__)
-
-class UpdaterError(Exception):
- pass
-
-
-class SchemaTooOldError(UpdaterError):
- pass
-
-
-def _release_to_version(release):
- """
- Build a release base for a version
-
- Expects a string of the form 00.00
-
- returns an integer of the form MMmm00
- """
- regex = re.compile("^(\d+)\.(\d+)$")
- m = regex.match(release)
- if not m:
- logger.error("Invalid release string: %s" % release)
- raise TypeError
- return int("%02d%02d00" % (int(m.group(1)), int(m.group(2))))
-
-
-class Updater(CmpMixin):
- """Database updater to standardize updates"""
-
- def __init__(self, release):
- CmpMixin.__init__(self)
-
- self._cursor = None
- self._release = release
- try:
- self._base_version = _release_to_version(release)
- except:
- err = "Invalid release string: %s" % release
- logger.error(err)
- raise UpdaterError(err)
-
- self._fixes = []
- self._version = -1
-
- def __cmp__(self, other):
- return self._base_version - other._base_version
-
- @property
- def release(self):
- return self._release
-
- @property
- def version(self):
- if self._version < 0:
- try:
- iv = InternalDatabaseVersion.objects.latest()
- self._version = iv.version
- except InternalDatabaseVersion.DoesNotExist:
- raise UpdaterError("No database version stored internally")
- return self._version
-
- @property
- def cursor(self):
- if not self._cursor:
- self._cursor = connection.cursor()
- return self._cursor
-
- @property
- def target_version(self):
- if(len(self._fixes) == 0):
- return self._base_version
- else:
- return self._base_version + len(self._fixes) - 1
-
-
- def add(self, update):
- if type(update) == str or isinstance(update, UpdaterRoutine):
- self._fixes.append(update)
- else:
- raise TypeError
-
-
- def override_base_version(self, version):
- """Override our starting point for old releases. New code should
- not use this method"""
- self._base_version = int(version)
-
-
- @staticmethod
- def get_current_version():
- """Queries the db for the latest version. Returns 0 for a
- fresh install"""
-
- if "call_command" in dir(django.core.management):
- django.core.management.call_command("syncdb", interactive=False,
- verbosity=0)
- else:
- msg = "Unable to call syndb routine"
- logger.warning(msg)
- raise UpdaterError(msg)
-
- try:
- iv = InternalDatabaseVersion.objects.latest()
- version = iv.version
- except InternalDatabaseVersion.DoesNotExist:
- version = 0
-
- return version
-
-
- def syncdb(self):
- """Function to do the syncronisation for the models"""
-
- self._version = Updater.get_current_version()
- self._cursor = None
-
-
- def increment(self):
- """Increment schema version in the database"""
- if self._version < self._base_version:
- self._version = self._base_version
- else:
- self._version += 1
- InternalDatabaseVersion.objects.create(version=self._version)
-
- def apply(self):
- """Apply pending schema changes"""
-
- if self.version >= self.target_version:
- logger.debug("No updates for release %s" % self._release)
- return
-
- logger.debug("Applying updates for release %s" % self._release)
-
- if self.version < self._base_version:
- start = 0
- else:
- start = self.version - self._base_version + 1
-
- try:
- for fix in self._fixes[start:]:
- if type(fix) == str:
- self.cursor.execute(fix)
- elif isinstance(fix, UpdaterRoutine):
- fix.run()
- else:
- logger.error("Invalid schema change at %s" % \
- self._version + 1)
- self.increment()
- logger.debug("Applied schema change number %s: %s" % \
- (self.version, fix))
- logger.info("Applied schema changes for release %s" % self._release)
- except:
- msg = "Failed to perform db update %s (%s): %s" % \
- (self._version + 1, fix,
- traceback.format_exc().splitlines()[-1])
- logger.error(msg)
- raise UpdaterError(msg)
-
-
-class UnsupportedUpdate(Updater):
- """Handle an unsupported update"""
-
- def __init__(self, release, version):
- super(UnsupportedUpdate, self).__init__(release)
- self._base_version = version
-
- def apply(self):
- """Raise an exception if we're too old"""
-
- if self.version < self.target_version:
- logger.error("Upgrade from release %s unsupported" % self._release)
- raise SchemaTooOldError
-
-
-def update_database():
- """method to search where we are in the revision
- of the database models and update them"""
- try:
- logger.debug("Verifying database schema")
-
- updaters = []
- for loader, submodule, ispkg in walk_packages(path=Changes.__path__):
- if ispkg:
- continue
- try:
- updates = getattr(
- __import__("%s.%s" % (Changes.__name__, submodule),
- globals(), locals(), ['*']),
- "updates")
- updaters.append(updates())
- except ImportError:
- logger.error("Failed to import %s" % submodule)
- except AttributeError:
- logger.warning("Module %s does not have an updates function" %
- submodule)
- except:
- msg = "Failed to build updater for %s" % submodule
- logger.error(msg, exc_info=1)
- raise UpdaterError(msg)
-
- current_version = Updater.get_current_version()
- logger.debug("Database version at %s" % current_version)
-
- updaters.sort()
- if current_version > 0:
- [u.apply() for u in updaters]
- logger.debug("Database version at %s" %
- Updater.get_current_version())
- else:
- target = updaters[-1].target_version
- InternalDatabaseVersion.objects.create(version=target)
- logger.info("A new database was created")
-
- except UpdaterError:
- raise
- except ImproperlyConfigured:
- logger.error("Django is not properly configured: %s" %
- traceback.format_exc().splitlines()[-1])
- raise UpdaterError
- except:
- logger.error("Error while updating the database")
- for x in traceback.format_exc().splitlines():
- logger.error(x)
- raise UpdaterError