From 78d938a0bdddf385b1e887ff0a518009ab14b154 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 21 Mar 2014 15:33:30 -0400 Subject: Preliminary support for seperate reporting database This commit implements a Django database router which routes each Django application to a database whose name matches a key in the database dict, falling back to the default database if no matching key is found. This support is plumbed through to the config file via database.reporting_* database connection config options. These options mirror ones available for the default database config. If database.reporting_engine is not specified in the config, then the configuration falls back to the traditional single-database way of doing things with the database router becoming a no-op. --- src/lib/Bcfg2/DBSettings.py | 95 +++++++++++++++++++++++++++- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 6 +- src/lib/Bcfg2/Server/Admin.py | 13 ++-- src/lib/Bcfg2/Server/Core.py | 4 +- 4 files changed, 106 insertions(+), 12 deletions(-) diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 06ff173d4..9675e6d6b 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -63,7 +63,8 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.debug', 'django.core.context_processors.i18n', 'django.core.context_processors.media', - 'django.core.context_processors.request')) + 'django.core.context_processors.request'), + DATABASE_ROUTERS = ['Bcfg2.DBSettings.PerApplicationRouter']) if HAS_SOUTH: settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') @@ -95,6 +96,17 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) + if opts.reporting_db_engine is not None: + settings['DATABASES']['Reporting'] = dict( + ENGINE="django.db.backends.%s" % opts.reporting_db_engine, + NAME=opts.reporting_db_name, + USER=opts.reporting_db_user, + PASSWORD=opts.reporting_db_password, + HOST=opts.reporting_db_host, + PORT=opts.reporting_db_port, + OPTIONS=opts.reporting_db_opts, + SCHEMA=opts.reporting_db_schema) + settings['TIME_ZONE'] = opts.timezone settings['TEMPLATE_DEBUG'] = settings['DEBUG'] = \ @@ -123,6 +135,57 @@ def finalize_django_config(opts=None, silent=False): sys.exc_info()[1]) +def sync_databases(**kwargs): + """ Synchronize all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Syncing database %s" % (database)) + django.core.management.call_command("syncdb", database=database, + **kwargs) + + +def migrate_databases(**kwargs): + """ Do South migrations on all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Migrating database %s" % (database)) + django.core.management.call_command("migrate", database=database, + **kwargs) + + +class PerApplicationRouter(object): + """ Django database router for redirecting different applications to their + own database """ + + def _db_per_app(self, model, **hints): + """ If a database with the same name as the application exists, use it. + Otherwise use the default """ + app = model._meta.app_label + if app in settings['DATABASES']: + return app + + return 'default' + + def db_for_read(self, model, **hints): + """ Called when Django wants to find out what database to read from """ + return self._db_per_app(model, **hints) + + def db_for_write(self, model, **hints): + """ Called when Django wants to find out what database to write to """ + return self._db_per_app(model, **hints) + + def allow_relation(self, obj1, obj2, **hints): + """ Called when Django wants to determine what relations to allow. Only + allow relations within an app """ + return obj1._meta.app_label == obj2._meta.app_label + + def allow_syncdb(self, db, model): + """ Called when Django wants to determine which models to sync to a + given database. Take the cowards way out and sync all models to all + databases to allow for easy migrations. """ + return True + + class _OptionContainer(object): """ Container for options loaded at import-time to configure databases """ @@ -134,6 +197,7 @@ class _OptionContainer(object): default="/etc/bcfg2-web.conf", action=Bcfg2.Options.ConfigFileAction, help='Web interface configuration file'), + # default database options Bcfg2.Options.Option( cf=('database', 'engine'), default='sqlite3', help='Database engine', dest='db_engine'), @@ -156,6 +220,35 @@ class _OptionContainer(object): cf=('database', 'options'), help='Database options', dest='db_opts', type=Bcfg2.Options.Types.comma_dict, default=dict()), + # reporting database options + Bcfg2.Options.Option( + cf=('database', 'reporting_engine'), + help='Reporting database engine', dest='reporting_db_engine'), + Bcfg2.Options.Option( + cf=('database', 'reporting_name'), + default='/etc/reporting.sqlite', + help="Reporting database name", dest="reporting_db_name"), + Bcfg2.Options.Option( + cf=('database', 'reporting_user'), + help='Reporting database username', dest='reporting_db_user'), + Bcfg2.Options.Option( + cf=('database', 'reporting_password'), + help='Reporting database password', dest='reporting_db_password'), + Bcfg2.Options.Option( + cf=('database', 'reporting_host'), + help='Reporting database host', dest='reporting_db_host'), + Bcfg2.Options.Option( + cf=('database', 'reporting_port'), + help='Reporting database port', dest='reporting_db_port'), + Bcfg2.Options.Option( + cf=('database', 'reporting_schema'), + help='Reporting database schema', dest='reporting_db_schema', + default='public'), + Bcfg2.Options.Option( + cf=('database', 'reporting_options'), + help='Reporting database options', dest='reporting_db_opts', + type=Bcfg2.Options.Types.comma_dict, default=dict()), + # Django options Bcfg2.Options.Option( cf=('reporting', 'timezone'), help='Django timezone'), Bcfg2.Options.BooleanOption( diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index c223c3c73..992687a85 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -380,9 +380,9 @@ class DjangoORM(StorageBase): vrb = 1 else: vrb = 0 - management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, - interactive=False) + Bcfg2.DBSettings.sync_databases(verbosity=vrb, interactive=False) + Bcfg2.DBSettings.migrate_databases(verbosity=vrb, + interactive=False) except: msg = "Failed to update database schema: %s" % sys.exc_info()[1] self.logger.error(msg) diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 27152b867..0807fb2b0 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -912,8 +912,9 @@ if HAS_DJANGO: def run(self, setup): Bcfg2.Server.models.load_models() try: - management.call_command("syncdb", interactive=False, - verbosity=setup.verbose + setup.debug) + Bcfg2.DBSettings.sync_databases( + interactive=False, + verbosity=setup.verbose + setup.debug) except ImproperlyConfigured: err = sys.exc_info()[1] self.logger.error("Django configuration problem: %s" % err) @@ -960,10 +961,10 @@ if HAS_REPORTS: def run(self, setup): verbose = setup.verbose + setup.debug try: - management.call_command("syncdb", interactive=False, - verbosity=verbose) - management.call_command("migrate", interactive=False, - verbosity=verbose) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=verbose) + Bcfg2.DBSettings.migrate_databases(interactive=False, + verbosity=verbose) except: # pylint: disable=W0702 self.errExit("%s failed: %s" % (self.__class__.__name__.title(), diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 23209448d..4f51ebe87 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -236,8 +236,8 @@ class Core(object): self._database_available = False if HAS_DJANGO: try: - management.call_command("syncdb", interactive=False, - verbosity=0) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=0) self._database_available = True except ImproperlyConfigured: self.logger.error("Django configuration problem: %s" % -- cgit v1.2.3-1-g7c22