From 78d938a0bdddf385b1e887ff0a518009ab14b154 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 21 Mar 2014 15:33:30 -0400 Subject: Preliminary support for seperate reporting database This commit implements a Django database router which routes each Django application to a database whose name matches a key in the database dict, falling back to the default database if no matching key is found. This support is plumbed through to the config file via database.reporting_* database connection config options. These options mirror ones available for the default database config. If database.reporting_engine is not specified in the config, then the configuration falls back to the traditional single-database way of doing things with the database router becoming a no-op. --- src/lib/Bcfg2/DBSettings.py | 95 +++++++++++++++++++++++++++- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 6 +- src/lib/Bcfg2/Server/Admin.py | 13 ++-- src/lib/Bcfg2/Server/Core.py | 4 +- 4 files changed, 106 insertions(+), 12 deletions(-) (limited to 'src/lib/Bcfg2') diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 06ff173d4..9675e6d6b 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -63,7 +63,8 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.debug', 'django.core.context_processors.i18n', 'django.core.context_processors.media', - 'django.core.context_processors.request')) + 'django.core.context_processors.request'), + DATABASE_ROUTERS = ['Bcfg2.DBSettings.PerApplicationRouter']) if HAS_SOUTH: settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') @@ -95,6 +96,17 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) + if opts.reporting_db_engine is not None: + settings['DATABASES']['Reporting'] = dict( + ENGINE="django.db.backends.%s" % opts.reporting_db_engine, + NAME=opts.reporting_db_name, + USER=opts.reporting_db_user, + PASSWORD=opts.reporting_db_password, + HOST=opts.reporting_db_host, + PORT=opts.reporting_db_port, + OPTIONS=opts.reporting_db_opts, + SCHEMA=opts.reporting_db_schema) + settings['TIME_ZONE'] = opts.timezone settings['TEMPLATE_DEBUG'] = settings['DEBUG'] = \ @@ -123,6 +135,57 @@ def finalize_django_config(opts=None, silent=False): sys.exc_info()[1]) +def sync_databases(**kwargs): + """ Synchronize all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Syncing database %s" % (database)) + django.core.management.call_command("syncdb", database=database, + **kwargs) + + +def migrate_databases(**kwargs): + """ Do South migrations on all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Migrating database %s" % (database)) + django.core.management.call_command("migrate", database=database, + **kwargs) + + +class PerApplicationRouter(object): + """ Django database router for redirecting different applications to their + own database """ + + def _db_per_app(self, model, **hints): + """ If a database with the same name as the application exists, use it. + Otherwise use the default """ + app = model._meta.app_label + if app in settings['DATABASES']: + return app + + return 'default' + + def db_for_read(self, model, **hints): + """ Called when Django wants to find out what database to read from """ + return self._db_per_app(model, **hints) + + def db_for_write(self, model, **hints): + """ Called when Django wants to find out what database to write to """ + return self._db_per_app(model, **hints) + + def allow_relation(self, obj1, obj2, **hints): + """ Called when Django wants to determine what relations to allow. Only + allow relations within an app """ + return obj1._meta.app_label == obj2._meta.app_label + + def allow_syncdb(self, db, model): + """ Called when Django wants to determine which models to sync to a + given database. Take the cowards way out and sync all models to all + databases to allow for easy migrations. """ + return True + + class _OptionContainer(object): """ Container for options loaded at import-time to configure databases """ @@ -134,6 +197,7 @@ class _OptionContainer(object): default="/etc/bcfg2-web.conf", action=Bcfg2.Options.ConfigFileAction, help='Web interface configuration file'), + # default database options Bcfg2.Options.Option( cf=('database', 'engine'), default='sqlite3', help='Database engine', dest='db_engine'), @@ -156,6 +220,35 @@ class _OptionContainer(object): cf=('database', 'options'), help='Database options', dest='db_opts', type=Bcfg2.Options.Types.comma_dict, default=dict()), + # reporting database options + Bcfg2.Options.Option( + cf=('database', 'reporting_engine'), + help='Reporting database engine', dest='reporting_db_engine'), + Bcfg2.Options.Option( + cf=('database', 'reporting_name'), + default='/etc/reporting.sqlite', + help="Reporting database name", dest="reporting_db_name"), + Bcfg2.Options.Option( + cf=('database', 'reporting_user'), + help='Reporting database username', dest='reporting_db_user'), + Bcfg2.Options.Option( + cf=('database', 'reporting_password'), + help='Reporting database password', dest='reporting_db_password'), + Bcfg2.Options.Option( + cf=('database', 'reporting_host'), + help='Reporting database host', dest='reporting_db_host'), + Bcfg2.Options.Option( + cf=('database', 'reporting_port'), + help='Reporting database port', dest='reporting_db_port'), + Bcfg2.Options.Option( + cf=('database', 'reporting_schema'), + help='Reporting database schema', dest='reporting_db_schema', + default='public'), + Bcfg2.Options.Option( + cf=('database', 'reporting_options'), + help='Reporting database options', dest='reporting_db_opts', + type=Bcfg2.Options.Types.comma_dict, default=dict()), + # Django options Bcfg2.Options.Option( cf=('reporting', 'timezone'), help='Django timezone'), Bcfg2.Options.BooleanOption( diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index c223c3c73..992687a85 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -380,9 +380,9 @@ class DjangoORM(StorageBase): vrb = 1 else: vrb = 0 - management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, - interactive=False) + Bcfg2.DBSettings.sync_databases(verbosity=vrb, interactive=False) + Bcfg2.DBSettings.migrate_databases(verbosity=vrb, + interactive=False) except: msg = "Failed to update database schema: %s" % sys.exc_info()[1] self.logger.error(msg) diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 27152b867..0807fb2b0 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -912,8 +912,9 @@ if HAS_DJANGO: def run(self, setup): Bcfg2.Server.models.load_models() try: - management.call_command("syncdb", interactive=False, - verbosity=setup.verbose + setup.debug) + Bcfg2.DBSettings.sync_databases( + interactive=False, + verbosity=setup.verbose + setup.debug) except ImproperlyConfigured: err = sys.exc_info()[1] self.logger.error("Django configuration problem: %s" % err) @@ -960,10 +961,10 @@ if HAS_REPORTS: def run(self, setup): verbose = setup.verbose + setup.debug try: - management.call_command("syncdb", interactive=False, - verbosity=verbose) - management.call_command("migrate", interactive=False, - verbosity=verbose) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=verbose) + Bcfg2.DBSettings.migrate_databases(interactive=False, + verbosity=verbose) except: # pylint: disable=W0702 self.errExit("%s failed: %s" % (self.__class__.__name__.title(), diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 23209448d..4f51ebe87 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -236,8 +236,8 @@ class Core(object): self._database_available = False if HAS_DJANGO: try: - management.call_command("syncdb", interactive=False, - verbosity=0) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=0) self._database_available = True except ImproperlyConfigured: self.logger.error("Django configuration problem: %s" % -- cgit v1.2.3-1-g7c22 From 852f0c680646d34ff9c17eabf9212af650633d6e Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 24 Mar 2014 13:33:18 -0400 Subject: Make indentation consistent with other elements of the dict --- src/lib/Bcfg2/DBSettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2') diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 9675e6d6b..f7a610759 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -64,7 +64,7 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.i18n', 'django.core.context_processors.media', 'django.core.context_processors.request'), - DATABASE_ROUTERS = ['Bcfg2.DBSettings.PerApplicationRouter']) + DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter']) if HAS_SOUTH: settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') -- cgit v1.2.3-1-g7c22 From e06030b424d3405cdf0072f036f41db43634f404 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 24 Mar 2014 16:28:59 -0400 Subject: Be more robust about checking for reporting_db_engine presence This matters during the test suite: ====================================================================== ERROR: Failure: AttributeError ('Namespace' object has no attribute 'reporting_db_engine') ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/lib/python2.6/site-packages/nose/loader.py", line 364, in loadTestsFromName addr.filename, addr.module) File "/usr/lib/python2.6/site-packages/nose/importer.py", line 39, in importFromPath return self.importFromDir(dir_path, fqname) File "/usr/lib/python2.6/site-packages/nose/importer.py", line 84, in importFromDir mod = load_module(part_fqname, fh, filename, desc) File "/d/en/fennm-0/bcfg2-dbrouter/bcfg2-dbrouter.git/testsuite/Testtools/__init__.py", line 14, in from common import * File "/d/en/fennm-0/bcfg2-dbrouter/bcfg2-dbrouter.git/testsuite/common.py", line 62, in Bcfg2.DBSettings.finalize_django_config() File "/d/en/fennm-0/bcfg2-dbrouter/bcfg2-dbrouter.git/src/lib/Bcfg2/DBSettings.py", line 99, in finalize_django_config if opts.reporting_db_engine is not None: AttributeError: 'Namespace' object has no attribute 'reporting_db_engine' --- src/lib/Bcfg2/DBSettings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2') diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index f7a610759..e45807094 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -96,7 +96,8 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) - if opts.reporting_db_engine is not None: + if hasattr(opts, "reporting_db_engine") and \ + opts.reporting_db_engine is not None: settings['DATABASES']['Reporting'] = dict( ENGINE="django.db.backends.%s" % opts.reporting_db_engine, NAME=opts.reporting_db_name, -- cgit v1.2.3-1-g7c22