From 78d938a0bdddf385b1e887ff0a518009ab14b154 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Fri, 21 Mar 2014 15:33:30 -0400 Subject: Preliminary support for seperate reporting database This commit implements a Django database router which routes each Django application to a database whose name matches a key in the database dict, falling back to the default database if no matching key is found. This support is plumbed through to the config file via database.reporting_* database connection config options. These options mirror ones available for the default database config. If database.reporting_engine is not specified in the config, then the configuration falls back to the traditional single-database way of doing things with the database router becoming a no-op. --- src/lib/Bcfg2/DBSettings.py | 95 +++++++++++++++++++++++++++- src/lib/Bcfg2/Reporting/Storage/DjangoORM.py | 6 +- src/lib/Bcfg2/Server/Admin.py | 13 ++-- src/lib/Bcfg2/Server/Core.py | 4 +- 4 files changed, 106 insertions(+), 12 deletions(-) diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 06ff173d4..9675e6d6b 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -63,7 +63,8 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.debug', 'django.core.context_processors.i18n', 'django.core.context_processors.media', - 'django.core.context_processors.request')) + 'django.core.context_processors.request'), + DATABASE_ROUTERS = ['Bcfg2.DBSettings.PerApplicationRouter']) if HAS_SOUTH: settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') @@ -95,6 +96,17 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) + if opts.reporting_db_engine is not None: + settings['DATABASES']['Reporting'] = dict( + ENGINE="django.db.backends.%s" % opts.reporting_db_engine, + NAME=opts.reporting_db_name, + USER=opts.reporting_db_user, + PASSWORD=opts.reporting_db_password, + HOST=opts.reporting_db_host, + PORT=opts.reporting_db_port, + OPTIONS=opts.reporting_db_opts, + SCHEMA=opts.reporting_db_schema) + settings['TIME_ZONE'] = opts.timezone settings['TEMPLATE_DEBUG'] = settings['DEBUG'] = \ @@ -123,6 +135,57 @@ def finalize_django_config(opts=None, silent=False): sys.exc_info()[1]) +def sync_databases(**kwargs): + """ Synchronize all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Syncing database %s" % (database)) + django.core.management.call_command("syncdb", database=database, + **kwargs) + + +def migrate_databases(**kwargs): + """ Do South migrations on all databases that we know about. """ + logger = logging.getLogger() + for database in settings['DATABASES']: + logger.debug("Migrating database %s" % (database)) + django.core.management.call_command("migrate", database=database, + **kwargs) + + +class PerApplicationRouter(object): + """ Django database router for redirecting different applications to their + own database """ + + def _db_per_app(self, model, **hints): + """ If a database with the same name as the application exists, use it. + Otherwise use the default """ + app = model._meta.app_label + if app in settings['DATABASES']: + return app + + return 'default' + + def db_for_read(self, model, **hints): + """ Called when Django wants to find out what database to read from """ + return self._db_per_app(model, **hints) + + def db_for_write(self, model, **hints): + """ Called when Django wants to find out what database to write to """ + return self._db_per_app(model, **hints) + + def allow_relation(self, obj1, obj2, **hints): + """ Called when Django wants to determine what relations to allow. Only + allow relations within an app """ + return obj1._meta.app_label == obj2._meta.app_label + + def allow_syncdb(self, db, model): + """ Called when Django wants to determine which models to sync to a + given database. Take the cowards way out and sync all models to all + databases to allow for easy migrations. """ + return True + + class _OptionContainer(object): """ Container for options loaded at import-time to configure databases """ @@ -134,6 +197,7 @@ class _OptionContainer(object): default="/etc/bcfg2-web.conf", action=Bcfg2.Options.ConfigFileAction, help='Web interface configuration file'), + # default database options Bcfg2.Options.Option( cf=('database', 'engine'), default='sqlite3', help='Database engine', dest='db_engine'), @@ -156,6 +220,35 @@ class _OptionContainer(object): cf=('database', 'options'), help='Database options', dest='db_opts', type=Bcfg2.Options.Types.comma_dict, default=dict()), + # reporting database options + Bcfg2.Options.Option( + cf=('database', 'reporting_engine'), + help='Reporting database engine', dest='reporting_db_engine'), + Bcfg2.Options.Option( + cf=('database', 'reporting_name'), + default='/etc/reporting.sqlite', + help="Reporting database name", dest="reporting_db_name"), + Bcfg2.Options.Option( + cf=('database', 'reporting_user'), + help='Reporting database username', dest='reporting_db_user'), + Bcfg2.Options.Option( + cf=('database', 'reporting_password'), + help='Reporting database password', dest='reporting_db_password'), + Bcfg2.Options.Option( + cf=('database', 'reporting_host'), + help='Reporting database host', dest='reporting_db_host'), + Bcfg2.Options.Option( + cf=('database', 'reporting_port'), + help='Reporting database port', dest='reporting_db_port'), + Bcfg2.Options.Option( + cf=('database', 'reporting_schema'), + help='Reporting database schema', dest='reporting_db_schema', + default='public'), + Bcfg2.Options.Option( + cf=('database', 'reporting_options'), + help='Reporting database options', dest='reporting_db_opts', + type=Bcfg2.Options.Types.comma_dict, default=dict()), + # Django options Bcfg2.Options.Option( cf=('reporting', 'timezone'), help='Django timezone'), Bcfg2.Options.BooleanOption( diff --git a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py index c223c3c73..992687a85 100644 --- a/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py +++ b/src/lib/Bcfg2/Reporting/Storage/DjangoORM.py @@ -380,9 +380,9 @@ class DjangoORM(StorageBase): vrb = 1 else: vrb = 0 - management.call_command("syncdb", verbosity=vrb, interactive=False) - management.call_command("migrate", verbosity=vrb, - interactive=False) + Bcfg2.DBSettings.sync_databases(verbosity=vrb, interactive=False) + Bcfg2.DBSettings.migrate_databases(verbosity=vrb, + interactive=False) except: msg = "Failed to update database schema: %s" % sys.exc_info()[1] self.logger.error(msg) diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 27152b867..0807fb2b0 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -912,8 +912,9 @@ if HAS_DJANGO: def run(self, setup): Bcfg2.Server.models.load_models() try: - management.call_command("syncdb", interactive=False, - verbosity=setup.verbose + setup.debug) + Bcfg2.DBSettings.sync_databases( + interactive=False, + verbosity=setup.verbose + setup.debug) except ImproperlyConfigured: err = sys.exc_info()[1] self.logger.error("Django configuration problem: %s" % err) @@ -960,10 +961,10 @@ if HAS_REPORTS: def run(self, setup): verbose = setup.verbose + setup.debug try: - management.call_command("syncdb", interactive=False, - verbosity=verbose) - management.call_command("migrate", interactive=False, - verbosity=verbose) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=verbose) + Bcfg2.DBSettings.migrate_databases(interactive=False, + verbosity=verbose) except: # pylint: disable=W0702 self.errExit("%s failed: %s" % (self.__class__.__name__.title(), diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 23209448d..4f51ebe87 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -236,8 +236,8 @@ class Core(object): self._database_available = False if HAS_DJANGO: try: - management.call_command("syncdb", interactive=False, - verbosity=0) + Bcfg2.DBSettings.sync_databases(interactive=False, + verbosity=0) self._database_available = True except ImproperlyConfigured: self.logger.error("Django configuration problem: %s" % -- cgit v1.2.3-1-g7c22 From 852f0c680646d34ff9c17eabf9212af650633d6e Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 24 Mar 2014 13:33:18 -0400 Subject: Make indentation consistent with other elements of the dict --- src/lib/Bcfg2/DBSettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index 9675e6d6b..f7a610759 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -64,7 +64,7 @@ settings = dict( # pylint: disable=C0103 'django.core.context_processors.i18n', 'django.core.context_processors.media', 'django.core.context_processors.request'), - DATABASE_ROUTERS = ['Bcfg2.DBSettings.PerApplicationRouter']) + DATABASE_ROUTERS=['Bcfg2.DBSettings.PerApplicationRouter']) if HAS_SOUTH: settings['INSTALLED_APPS'] += ('south', 'Bcfg2.Reporting') -- cgit v1.2.3-1-g7c22 From e06030b424d3405cdf0072f036f41db43634f404 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Mon, 24 Mar 2014 16:28:59 -0400 Subject: Be more robust about checking for reporting_db_engine presence This matters during the test suite: ====================================================================== ERROR: Failure: AttributeError ('Namespace' object has no attribute 'reporting_db_engine') ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/lib/python2.6/site-packages/nose/loader.py", line 364, in loadTestsFromName addr.filename, addr.module) File "/usr/lib/python2.6/site-packages/nose/importer.py", line 39, in importFromPath return self.importFromDir(dir_path, fqname) File "/usr/lib/python2.6/site-packages/nose/importer.py", line 84, in importFromDir mod = load_module(part_fqname, fh, filename, desc) File "/d/en/fennm-0/bcfg2-dbrouter/bcfg2-dbrouter.git/testsuite/Testtools/__init__.py", line 14, in from common import * File "/d/en/fennm-0/bcfg2-dbrouter/bcfg2-dbrouter.git/testsuite/common.py", line 62, in Bcfg2.DBSettings.finalize_django_config() File "/d/en/fennm-0/bcfg2-dbrouter/bcfg2-dbrouter.git/src/lib/Bcfg2/DBSettings.py", line 99, in finalize_django_config if opts.reporting_db_engine is not None: AttributeError: 'Namespace' object has no attribute 'reporting_db_engine' --- src/lib/Bcfg2/DBSettings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/DBSettings.py b/src/lib/Bcfg2/DBSettings.py index f7a610759..e45807094 100644 --- a/src/lib/Bcfg2/DBSettings.py +++ b/src/lib/Bcfg2/DBSettings.py @@ -96,7 +96,8 @@ def finalize_django_config(opts=None, silent=False): OPTIONS=opts.db_opts, SCHEMA=opts.db_schema)) - if opts.reporting_db_engine is not None: + if hasattr(opts, "reporting_db_engine") and \ + opts.reporting_db_engine is not None: settings['DATABASES']['Reporting'] = dict( ENGINE="django.db.backends.%s" % opts.reporting_db_engine, NAME=opts.reporting_db_name, -- cgit v1.2.3-1-g7c22 From 5f2e902a4dc8482a91adde7e2dffd7703051e968 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 26 Mar 2014 16:13:45 -0400 Subject: doc: Document new reporting database separation feature --- doc/reports/dynamic.txt | 21 +++++++++++-- doc/server/database.txt | 82 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 78 insertions(+), 25 deletions(-) diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 6b8a1f467..53bdef24e 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -25,6 +25,7 @@ configuration. Specific features in the new system include: users to drill down to find out about a :ref:`specific host `, rather than only having one huge page with too much information. +* Ability to store reporting data separately from other server data. Installation ============ @@ -214,8 +215,8 @@ database ^^^^^^^^ If you choose to use a different database, you'll need to edit -``/etc/bcfg2.conf``. These fields should be updated in the [database] -section: +``/etc/bcfg2.conf``. These fields should be updated in the +``[database]`` section: * engine @@ -228,11 +229,27 @@ section: * host * port (optional) +To store reporting data separately from the main server data, use +the following options: + +* reporting_engine + + * ex: reporting_engine = mysql + * ex: reporting_engine = postgresql_psycopg2 + +* reporting_name +* reporting_user +* reporting_password +* reporting_host +* reporting_port (optional) + .. warning:: If mysql is used as a backend, it is recommended to use InnoDB for the `storage engine `_. +Refer to :ref:`server-database` for a full listing of +available options. statistics ^^^^^^^^^^ diff --git a/doc/server/database.txt b/doc/server/database.txt index 3c8970f68..fdb3ab04c 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-database: @@ -25,35 +26,70 @@ settings in ``bcfg2.conf``, described in this document, to connect. recommended that you use a higher performance database backend. +Separate Reporting Database +=========================== + +.. versionadded:: 1.4.0 + +Bcfg2 supports storing the data generated by the +:ref:`server-plugins-statistics-reporting` in a separate database +from the data generated by the other plugins (e.g. +:ref:`server-plugins-grouping-metadata` and +:ref:`server-plugins-probes-index`). To activate this support, set +the ``reporting_engine``, ``reporting_name``, ``reporting_user``, etc. +options in the ``[database]`` section of the config file. The valid values +for the ``reporting_*`` options are the same as for the standard database +options. See :ref:`server-database-configuration-options` for a full listing. + +.. _server-database-configuration-options: + Configuration Options ===================== All of the following options should go in the ``[database]`` section of ``/etc/bcfg2.conf``. -+-------------+------------------------------------------------------------+-------------------------------+ -| Option name | Description | Default | -+=============+============================================================+===============================+ -| engine | The name of the Django database backend to use. See | "sqlite3" | -| | https://docs.djangoproject.com/en/dev/ref/settings/#engine | | -| | for available options (note that django.db.backends is not | | -| | included in the engine name) | | -+-------------+------------------------------------------------------------+-------------------------------+ -| name | The name of the database | "/var/lib/bcfg2/bcfg2.sqlite" | -+-------------+------------------------------------------------------------+-------------------------------+ -| user | The user to connect to the database as | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| password | The password to connect to the database with | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| host | The host to connect to | "localhost" | -+-------------+------------------------------------------------------------+-------------------------------+ -| port | The port to connect to | None | -+-------------+------------------------------------------------------------+-------------------------------+ -| options | Extra parameters to use when connecting to the database. | None | -| | Available parameters vary depending on your database | | -| | backend. The parameters are supplied as comma separated | | -| | key=value pairs. | | -+-------------+------------------------------------------------------------+-------------------------------+ ++--------------------+------------------------------------------------------------+---------------------------------------+ +| Option name | Description | Default | ++====================+============================================================+=======================================+ +| engine | The name of the Django database backend to use. See | "sqlite3" | +| | https://docs.djangoproject.com/en/dev/ref/settings/#engine | | +| | for available options (note that django.db.backends is not | | +| | included in the engine name) | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| name | The name of the database | "/var/lib/bcfg2/etc/bcfg2.sqlite" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| user | The user to connect to the database as | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| password | The password to connect to the database with | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| host | The host to connect to | "localhost" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| port | The port to connect to | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as comma separated | | +| | key=value pairs. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_engine | The name of the Django database backend to use for the | None | +| | reporting database. Takes the same values as ``engine``. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_name | The name of the reporting database | "/var/lib/bcfg2/etc/reporting.sqlite" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_user | The user to connect to the reporting database as | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_password | The password to connect to the reporting database with | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_host | The host to connect to for the reporting database | "localhost" | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_port | The port to connect to for the reporting database | None | ++--------------------+------------------------------------------------------------+---------------------------------------+ +| reporting_options | Extra parameters to use when connecting to the reporting | None | +| | database. Available parameters vary depending on your | | +| | database backend. The parameters are supplied as comma | | +| | separated key=value pairs. | | ++--------------------+------------------------------------------------------------+---------------------------------------+ Database Schema Sync -- cgit v1.2.3-1-g7c22 From 9c367cade3afc0738811eda7f854e5f08bbd9e23 Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 26 Mar 2014 16:26:04 -0400 Subject: doc: also update the bcfg2.conf man page for separate reporting --- doc/man/bcfg2.conf.txt | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index df49f3d4a..36776b5cb 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -632,7 +632,7 @@ Server-only, specified in the **[database]** section. These options control the database connection of the server. engine - The database engine used by the statistics module. One of the + The database engine used by server plugins. One of the following:: postgresql @@ -641,9 +641,9 @@ control the database connection of the server. ado_mssql name - The name of the database to use for statistics data. If + The name of the database to use for server data. If 'database_engine' is set to 'sqlite3' this is a file path to - the sqlite file and defaults to ``$REPOSITORY_DIR/etc/brpt.sqlite``. + the sqlite file and defaults to ``$REPOSITORY_DIR/etc/bcfg2.sqlite``. user User for database connections. Not used for sqlite3. @@ -662,6 +662,41 @@ control the database connection of the server. expected as multiple key=value pairs, separated with commas. The concrete value depends on the database engine. + reporting_engine + The database engine used by the Reporting plugin. One of the + following:: + + postgresql + mysql + sqlite3 + ado_mssql + + If reporting_engine is not specified, the Reporting plugin uses + the same database as the other server plugins. + + reporting_name + The name of the database to use for reporting data. If + 'database_engine' is set to 'sqlite3' this is a file path to + the sqlite file and defaults to + ``$REPOSITORY_DIR/etc/reporting.sqlite``. + + reporting_user + User for reporting database connections. Not used for sqlite3. + + reporting_password + Password for reporting database connections. Not used for sqlite3. + + reporting_host + Host for reporting database connections. Not used for sqlite3. + + reporting_port + Port for reporting database connections. Not used for sqlite3. + + reporting_options + Various options for the reporting database connection. The value + is expected as multiple key=value pairs, separated with commas. + The concrete value depends on the database engine. + Reporting options ----------------- -- cgit v1.2.3-1-g7c22 From f94aa46296a8cf1cdd66c7f5802b7bd8a5c70c0a Mon Sep 17 00:00:00 2001 From: Michael Fenn Date: Wed, 26 Mar 2014 16:30:02 -0400 Subject: Vanity copyright update --- COPYRIGHT | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/COPYRIGHT b/COPYRIGHT index 379ddaa4b..32c5705ef 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -157,7 +157,8 @@ add themselves to this file. See LICENSE for the full license. - Zach Lowry wrote Solaris support and general hardening. -- Michael Fenn fixed various small bugs +- Michael Fenn implemented the database router + for separately storing the reporting database and fixed various small bugs related to bcfg2 on CentOS 5 - Alexander Sulfrian fixed various bugs. -- cgit v1.2.3-1-g7c22