Merge reporting configuration with main server configuration

Admin/Syncdb: Use SchemaUpdater
Move the schema update routines from reports to Bcfg2.Server
Move Reports.settings into Bcfg2.settings

15 files changed, 65 insertions, 223 deletions

diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py
index 8201ad104..175b99d1d 100644
--- a/src/lib/Bcfg2/Server/Admin/Reports.py
+++ b/src/lib/Bcfg2/Server/Admin/Reports.py
@@ -19,17 +19,15 @@ if sys.version_info >= (2, 5):
 else:
     from md5 import md5
 
-# Prereq issues can be signaled with ImportError, so no try needed
-# FIXME - settings file uses a hardcoded path for /etc/bcfg2.conf
-import Bcfg2.Server.Reports.settings
+import Bcfg2.settings
 
 # Load django and reports stuff _after_ we know we can load settings
 import django.core.management
 from Bcfg2.Server.Reports.importscript import load_stats
-from Bcfg2.Server.Reports.Updater import update_database, UpdaterError
+from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError
 from Bcfg2.Server.Reports.utils import *
 
-project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__)
+project_directory = os.path.dirname(Bcfg2.settings.__file__)
 project_name = os.path.basename(project_directory)
 sys.path.append(os.path.join(project_directory, '..'))
 project_module = __import__(project_name, '', '', [''])
@@ -281,7 +279,7 @@ class Reports(Bcfg2.Server.Admin.Mode):
             self.log.debug("Filtering by maxdate: %s" % maxdate)
             ipurge = ipurge.filter(timestamp__lt=maxdate)
 
-        if Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
+        if Bcfg2.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
             grp_limit = 100
         else:
             grp_limit = 1000
diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py
index 73dc5b8b2..c9bba0810 100644
--- a/src/lib/Bcfg2/Server/Admin/Syncdb.py
+++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py
@@ -1,6 +1,7 @@
 import Bcfg2.settings
 import Bcfg2.Options
 import Bcfg2.Server.Admin
+from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError
 from django.core.management import setup_environ
 
 class Syncdb(Bcfg2.Server.Admin.Mode):
@@ -10,7 +11,11 @@ class Syncdb(Bcfg2.Server.Admin.Mode):
     options = {'configfile': Bcfg2.Options.CFILE,
                'repo': Bcfg2.Options.SERVER_REPOSITORY}
 
+    def __init__(self, setup):
+        Bcfg2.Server.Admin.Mode.__init__(self, setup)
+
     def __call__(self, args):
+        import Bcfg2.Server.Admin
         Bcfg2.Server.Admin.Mode.__call__(self, args)
 
         # Parse options
@@ -27,7 +32,8 @@ class Syncdb(Bcfg2.Server.Admin.Mode):
         import Bcfg2.Server.models
         Bcfg2.Server.models.load_models(cfile=self.opts['configfile'])
 
-        from django.core.management.commands import syncdb
-
-        cmd = syncdb.Command()
-        cmd.handle_noargs(interactive=False)
+        try:
+            update_database()
+        except UpdaterError:
+            print "Update failed"
+            raise SystemExit(-1)
diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py
index 34e0d4889..7e0e33024 100644
--- a/src/lib/Bcfg2/Server/Core.py
+++ b/src/lib/Bcfg2/Server/Core.py
@@ -105,6 +105,23 @@ class BaseCore(object):
         # load plugins
         Bcfg2.settings.read_config(cfile=self.cfile, repo=self.datastore)
 
+        # verify our database schema
+        try:
+            from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError
+            update_database()
+        except ImportError:
+            # assume django is not installed
+            pass
+        except UpdaterError:
+            self.logger.error("Failed to update database schema")
+            raise CoreInitError
+        except Exception:
+            inst = sys.exc_info()[1]
+            self.logger.error("Failed to update database schema")
+            self.logger.error(str(inst))
+            self.logger.error(str(type(inst)))
+            raise CoreInitError
+
         if '' in setup['plugins']:
             setup['plugins'].remove('')
 
diff --git a/src/lib/Bcfg2/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py
index ca948aabd..40ab11208 100644
--- a/src/lib/Bcfg2/Server/Plugins/DBStats.py
+++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py
@@ -14,8 +14,6 @@ except ImportError:
 import Bcfg2.Server.Plugin
 from Bcfg2.Server.Reports.importscript import load_stat
 from Bcfg2.Server.Reports.reports.models import Client
-import Bcfg2.Server.Reports.settings
-from Bcfg2.Server.Reports.Updater import update_database, UpdaterError
 # for debugging output only
 logger = logging.getLogger('Bcfg2.Plugins.DBStats')
 
@@ -33,14 +31,6 @@ class DBStats(Bcfg2.Server.Plugin.Plugin,
         self.core = core
         logger.debug("Searching for new models to "
                      "add to the statistics database")
-        try:
-            update_database()
-        except UpdaterError:
-            raise Bcfg2.Server.Plugin.PluginInitError
-        except Exception:
-            inst = sys.exc_info()[1]
-            logger.debug(str(inst))
-            logger.debug(str(type(inst)))
 
     def handle_statistic(self, metadata, data):
         newstats = data.find("Statistics")
diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py
index 14f2bb1f9..289970244 100755
--- a/src/lib/Bcfg2/Server/Reports/importscript.py
+++ b/src/lib/Bcfg2/Server/Reports/importscript.py
@@ -9,13 +9,13 @@ import os
 import sys
 import traceback
 try:
-    import Bcfg2.Server.Reports.settings
+    import Bcfg2.settings
 except Exception:
     e = sys.exc_info()[1]
     sys.stderr.write("Failed to load configuration settings. %s\n" % e)
     sys.exit(1)
 
-project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__)
+project_directory = os.path.dirname(Bcfg2.settings.__file__)
 project_name = os.path.basename(project_directory)
 sys.path.append(os.path.join(project_directory, '..'))
 project_module = __import__(project_name, '', '', [''])
@@ -30,7 +30,6 @@ from datetime import datetime
 from time import strptime
 from django.db import connection, transaction
 from Bcfg2.Server.Plugins.Metadata import ClientMetadata
-from Bcfg2.Server.Reports.Updater import update_database, UpdaterError
 import logging
 import Bcfg2.Logger
 import platform
@@ -321,6 +320,9 @@ if __name__ == '__main__':
         encoding = 'UTF-8'
 
     q = '-O3' in sys.argv
+
+    # don't load this at the top.  causes a circular import error
+    from Bcfg2.Server.SchemaUpdater import update_database, UpdaterError
     # Be sure the database is ready for new schema
     try:
         update_database()
diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py
index 4b078eb2c..73adaaaaf 100644
--- a/src/lib/Bcfg2/Server/Reports/reports/models.py
+++ b/src/lib/Bcfg2/Server/Reports/reports/models.py
@@ -342,18 +342,6 @@ class Performance(models.Model):
         transaction.set_dirty()
 
 
-class InternalDatabaseVersion(models.Model):
-    """Object that tell us to witch version is the database."""
-    version = models.IntegerField()
-    updated = models.DateTimeField(auto_now_add=True)
-
-    def __str__(self):
-        return "version %d updated the %s" % (self.version, self.updated.isoformat())
-
-    class Meta:
-        get_latest_by = "version"
-
-
 class Group(models.Model):
     """
     Groups extracted from interactions
diff --git a/src/lib/Bcfg2/Server/Reports/settings.py b/src/lib/Bcfg2/Server/Reports/settings.py
deleted file mode 100644
index 26138cddb..000000000
--- a/src/lib/Bcfg2/Server/Reports/settings.py
+++ /dev/null
@@ -1,173 +0,0 @@
-import os
-import sys
-import getopt
-import Bcfg2.Options
-
-try:
-    import django
-except ImportError:
-    raise ImportError('Import of Django module failed. Is Django installed?')
-
-cfile_opt=Bcfg2.Options.CFILE
-cfiles=[cfile_opt.default, '/etc/bcfg2-web.conf']
-for i in range(1, len(sys.argv)):
-    if sys.argv[i] == cfile_opt.cmd:
-        cfiles = sys.argv[i+1]
-        break
-
-# Compatibility import
-from Bcfg2.Bcfg2Py3k import ConfigParser
-# Django settings for bcfg2 reports project.
-c = ConfigParser.ConfigParser()
-if len(c.read(cfiles)) == 0:
-    raise ImportError("Please check that bcfg2.conf or bcfg2-web.conf exists "
-                      "and is readable by your web server.")
-
-try:
-    DEBUG = c.getboolean('statistics', 'web_debug')
-except:
-    DEBUG = False
-
-if DEBUG:
-    print("Warning: Setting web_debug to True causes extraordinary memory "
-          "leaks.  Only use this setting if you know what you're doing.")
-
-TEMPLATE_DEBUG = DEBUG
-
-ADMINS = (
-     ('Root', 'root'),
-)
-
-MANAGERS = ADMINS
-try:
-    db_engine = c.get('statistics', 'database_engine')
-except ConfigParser.NoSectionError:
-    e = sys.exc_info()[1]
-    raise ImportError("Failed to determine database engine for reports: %s" % e)
-db_name = ''
-if c.has_option('statistics', 'database_name'):
-    db_name = c.get('statistics', 'database_name')
-if db_engine == 'sqlite3' and db_name == '':
-    db_name = "%s/etc/brpt.sqlite" % c.get('server', 'repository')
-
-DATABASES = {
-    'default': {
-        'ENGINE': "django.db.backends.%s" % db_engine,
-        'NAME': db_name
-    }
-}
-
-if db_engine == 'ibm_db_django':
-    DATABASES['default']['ENGINE'] = db_engine
-
-if db_engine != 'sqlite3':
-    DATABASES['default']['USER'] =  c.get('statistics', 'database_user')
-    DATABASES['default']['PASSWORD'] = c.get('statistics', 'database_password')
-    DATABASES['default']['HOST'] = c.get('statistics', 'database_host')
-    try:
-        DATABASES['default']['PORT'] = c.get('statistics', 'database_port')
-    except: # An empty string tells Django to use the default port.
-        DATABASES['default']['PORT'] = ''
-
-if django.VERSION[0] == 1 and django.VERSION[1] < 2:
-    DATABASE_ENGINE = db_engine
-    DATABASE_NAME = DATABASES['default']['NAME']
-    if DATABASE_ENGINE != 'sqlite3':
-        DATABASE_USER = DATABASES['default']['USER']
-        DATABASE_PASSWORD = DATABASES['default']['PASSWORD']
-        DATABASE_HOST = DATABASES['default']['HOST']
-        DATABASE_PORT = DATABASES['default']['PORT']
-
-
-# Local time zone for this installation. All choices can be found here:
-# http://docs.djangoproject.com/en/dev/ref/settings/#time-zone
-if django.VERSION[0] == 1 and django.VERSION[1] > 2:
-    try:
-        TIME_ZONE = c.get('statistics', 'time_zone')
-    except:
-        TIME_ZONE = None
-
-# Language code for this installation. All choices can be found here:
-# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes
-# http://blogs.law.harvard.edu/tech/stories/storyReader$15
-LANGUAGE_CODE = 'en-us'
-
-SITE_ID = 1
-
-# Absolute path to the directory that holds media.
-# Example: "/home/media/media.lawrence.com/"
-MEDIA_ROOT = ''
-
-# URL that handles the media served from MEDIA_ROOT.
-# Example: "http://media.lawrence.com"
-MEDIA_URL = '/site_media'
-if c.has_option('statistics', 'web_prefix'):
-    MEDIA_URL = c.get('statistics', 'web_prefix').rstrip('/') + MEDIA_URL
-
-# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
-# trailing slash.
-# Examples: "http://foo.com/media/", "/media/".
-ADMIN_MEDIA_PREFIX = '/media/'
-
-# Make this unique, and don't share it with anybody.
-SECRET_KEY = 'eb5+y%oy-qx*2+62vv=gtnnxg1yig_odu0se5$h0hh#pc*lmo7'
-
-# List of callables that know how to import templates from various sources.
-TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.load_template_source',
-    'django.template.loaders.app_directories.load_template_source',
-)
-
-MIDDLEWARE_CLASSES = (
-    'django.middleware.common.CommonMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.middleware.doc.XViewMiddleware',
-)
-
-ROOT_URLCONF = 'Bcfg2.Server.Reports.urls'
-
-# Authentication Settings
-# Use NIS authentication backend defined in backends.py
-AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend',
-                           'Bcfg2.Server.Reports.backends.NISBackend')
-# The NIS group authorized to login to BCFG2's reportinvg system
-AUTHORIZED_GROUP = ''
-#create login url area:
-LOGIN_URL = '/login'
-
-SESSION_EXPIRE_AT_BROWSER_CLOSE = True
-
-
-
-TEMPLATE_DIRS = (
-    # Put strings here, like "/home/html/django_templates".
-    # Always use forward slashes, even on Windows.
-    '/usr/share/python-support/python-django/django/contrib/admin/templates/',
-)
-
-if django.VERSION[0] == 1 and django.VERSION[1] < 2:
-    TEMPLATE_CONTEXT_PROCESSORS = (
-        'django.core.context_processors.auth',
-        'django.core.context_processors.debug',
-        'django.core.context_processors.i18n',
-        'django.core.context_processors.media',
-        'django.core.context_processors.request'
-    )
-else:
-    TEMPLATE_CONTEXT_PROCESSORS = (
-        'django.contrib.auth.context_processors.auth',
-        'django.core.context_processors.debug',
-        'django.core.context_processors.i18n',
-        'django.core.context_processors.media',
-        'django.core.context_processors.request'
-    )
-
-INSTALLED_APPS = (
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.sites',
-    'django.contrib.admin',
-    'Bcfg2.Server.Reports.reports'
-)
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py
index 54ba07554..ff4c24328 100644
--- a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_0_x.py
@@ -4,7 +4,7 @@
 This file should contain updates relevant to the 1.0.x branches ONLY.
 The updates() method must be defined and it should return an Updater object
 """
-from Bcfg2.Server.Reports.Updater import UnsupportedUpdate
+from Bcfg2.Server.SchemaUpdater import UnsupportedUpdate
 
 def updates():
     return UnsupportedUpdate("1.0", 10)
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py
index 26194cb67..0d28786fd 100644
--- a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_1_x.py
@@ -4,12 +4,12 @@
 This file should contain updates relevant to the 1.1.x branches ONLY.
 The updates() method must be defined and it should return an Updater object
 """
-from Bcfg2.Server.Reports.Updater import Updater
-from Bcfg2.Server.Reports.Updater.Routines import updatercallable
+from Bcfg2.Server.SchemaUpdater import Updater
+from Bcfg2.Server.SchemaUpdater.Routines import updatercallable
 
 from django.db import connection
 import sys
-import Bcfg2.Server.Reports.settings
+import Bcfg2.settings
 from Bcfg2.Server.Reports.reports.models import \
                 TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA
 
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py
index 22bd937c2..024965bd5 100644
--- a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_2_x.py
@@ -4,8 +4,8 @@
 This file should contain updates relevant to the 1.2.x branches ONLY.
 The updates() method must be defined and it should return an Updater object
 """
-from Bcfg2.Server.Reports.Updater import Updater
-from Bcfg2.Server.Reports.Updater.Routines import updatercallable
+from Bcfg2.Server.SchemaUpdater import Updater
+from Bcfg2.Server.SchemaUpdater.Routines import updatercallable
 
 def updates():
     fixes = Updater("1.2")
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py
index 1a2fff1ea..4fc57c653 100644
--- a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/1_3_0.py
@@ -4,8 +4,8 @@
 This file should contain updates relevant to the 1.3.x branches ONLY.
 The updates() method must be defined and it should return an Updater object
 """
-from Bcfg2.Server.Reports.Updater import Updater, UpdaterError
-from Bcfg2.Server.Reports.Updater.Routines import AddColumns, \
+from Bcfg2.Server.SchemaUpdater import Updater, UpdaterError
+from Bcfg2.Server.SchemaUpdater.Routines import AddColumns, \
         RemoveColumns, RebuildTable, DropTable
 
 from Bcfg2.Server.Reports.reports.models import Reason, Interaction
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py
index e69de29bb..e69de29bb 100644
--- a/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/Changes/__init__.py
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Routines.py b/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py
index edb21c321..542a1302e 100644
--- a/src/lib/Bcfg2/Server/Reports/Updater/Routines.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/Routines.py
@@ -6,7 +6,7 @@ from django.core.management.color import no_style
 from django.core.management.sql import sql_create
 import django.core.management
 
-import Bcfg2.Server.Reports.settings
+import Bcfg2.settings
 
 logger = logging.getLogger(__name__)
 
@@ -116,7 +116,7 @@ class AddColumns(UpdaterRoutine):
             sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % (
                     _quote(self.model._meta.db_table),
                     _quote(field.column), field.db_type(), )
-            db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
+            db_engine = Bcfg2.settings.DATABASES['default']['ENGINE']
             if db_engine == 'django.db.backends.sqlite3':
                 sql += _quote(field.default)
                 sql_values = ()
@@ -159,7 +159,7 @@ class RebuildTable(UpdaterRoutine):
             logger.error("Failed to connect to the db")
             raise UpdaterRoutineException
 
-        db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
+        db_engine = Bcfg2.settings.DATABASES['default']['ENGINE']
         if db_engine == 'django.db.backends.sqlite3':
             """ Sqlite is a special case.  Altering columns is not supported. """
             _rebuild_sqlite_table(self.model)
@@ -219,7 +219,7 @@ class RemoveColumns(RebuildTable):
 
             logger.debug("Dropping column %s" % column)
 
-            db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE']
+            db_engine = Bcfg2.DATABASES['default']['ENGINE']
             if db_engine == 'django.db.backends.sqlite3':
                 _rebuild_sqlite_table(self.model)
             else:
diff --git a/src/lib/Bcfg2/Server/Reports/Updater/__init__.py b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py
index 3038e9691..ac33724a0 100644
--- a/src/lib/Bcfg2/Server/Reports/Updater/__init__.py
+++ b/src/lib/Bcfg2/Server/SchemaUpdater/__init__.py
@@ -6,10 +6,10 @@ import re
 import sys
 import traceback
 
-from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion
-from Bcfg2.Server.Reports.Updater.Routines import UpdaterRoutineException, \
+from Bcfg2.Server.models import InternalDatabaseVersion
+from Bcfg2.Server.SchemaUpdater.Routines import UpdaterRoutineException, \
                 UpdaterRoutine
-from Bcfg2.Server.Reports.Updater import Changes
+from Bcfg2.Server.SchemaUpdater import Changes
 
 logger = logging.getLogger(__name__)
 
diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py
index ba9ea761c..96a7261fd 100644
--- a/src/lib/Bcfg2/Server/models.py
+++ b/src/lib/Bcfg2/Server/models.py
@@ -60,3 +60,17 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True):
 # basic invocation to ensure that a default set of models is loaded,
 # and thus that this module will always work.
 load_models(quiet=True)
+
+# Monitor our internal db version
+class InternalDatabaseVersion(models.Model):
+    """Object that tell us to witch version is the database."""
+    version = models.IntegerField()
+    updated = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return "version %d updated the %s" % (self.version, self.updated.isoformat())
+
+    class Meta:
+        app_label = "reports"
+        get_latest_by = "version"
+