From fe2e0a3ddbe05e5eace11268ddf909ed386438d0 Mon Sep 17 00:00:00 2001 From: Jonas Jochmaring Date: Fri, 22 May 2015 15:29:02 +0200 Subject: make Bcfg2-web compatible with django 1.7 - reports.wsgi uses get_wsgi_application() now - old south-based migrations have been moved - manage.py has been updated --- src/lib/Bcfg2/Server/Admin.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index c294e6be5..0eba843c7 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -25,15 +25,19 @@ import Bcfg2.Server.Plugins.Metadata try: from django.core.exceptions import ImproperlyConfigured from django.core import management + import django import django.conf import Bcfg2.Server.models HAS_DJANGO = True - try: - import south # pylint: disable=W0611 + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: HAS_REPORTS = True - except ImportError: - HAS_REPORTS = False + elif django.VERSION[0] == 1 and django.VERSION[1] <= 6: + try: + import south # pylint: disable=W0611 + HAS_REPORTS = True + except ImportError: + HAS_REPORTS = False except ImportError: HAS_DJANGO = False HAS_REPORTS = False @@ -1194,6 +1198,10 @@ class CLI(Bcfg2.Options.CommandRegistry): components=[self]) parser.add_options(self.subcommand_options) parser.parse() + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + # this has been introduced in django 1.7, so pylint fails with + # older django releases + django.setup() # pylint disable=E1101 def run(self): """ Run bcfg2-admin """ -- cgit v1.2.3-1-g7c22 From 07672ad6e6985b029c0ccf750f63576b488b95d7 Mon Sep 17 00:00:00 2001 From: Jonas Jochmaring Date: Mon, 6 Jul 2015 14:51:48 +0200 Subject: some more django 1.7 compatibility fixes --- src/lib/Bcfg2/Server/Admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 0eba843c7..b7ee0c0ed 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -32,7 +32,7 @@ try: HAS_DJANGO = True if django.VERSION[0] == 1 and django.VERSION[1] >= 7: HAS_REPORTS = True - elif django.VERSION[0] == 1 and django.VERSION[1] <= 6: + else: try: import south # pylint: disable=W0611 HAS_REPORTS = True @@ -904,6 +904,9 @@ if HAS_DJANGO: class Syncdb(AdminCmd): """ Sync the Django ORM with the configured database """ + if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 7: + django.setup() + def run(self, setup): Bcfg2.Server.models.load_models() try: -- cgit v1.2.3-1-g7c22 From bda4daf4c2c5dad964c8f2e9c777fdea000d0731 Mon Sep 17 00:00:00 2001 From: Jonas Jochmaring Date: Thu, 9 Jul 2015 16:14:08 +0200 Subject: added django.setup() calls to src/lib/Bcfg2/DBSettings.py --- src/lib/Bcfg2/Server/Admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index b7ee0c0ed..1b2a2fe14 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -905,7 +905,7 @@ if HAS_DJANGO: """ Sync the Django ORM with the configured database """ if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 7: - django.setup() + django.setup() # pylint: disable=E1101 def run(self, setup): Bcfg2.Server.models.load_models() @@ -1204,7 +1204,7 @@ class CLI(Bcfg2.Options.CommandRegistry): if django.VERSION[0] == 1 and django.VERSION[1] >= 7: # this has been introduced in django 1.7, so pylint fails with # older django releases - django.setup() # pylint disable=E1101 + django.setup() # pylint: disable=E1101 def run(self): """ Run bcfg2-admin """ -- cgit v1.2.3-1-g7c22 From cf740e06f4131c8cff33727a5e98d5d3eb6f3e21 Mon Sep 17 00:00:00 2001 From: Jonas Jochmaring Date: Thu, 9 Jul 2015 18:12:20 +0200 Subject: fix check for reporting in src/lib/Bcfg2/Server/Plugins/Reporting.py --- src/lib/Bcfg2/Server/Plugins/Reporting.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index 5c73546b4..b9dcdcc81 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -8,13 +8,17 @@ import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ PluginInitError, PluginExecutionError +import django # required for reporting -try: - import south # pylint: disable=W0611 - HAS_SOUTH = True -except ImportError: - HAS_SOUTH = False +if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + HAS_REPORTING = True +else: + try: + import south # pylint: disable=W0611 + HAS_REPORTING = True + except ImportError: + HAS_REPORTING = False def _rpc_call(method): @@ -48,8 +52,8 @@ class Reporting(Statistics, Threaded, PullSource): self.whoami = platform.node() self.transport = None - if not HAS_SOUTH: - msg = "Django south is required for Reporting" + if not HAS_REPORTING: + msg = "Django 1.7+ or Django south is required for Reporting" self.logger.error(msg) raise PluginInitError(msg) -- cgit v1.2.3-1-g7c22 From 2d10367098b1e7424a48de670c6a936f2b4dbb0f Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 16 Sep 2015 19:06:59 +0200 Subject: Reporting: Clear error message if django is missing --- src/lib/Bcfg2/Server/Plugins/Reporting.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Plugins/Reporting.py b/src/lib/Bcfg2/Server/Plugins/Reporting.py index b9dcdcc81..e372006c7 100644 --- a/src/lib/Bcfg2/Server/Plugins/Reporting.py +++ b/src/lib/Bcfg2/Server/Plugins/Reporting.py @@ -8,17 +8,16 @@ import Bcfg2.Options from Bcfg2.Reporting.Transport.base import TransportError from Bcfg2.Server.Plugin import Statistics, PullSource, Threaded, \ PluginInitError, PluginExecutionError -import django -# required for reporting -if django.VERSION[0] == 1 and django.VERSION[1] >= 7: - HAS_REPORTING = True -else: - try: +try: + import django + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + HAS_REPORTING = True + else: import south # pylint: disable=W0611 HAS_REPORTING = True - except ImportError: - HAS_REPORTING = False +except ImportError: + HAS_REPORTING = False def _rpc_call(method): -- cgit v1.2.3-1-g7c22 From 96bdd192700aa1f638f47e96170c9ccd2821c615 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 21 Sep 2015 19:46:26 +0200 Subject: Server/Admin: Remove call of django.setup django.setup() is already called from the option parsing before. --- src/lib/Bcfg2/Server/Admin.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index 1b2a2fe14..c6924ef6c 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -904,9 +904,6 @@ if HAS_DJANGO: class Syncdb(AdminCmd): """ Sync the Django ORM with the configured database """ - if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] >= 7: - django.setup() # pylint: disable=E1101 - def run(self, setup): Bcfg2.Server.models.load_models() try: -- cgit v1.2.3-1-g7c22 From b5a966bdf8bed555e7c79b2cb77c14463aade22d Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Mon, 21 Sep 2015 20:45:52 +0200 Subject: Server/Admin: Remove load_models() call The models should be loaded during the options parsing and so this call issues a warning message from django, that the models are already registered. --- src/lib/Bcfg2/Server/Admin.py | 1 - 1 file changed, 1 deletion(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Admin.py b/src/lib/Bcfg2/Server/Admin.py index c6924ef6c..6bb973173 100644 --- a/src/lib/Bcfg2/Server/Admin.py +++ b/src/lib/Bcfg2/Server/Admin.py @@ -905,7 +905,6 @@ if HAS_DJANGO: """ Sync the Django ORM with the configured database """ def run(self, setup): - Bcfg2.Server.models.load_models() try: Bcfg2.DBSettings.sync_databases( interactive=False, -- cgit v1.2.3-1-g7c22 From 0304e24b8011b96d5d9d3efad16d45f10a4c7709 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Tue, 22 Sep 2015 23:51:28 +0200 Subject: Server: Use close_db_connection decorator everywhere --- src/lib/Bcfg2/Server/Core.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index dc9c91556..4592688e7 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -430,6 +430,7 @@ class Core(object): self.logger.error("Unexpected instantiation failure for plugin %s" % plugin, exc_info=1) + @close_db_connection def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ if not self._running: @@ -444,10 +445,6 @@ class Core(object): for plugin in list(self.plugins.values()): plugin.shutdown() self.logger.info("%s: All plugins shut down" % self.name) - if self._database_available: - from django import db - self.logger.info("%s: Closing database connection" % self.name) - db.close_connection() @property def metadata_cache_mode(self): -- cgit v1.2.3-1-g7c22 From 7dfe4ea2f6c62ffb662085cb4d62a13b8e902ce1 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Sep 2015 00:04:27 +0200 Subject: Replace close_connection() for newer django versions django.db.close_connection() is deprecated in django1.7 (and removed in 1.8). The new django.db.close_old_connections() does not seem to work like the old one (see http://stackoverflow.com/a/32614137), so we replace it with an own implementation. --- src/lib/Bcfg2/Server/Core.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 4592688e7..87bbe7e1b 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -27,6 +27,7 @@ from Bcfg2.Server.Statistics import track_statistics try: from django.core.exceptions import ImproperlyConfigured + import django import django.conf HAS_DJANGO = True except ImportError: @@ -83,10 +84,14 @@ def close_db_connection(func): """ The decorated function """ rv = func(self, *args, **kwargs) if self._database_available: # pylint: disable=W0212 - from django import db self.logger.debug("%s: Closing database connection" % threading.current_thread().getName()) - db.close_connection() + + if django.VERSION[0] == 1 and django.VERSION[1] >= 7: + for connection in django.db.connections.all(): + connection.close() + else: + django.db.close_connection() return rv return inner -- cgit v1.2.3-1-g7c22 From 98e7d57e54ea8122b705e643cce23ccc9aa7cd86 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Sep 2015 18:26:04 +0200 Subject: Reports: Fix InternalDatabaseVersion You could not import a class, that is defined in a function. We need to return the class, but we do not want to define it multiple times. So we have to save the class in a global variable. --- src/lib/Bcfg2/Server/Reports/updatefix.py | 8 ++++---- src/lib/Bcfg2/Server/models.py | 29 +++++++++++++++++++---------- 2 files changed, 23 insertions(+), 14 deletions(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py index 91c370994..09b218464 100644 --- a/src/lib/Bcfg2/Server/Reports/updatefix.py +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -4,7 +4,7 @@ import django.core.management import sys import logging import traceback -from Bcfg2.Server.models import InternalDatabaseVersion +from Bcfg2.Server.models import internal_database_version logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') @@ -138,7 +138,7 @@ def rollupdate(current_version): exc_info=1) # since array start at 0 but version start at 1 # we add 1 to the normal count - ret = InternalDatabaseVersion.objects.create(version=i + 1) + ret = internal_database_version().create(version=i + 1) return ret else: return None @@ -149,10 +149,10 @@ def update_database(): try: logger.debug("Running upgrade of models to the new one") django.core.management.call_command("syncdb", interactive=False, verbosity=0) - know_version = InternalDatabaseVersion.objects.order_by('-version') + know_version = internal_database_version().order_by('-version') if not know_version: logger.debug("No version, creating initial version") - know_version = InternalDatabaseVersion.objects.create(version=lastversion) + know_version = internal_database_version().create(version=lastversion) else: know_version = know_version[0] logger.debug("Presently at %s" % know_version) diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 8d6642a25..9c0153c74 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -8,6 +8,7 @@ import Bcfg2.Server.Plugins LOGGER = logging.getLogger(__name__) MODELS = [] +INTERNAL_DATABASE_VERSION = None class _OptionContainer(object): @@ -56,15 +57,23 @@ def load_models(plugins=None): setattr(sys.modules[__name__], sym, obj) MODELS.append(sym) - class InternalDatabaseVersion(models.Model): - """ Object that tell us to which version the database is """ - version = models.IntegerField() - updated = models.DateTimeField(auto_now_add=True) +def internal_database_version(): + global INTERNAL_DATABASE_VERSION - def __str__(self): - return "version %d updated %s" % (self.version, - self.updated.isoformat()) + if INTERNAL_DATABASE_VERSION is None: + from django.db import models + class InternalDatabaseVersion(models.Model): + """ Object that tell us to which version the database is """ + version = models.IntegerField() + updated = models.DateTimeField(auto_now_add=True) - class Meta: # pylint: disable=C0111,W0232 - app_label = "reports" - get_latest_by = "version" + def __str__(self): + return "version %d updated %s" % (self.version, + self.updated.isoformat()) + + class Meta: # pylint: disable=C0111,W0232 + app_label = "reports" + get_latest_by = "version" + INTERNAL_DATABASE_VERSION = InternalDatabaseVersion + + return INTERNAL_DATABASE_VERSION.objects -- cgit v1.2.3-1-g7c22 From e438e6f1003389c290a4811190f8b83a1abc743e Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Wed, 23 Sep 2015 18:31:30 +0200 Subject: Reports: Add explicit default value Django changed the default value for BooleanFields some time ago, so we add an explicit default value to remove the warnings. --- src/lib/Bcfg2/Server/Reports/reports/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib/Bcfg2/Server') diff --git a/src/lib/Bcfg2/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py index ac4c8eac4..67aa425d9 100644 --- a/src/lib/Bcfg2/Server/Reports/reports/models.py +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -266,7 +266,7 @@ class Reason(models.Model): current_to = models.CharField(max_length=1024, blank=True) version = models.CharField(max_length=1024, blank=True) current_version = models.CharField(max_length=1024, blank=True) - current_exists = models.BooleanField() # False means its missing. Default True + current_exists = models.BooleanField(default=True) # False means its missing. current_diff = models.TextField(max_length=1024*1024, blank=True) is_binary = models.BooleanField(default=False) is_sensitive = models.BooleanField(default=False) -- cgit v1.2.3-1-g7c22