From f124f7f6ee5dc8710ba354e0e978a82245c11fbf Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Mon, 4 Apr 2011 13:15:14 -0500 Subject: Reports: PY3K + PEP8 fixes Signed-off-by: Sol Jerome --- src/lib/Server/Reports/backends.py | 21 ++-- src/lib/Server/Reports/importscript.py | 116 +++++++++++++--------- src/lib/Server/Reports/manage.py | 2 +- src/lib/Server/Reports/nisauth.py | 19 ++-- src/lib/Server/Reports/reports/models.py | 102 +++++++++++--------- src/lib/Server/Reports/reports/views.py | 161 +++++++++++++++++++------------ src/lib/Server/Reports/updatefix.py | 30 +++--- src/lib/Server/Reports/utils.py | 20 ++-- src/lib/Server/Snapshots/__init__.py | 2 +- src/lib/Server/Snapshots/model.py | 94 +++++++++++++----- src/lib/Statistics.py | 2 +- 11 files changed, 345 insertions(+), 224 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Reports/backends.py b/src/lib/Server/Reports/backends.py index 9207038ed..9e37a2e6f 100644 --- a/src/lib/Server/Reports/backends.py +++ b/src/lib/Server/Reports/backends.py @@ -1,35 +1,32 @@ from django.contrib.auth.models import User from nisauth import * + class NISBackend(object): def authenticate(self, username=None, password=None): try: - print "start nis authenticate" + print("start nis authenticate") n = nisauth(username, password) temp_pass = User.objects.make_random_password(100) nis_user = dict(username=username, ) - user_session_obj = dict( - email = username, - first_name = None, - last_name = None, - uid = n.uid - ) + user_session_obj = dict(email=username, + first_name=None, + last_name=None, + uid=n.uid) user, created = User.objects.get_or_create(username=username) - + return user except NISAUTHError, e: - print str(e) + print(str(e)) return None - def get_user(self, user_id): try: return User.objects.get(pk=user_id) except User.DoesNotExist, e: - print str(e) + print(str(e)) return None - diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index cdfd8079c..86e176394 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -1,8 +1,13 @@ #! /usr/bin/env python -'''Imports statistics.xml and clients.xml files in to database backend for new statistics engine''' +""" +Imports statistics.xml and clients.xml files in to database backend for +new statistics engine +""" __revision__ = '$Revision$' -import os, sys, binascii +import binascii +import os +import sys try: import Bcfg2.Server.Reports.settings except Exception, e: @@ -29,16 +34,17 @@ import logging import Bcfg2.Logger import platform + def build_reason_kwargs(r_ent): - binary_file=False + binary_file = False if r_ent.get('current_bfile', False): - binary_file=True + binary_file = True rc_diff = r_ent.get('current_bfile') - if len(rc_diff) > 1024*1024: + if len(rc_diff) > 1024 * 1024: rc_diff = '' elif len(rc_diff) == 0: # No point in flagging binary if we have no data - binary_file=False + binary_file = False elif r_ent.get('current_bdiff', False): rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) elif r_ent.get('current_diff', False): @@ -57,7 +63,7 @@ def build_reason_kwargs(r_ent): current_to=r_ent.get('current_to', default=""), version=r_ent.get('version', default=""), current_version=r_ent.get('current_version', default=""), - current_exists=r_ent.get('current_exists', default="True").capitalize()=="True", + current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", current_diff=rc_diff, is_binary=binary_file) @@ -75,7 +81,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): name = node.get('name') c_inst, created = Client.objects.get_or_create(name=name) if vlevel > 0: - logger.info("Client %s added to db" % name) + logger.info("Client %s added to db" % name) clients[name] = c_inst try: pingability[name] @@ -93,24 +99,30 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): continue else: newint = Interaction(client=c_inst, - timestamp = timestamp, - state = statistics.get('state', default="unknown"), - repo_rev_code = statistics.get('revision',default="unknown"), - client_version = statistics.get('client_version',default="unknown"), - goodcount = statistics.get('good',default="0"), - totalcount = statistics.get('total',default="0"), - server = location) + timestamp=timestamp, + state=statistics.get('state', + default="unknown"), + repo_rev_code=statistics.get('revision', + default="unknown"), + client_version=statistics.get('client_version', + default="unknown"), + goodcount=statistics.get('good', + default="0"), + totalcount=statistics.get('total', + default="0"), + server=location) newint.save() current_interaction = newint if vlevel > 0: - logger.info("Interaction for %s at %s with id %s INSERTED in to db"%(c_inst.id, + logger.info("Interaction for %s at %s with id %s INSERTED in to db" % (c_inst.id, timestamp, current_interaction.id)) - - counter_fields = { TYPE_CHOICES[0]: 0, TYPE_CHOICES[1]: 0, TYPE_CHOICES[2]: 0 } + counter_fields = {TYPE_CHOICES[0]: 0, + TYPE_CHOICES[1]: 0, + TYPE_CHOICES[2]: 0} pattern = [('Bad/*', TYPE_CHOICES[0]), ('Extra/*', TYPE_CHOICES[2]), - ('Modified/*', TYPE_CHOICES[1]),] + ('Modified/*', TYPE_CHOICES[1])] for (xpath, type) in pattern: for x in statistics.findall(xpath): counter_fields[type] = counter_fields[type] + 1 @@ -131,12 +143,12 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): except Exception, ex: logger.error("Failed to create reason for %s: %s" % (x.get('name'), ex)) rr = Reason(current_exists=x.get('current_exists', - default="True").capitalize()=="True") + default="True").capitalize() == "True") rr.save() entry, created = Entries.objects.get_or_create(\ name=x.get('name'), kind=x.tag) - + Entries_interactions(entry=entry, reason=rr, interaction=current_interaction, type=type[0]).save() @@ -151,7 +163,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): mperfs = [] for times in statistics.findall('OpStamps'): - for metric, value in times.items(): + for metric, value in list(times.items()): mmatch = [] if not quick: mmatch = Performance.objects.filter(metric=metric, value=value) @@ -164,7 +176,7 @@ def load_stats(cdata, sdata, vlevel, logger, quick=False, location=''): mperfs.append(mperf) current_interaction.performance_items.add(*mperfs) - for key in pingability.keys(): + for key in list(pingability.keys()): if key not in clients: continue try: @@ -191,27 +203,32 @@ if __name__ == '__main__': clientpath = False statpath = False syslog = False - + try: - opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", "verbose", "updates" , - "debug", "clients=", "stats=", - "config=", "syslog"]) + opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", + "verbose", + "updates", + "debug", + "clients=", + "stats=", + "config=", + "syslog"]) except GetoptError, mesg: # print help information and exit: - print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) - raise SystemExit, 2 + print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg)) + raise SystemExit(2) for o, a in opts: if o in ("-h", "--help"): - print "Usage:\nimportscript.py [-h] [-v] -c -s \n" - print "h : help; this message" - print "v : verbose; print messages on record insertion/skip" - print "u : updates; print status messages as items inserted semi-verbose" - print "d : debug; print most SQL used to manipulate database" - print "C : path to bcfg2.conf config file." - print "c : clients.xml file" - print "s : statistics.xml file" - print "S : syslog; output to syslog" + print("Usage:\nimportscript.py [-h] [-v] -c -s \n") + print("h : help; this message") + print("v : verbose; print messages on record insertion/skip") + print("u : updates; print status messages as items inserted semi-verbose") + print("d : debug; print most SQL used to manipulate database") + print("C : path to bcfg2.conf config file.") + print("c : clients.xml file") + print("s : statistics.xml file") + print("S : syslog; output to syslog") raise SystemExit if o in ["-C", "--config"]: cpath = a @@ -243,28 +260,33 @@ if __name__ == '__main__': try: statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 + print("Could not read bcfg2.conf; exiting") + raise SystemExit(1) try: statsdata = XML(open(statpath).read()) except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(statpath)) - raise SystemExit, 1 + print("StatReports: Failed to parse %s" % (statpath)) + raise SystemExit(1) if not clientpath: try: clientspath = "%s/Metadata/clients.xml" % \ cf.get('server', 'repository') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 + print("Could not read bcfg2.conf; exiting") + raise SystemExit(1) try: clientsdata = XML(open(clientspath).read()) except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(clientspath)) - raise SystemExit, 1 + print("StatReports: Failed to parse %s" % (clientspath)) + raise SystemExit(1) q = '-O3' in sys.argv # Be sure the database is ready for new schema update_database() - load_stats(clientsdata, statsdata, verb, logger, quick=q, location=platform.node()) + load_stats(clientsdata, + statsdata, + verb, + logger, + quick=q, + location=platform.node()) diff --git a/src/lib/Server/Reports/manage.py b/src/lib/Server/Reports/manage.py index 5e78ea979..4f84f107a 100755 --- a/src/lib/Server/Reports/manage.py +++ b/src/lib/Server/Reports/manage.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from django.core.management import execute_manager try: - import settings # Assumed to be in the same directory. + from . import settings # Assumed to be in the same directory. except ImportError: import sys sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) diff --git a/src/lib/Server/Reports/nisauth.py b/src/lib/Server/Reports/nisauth.py index b4be0e391..6fc346f1e 100644 --- a/src/lib/Server/Reports/nisauth.py +++ b/src/lib/Server/Reports/nisauth.py @@ -1,15 +1,17 @@ -import os -import crypt, nis +import crypt +import nis from Bcfg2.Server.Reports.settings import AUTHORIZED_GROUP """Checks with NIS to see if the current user is in the support group""" __revision__ = "$Revision: $" + class NISAUTHError(Exception): """NISAUTHError is raised when somehting goes boom.""" pass + class nisauth(object): group_test = False samAcctName = None @@ -18,26 +20,27 @@ class nisauth(object): telephoneNumber = None title = None memberOf = None - department = None #this will be a list + department = None # this will be a list mail = None - extensionAttribute1 = None #badgenumber + extensionAttribute1 = None # badgenumber badge_no = None uid = None - def __init__(self,login,passwd=None): + def __init__(self, login, passwd=None): """get user profile from NIS""" try: p = nis.match(login, 'passwd.byname').split(":") - print p + print(p) except: raise NISAUTHError('username') # check user password using crypt and 2 character salt from passwd file if p[1] == crypt.crypt(passwd, p[1][:2]): # check to see if user is in valid support groups # will have to include these groups in a settings file eventually - if not login in nis.match(AUTHORIZED_GROUP, 'group.byname').split(':')[-1].split(','): + if not login in nis.match(AUTHORIZED_GROUP, + 'group.byname').split(':')[-1].split(','): raise NISAUTHError('group') self.uid = p[2] - print self.uid + print(self.uid) else: raise NISAUTHError('password') diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Server/Reports/reports/models.py index 1963a9090..c2be40870 100644 --- a/src/lib/Server/Reports/reports/models.py +++ b/src/lib/Server/Reports/reports/models.py @@ -29,6 +29,7 @@ TYPE_CHOICES = ( (TYPE_EXTRA, 'Extra'), ) + def convert_entry_type_to_id(type_name): """Convert a entry type to its entry id""" for e_id, e_name in TYPE_CHOICES: @@ -36,23 +37,25 @@ def convert_entry_type_to_id(type_name): return e_id return -1 + class ClientManager(models.Manager): """Extended client manager functions.""" def active(self, timestamp=None): - """returns a set of clients that have been created and have not yet been - expired as of optional timestmamp argument. Timestamp should be a - datetime object.""" - + """returns a set of clients that have been created and have not + yet been expired as of optional timestmamp argument. Timestamp + should be a datetime object.""" + if timestamp == None: timestamp = datetime.now() elif not isinstance(timestamp, datetime): raise ValueError, 'Expected a datetime object' else: try: - timestamp = datetime(*strptime(timestamp, "%Y-%m-%d %H:%M:%S")[0:6]) + timestamp = datetime(*strptime(timestamp, + "%Y-%m-%d %H:%M:%S")[0:6]) except ValueError: return self.none() - + return self.filter(Q(expiration__gt=timestamp) | Q(expiration__isnull=True), creation__lt=timestamp) @@ -65,25 +68,27 @@ class Client(models.Model): null=True, blank=True, related_name="parent_client") expiration = models.DateTimeField(blank=True, null=True) - + def __str__(self): return self.name objects = ClientManager() - + class Admin: pass + class Ping(models.Model): """Represents a ping of a client (sparsely).""" client = models.ForeignKey(Client, related_name="pings") starttime = models.DateTimeField() endtime = models.DateTimeField() - status = models.CharField(max_length=4, choices=PING_CHOICES)#up/down + status = models.CharField(max_length=4, choices=PING_CHOICES) # up/down class Meta: get_latest_by = 'endtime' - + + class InteractiveManager(models.Manager): """Manages interactions objects.""" @@ -94,31 +99,31 @@ class InteractiveManager(models.Manager): This method uses aggregated queries to return a ValuesQueryDict object. Faster then raw sql since this is executed as a single query. """ - - return self.values('client').annotate(max_timestamp=Max('timestamp')).values() - def interaction_per_client(self, maxdate = None, active_only=True): + return list(self.values('client').annotate(max_timestamp=Max('timestamp')).values()) + + def interaction_per_client(self, maxdate=None, active_only=True): """ Returns the most recent interactions for clients as of a date Arguments: maxdate -- datetime object. Most recent date to pull. (dafault None) active_only -- Include only active clients (default True) - + """ - if maxdate and not isinstance(maxdate,datetime): + if maxdate and not isinstance(maxdate, datetime): raise ValueError, 'Expected a datetime object' - return self.filter(id__in = self.get_interaction_per_client_ids(maxdate, active_only)) + return self.filter(id__in=self.get_interaction_per_client_ids(maxdate, active_only)) - def get_interaction_per_client_ids(self, maxdate = None, active_only=True): + def get_interaction_per_client_ids(self, maxdate=None, active_only=True): """ Returns the ids of most recent interactions for clients as of a date. Arguments: maxdate -- datetime object. Most recent date to pull. (dafault None) active_only -- Include only active clients (default True) - + """ from django.db import connection cursor = connection.cursor() @@ -127,10 +132,10 @@ class InteractiveManager(models.Manager): sql = 'select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) ' + \ 'as timer from reports_interaction' if maxdate: - if not isinstance(maxdate,datetime): + if not isinstance(maxdate, datetime): raise ValueError, 'Expected a datetime object' sql = sql + " where timestamp <= '%s' " % maxdate - cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate,maxdate) + cfilter = "(expiration is null or expiration > '%s') and creation <= '%s'" % (maxdate, maxdate) sql = sql + ' GROUP BY client_id) x, reports_interaction where ' + \ 'reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer' if active_only: @@ -144,16 +149,17 @@ class InteractiveManager(models.Manager): pass return [] + class Interaction(models.Model): """Models each reconfiguration operation interaction between client and server.""" client = models.ForeignKey(Client, related_name="interactions",) - timestamp = models.DateTimeField()#Timestamp for this record - state = models.CharField(max_length=32)#good/bad/modified/etc - repo_rev_code = models.CharField(max_length=64)#repo revision at time of interaction - client_version = models.CharField(max_length=32)#Client Version - goodcount = models.IntegerField()#of good config-items - totalcount = models.IntegerField()#of total config-items - server = models.CharField(max_length=256) # Name of the server used for the interaction + timestamp = models.DateTimeField() # Timestamp for this record + state = models.CharField(max_length=32) # good/bad/modified/etc + repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction + client_version = models.CharField(max_length=32) # Client Version + goodcount = models.IntegerField() # of good config-items + totalcount = models.IntegerField() # of total config-items + server = models.CharField(max_length=256) # Name of the server used for the interaction bad_entries = models.IntegerField(default=-1) modified_entries = models.IntegerField(default=-1) extra_entries = models.IntegerField(default=-1) @@ -163,25 +169,25 @@ class Interaction(models.Model): def percentgood(self): if not self.totalcount == 0: - return (self.goodcount/float(self.totalcount))*100 + return (self.goodcount / float(self.totalcount)) * 100 else: return 0 def percentbad(self): if not self.totalcount == 0: - return ((self.totalcount-self.goodcount)/(float(self.totalcount)))*100 + return ((self.totalcount - self.goodcount) / (float(self.totalcount))) * 100 else: return 0 - + def isclean(self): if (self.bad_entry_count() == 0 and self.goodcount == self.totalcount): return True else: return False - + def isstale(self): - if (self == self.client.current_interaction):#Is Mostrecent - if(datetime.now()-self.timestamp > timedelta(hours=25) ): + if (self == self.client.current_interaction): # Is Mostrecent + if(datetime.now() - self.timestamp > timedelta(hours=25)): return True else: return False @@ -194,10 +200,11 @@ class Interaction(models.Model): return True else: return False + def save(self): - super(Interaction, self).save() #call the real save... + super(Interaction, self).save() # call the real save... self.client.current_interaction = self.client.interactions.latest() - self.client.save()#save again post update + self.client.save() # save again post update def delete(self): '''Override the default delete. Allows us to remove Performance items''' @@ -239,35 +246,38 @@ class Interaction(models.Model): self.extra_entries = Entries_interactions.objects.filter(interaction=self, type=TYPE_EXTRA).count() self.save() return self.extra_entries - + objects = InteractiveManager() class Admin: list_display = ('client', 'timestamp', 'state') list_filter = ['client', 'timestamp'] pass + class Meta: get_latest_by = 'timestamp' ordering = ['-timestamp'] unique_together = ("client", "timestamp") + class Reason(models.Model): """reason why modified or bad entry did not verify, or changed.""" owner = models.TextField(max_length=128, blank=True) current_owner = models.TextField(max_length=128, blank=True) group = models.TextField(max_length=128, blank=True) current_group = models.TextField(max_length=128, blank=True) - perms = models.TextField(max_length=4, blank=True)#txt fixes typing issue + perms = models.TextField(max_length=4, blank=True) # txt fixes typing issue current_perms = models.TextField(max_length=4, blank=True) - status = models.TextField(max_length=3, blank=True)#on/off/(None) - current_status = models.TextField(max_length=1, blank=True)#on/off/(None) + status = models.TextField(max_length=3, blank=True) # on/off/(None) + current_status = models.TextField(max_length=1, blank=True) # on/off/(None) to = models.TextField(max_length=256, blank=True) current_to = models.TextField(max_length=256, blank=True) version = models.TextField(max_length=128, blank=True) current_version = models.TextField(max_length=128, blank=True) - current_exists = models.BooleanField()#False means its missing. Default True + current_exists = models.BooleanField() # False means its missing. Default True current_diff = models.TextField(max_length=1280, blank=True) is_binary = models.BooleanField(default=False) + def _str_(self): return "Reason" @@ -278,7 +288,7 @@ class Reason(models.Model): cursor = connection.cursor() cursor.execute('delete from reports_reason where not exists (select rei.id from reports_entries_interactions rei where rei.reason_id = reports_reason.id)') transaction.set_dirty() - + class Entries(models.Model): """Contains all the entries feed by the client.""" @@ -295,19 +305,22 @@ class Entries(models.Model): cursor = connection.cursor() cursor.execute('delete from reports_entries where not exists (select rei.id from reports_entries_interactions rei where rei.entry_id = reports_entries.id)') transaction.set_dirty() - + + class Entries_interactions(models.Model): """Define the relation between the reason, the interaction and the entry.""" entry = models.ForeignKey(Entries) reason = models.ForeignKey(Reason) interaction = models.ForeignKey(Interaction) type = models.IntegerField(choices=TYPE_CHOICES) - + + class Performance(models.Model): """Object representing performance data for any interaction.""" interaction = models.ManyToManyField(Interaction, related_name="performance_items") metric = models.CharField(max_length=128) value = models.DecimalField(max_digits=32, decimal_places=16) + def __str__(self): return self.metric @@ -318,7 +331,8 @@ class Performance(models.Model): cursor = connection.cursor() cursor.execute('delete from reports_performance where not exists (select ri.id from reports_performance_interaction ri where ri.performance_id = reports_performance.id)') transaction.set_dirty() - + + class InternalDatabaseVersion(models.Model): """Object that tell us to witch version is the database.""" version = models.IntegerField() diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Server/Reports/reports/views.py index 00d35c092..3cffa68dd 100644 --- a/src/lib/Server/Reports/reports/views.py +++ b/src/lib/Server/Reports/reports/views.py @@ -3,22 +3,26 @@ Report views Functions to handle all of the reporting views. """ -from django.template import Context, RequestContext, loader -from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError, Http404 +from datetime import datetime, timedelta +import sys +from time import strptime + +from django.template import Context, RequestContext +from django.http import \ + HttpResponse, HttpResponseRedirect, HttpResponseServerError, Http404 from django.shortcuts import render_to_response, get_object_or_404 -from django.core.urlresolvers import resolve, reverse, Resolver404, NoReverseMatch +from django.core.urlresolvers import \ + resolve, reverse, Resolver404, NoReverseMatch from django.db import connection -from django.db.backends import util from Bcfg2.Server.Reports.reports.models import * -from datetime import datetime, timedelta -from time import strptime -import sys + class PaginationError(Exception): """This error is raised when pagination cannot be completed.""" pass + def server_error(request): """ 500 error handler. @@ -29,6 +33,7 @@ def server_error(request): from django.views import debug return debug.technical_500_response(request, *sys.exc_info()) + def timeview(fn): """ Setup a timeview view @@ -53,28 +58,32 @@ def timeview(fn): if cal_date.find(' ') > -1: kw['hour'] = timestamp.hour kw['minute'] = timestamp.minute - return HttpResponseRedirect(reverse(view, args=args, kwargs=kw)) + return HttpResponseRedirect(reverse(view, + args=args, + kwargs=kw)) except KeyError: pass except: pass # FIXME - Handle this - + """Extract timestamp from args.""" timestamp = None try: - timestamp = datetime(int(kwargs.pop('year')), int(kwargs.pop('month')), + timestamp = datetime(int(kwargs.pop('year')), + int(kwargs.pop('month')), int(kwargs.pop('day')), int(kwargs.pop('hour', 0)), int(kwargs.pop('minute', 0)), 0) kwargs['timestamp'] = timestamp except KeyError: pass - except: + except: raise return fn(request, **kwargs) return _handle_timeview - + + def config_item(request, pk, type="bad"): """ Display a single entry. @@ -83,30 +92,33 @@ def config_item(request, pk, type="bad"): """ item = get_object_or_404(Entries_interactions, id=pk) - timestamp=item.interaction.timestamp - time_start=item.interaction.timestamp.replace(\ - hour=0, minute=0, second=0, microsecond=0) - time_end=time_start + timedelta(days=1) - - todays_data = Interaction.objects.filter(\ - timestamp__gte=time_start,\ - timestamp__lt=time_end) - shared_entries = Entries_interactions.objects.filter(entry=item.entry,\ - reason=item.reason, type=item.type, - interaction__in=[x['id']\ - for x in todays_data.values('id')]) + timestamp = item.interaction.timestamp + time_start = item.interaction.timestamp.replace(hour=0, + minute=0, + second=0, + microsecond=0) + time_end = time_start + timedelta(days=1) + + todays_data = Interaction.objects.filter(timestamp__gte=time_start, + timestamp__lt=time_end) + shared_entries = Entries_interactions.objects.filter(entry=item.entry, + reason=item.reason, + type=item.type, + interaction__in=[x['id']\ + for x in todays_data.values('id')]) associated_list = Interaction.objects.filter(id__in=[x['interaction']\ for x in shared_entries.values('interaction')])\ - .order_by('client__name','timestamp').select_related().all() + .order_by('client__name', 'timestamp').select_related().all() return render_to_response('config_items/item.html', - {'item':item, - 'isextra': item.type == TYPE_EXTRA, - 'mod_or_bad': type, - 'associated_list':associated_list, - 'timestamp' : timestamp}, - context_instance=RequestContext(request)) + {'item': item, + 'isextra': item.type == TYPE_EXTRA, + 'mod_or_bad': type, + 'associated_list': associated_list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + @timeview def config_item_list(request, type, timestamp=None): @@ -115,11 +127,12 @@ def config_item_list(request, type, timestamp=None): type = convert_entry_type_to_id(type) if type < 0: raise Http404 - + current_clients = Interaction.objects.get_interaction_per_client_ids(timestamp) item_list_dict = {} seen = dict() - for x in Entries_interactions.objects.filter(interaction__in=current_clients, type=type).select_related(): + for x in Entries_interactions.objects.filter(interaction__in=current_clients, + type=type).select_related(): if (x.entry, x.reason) in seen: continue seen[(x.entry, x.reason)] = 1 @@ -129,13 +142,15 @@ def config_item_list(request, type, timestamp=None): item_list_dict[x.entry.kind] = [x] for kind in item_list_dict: - item_list_dict[kind].sort(lambda a,b: cmp(a.entry.name, b.entry.name)) + item_list_dict[kind].sort(lambda a, b: cmp(a.entry.name, b.entry.name)) - return render_to_response('config_items/listing.html', {'item_list_dict':item_list_dict, - 'mod_or_bad':mod_or_bad, - 'timestamp' : timestamp}, + return render_to_response('config_items/listing.html', + {'item_list_dict': item_list_dict, + 'mod_or_bad': mod_or_bad, + 'timestamp': timestamp}, context_instance=RequestContext(request)) + @timeview def client_index(request, timestamp=None): """ @@ -149,8 +164,10 @@ def client_index(request, timestamp=None): .order_by("client__name").all() return render_to_response('clients/index.html', - { 'inter_list': list, 'timestamp' : timestamp}, - context_instance=RequestContext(request)) + {'inter_list': list, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + @timeview def client_detailed_list(request, timestamp=None, **kwargs): @@ -165,7 +182,8 @@ def client_detailed_list(request, timestamp=None, **kwargs): kwargs['page_limit'] = 0 return render_history_view(request, 'clients/detailed-list.html', **kwargs) -def client_detail(request, hostname = None, pk = None): + +def client_detail(request, hostname=None, pk=None): context = dict() client = get_object_or_404(Client, name=hostname) if(pk == None): @@ -177,6 +195,7 @@ def client_detail(request, hostname = None, pk = None): return render_history_view(request, 'clients/detail.html', page_limit=5, client=client, maxdate=context['interaction'].timestamp, context=context) + def client_manage(request): """Manage client expiration""" message = '' @@ -186,12 +205,12 @@ def client_manage(request): client_action = request.POST.get('client_action', None) client = Client.objects.get(name=client_name) if client_action == 'expire': - client.expiration = datetime.now(); + client.expiration = datetime.now() client.save() message = "Expiration for %s set to %s." % \ (client_name, client.expiration.strftime("%Y-%m-%d %H:%M:%S")) elif client_action == 'unexpire': - client.expiration = None; + client.expiration = None client.save() message = "%s is now active." % client_name else: @@ -205,6 +224,7 @@ def client_manage(request): {'clients': Client.objects.order_by('name').all(), 'message': message}, context_instance=RequestContext(request)) + @timeview def display_summary(request, timestamp=None): """ @@ -216,7 +236,12 @@ def display_summary(request, timestamp=None): if not timestamp: timestamp = datetime.now() - collected_data = dict(clean=[],bad=[],modified=[],extra=[],stale=[],pings=[]) + collected_data = dict(clean=[], + bad=[], + modified=[], + extra=[], + stale=[], + pings=[]) for node in recent_data: if timestamp - node.timestamp > timedelta(hours=24): collected_data['stale'].append(node) @@ -238,42 +263,47 @@ def display_summary(request, timestamp=None): # label, header_text, node_list summary_data = [] - get_dict = lambda name, label: { 'name': name, - 'nodes': collected_data[name], - 'label': label } + get_dict = lambda name, label: {'name': name, + 'nodes': collected_data[name], + 'label': label} if len(collected_data['clean']) > 0: - summary_data.append( get_dict('clean', 'nodes are clean.') ) + summary_data.append(get_dict('clean', + 'nodes are clean.')) if len(collected_data['bad']) > 0: - summary_data.append( get_dict('bad', 'nodes are bad.') ) + summary_data.append(get_dict('bad', + 'nodes are bad.')) if len(collected_data['modified']) > 0: - summary_data.append( get_dict('modified', 'nodes were modified.') ) + summary_data.append(get_dict('modified', + 'nodes were modified.')) if len(collected_data['extra']) > 0: - summary_data.append( get_dict('extra', - 'nodes have extra configurations.') ) + summary_data.append(get_dict('extra', + 'nodes have extra configurations.')) if len(collected_data['stale']) > 0: - summary_data.append( get_dict('stale', - 'nodes did not run within the last 24 hours.') ) + summary_data.append(get_dict('stale', + 'nodes did not run within the last 24 hours.')) if len(collected_data['pings']) > 0: - summary_data.append( get_dict('pings', - 'are down.') ) + summary_data.append(get_dict('pings', + 'are down.')) return render_to_response('displays/summary.html', {'summary_data': summary_data, 'node_count': node_count, 'timestamp': timestamp}, context_instance=RequestContext(request)) + @timeview def display_timing(request, timestamp=None): mdict = dict() inters = Interaction.objects.interaction_per_client(timestamp).select_related().all() [mdict.__setitem__(inter, {'name': inter.client.name}) \ for inter in inters] - for metric in Performance.objects.filter(interaction__in=mdict.keys()).all(): + for metric in Performance.objects.filter(interaction__in=list(mdict.keys())).all(): for i in metric.interaction.all(): mdict[i][metric.metric] = metric.value return render_to_response('displays/timing.html', - {'metrics': mdict.values(), 'timestamp': timestamp}, - context_instance=RequestContext(request)) + {'metrics': list(mdict.values()), + 'timestamp': timestamp}, + context_instance=RequestContext(request)) def render_history_view(request, template='clients/history.html', **kwargs): @@ -303,7 +333,7 @@ def render_history_view(request, template='clients/history.html', **kwargs): max_results = int(kwargs.get('page_limit', 25)) page = int(kwargs.get('page_number', 1)) - client=kwargs.get('client', None) + client = kwargs.get('client', None) if not client and 'hostname' in kwargs: client = get_object_or_404(Client, name=kwargs['hostname']) if client: @@ -333,7 +363,11 @@ def render_history_view(request, template='clients/history.html', **kwargs): entry_list = [] if max_results > 0: try: - rec_start, rec_end = prepare_paginated_list(request, context, iquery, page, max_results) + rec_start, rec_end = prepare_paginated_list(request, + context, + iquery, + page, + max_results) except PaginationError, page_error: if isinstance(page_error[0], HttpResponse): return page_error[0] @@ -345,6 +379,7 @@ def render_history_view(request, template='clients/history.html', **kwargs): return render_to_response(template, context, context_instance=RequestContext(request)) + def prepare_paginated_list(request, context, paged_list, page=1, max_results=25): """ Prepare context and slice an object for pagination. @@ -358,7 +393,7 @@ def prepare_paginated_list(request, context, paged_list, page=1, max_results=25) nitems = paged_list.count() except TypeError: nitems = len(paged_list) - + rec_start = (page - 1) * int(max_results) try: total_pages = (nitems / int(max_results)) + 1 @@ -369,11 +404,11 @@ def prepare_paginated_list(request, context, paged_list, page=1, max_results=25) try: view, args, kwargs = resolve(request.META['PATH_INFO']) kwargs['page_number'] = total_pages - raise PaginationError, HttpResponseRedirect( reverse(view, kwargs=kwargs) ) + raise PaginationError, HttpResponseRedirect(reverse(view, + kwargs=kwargs)) except (Resolver404, NoReverseMatch, ValueError): raise "Accessing beyond last page. Unable to resolve redirect." context['total_pages'] = total_pages context['records_per_page'] = max_results return (rec_start, rec_start + int(max_results)) - diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py index f8fca1f90..5c61f599f 100644 --- a/src/lib/Server/Reports/updatefix.py +++ b/src/lib/Server/Reports/updatefix.py @@ -2,12 +2,13 @@ import Bcfg2.Server.Reports.settings from django.db import connection import django.core.management +import logging +import traceback from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion, \ TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA - -import logging, traceback logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') + # all update function should go here def _merge_database_table_entries(): cursor = connection.cursor() @@ -21,7 +22,7 @@ def _merge_database_table_entries(): select name, kind from reports_extra """) # this fetch could be better done - entries_map={} + entries_map = {} for row in cursor.fetchall(): insert_cursor.execute("insert into reports_entries (name, kind) \ values (%s, %s)", (row[0], row[1])) @@ -48,6 +49,7 @@ def _merge_database_table_entries(): insert_cursor.execute("insert into reports_entries_interactions \ (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) + def _interactions_constraint_or_idx(): '''sqlite doesn't support alter tables.. or constraints''' cursor = connection.cursor() @@ -55,17 +57,17 @@ def _interactions_constraint_or_idx(): cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') except: cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') - + def _populate_interaction_entry_counts(): '''Populate up the type totals for the interaction table''' cursor = connection.cursor() - count_field = { TYPE_BAD: 'bad_entries', - TYPE_MODIFIED: 'modified_entries', - TYPE_EXTRA: 'extra_entries' } + count_field = {TYPE_BAD: 'bad_entries', + TYPE_MODIFIED: 'modified_entries', + TYPE_EXTRA: 'extra_entries'} - for type in count_field.keys(): - cursor.execute("select count(type), interaction_id "+ + for type in list(count_field.keys()): + cursor.execute("select count(type), interaction_id " + "from reports_entries_interactions where type = %s group by interaction_id" % type) updates = [] for row in cursor.fetchall(): @@ -73,9 +75,9 @@ def _populate_interaction_entry_counts(): try: cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) except Exception, e: - print e + print(e) cursor.close() - + # be sure to test your upgrade query before reflecting the change in the models # the list of function and sql command to do should go here @@ -104,6 +106,7 @@ _fixes = [_merge_database_table_entries, # this will calculate the last possible version of the database lastversion = len(_fixes) + def rollupdate(current_version): """ function responsible to coordinates all the updates need current_version as integer @@ -119,11 +122,12 @@ def rollupdate(current_version): except: logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) # since array start at 0 but version start at 1 we add 1 to the normal count - ret = InternalDatabaseVersion.objects.create(version=i+1) + ret = InternalDatabaseVersion.objects.create(version=i + 1) return ret else: return None + def dosync(): """Function to do the syncronisation for the models""" # try to detect if it's a fresh new database @@ -164,7 +168,7 @@ def dosync(): def update_database(): ''' methode to search where we are in the revision of the database models and update them ''' - try : + try: logger.debug("Running upgrade of models to the new one") dosync() know_version = InternalDatabaseVersion.objects.order_by('-version') diff --git a/src/lib/Server/Reports/utils.py b/src/lib/Server/Reports/utils.py index b74f09e74..6010f366b 100755 --- a/src/lib/Server/Reports/utils.py +++ b/src/lib/Server/Reports/utils.py @@ -1,11 +1,11 @@ """Helper functions for reports""" -from Bcfg2.Server.Reports.reports.models import TYPE_CHOICES from django.conf.urls.defaults import * import re """List of filters provided by filteredUrls""" filter_list = ('server', 'state') + class BatchFetch(object): """Fetch Django objects in smaller batches to save memory""" @@ -20,7 +20,7 @@ class BatchFetch(object): def __iter__(self): return self - def next(self): + def __next__(self): """Return the next object from our array and fetch from the database when needed""" if self.block_count + self.count - self.step == self.max: @@ -34,11 +34,12 @@ class BatchFetch(object): self.count += 1 return self.data[self.count - 1] + def generateUrls(fn): """ Parse url tuples and send to functions. - Decorator for url generators. Handles url tuple parsing + Decorator for url generators. Handles url tuple parsing before the actual function is called. """ def url_gen(*urls): @@ -51,13 +52,14 @@ def generateUrls(fn): return results return url_gen + @generateUrls def paginatedUrls(pattern, view, kwargs=None, name=None): """ Takes a group of url tuples and adds paginated urls. - Extends a url tuple to include paginated urls. Currently doesn't handle url() compiled - patterns. + Extends a url tuple to include paginated urls. + Currently doesn't handle url() compiled patterns. """ results = [(pattern, view, kwargs, name)] @@ -67,13 +69,15 @@ def paginatedUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] results += [(pattern + "/(?P\d+)" + tail, view, kwargs)] - results += [(pattern + "/(?P\d+)\|(?P\d+)" + tail, view, kwargs)] + results += [(pattern + "/(?P\d+)\|(?P\d+)" + + tail, view, kwargs)] if not kwargs: kwargs = dict() kwargs['page_limit'] = 0 results += [(pattern + "/?\|(?Pall)" + tail, view, kwargs)] return results + @generateUrls def filteredUrls(pattern, view, kwargs=None, name=None): """ @@ -93,7 +97,8 @@ def filteredUrls(pattern, view, kwargs=None, name=None): '/server/(?P[\w\-\.]+)/(?P[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] return results - + + @generateUrls def timeviewUrls(pattern, view, kwargs=None, name=None): """ @@ -113,4 +118,3 @@ def timeviewUrls(pattern, view, kwargs=None, name=None): '/(?P\d{4})-(?P\d{2})-(?P\d{2})'): results += [(pattern + filter + tail, view, kwargs)] return results - diff --git a/src/lib/Server/Snapshots/__init__.py b/src/lib/Server/Snapshots/__init__.py index 6018377cb..a4d8fadbc 100644 --- a/src/lib/Server/Snapshots/__init__.py +++ b/src/lib/Server/Snapshots/__init__.py @@ -19,7 +19,7 @@ def db_from_config(cfile): db = cp.get('snapshots', 'database') return '%s://%s:%s@%s/%s' % (driver, user, password, host, db) else: - raise Exception, "unsupported db driver %s" % driver + raise Exception("unsupported db driver %s" % driver) def setup_session(cfile, debug=False): diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index cbb14be79..130d3e8c2 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -33,12 +33,20 @@ class Administrator(Uniquer, Base): email = Column(Unicode(64)) admin_client = Table('admin_client', Base.metadata, - Column('admin_id', Integer, ForeignKey('administrator.id')), - Column('client_id', Integer, ForeignKey('client.id'))) + Column('admin_id', + Integer, + ForeignKey('administrator.id')), + Column('client_id', + Integer, + ForeignKey('client.id'))) admin_group = Table('admin_group', Base.metadata, - Column('admin_id', Integer, ForeignKey('administrator.id')), - Column('group_id', Integer, ForeignKey('group.id'))) + Column('admin_id', + Integer, + ForeignKey('administrator.id')), + Column('group_id', + Integer, + ForeignKey('group.id'))) class Client(Uniquer, Base): @@ -68,12 +76,20 @@ class ConnectorKeyVal(Uniquer, Base): value = Column(UnicodeText) meta_group = Table('meta_group', Base.metadata, - Column('metadata_id', Integer, ForeignKey('metadata.id')), - Column('group_id', Integer, ForeignKey('group.id'))) + Column('metadata_id', + Integer, + ForeignKey('metadata.id')), + Column('group_id', + Integer, + ForeignKey('group.id'))) meta_conn = Table('meta_conn', Base.metadata, - Column('metadata_id', Integer, ForeignKey('metadata.id')), - Column('connkeyval_id', Integer, ForeignKey('connkeyval.id'))) + Column('metadata_id', + Integer, + ForeignKey('metadata.id')), + Column('connkeyval_id', + Integer, + ForeignKey('connkeyval.id'))) class Metadata(Base): @@ -95,7 +111,7 @@ class Metadata(Base): data = getattr(mymetadata, connector) if not isinstance(data, dict): continue - for key, value in data.iteritems(): + for key, value in list(data.items()): if not isinstance(value, str): continue m.keyvals.append(ConnectorKeyVal.by_value(mysession, @@ -143,8 +159,12 @@ class PackageCorrespondence(Base, CorrespondenceType): correct = Column(Boolean) package_snap = Table('package_snap', Base.metadata, - Column('ppair_id', Integer, ForeignKey('package_pair.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('ppair_id', + Integer, + ForeignKey('package_pair.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) class Service(Base, Uniquer): @@ -167,8 +187,12 @@ class ServiceCorrespondence(Base, CorrespondenceType): correct = Column(Boolean) service_snap = Table('service_snap', Base.metadata, - Column('spair_id', Integer, ForeignKey('service_pair.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('spair_id', + Integer, + ForeignKey('service_pair.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) class File(Base, Uniquer): @@ -194,20 +218,36 @@ class FileCorrespondence(Base, CorrespondenceType): correct = Column(Boolean) file_snap = Table('file_snap', Base.metadata, - Column('fpair_id', Integer, ForeignKey('file_pair.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('fpair_id', + Integer, + ForeignKey('file_pair.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) extra_pkg_snap = Table('extra_pkg_snap', Base.metadata, - Column('package_id', Integer, ForeignKey('package.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('package_id', + Integer, + ForeignKey('package.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) extra_file_snap = Table('extra_file_snap', Base.metadata, - Column('file_id', Integer, ForeignKey('file.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('file_id', + Integer, + ForeignKey('file.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) extra_service_snap = Table('extra_service_snap', Base.metadata, - Column('service_id', Integer, ForeignKey('service.id')), - Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) + Column('service_id', + Integer, + ForeignKey('service.id')), + Column('snapshot_id', + Integer, + ForeignKey('snapshot.id'))) class Action(Base): @@ -228,7 +268,7 @@ class Snapshot(Base): correct = Column(Boolean) revision = Column(Unicode(36)) metadata_id = Column(Integer, ForeignKey('metadata.id')) - client_metadata = relation(Metadata, primaryjoin=metadata_id==Metadata.id) + client_metadata = relation(Metadata, primaryjoin=metadata_id == Metadata.id) timestamp = Column(DateTime, default=datetime.datetime.now) client_id = Column(Integer, ForeignKey('client.id')) client = relation(Client, backref=backref('snapshots')) @@ -256,23 +296,25 @@ class Snapshot(Base): (cls.e_dispatch, extra)]: for key in dispatch: dest, ecls = dispatch[key] - for edata in data[key].values(): + for edata in list(data[key].values()): getattr(snap, dest).append(ecls.from_record(session, edata)) return snap @classmethod def by_client(cls, session, clientname): - return session.query(cls).join(cls.client_metadata, Metadata.client).filter(Client.name==clientname) + return session.query(cls).join(cls.client_metadata, + Metadata.client).filter(Client.name == clientname) @classmethod def get_current(cls, session, clientname): - return session.query(Snapshot).join(Snapshot.client_metadata, Metadata.client).filter(Client.name==clientname).order_by(desc(Snapshot.timestamp)).first() + return session.query(Snapshot).join(Snapshot.client_metadata, + Metadata.client).filter(Client.name == clientname).order_by(desc(Snapshot.timestamp)).first() @classmethod def get_by_date(cls, session, clientname, timestamp): return session.query(Snapshot)\ .join(Snapshot.client_metadata, Metadata.client)\ .filter(Snapshot.timestamp < timestamp)\ - .filter(Client.name==clientname)\ + .filter(Client.name == clientname)\ .order_by(desc(Snapshot.timestamp))\ .first() diff --git a/src/lib/Statistics.py b/src/lib/Statistics.py index b2240db98..a0cb8f39b 100644 --- a/src/lib/Statistics.py +++ b/src/lib/Statistics.py @@ -29,4 +29,4 @@ class Statistics(object): self.data[name].add_value(value) def display(self): - return dict([value.get_value() for value in self.data.values()]) + return dict([value.get_value() for value in list(self.data.values())]) -- cgit v1.2.3-1-g7c22