From 5d36c165a931fb9d9b49e1123d8d0647942339c1 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Wed, 20 Aug 2008 15:50:31 +0000 Subject: Add reporting system schema evolution support (from Stousignant) git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4884 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Server/Reports/importscript.py | 70 +++++------ src/lib/Server/Reports/reports/models.py | 78 ++++++------ .../reports/templates/clients/client-nodebox.html | 26 ++-- .../reports/templates/config_items/listing.html | 4 +- .../reports/templatetags/django_templating_sigh.py | 4 +- src/lib/Server/Reports/reports/views.py | 136 ++++++--------------- src/lib/Server/Reports/updatefix.py | 126 +++++++++++++++++++ 7 files changed, 252 insertions(+), 192 deletions(-) create mode 100644 src/lib/Server/Reports/updatefix.py (limited to 'src/lib/Server/Reports') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index 30657fd07..e1ae52efb 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -17,12 +17,13 @@ sys.path.pop() # Set DJANGO_SETTINGS_MODULE appropriately. os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name -from Bcfg2.Server.Reports.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason, Ping +from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, Entries_interactions, Performance, Reason, Ping, TYPE_CHOICES, InternalDatabaseVersion from lxml.etree import XML, XMLSyntaxError from getopt import getopt, GetoptError from datetime import datetime from time import strptime from django.db import connection +from Bcfg2.Server.Reports.updatefix import update_database import ConfigParser import difflib @@ -52,7 +53,7 @@ def build_reason_kwargs(r_ent): current_diff=rc_diff) -def load_stats(cdata, sdata, vlevel, quick=False): +def load_stats(cdata, sdata, vlevel, quick=False, location=''): cursor = connection.cursor() clients = {} cursor.execute("SELECT name, id from reports_client;") @@ -90,10 +91,10 @@ def load_stats(cdata, sdata, vlevel, quick=False): ilist = Interaction.objects.filter(client=c_inst, timestamp=timestamp) if ilist: - current_interaction_id = ilist[0].id + current_interaction = ilist[0] if vlevel > 0: print("Interaction for %s at %s with id %s already exists"%(clients[name], - datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id)) + datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction.id)) continue else: newint = Interaction(client=c_inst, @@ -102,18 +103,19 @@ def load_stats(cdata, sdata, vlevel, quick=False): repo_revision=statistics.get('revision',default="unknown"), client_version=statistics.get('client_version',default="unknown"), goodcount=statistics.get('good',default="0"), - totalcount=statistics.get('total',default="0")) + totalcount=statistics.get('total',default="0"), + server=location) newint.save() - current_interaction_id = newint.id + current_interaction = newint if vlevel > 0: print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], - timestamp, current_interaction_id)) + timestamp, current_interaction.id)) - pattern = [('Bad/*', Bad, 'reports_bad'), - ('Extra/*', Extra, 'reports_extra'), - ('Modified/*', Modified, 'reports_modified')] - for (xpath, obj, tablename) in pattern: + pattern = [('Bad/*', TYPE_CHOICES[0]), + ('Extra/*', TYPE_CHOICES[2]), + ('Modified/*', TYPE_CHOICES[1]),] + for (xpath, type) in pattern: for x in statistics.findall(xpath): kargs = build_reason_kwargs(x) if not quick: @@ -130,30 +132,23 @@ def load_stats(cdata, sdata, vlevel, quick=False): rr.save() if vlevel > 0: print "Created reason: %s" % rr.id - if not quick: - links = obj.objects.filter(name=x.get('name'), - kind=x.tag, - reason=rr) - else: - links = [] + + links = Entries.objects.filter(name=x.get('name'), + kind=x.tag) if links: - item_id = links[0].id - if vlevel > 0: - print "%s item exists, has reason id %s and ID %s" % (xpath, rr.id, item_id) + entry = links[0] else: - newitem = obj(name=x.get('name'), - kind=x.tag, critical=False, - reason=rr) - newitem.save() - item_id = newitem.id - if vlevel > 0: - print "%s item INSERTED having reason id %s and ID %s" % (xpath, rr.id, item_id) - try: - cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);", - [item_id, current_interaction_id]) - except: - pass + entry = Entries(name=x.get('name'), + kind=x.tag) + entry.save() + + interaction = Entries_interactions(entry=entry, reason=rr, + interaction=current_interaction, + type=type[0]) + interaction.save() + if vlevel > 0: + print "%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id) for times in statistics.findall('OpStamps'): for metric, value in times.items(): @@ -170,7 +165,7 @@ def load_stats(cdata, sdata, vlevel, quick=False): item_id = mperf.id try: cursor.execute("INSERT INTO reports_performance_interaction VALUES (NULL, %s, %s);", - [item_id, current_interaction_id]) + [item_id, current_interaction.id]) except: pass @@ -280,4 +275,11 @@ if __name__ == '__main__': raise SystemExit, 1 q = '-O3' in sys.argv - load_stats(clientsdata, statsdata, verb, quick=q) + try: + location = cf.get('components', 'bcfg2') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print "Could not read bcfg2 from bcfg2.conf; exiting" + raise SystemExit, 1 + # Be sure the database is ready for new schema + update_database() + load_stats(clientsdata, statsdata, verb, quick=q, location=location) diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Server/Reports/reports/models.py index b54d83912..333618a56 100644 --- a/src/lib/Server/Reports/reports/models.py +++ b/src/lib/Server/Reports/reports/models.py @@ -18,6 +18,15 @@ PING_CHOICES = ( ('Up (Y)', 'Y'), ('Down (N)', 'N') ) +TYPE_BAD = 1 +TYPE_MODIFIED = 2 +TYPE_EXTRA = 3 + +TYPE_CHOICES = ( + (TYPE_BAD, 'Bad'), + (TYPE_MODIFIED, 'Modified'), + (TYPE_EXTRA, 'Extra'), +) class ClientManager(models.Manager): '''extended client manager functions''' def active(self,timestamp='now'): @@ -57,19 +66,6 @@ class Client(models.Model): class Admin: pass -class Metadata(models.Model): - '''insert magical interface to client metadata here''' - client = models.ForeignKey(Client) - timestamp = models.DateTimeField() - def __str__(self): - return self.timestamp - -class Repository(models.Model): - '''insert magical interface to subversioned repository here''' - timestamp = models.DateTimeField() - def __str__(self): - return self.timestamp - class Ping(models.Model): '''represents a ping of a client (sparsely)''' client = models.ForeignKey(Client, related_name="pings") @@ -116,6 +112,7 @@ class Interaction(models.Model): client_version = models.CharField(maxlength=32)#Client Version goodcount = models.IntegerField()#of good config-items totalcount = models.IntegerField()#of total config-items + server = models.CharField(maxlength=256) # Name of the server used for the interaction def __str__(self): return "With " + self.client.name + " @ " + self.timestamp.isoformat() @@ -158,7 +155,16 @@ class Interaction(models.Model): super(Interaction, self).save() #call the real save... self.client.current_interaction = self.client.interactions.latest() self.client.save()#save again post update - + + def bad(self): + return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_BAD) + + def modified(self): + return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_MODIFIED) + + def extra(self): + return Entries_interactions.objects.select_related().filter(interaction=self, type=TYPE_EXTRA) + objects = InteractiveManager() class Admin: @@ -187,36 +193,21 @@ class Reason(models.Model): def _str_(self): return "Reason" -class Modified(models.Model): - '''Modified configuration element''' - interactions = models.ManyToManyField(Interaction, related_name="modified_items") - name = models.CharField(maxlength=128, core=True) - kind = models.CharField(maxlength=16, choices=KIND_CHOICES) - critical = models.BooleanField() - reason = models.ForeignKey(Reason) +class Entries(models.Model): + """ Contains all the entries feed by the client """ + name = models.CharField(maxlength=128, core=True, db_index=True) + kind = models.CharField(maxlength=16, choices=KIND_CHOICES, db_index=True) + def __str__(self): return self.name - -class Extra(models.Model): - '''Extra configuration element''' - interactions = models.ManyToManyField(Interaction, related_name="extra_items") - name = models.CharField(maxlength=128, core=True) - kind = models.CharField(maxlength=16, choices=KIND_CHOICES) - critical = models.BooleanField() + +class Entries_interactions(models.Model): + """ Define the relation between the reason, the interaction and the entry """ + entry = models.ForeignKey(Entries) reason = models.ForeignKey(Reason) - def __str__(self): - return self.name + interaction = models.ForeignKey(Interaction) + type = models.IntegerField(choices=TYPE_CHOICES) -class Bad(models.Model): - '''Bad configuration element''' - interactions = models.ManyToManyField(Interaction, related_name="bad_items") - name = models.CharField(maxlength=128, core=True) - kind = models.CharField(maxlength=16, choices=KIND_CHOICES) - critical = models.BooleanField() - reason = models.ForeignKey(Reason) - def __str__(self): - return self.name - class PerformanceManager(models.Manager): '''Provides ability to effectively query for performance information It is possible this should move to the view''' @@ -260,3 +251,10 @@ class Performance(models.Model): objects = PerformanceManager() +class InternalDatabaseVersion(models.Model): + '''Object that tell us to witch version is the database''' + version = models.IntegerField() + updated = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return "version %d updated the %s" % (self.version, self.updated.isoformat()) diff --git a/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html b/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html index 77621cace..88cf05345 100644 --- a/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html +++ b/src/lib/Server/Reports/reports/templates/clients/client-nodebox.html @@ -26,33 +26,33 @@
This node did not run within the last 24 hours-- it may be out of date.
- {% endif %} - {% if interaction.bad_items.all %} + {% endif %} + {% if interaction.bad %}
- {{interaction.bad_items.count}} items did not verify and are considered Dirty.
 + {{interaction.bad.count}} items did not verify and are considered Dirty.
    - {% for bad in interaction.bad_items.all|sortwell %} -
  • {{bad.kind}}: {{bad.name}}
    • + {% for bad in interaction.bad|sortwell %} +
  • {{bad.entry.kind}}: {{bad.entry.name}}
    • {% endfor %}
{% endif %} - {% if interaction.modified_items.all %} + {% if interaction.modified %}
- {{interaction.modified_items.count}} items were modified in the last run.
 + {{interaction.modified.count}} items were modified in the last run.
    - {% for modified in interaction.modified_items.all|sortwell %} -
  • {{modified.kind}}: {{modified.name}}
    • + {% for modified in interaction.modified|sortwell %} +
  • {{modified.entry.kind}}: {{modified.name}}
    • {% endfor %}
{% endif %} - {% if interaction.extra_items.all %} + {% if interaction.extra %}
- {{interaction.extra_items.count}} extra configuration elements on the node.
 + {{interaction.extra.count}} extra configuration elements on the node.
    - {% for extra in interaction.extra_items.all|sortwell %} -
  • {{extra.kind}}: {{extra.name}}
    • + {% for extra in interaction.extra|sortwell %} +
  • {{extra.entry.kind}}: {{extra.entry.name}}
    • {% endfor %}
diff --git a/src/lib/Server/Reports/reports/templates/config_items/listing.html b/src/lib/Server/Reports/reports/templates/config_items/listing.html index f3be1e4d5..1dba4bd6d 100644 --- a/src/lib/Server/Reports/reports/templates/config_items/listing.html +++ b/src/lib/Server/Reports/reports/templates/config_items/listing.html @@ -15,7 +15,7 @@ YAHOO.example.init = function( ){ {% for item_list in item_list_pseudodict %} tabView.addTab( new YAHOO.widget.Tab({ label: '{{item_list.0}}', - content: '

', + content: '

', active: 'True' })); {% endfor %} @@ -48,4 +48,4 @@ YAHOO.example.init(); {% else %}

There are currently no inconsistent configuration entries.

{% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py b/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py index 5591e065e..c0d05d2c1 100644 --- a/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py +++ b/src/lib/Server/Reports/reports/templatetags/django_templating_sigh.py @@ -17,8 +17,8 @@ def sortwell(value): "sorts a list(or evaluates queryset to list) of bad, extra, or modified items in the best" "way for presentation" configItems = list(value) - configItems.sort(lambda x,y: cmp(x.name, y.name)) - configItems.sort(lambda x,y: cmp(x.kind, y.kind)) + configItems.sort(lambda x,y: cmp(x.entry.name, y.entry.name)) + configItems.sort(lambda x,y: cmp(x.entry.kind, y.entry.kind)) return configItems def sortname(value): "sorts a list( or evaluates queryset) by name" diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Server/Reports/reports/views.py index 9934a0dc3..d5367b168 100644 --- a/src/lib/Server/Reports/reports/views.py +++ b/src/lib/Server/Reports/reports/views.py @@ -3,7 +3,8 @@ from django.template import Context, loader from django.http import HttpResponseRedirect, HttpResponse from django.shortcuts import render_to_response, get_object_or_404 -from Bcfg2.Server.Reports.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason +from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, Entries_interactions, Performance, Reason +from Bcfg2.Server.Reports.reports.models import TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA from datetime import datetime, timedelta from time import strptime from django.db import connection @@ -14,29 +15,34 @@ from sets import Set def index(request): return render_to_response('index.html') -def config_item_modified(request, eyedee =None, timestamp = 'now'): +def config_item_modified(request, eyedee =None, timestamp = 'now', type=TYPE_MODIFIED): #if eyedee = None, dump with a 404 timestamp = timestamp.replace("@"," ") - mod_or_bad = "modified" + if type == TYPE_MODIFIED: + mod_or_bad = "modified" + else: + mod_or_bad = "bad" - item = Modified.objects.get(id=eyedee) + item = Entries_interactions.objects.get(id=eyedee).entry #if everything is blank except current_exists, do something special cursor = connection.cursor() if timestamp == 'now': - cursor.execute("select client_id from reports_interaction, reports_modified_interactions, reports_client "+ - "WHERE reports_client.current_interaction_id = reports_modified_interactions.interaction_id "+ - "AND reports_modified_interactions.interaction_id = reports_interaction.id "+ - "AND reports_modified_interactions.modified_id = %s", [eyedee]) + cursor.execute("select client_id from reports_interaction, reports_entries_interactions, reports_client "+ + "WHERE reports_client.current_interaction_id = reports_entries_interactions.interaction_id "+ + "AND reports_entries_interactions.interaction_id = reports_interaction.id "+ + "AND reports_entries_interactions.entry_id = %s " + + "AND reports_entries_interactions.type = %s", [eyedee, type]) associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()]) else: interact_queryset = Interaction.objects.interaction_per_client(timestamp) interactionlist = [] [interactionlist.append(x.id) for x in interact_queryset] if not interactionlist == []: - cursor.execute("select client_id from reports_interaction, reports_modified_interactions, reports_client "+ - "WHERE reports_modified_interactions.interaction_id IN %s "+ - "AND reports_modified_interactions.interaction_id = reports_interaction.id "+ - "AND reports_modified_interactions.modified_id = %s", [interactionlist, eyedee]) + cursor.execute("select client_id from reports_interaction, reports_entries_interactions, reports_client "+ + "WHERE reports_entries_interactions.interaction_id IN %s "+ + "AND reports_entries_interactions.interaction_id = reports_interaction.id "+ + "AND reports_entries_interactions.modified_id = %s " + + "AND reports_entries_interactions.type = %s ", [interactionlist, eyedee, type]) associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()]) else: associated_client_list = [] @@ -50,64 +56,25 @@ def config_item_modified(request, eyedee =None, timestamp = 'now'): 'timestamp' : timestamp, 'timestamp_date' : timestamp[:10], 'timestamp_time' : timestamp[11:19]}) - + def config_item_bad(request, eyedee = None, timestamp = 'now'): + return config_item_modified(request, eyedee, timestamp, TYPE_BAD) + +def bad_item_index(request, timestamp = 'now', type=TYPE_BAD): timestamp = timestamp.replace("@"," ") - mod_or_bad = "bad" - item = Bad.objects.get(id=eyedee) - cursor = connection.cursor() - if timestamp == 'now': - cursor.execute("select client_id from reports_interaction, reports_bad_interactions, reports_client "+ - "WHERE reports_client.current_interaction_id = reports_bad_interactions.interaction_id "+ - "AND reports_bad_interactions.interaction_id = reports_interaction.id "+ - "AND reports_bad_interactions.bad_id = %s", [eyedee]) - associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()]) + if type == TYPE_BAD: + mod_or_bad = "bad" else: - interact_queryset = Interaction.objects.interaction_per_client(timestamp) - interactionlist = [] - [interactionlist.append(x.id) for x in interact_queryset] - if not interactionlist == []: - cursor.execute("SELECT DISTINCT client_id from reports_interaction, reports_bad_interactions, reports_client "+ - "WHERE reports_bad_interactions.interaction_id IN %s "+ - "AND reports_bad_interactions.interaction_id = reports_interaction.id "+ - "AND reports_bad_interactions.bad_id = %s", [interactionlist, eyedee]) - associated_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()]) + mod_or_bad = "modified" + + current_clients = [c.current_interaction for c in Client.objects.active(timestamp)] + item_list_dict = {} + for x in Entries_interactions.objects.select_related().filter(interaction__in=current_clients, type=type).distinct(): + if item_list_dict.get(x.entry.kind, None): + item_list_dict[x.entry.kind].append(x) else: - associated_client_list = None - - if timestamp == 'now': - timestamp = datetime.now().isoformat('@') - - return render_to_response('config_items/index.html', {'item':item, - 'mod_or_bad':mod_or_bad, - 'associated_client_list':associated_client_list, - 'timestamp' : timestamp, - 'timestamp_date' : timestamp[:10], - 'timestamp_time' : timestamp[11:19]}) - -def bad_item_index(request, timestamp = 'now'): - timestamp = timestamp.replace("@"," ") - mod_or_bad = "bad" - cursor = connection.cursor() - - if timestamp == 'now': - bad_kinds = dict([(x,x.kind) for x in Bad.objects.filter(interactions__in= - [c.current_interaction - for c in Client.objects.active(timestamp)]).distinct()]) - kinds = list(Set(bad_kinds.values())) - item_list_dict = dict([(x,[]) for x in kinds]) - for obj in bad_kinds: - item_list_dict[obj.kind].append(obj) - - else: #this isn't done yet - bad_kinds = dict([(x,x.kind) for x in Bad.objects.filter(interactions__in= - [c.current_interaction - for c in Client.objects.active(timestamp)]).distinct()]) - kinds = list(Set(bad_kinds.values())) - item_list_dict = dict([(x,[]) for x in kinds]) - for obj in bad_kinds: - item_list_dict[obj.kind].append(obj) + item_list_dict[x.entry.kind] = [x] item_list_pseudodict = item_list_dict.items() if timestamp == 'now': @@ -119,40 +86,7 @@ def bad_item_index(request, timestamp = 'now'): 'timestamp_date' : timestamp[:10], 'timestamp_time' : timestamp[11:19]}) def modified_item_index(request, timestamp = 'now'): - timestamp = timestamp.replace("@"," ") - mod_or_bad = "modified" - cursor = connection.cursor() - - if timestamp == 'now': - mod_kinds = dict([(x,x.kind) for x in Modified.objects.filter(interactions__in= - [c.current_interaction - for c in Client.objects.active(timestamp)]).distinct()]) - #this will need expiration support - kinds = list(Set(mod_kinds.values())) - item_list_dict = dict([(x,[]) for x in kinds]) - for obj in mod_kinds: - item_list_dict[obj.kind].append(obj) - - else: #this isn't done yet - mod_kinds = dict([(x,x.kind) for x in Modified.objects.filter(interactions__in= - [c.current_interaction - for c in Client.objects.active(timestamp)]).distinct()]) - #this will need expiration support - kinds = list(Set(mod_kinds.values())) - item_list_dict = dict([(x,[]) for x in kinds]) - for obj in mod_kinds: - item_list_dict[obj.kind].append(obj) - - item_list_pseudodict = item_list_dict.items() - if timestamp == 'now': - timestamp = datetime.now().isoformat('@') - - return render_to_response('config_items/listing.html', {'item_list_pseudodict':item_list_pseudodict, - 'mod_or_bad':mod_or_bad, - 'timestamp' : timestamp, - 'timestamp_date' : timestamp[:10], - 'timestamp_time' : timestamp[11:19]}) - + return bad_item_index(request, timestamp, TYPE_MODIFIED) def client_index(request, timestamp = 'now'): timestamp = timestamp.replace("@"," ") @@ -332,10 +266,10 @@ def prepare_client_lists(request, timestamp = 'now'): [stale_up_client_list.append(x) for x in stale_all_client_list if not client_ping_dict[x.id]=='N'] - cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_modified_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_modified_interactions.interaction_id GROUP BY reports_client.id") + cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_entries_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_entries_interactions.interaction_id and reports_entries_interactions.type=%s GROUP BY reports_client.id", [TYPE_MODIFIED]) modified_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()]) - cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_extra_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_extra_interactions.interaction_id GROUP BY reports_client.id") + cursor.execute("SELECT reports_client.id FROM reports_client, reports_interaction, reports_entries_interactions WHERE reports_client.id=reports_interaction.client_id AND reports_interaction.id = reports_entries_interactions.interaction_id and reports_entries_interactions.type=%s GROUP BY reports_client.id", [TYPE_EXTRA]) extra_client_list = Client.objects.active(timestamp).filter(id__in=[x[0] for x in cursor.fetchall()]) if timestamp == 'now': diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py new file mode 100644 index 000000000..48077e44b --- /dev/null +++ b/src/lib/Server/Reports/updatefix.py @@ -0,0 +1,126 @@ +import Bcfg2.Server.Reports.settings + +from django.db import connection +import django.core.management +from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion + +import logging, traceback +logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') + +# all update function should go here +def _merge_database_table_entries(): + cursor = connection.cursor() + insert_cursor = connection.cursor() + cursor.execute(""" + Select name, kind from reports_bad + union + select name, kind from reports_modified + union + select name, kind from reports_extra + """) + # this fetch could be better done + entries_map={} + for row in cursor.fetchall(): + insert_cursor.execute("insert into reports_entries (name, kind) \ + values (%s, %s)", (row[0], row[1])) + entries_map[(row[0], row[1])] = insert_cursor.lastrowid + + cursor.execute(""" + Select name, kind, reason_id, interaction_id, 1 from reports_bad + inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id + union + Select name, kind, reason_id, interaction_id, 2 from reports_modified + inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id + union + Select name, kind, reason_id, interaction_id, 3 from reports_extra + inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id + """) + for row in cursor.fetchall(): + entry_id = entries_map[(row[0], row[1])] + insert_cursor.execute("insert into reports_entries_interactions \ + (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) + +# be sure to test your upgrade query before reflecting the change in the models +# the list of function and sql command to do should go here +_fixes = [_merge_database_table_entries, + # this will remove unused tables + "drop table reports_bad;", + "drop table reports_bad_interactions;", + "drop table reports_extra;", + "drop table reports_extra_interactions;", + "drop table reports_modified;", + "drop table reports_modified_interactions;", + "drop table reports_repository;", + "drop table reports_metadata;", + "alter table reports_interaction add server varchar(256) not null default 'N/A';", +] + +# this will calculate the last possible version of the database +lastversion = len(_fixes) + +def rollupdate(current_version): + """ function responsible to coordinates all the updates + need current_version as integer + """ + if current_version < lastversion: + for i in range(current_version, lastversion): + if type(_fixes[i]) == str: + connection.cursor().execute(_fixes[i]) + else: + _fixes[i]() + # since array start at 0 but version start at 1 we add 1 to the normal count + ret = InternalDatabaseVersion.objects.create(version=i+1) + return ret + else: + return None + +def dosync(): + """Function to do the syncronisation for the models""" + # try to detect if it's a fresh new database + try: + cursor = connection.cursor() + # If this table goes missing then don't forget to change it to the new one + cursor.execute("Select * from reports_client") + # if we get here with no error then the database has existing tables + fresh = False + except: + logger.debug("there was an error while detecting the freshnest of the database") + #we should get here if the database is new + fresh = True + + if "call_command" in dir(django.core.management): + # this is available since django 1.0 alpha. + # not yet tested for full functionnality + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + if fresh: + django.core.management.call_command("loaddata", fixture_labels=['initial_version'], verbosity=0) + elif "syncdb" in dir(django.core.management): + # this exist only for django 0.96.* + django.core.management.syncdb(interactive=False, verbosity=0) + if fresh: + logger.debug("loading the initial_version fixtures") + django.core.management.load_data(fixture_labels=['initial_version'], verbosity=0) + else: + logger.warning("Don't forget to run syncdb") + + +def update_database(): + ''' methode to search where we are in the revision of the database models and update them ''' + try : + logger.debug("Running upgrade of models to the new one") + dosync() + know_version = InternalDatabaseVersion.objects.order_by('-version') + if not know_version: + logger.debug("No version, creating initial version") + know_version = InternalDatabaseVersion.objects.create(version=0) + else: + know_version = know_version[0] + logger.debug("Presently at %s" % know_version) + if know_version.version < lastversion: + new_version = rollupdate(know_version.version) + if new_version: + logger.debug("upgraded to %s" % new_version) + except: + logger.error("Error while updating the database") + for x in traceback.format_exc().splitlines(): + logger.error(x) -- cgit v1.2.3-1-g7c22