From 3cf149d0e05d91305a6e6e8957f8da9d3b28ac7c Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Tue, 22 Jul 2008 03:22:47 +0000 Subject: Get importscript properly function-ified git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4819 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Server/Reports/importscript.py | 180 ++++++++++++++++----------------- 1 file changed, 87 insertions(+), 93 deletions(-) (limited to 'src') diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py index a37f64214..fbc3c81f9 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Server/Reports/importscript.py @@ -19,7 +19,6 @@ os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name from Bcfg2.Server.Reports.reports.models import Client, Interaction, Bad, Modified, Extra, Performance, Reason, Ping from lxml.etree import XML, XMLSyntaxError -from sys import argv from getopt import getopt, GetoptError from datetime import datetime from time import strptime @@ -46,100 +45,31 @@ def build_reason_kwargs(r_ent): current_exists=r_ent.get('current_exists', default="True").capitalize()=="True", current_diff=rc_diff) -if __name__ == '__main__': - somewhatverbose = False - verbose = False - veryverbose = False - cpath = "/etc/bcfg2.conf" - clientpath = False - statpath = False - - try: - opts, args = getopt(argv[1:], "hvudc:s:", ["help", "verbose", "updates" , - "debug", "clients=", "stats=", - "config="]) - except GetoptError, mesg: - # print help information and exit: - print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) - raise SystemExit, 2 - - for o, a in opts: - if o in ("-h", "--help"): - print "Usage:\nimportscript.py [-h] [-v] -c -s \n" - print "h : help; this message" - print "v : verbose; print messages on record insertion/skip" - print "u : updates; print status messages as items inserted semi-verbose" - print "d : debug; print most SQL used to manipulate database" - print "C : path to bcfg2.conf config file." - print "c : clients.xml file" - print "s : statistics.xml file" - raise SystemExit - if o in ["-C", "--config"]: - cpath = a - - if o in ("-v", "--verbose"): - verbose = True - if o in ("-u", "--updates"): - somewhatverbose = True - if o in ("-d", "--debug"): - veryverbose = True - if o in ("-c", "--clients"): - clientspath = a - - if o in ("-s", "--stats"): - statpath = a - - cf = ConfigParser.ConfigParser() - cf.read([cpath]) - - if not statpath: - try: - statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(statpath)) - raise SystemExit, 1 - - if not clientpath: - try: - clientspath = "%s/Metadata/clients.xml" % \ - cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print "Could not read bcfg2.conf; exiting" - raise SystemExit, 1 - try: - clientsdata = XML(open(clientspath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(clientspath)) - raise SystemExit, 1 +def load_stats(cdata, sdata, vlevel): cursor = connection.cursor() clients = {} cursor.execute("SELECT name, id from reports_client;") [clients.__setitem__(a, b) for a, b in cursor.fetchall()] - for node in statsdata.findall('Node'): + for node in sdata.findall('Node'): name = node.get('name') if not clients.has_key(name): cursor.execute(\ "INSERT INTO reports_client VALUES (NULL, %s, %s, NULL, NULL)", [datetime.now(), name]) clients[name] = cursor.lastrowid - if verbose: + if vlevel > 0: print("Client %s added to db" % name) else: - if verbose: + if vlevel > 0: print("Client %s already exists in db" % name) pingability = {} [pingability.__setitem__(n.get('name'), n.get('pingable', default='N')) \ - for n in clientsdata.findall('Client')] + for n in cdata.findall('Client')] - for node in statsdata.findall('Node'): + for node in sdata.findall('Node'): name = node.get('name') c_inst = Client.objects.filter(id=clients[name])[0] try: @@ -155,7 +85,7 @@ if __name__ == '__main__': timestamp=timestamp) if ilist: current_interaction_id = ilist[0].id - if verbose: + if vlevel > 0: print("Interaction for %s at %s with id %s already exists"%(clients[name], datetime(t[0],t[1],t[2],t[3],t[4],t[5]),current_interaction_id)) continue @@ -169,7 +99,7 @@ if __name__ == '__main__': totalcount=statistics.get('total',default="0")) newint.save() current_interaction_id = newint.id - if verbose: + if vlevel > 0: print("Interaction for %s at %s with id %s INSERTED in to db"%(clients[name], timestamp, current_interaction_id)) @@ -187,12 +117,12 @@ if __name__ == '__main__': if rls: rr = rls[0] - if verbose: + if vlevel > 0: print "Reason exists: %s"% (rr.id) else: rr = Reason(**kargs) rr.save() - if verbose: + if vlevel > 0: print "Created reason: %s" % rr.id if '-O3' not in sys.argv: links = obj.objects.filter(name=x.get('name'), @@ -203,7 +133,7 @@ if __name__ == '__main__': if links: item_id = links[0].id - if verbose: + if vlevel > 0: print "%s item exists, has reason id %s and ID %s" % (xpath, rr.id, item_id) else: newitem = obj(name=x.get('name'), @@ -211,7 +141,7 @@ if __name__ == '__main__': reason=rr) newitem.save() item_id = newitem.id - if verbose: + if vlevel > 0: print "%s item INSERTED having reason id %s and ID %s" % (xpath, rr.id, item_id) try: cursor.execute("INSERT INTO "+tablename+"_interactions VALUES (NULL, %s, %s);", @@ -238,13 +168,13 @@ if __name__ == '__main__': except: pass - if (somewhatverbose or verbose): + if vlevel > 1: print("----------------INTERACTIONS SYNCED----------------") cursor.execute("select reports_interaction.id, x.client_id from (select client_id, MAX(timestamp) as timer from reports_interaction Group BY client_id) x, reports_interaction where reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer") for row in cursor.fetchall(): cursor.execute("UPDATE reports_client SET current_interaction_id = %s where reports_client.id = %s", [row[0],row[1]]) - if (somewhatverbose or verbose): + if vlevel > 1: print("------------LATEST INTERACTION SET----------------") for key in pingability.keys(): @@ -267,18 +197,82 @@ if __name__ == '__main__': starttime=datetime.now(), endtime=datetime.now()) newp.save() - if (somewhatverbose or verbose): + if vlevel > 1: print "---------------PINGDATA SYNCED---------------------" connection._commit() #Clients are consistent - if veryverbose: - for q in connection.queries: - if not (q['sql'].startswith('INSERT INTO reports_bad_interactions')| - q['sql'].startswith('INSERT INTO reports_extra_interactions')| - q['sql'].startswith('INSERT INTO reports_performance_interaction')| - q['sql'].startswith('INSERT INTO reports_modified_interactions')| - q['sql'].startswith('UPDATE reports_client SET current_interaction_id ')): - print q raise SystemExit, 0 + +if __name__ == '__main__': + from sys import argv + verb = 0 + cpath = "/etc/bcfg2.conf" + clientpath = False + statpath = False + + try: + opts, args = getopt(argv[1:], "hvudc:s:", ["help", "verbose", "updates" , + "debug", "clients=", "stats=", + "config="]) + except GetoptError, mesg: + # print help information and exit: + print "%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg) + raise SystemExit, 2 + + for o, a in opts: + if o in ("-h", "--help"): + print "Usage:\nimportscript.py [-h] [-v] -c -s \n" + print "h : help; this message" + print "v : verbose; print messages on record insertion/skip" + print "u : updates; print status messages as items inserted semi-verbose" + print "d : debug; print most SQL used to manipulate database" + print "C : path to bcfg2.conf config file." + print "c : clients.xml file" + print "s : statistics.xml file" + raise SystemExit + if o in ["-C", "--config"]: + cpath = a + + if o in ("-v", "--verbose"): + verb = 1 + if o in ("-u", "--updates"): + verb = 2 + if o in ("-d", "--debug"): + verb = 3 + if o in ("-c", "--clients"): + clientspath = a + + if o in ("-s", "--stats"): + statpath = a + + cf = ConfigParser.ConfigParser() + cf.read([cpath]) + + if not statpath: + try: + statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print "Could not read bcfg2.conf; exiting" + raise SystemExit, 1 + try: + statsdata = XML(open(statpath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s"%(statpath)) + raise SystemExit, 1 + + if not clientpath: + try: + clientspath = "%s/Metadata/clients.xml" % \ + cf.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print "Could not read bcfg2.conf; exiting" + raise SystemExit, 1 + try: + clientsdata = XML(open(clientspath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s"%(clientspath)) + raise SystemExit, 1 + + load_stats(clientsdata, statsdata, verb) -- cgit v1.2.3-1-g7c22