""" This file provides the Hostbase plugin. It manages dns/dhcp/nis host information """ import os os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' from lxml.etree import Element, SubElement import Bcfg2.Server.Plugin from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError from time import strftime from sets import Set from django.template import Context, loader from django.db import connection import re # Compatibility imports from Bcfg2.Bcfg2Py3k import StringIO class Hostbase(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Structure, Bcfg2.Server.Plugin.Generator): """The Hostbase plugin handles host/network info.""" name = 'Hostbase' __author__ = 'bcfg-dev@mcs.anl.gov' filepath = '/my/adm/hostbase/files/bind' def __init__(self, core, datastore): self.ready = False Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Structure.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) files = ['zone.tmpl', 'reversesoa.tmpl', 'named.tmpl', 'reverseappend.tmpl', 'dhcpd.tmpl', 'hosts.tmpl', 'hostsappend.tmpl'] self.filedata = {} self.dnsservers = [] self.dhcpservers = [] self.templates = {'zone': loader.get_template('zone.tmpl'), 'reversesoa': loader.get_template('reversesoa.tmpl'), 'named': loader.get_template('named.tmpl'), 'namedviews': loader.get_template('namedviews.tmpl'), 'reverseapp': loader.get_template('reverseappend.tmpl'), 'dhcp': loader.get_template('dhcpd.tmpl'), 'hosts': loader.get_template('hosts.tmpl'), 'hostsapp': loader.get_template('hostsappend.tmpl'), } self.Entries['ConfigFile'] = {} self.__rmi__ = ['rebuildState'] try: self.rebuildState(None) except: raise PluginInitError def FetchFile(self, entry, metadata): """Return prebuilt file data.""" fname = entry.get('name').split('/')[-1] if not fname in self.filedata: raise PluginExecutionError perms = {'owner': 'root', 'group': 'root', 'perms': '644'} [entry.attrib.__setitem__(key, value) for (key, value) in list(perms.items())] entry.text = self.filedata[fname] def BuildStructures(self, metadata): """Build hostbase bundle.""" if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers: return [] output = Element("Bundle", name='hostbase') if metadata.hostname in self.dnsservers: for configfile in self.Entries['ConfigFile']: if re.search('/etc/bind/', configfile): SubElement(output, "ConfigFile", name=configfile) if metadata.hostname in self.dhcpservers: SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf") return [output] def rebuildState(self, _): """Pre-cache all state information for hostbase config files callable as an XMLRPC function. """ self.buildZones() self.buildDHCP() self.buildHosts() self.buildHostsLPD() self.buildPrinters() self.buildNetgroups() return True def buildZones(self): """Pre-build and stash zone files.""" cursor = connection.cursor() cursor.execute("SELECT id, serial FROM hostbase_zone") zones = cursor.fetchall() for zone in zones: # update the serial number for all zone files todaydate = (strftime('%Y%m%d')) try: if todaydate == str(zone[1])[:8]: serial = zone[1] + 1 else: serial = int(todaydate) * 100 except (KeyError): serial = int(todaydate) * 100 cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0])) cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'") zones = cursor.fetchall() iplist = [] hosts = {} for zone in zones: zonefile = StringIO() externalzonefile = StringIO() cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id WHERE z.zone_id = \'%s\'""" % zone[0]) nameservers = cursor.fetchall() cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id WHERE z.zone_id = \'%s\'""" % zone[0]) addresses = cursor.fetchall() cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z INNER JOIN hostbase_mx m ON z.mx_id = m.id WHERE z.zone_id = \'%s\'""" % zone[0]) mxs = cursor.fetchall() context = Context({ 'zone': zone, 'nameservers': nameservers, 'addresses': addresses, 'mxs': mxs }) zonefile.write(self.templates['zone'].render(context)) externalzonefile.write(self.templates['zone'].render(context)) querystring = """SELECT h.hostname, p.ip_addr, n.name, c.cname, m.priority, m.mx, n.dns_view FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id) INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) INNER JOIN hostbase_mx m ON m.id = x.mx_id) LEFT JOIN hostbase_cname c ON n.id = c.name_id WHERE n.name LIKE '%%%%%s' AND h.status = 'active' ORDER BY h.hostname, n.name, p.ip_addr """ % zone[1] cursor.execute(querystring) zonehosts = cursor.fetchall() prevhost = (None, None, None, None) cnames = StringIO() cnamesexternal = StringIO() for host in zonehosts: if not host[2].split(".", 1)[1] == zone[1]: zonefile.write(cnames.getvalue()) externalzonefile.write(cnamesexternal.getvalue()) cnames = StringIO() cnamesexternal = StringIO() continue if not prevhost[1] == host[1] or not prevhost[2] == host[2]: zonefile.write(cnames.getvalue()) externalzonefile.write(cnamesexternal.getvalue()) cnames = StringIO() cnamesexternal = StringIO() zonefile.write("%-32s%-10s%-32s\n" % (host[2].split(".", 1)[0], 'A', host[1])) zonefile.write("%-32s%-10s%-3s%s.\n" % ('', 'MX', host[4], host[5])) if host[6] == 'global': externalzonefile.write("%-32s%-10s%-32s\n" % (host[2].split(".", 1)[0], 'A', host[1])) externalzonefile.write("%-32s%-10s%-3s%s.\n" % ('', 'MX', host[4], host[5])) elif not prevhost[5] == host[5]: zonefile.write("%-32s%-10s%-3s%s.\n" % ('', 'MX', host[4], host[5])) if host[6] == 'global': externalzonefile.write("%-32s%-10s%-3s%s.\n" % ('', 'MX', host[4], host[5])) if host[3]: try: if host[3].split(".", 1)[1] == zone[1]: cnames.write("%-32s%-10s%-32s\n" % (host[3].split(".", 1)[0], 'CNAME', host[2].split(".", 1)[0])) if host[6] == 'global': cnamesexternal.write("%-32s%-10s%-32s\n" % (host[3].split(".", 1)[0], 'CNAME', host[2].split(".", 1)[0])) else: cnames.write("%-32s%-10s%-32s\n" % (host[3] + ".", 'CNAME', host[2].split(".", 1)[0])) if host[6] == 'global': cnamesexternal.write("%-32s%-10s%-32s\n" % (host[3] + ".", 'CNAME', host[2].split(".", 1)[0])) except: pass prevhost = host zonefile.write(cnames.getvalue()) externalzonefile.write(cnamesexternal.getvalue()) zonefile.write("\n\n%s" % zone[9]) externalzonefile.write("\n\n%s" % zone[9]) self.filedata[zone[1]] = zonefile.getvalue() self.filedata[zone[1] + ".external"] = externalzonefile.getvalue() zonefile.close() externalzonefile.close() self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'") reversezones = cursor.fetchall() reversenames = [] for reversezone in reversezones: cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id WHERE z.zone_id = \'%s\'""" % reversezone[0]) reverse_nameservers = cursor.fetchall() context = Context({ 'inaddr': reversezone[1].rstrip('.rev'), 'zone': reversezone, 'nameservers': reverse_nameservers, }) self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context) self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context) self.filedata[reversezone[1]] += reversezone[9] self.filedata[reversezone[1] + '.external'] += reversezone[9] subnet = reversezone[1].split(".") subnet.reverse() reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:]))) for filename in reversenames: cursor.execute(""" SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON n.ip_id = p.id WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr """ % filename[1]) reversehosts = cursor.fetchall() zonefile = StringIO() externalzonefile = StringIO() if len(filename[0].split(".")) == 2: originlist = [] [originlist.append((".".join([ip[1].split(".")[2], filename[0]]), ".".join([filename[1], ip[1].split(".")[2]]))) for ip in reversehosts if (".".join([ip[1].split(".")[2], filename[0]]), ".".join([filename[1], ip[1].split(".")[2]])) not in originlist] for origin in originlist: hosts = [(host[1].split("."), host[0]) for host in reversehosts if host[1].rstrip('0123456789').rstrip('.') == origin[1]] hosts_external = [(host[1].split("."), host[0]) for host in reversehosts if (host[1].rstrip('0123456789').rstrip('.') == origin[1] and host[2] == 'global')] context = Context({ 'hosts': hosts, 'inaddr': origin[0], 'fileorigin': filename[0], }) zonefile.write(self.templates['reverseapp'].render(context)) context = Context({ 'hosts': hosts_external, 'inaddr': origin[0], 'fileorigin': filename[0], }) externalzonefile.write(self.templates['reverseapp'].render(context)) else: originlist = [filename[0]] hosts = [(host[1].split("."), host[0]) for host in reversehosts if (host[1].split("."), host[0]) not in hosts] hosts_external = [(host[1].split("."), host[0]) for host in reversehosts if ((host[1].split("."), host[0]) not in hosts_external and host[2] == 'global')] context = Context({ 'hosts': hosts, 'inaddr': filename[0], 'fileorigin': None, }) zonefile.write(self.templates['reverseapp'].render(context)) context = Context({ 'hosts': hosts_external, 'inaddr': filename[0], 'fileorigin': None, }) externalzonefile.write(self.templates['reverseapp'].render(context)) self.filedata['%s.rev' % filename[0]] += zonefile.getvalue() self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue() zonefile.close() externalzonefile.close() self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile ## here's where the named.conf file gets written context = Context({ 'zones': zones, 'reverses': reversenames, }) self.filedata['named.conf'] = self.templates['named'].render(context) self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile self.filedata['named.conf.views'] = self.templates['namedviews'].render(context) self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile def buildDHCP(self): """Pre-build dhcpd.conf and stash in the filedata table.""" # fetches all the hosts with DHCP == True cursor = connection.cursor() cursor.execute(""" SELECT hostname, mac_addr, ip_addr FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) INNER JOIN hostbase_ip ip ON i.id = ip.interface_id WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> '' AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown' ORDER BY h.hostname, i.mac_addr """) dhcphosts = cursor.fetchall() count = 0 hosts = [] hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]] if len(dhcphosts) > 1: for x in range(1, len(dhcphosts)): # if an interface has 2 or more ip addresses # adds the ip to the current interface if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]: hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]]) # if a host has 2 or more interfaces # writes the current one and grabs the next elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]: hosts.append(hostdata) count += 1 hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]] # new host found, writes current data to the template else: hosts.append(hostdata) count = 0 hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]] #makes sure the last of the data gets written out if hostdata not in hosts: hosts.append(hostdata) context = Context({ 'hosts': hosts, 'numips': len(hosts), }) self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context) self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile def buildHosts(self): """Pre-build and stash /etc/hosts file.""" append_data = [] cursor = connection.cursor() cursor.execute(""" SELECT hostname FROM hostbase_host ORDER BY hostname """) hostbase = cursor.fetchall() domains = [host[0].split(".", 1)[1] for host in hostbase] domains_set = Set(domains) domain_data = [(domain, domains.count(domain)) for domain in domains_set] domain_data.sort() cursor.execute(""" SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr """) ips = cursor.fetchall() three_octets = [ip[0].rstrip('0123456789').rstrip('.') \ for ip in ips] three_octets_set = Set(three_octets) three_octets_data = [(octet, three_octets.count(octet)) \ for octet in three_octets_set] three_octets_data.sort() for three_octet in three_octets_data: querystring = """SELECT h.hostname, h.primary_user, p.ip_addr, n.name, c.cname FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id) LEFT JOIN hostbase_cname c ON n.id = c.name_id WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0] cursor.execute(querystring) tosort = list(cursor.fetchall()) tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1]))) append_data.append((three_octet, tuple(tosort))) two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] two_octets_set = Set(two_octets) two_octets_data = [(octet, two_octets.count(octet)) for octet in two_octets_set] two_octets_data.sort() context = Context({ 'domain_data': domain_data, 'three_octets_data': three_octets_data, 'two_octets_data': two_octets_data, 'three_octets': three_octets, 'num_ips': len(three_octets), }) self.filedata['hosts'] = self.templates['hosts'].render(context) for subnet in append_data: ips = [] simple = True namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]] cnamelist = [] if subnet[1][0][4]: cnamelist.append(subnet[1][0][4].split('.', 1)[0]) simple = False appenddata = subnet[1][0] for ip in subnet[1][1:]: if appenddata[2] == ip[2]: namelist.append(ip[3].split('.', 1)[0]) if ip[4]: cnamelist.append(ip[4].split('.', 1)[0]) simple = False appenddata = ip else: if appenddata[0] == ip[0]: simple = False ips.append((appenddata[2], appenddata[0], Set(namelist), cnamelist, simple, appenddata[1])) appenddata = ip simple = True namelist = [ip[3].split('.', 1)[0]] cnamelist = [] if ip[4]: cnamelist.append(ip[4].split('.', 1)[0]) simple = False ips.append((appenddata[2], appenddata[0], Set(namelist), cnamelist, simple, appenddata[1])) context = Context({ 'subnet': subnet[0], 'ips': ips, }) self.filedata['hosts'] += self.templates['hostsapp'].render(context) self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile def buildPrinters(self): """The /mcs/etc/printers.data file""" header = """# This file is automatically generated. DO NOT EDIT IT! # Name Room User Type Notes ============== ========== ============================== ======================== ==================== """ cursor = connection.cursor() # fetches all the printers from the database cursor.execute(""" SELECT printq, location, primary_user, comments FROM hostbase_host WHERE whatami='printer' AND printq <> '' AND status = 'active' ORDER BY printq """) printers = cursor.fetchall() printersfile = header for printer in printers: # splits up the printq line and gets the # correct description out of the comments section temp = printer[3].split('\n') for printq in re.split(',[ ]*', printer[0]): if len(temp) > 1: printersfile += ("%-16s%-12s%-32s%-26s%s\n" % (printq, printer[1], printer[2], temp[1], temp[0])) else: printersfile += ("%-16s%-12s%-32s%-26s%s\n" % (printq, printer[1], printer[2], '', printer[3])) self.filedata['printers.data'] = printersfile self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile def buildHostsLPD(self): """Creates the /mcs/etc/hosts.lpd file""" # this header needs to be changed to be more generic header = """+@machines +@all-machines achilles.ctd.anl.gov raven.ops.anl.gov seagull.hr.anl.gov parrot.ops.anl.gov condor.ops.anl.gov delphi.esh.anl.gov anlcv1.ctd.anl.gov anlvms.ctd.anl.gov olivia.ctd.anl.gov\n\n""" cursor = connection.cursor() cursor.execute(""" SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active' ORDER BY hostname""") redmachines = list(cursor.fetchall()) cursor.execute(""" SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active' """) redmachines.extend(list(cursor.fetchall())) cursor.execute(""" SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active' ORDER BY hostname""") winmachines = list(cursor.fetchall()) cursor.execute(""" SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active' """) winmachines.__add__(list(cursor.fetchall())) hostslpdfile = header for machine in redmachines: hostslpdfile += machine[0] + "\n" hostslpdfile += "\n" for machine in winmachines: hostslpdfile += machine[0] + "\n" self.filedata['hosts.lpd'] = hostslpdfile self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile def buildNetgroups(self): """Makes the *-machine files""" header = """################################################################### # This file lists hosts in the '%s' machine netgroup, it is # automatically generated. DO NOT EDIT THIS FILE! # # Number of hosts in '%s' machine netgroup: %i #\n\n""" cursor = connection.cursor() # fetches all the hosts that with valid netgroup entries cursor.execute(""" SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active' ORDER BY h.netgroup, h.hostname """) nameslist = cursor.fetchall() # gets the first host and initializes the hash hostdata = nameslist[0] netgroups = {hostdata[2]: [hostdata[0]]} for row in nameslist: # if new netgroup, create it if row[2] not in netgroups: netgroups.update({row[2]: []}) # if it belongs in the netgroup and has multiple interfaces, put them in if hostdata[0] == row[0] and row[3]: netgroups[row[2]].append(row[1]) hostdata = row # if its a new host, write the old one to the hash elif hostdata[0] != row[0]: netgroups[row[2]].append(row[0]) hostdata = row for netgroup in netgroups: fileoutput = StringIO() fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup]))) for each in netgroups[netgroup]: fileoutput.write(each + "\n") self.filedata['%s-machines' % netgroup] = fileoutput.getvalue() fileoutput.close() self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile cursor.execute(""" UPDATE hostbase_host SET dirty=0 """)