Rework bcfg2-admin pull

 - forward port Cfg and SSHbase support
 - reimplement admin mode 
 - add verbose flag, and implement initial interactive mode, also force mode


git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@4446 ce84e21b-d406-0410-9b95-82705330c041

4 files changed, 156 insertions, 149 deletions

diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py
index 1071b5100..ede4271fb 100644
--- a/src/lib/Server/Admin/Pull.py
+++ b/src/lib/Server/Admin/Pull.py
@@ -1,89 +1,110 @@
 
-import binascii, lxml.etree, time
+import binascii, difflib, getopt, lxml.etree, time, ConfigParser
 import Bcfg2.Server.Admin
 
 class Pull(Bcfg2.Server.Admin.Mode):
     '''Pull mode retrieves entries from clients and integrates the information into the repository'''
-    __shorthelp__ = 'bcfg2-admin pull <client> <entry type> <entry name>'
+    __shorthelp__ = 'bcfg2-admin pull [-v] [-f] [-I] <client> <entry type> <entry name>'
     __longhelp__ = __shorthelp__ + '\n\tIntegrate configuration information from clients into the server repository'
-    def __init__(self):
-        Bcfg2.Server.Admin.Mode.__init__(self)
-
+    def __init__(self, configfile):
+        Bcfg2.Server.Admin.Mode.__init__(self, configfile)
+        cp = ConfigParser.ConfigParser()
+        cp.read([configfile])
+        self.repo = cp.get('server', 'repository')
+        self.log = False
+        self.mode = 'interactive'
+        
     def __call__(self, args):
         Bcfg2.Server.Admin.Mode.__call__(self, args)
-        self.PullEntry(args[0], args[1], args[2])
+        try:
+            opts, gargs = getopt.getopt(args, 'vfI')
+        except:
+            print self.__shorthelp__
+            raise SystemExit(0)
+        for opt in opts:
+            if opt[0] == '-v':
+                self.log = True
+            elif opt[0] == '-f':
+                self.mode = 'force'
+            elif opt[0] == '-I':
+                self.mode == 'interactive'
+        self.PullEntry(gargs[0], gargs[1], gargs[2])
 
-    def PullEntry(self, client, etype, ename):
-        '''Make currently recorded client state correct for entry'''
-        # FIXME Pull.py is _way_ too interactive
+    def BuildNewEntry(self, client, etype, ename):
+        '''construct a new full entry for given client/entry from statistics'''
+        new_entry = {'type':etype, 'name':ename}
         sdata = self.load_stats(client)
-        if sdata.xpath('.//Statistics[@state="dirty"]'):
-            state = 'dirty'
-        else:
-            state = 'clean'
-        # need to pull entry out of statistics
-        sxpath = ".//Statistics[@state='%s']/Bad/ConfigFile[@name='%s']/../.." % (state, ename)
+        # no entries if state != dirty
+        sxpath = ".//Statistics[@state='dirty']/Bad/ConfigFile[@name='%s']/../.." % \
+                 (ename)
         sentries = sdata.xpath(sxpath)
         if not len(sentries):
             self.errExit("Found %d entries for %s:%s:%s" % \
                          (len(sentries), client, etype, ename))
         else:
-            print "Found %d entries for %s:%s:%s" % \
-                  (len(sentries), client, etype, ename)
+            if self.log:
+                print "Found %d entries for %s:%s:%s" % \
+                      (len(sentries), client, etype, ename)
         maxtime = max([time.strptime(stat.get('time')) for stat in sentries])
-        print "Found entry from", time.strftime("%c", maxtime)
+        if self.log:
+            print "Found entry from", time.strftime("%c", maxtime)
         statblock = [stat for stat in sentries \
                      if time.strptime(stat.get('time')) == maxtime]
         entry = statblock[0].xpath('.//Bad/ConfigFile[@name="%s"]' % ename)
         if not entry:
-            self.errExit("Could not find state data for entry; rerun bcfg2 on client system")
+            self.errExit("Could not find state data for entry\n" \
+                         "rerun bcfg2 on client system")
         cfentry = entry[-1]
 
         badfields = [field for field in ['perms', 'owner', 'group'] \
                      if cfentry.get(field) != cfentry.get('current_' + field) and \
                      cfentry.get('current_' + field)]
         if badfields:
-            m_updates = dict([(field, cfentry.get('current_' + field)) \
-                              for field in badfields])
-            print "got metadata_updates", m_updates
-        else:
-            m_updates = {}
-
-        if 'current_bdiff' in cfentry.attrib:
-            data = False
+            for field in badfields:
+                new_entry[field] = cfentry.get('current_%s' % field)
+        # now metadata updates are in place
+        if 'current_bfile' in cfentry.attrib:
+            new_entry['text'] = binascii.a2b_base64(cfentry.get('current_bfile'))
+        elif 'current_bdiff' in cfentry.attrib:
             diff = binascii.a2b_base64(cfentry.get('current_bdiff'))
-        elif 'current_diff' in cfentry.attrib:
-            data = False
-            diff = cfentry.get('current_diff')
-        elif 'current_bfile' in cfentry.attrib:
-            data = binascii.a2b_base64(cfentry.get('current_bfile'))
-            diff = False
+            new_entry['text'] = '\n'.join(difflib.restore(diff.split('\n'), 1))
         else:
-            if not m_updates:
-                self.errExit("having trouble processing entry. Entry is:\n" \
-                             + lxml.etree.tostring(cfentry))
-            else:
-                data = False
-                diff = False
+            print "found no data::"
+            print lxml.etree.tostring(cfentry)
+            raise SystemExit(1)
+        return new_entry
 
-        if diff:
-            print "Located diff:\n %s" % diff
-        elif data:
-            print "Found full (binary) file data"
-        if m_updates:
-            print "Found metadata updates"
+    def Choose(self, choices):
+        '''Determine where to put pull data'''
+        if self.mode == 'interactive':
+            # FIXME improve bcfg2-admin pull interactive mode to add new entries
+            print "Plugin returned choice:"
+            if choices[0].all:
+                print " => global entry"
+            elif choices[0].group:
+                print " => group entry: %s (prio %d)" % (choices[0].group, choices[0].prio)
+            else:
+                print " => host entry: %s" % (choices[0].hostname)
+            if raw_input("Use this entry? [yN]: ") in ['y', 'Y']:
+                return choices[0]
+            return False
+        else:
+            # mode == 'force'
+            return choices[0]
 
-        if not diff and not data and not m_updates:
-            self.errExit("Failed to locate diff or full data or metadata updates\nStatistics entry was:\n%s" % lxml.etree.tostring(cfentry))
+    def PullEntry(self, client, etype, ename):
+        '''Make currently recorded client state correct for entry'''
+        new_entry = self.BuildNewEntry(client, etype, ename)
 
         try:
-            bcore = Bcfg2.Server.Core.Core({}, self.configfile)
+            bcore = Bcfg2.Server.Core.Core(self.repo, [], ['Cfg', 'SSHbase', 'Metadata'],
+                                           'foo', False)
         except Bcfg2.Server.Core.CoreInitError, msg:
             self.errExit("Core load failed because %s" % msg)
-        [bcore.fam.Service() for _ in range(10)]
+        [bcore.fam.Service() for _ in range(5)]
         while bcore.fam.Service():
             pass
-        m = bcore.metadata.get_metadata(client)
+        meta = bcore.metadata.get_metadata(client)
         # find appropriate plugin in bcore
         glist = [gen for gen in bcore.generators if
                  gen.Entries.get(etype, {}).has_key(ename)]
@@ -92,8 +113,11 @@ class Pull(Bcfg2.Server.Admin.Mode):
                          + "%s" % ([g.__name__ for g in glist]))
         plugin = glist[0]
         try:
-            plugin.AcceptEntry(m, 'ConfigFile', ename, diff, data, m_updates)
+            choices = plugin.AcceptChoices(new_entry, meta)
+            specific = self.Choose(choices)
+            if specific:
+                plugin.AcceptPullData(specific, new_entry, self.log)
         except Bcfg2.Server.Plugin.PluginExecutionError:
             self.errExit("Configuration upload not supported by plugin %s" \
                          % (plugin.__name__))
-        # svn commit if running under svn
+        # FIXME svn commit if running under svn
diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py
index 4b1d9d8d1..10bc4f1fc 100644
--- a/src/lib/Server/Plugin.py
+++ b/src/lib/Server/Plugin.py
@@ -70,7 +70,7 @@ class Plugin(object):
     def AcceptChoices(self, entry, metadata):
         raise PluginExecutionError
     
-    def AcceptPullData(self, specific, new_entry):
+    def AcceptPullData(self, specific, new_entry, verbose):
         '''This is the null per-plugin implementation
         of bcfg2-admin pull'''
         raise PluginExecutionError
diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py
index aebce6188..f3e485517 100644
--- a/src/lib/Server/Plugins/Cfg.py
+++ b/src/lib/Server/Plugins/Cfg.py
@@ -1,8 +1,7 @@
 '''This module implements a config file repository'''
 __revision__ = '$Revision$'
 
-import binascii, difflib, logging, os, re, tempfile, \
-       xml.sax.saxutils, Bcfg2.Server.Plugin, lxml.etree
+import binascii, logging, os, re, tempfile, Bcfg2.Server.Plugin
 
 logger = logging.getLogger('Bcfg2.Plugins.Cfg')
 
@@ -53,13 +52,15 @@ class CfgEntry(object):
         if entry.get('encoding') == 'base64':
             entry.text = binascii.b2a_base64(self.data)
         else:
-            entry.text = self.data            
+            entry.text = self.data
+        if not entry.text:
+            entry.set('empty', 'true')
 
 class CfgMatcher:
     def __init__(self, fname):
         name = re.escape(fname)
-        self.basefile_reg = re.compile('^%s(|\\.H_(?P<hostname>\S+)|.G(?P<prio>\d+)_(?P<group>\S+))$' % name)
-        self.delta_reg = re.compile('^%s(|\\.H_(?P<hostname>\S+)|\\.G(?P<prio>\d+)_(?P<group>\S+))\\.(?P<delta>(cat|diff))$' % fname)
+        self.basefile_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+)|.G(?P<prio>\d+)_(?P<group>\S+))$' % name)
+        self.delta_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+)|\\.G(?P<prio>\d+)_(?P<group>\S+))\\.(?P<delta>(cat|diff))$' % fname)
         self.cat_count = fname.count(".cat")
         self.diff_count = fname.count(".diff")
 
@@ -76,28 +77,66 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet):
 
     def sort_by_specific(self, one, other):
         return cmp(one.specific, other.specific)
-                
-    def bind_entry(self, entry, metadata):
+
+    def get_pertinent_entries(self, metadata):
+        '''return a list of all entries pertinent to a client => [base, delta1, delta2]'''
         matching = [ent for ent in self.entries.values() if \
                     ent.specific.matches(metadata)]
-        if [ent for ent in matching if ent.specific.delta]:
-            self.bind_info_to_entry(entry, metadata)
-            matching.sort(self.sort_by_specific)
-            base = min([matching.index(ent) for ent in matching
-                        if not ent.specific.delta])
-            used = matching[:base+1]
-            used.reverse()
-            # used is now [base, delta1, delta2]            
-            basefile = used.pop()
-            data = basefile.data
-            for delta in used:
-                data = process_delta(data, delta)
-            if entry.get('encoding') == 'base64':
-                entry.text = binascii.b2a_base64(data)
-            else:
-                entry.text = data            
+        matching.sort(self.sort_by_specific)
+        base = min([matching.index(ent) for ent in matching
+                    if not ent.specific.delta])
+        used = matching[:base+1]
+        used.reverse()
+        return used
+
+    def bind_entry(self, entry, metadata):
+        self.bind_info_to_entry(entry, metadata)
+        used = self.get_pertinent_entries(metadata)
+        basefile = used.pop()
+        data = basefile.data
+        for delta in used:
+            data = process_delta(data, delta)
+        if entry.get('encoding') == 'base64':
+            entry.text = binascii.b2a_base64(data)
         else:
-            Bcfg2.Server.Plugin.EntrySet.bind_entry(self, entry, metadata)
+            entry.text = data            
+
+    def list_accept_choices(self, metadata):
+        '''return a list of candidate pull locations'''
+        used = self.get_pertinent_entries(metadata)
+        if len(used) > 1:
+            return []
+        return [used[0].specific]
+
+    def build_filename(self, specific):
+        bfname = self.path + '/' + self.path.split('/')[-1]
+        if specific.all:
+            return bfname
+        elif specific.group:
+            return "%s.G%d_%s" % (bfname, specific.group, specific.prio)
+        elif specific.hostname:
+            return "%s.H_%s" % (bfname, specific.hostname)
+
+    def write_update(self, specific, new_entry, log):
+        name = self.build_filename(specific)
+        open(name, 'w').write(new_entry['text'])
+        if log:
+            logger.info("Wrote file %s" % name)
+        badattr = [attr for attr in ['owner', 'group', 'perms'] if attr in new_entry]
+        if badattr:
+            if hasattr(self.entries[name.split('/')[-1]], 'infoxml'):
+                print "InfoXML support not yet implemented"
+                return
+            metadata_updates = {}
+            metadata_updates.update(self.metadata)
+            for attr in badattr:
+                metadata_updates[attr] = new_entry.get('attr')
+            infofile = open(self.path + '/:info', 'w')
+            for x in metadata_updates.iteritems():
+                infofile.write("%s: %s\n" % x)
+            infofile.close()
+            if log:
+                logger.info("Wrote file %s" % infofile.name)
 
 class Cfg(Bcfg2.Server.Plugin.GroupSpool):
     '''This generator in the configuration file repository for bcfg2'''
@@ -108,56 +147,9 @@ class Cfg(Bcfg2.Server.Plugin.GroupSpool):
     es_cls = CfgEntrySet
     es_child_cls = CfgEntry
 
-    def AcceptEntry(self, meta, _, entry_name, diff, fulldata, metadata_updates={}):
-        '''per-plugin bcfg2-admin pull support'''
-        if metadata_updates:
-            if hasattr(self.Entries['ConfigFile'][entry_name], 'infoxml'):
-                print "InfoXML support not yet implemented"
-            elif raw_input("Should metadata updates apply to all hosts? (n/Y) ") in ['Y', 'y']:
-                self.entries[entry_name].metadata.update(metadata_updates)
-                infofile = open(self.entries[entry_name].repopath + '/:info', 'w')
-                for x in self.entries[entry_name].metadata.iteritems():
-                    infofile.write("%s: %s\n" % x)
-                infofile.close()
-        if not diff and not fulldata:
-            raise SystemExit, 0
-                
-        hsq = "Found host-specific file %s; Should it be updated (n/Y): "
-        repo_vers = lxml.etree.Element('ConfigFile', name=entry_name)
-        self.Entries['ConfigFile'][entry_name](repo_vers, meta)
-        repo_curr = repo_vers.text
-        # find the file fragment
-        basefile = [frag for frag in \
-                    self.entries[entry_name].fragments \
-                    if frag.applies(meta)][-1]
-        gsq = "Should this change apply to all hosts effected by file %s? (N/y): " % (basefile.name)
-        if ".H_%s" % (meta.hostname) in basefile.name:
-            answer = raw_input(hsq % basefile.name)
-        else:
-            answer = raw_input(gsq)
-        
-        if answer in ['Y', 'y']:
-            print "writing file, %s" % basefile.name
-            if fulldata:
-                newdata = fulldata
-            else:
-                newdata = '\n'.join(difflib.restore(diff.split('\n'), 1))
-            open(basefile.name, 'w').write(newdata)
-            return
+    def AcceptChoices(self, entry, metadata):
+        return self.entries[entry.get('name')].list_accept_choices(metadata)
+
+    def AcceptPullData(self, specific, new_entry, log):
+        return self.entries[new_entry.get('name')].write_update(specific, new_entry, log)
 
-        if ".H_%s" % (meta.hostname) in basefile.name:
-            raise SystemExit, 1
-        # figure out host-specific filename
-        reg = re.compile("(.*)\.G\d+.*")
-        if reg.match(basefile.name):
-            newname = reg.match(basefile.name).group(1) + ".H_%s" % (meta.hostname)
-        else:
-            newname = basefile.name + ".H_%s" % (meta.hostname)
-        print "This file will be installed as file %s" % newname
-        if raw_input("Should it be installed? (N/y): ") in ['Y', 'y']:
-            print "writing file, %s" % newname
-            if fulldata:
-                newdata = fulldata
-            else:
-                newdata = '\n'.join(difflib.restore(diff.split('\n'), 1))
-            open(newname, 'w').write(newdata)
diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py
index 4254ad6d9..89767cf85 100644
--- a/src/lib/Server/Plugins/SSHbase.py
+++ b/src/lib/Server/Plugins/SSHbase.py
@@ -1,15 +1,9 @@
 '''This module manages ssh key files for bcfg2'''
 __revision__ = '$Revision$'
 
-import binascii, difflib, os, socket, xml.sax.saxutils
+import binascii, os, socket
 import Bcfg2.Server.Plugin
 
-def update_file(path, diff):
-    '''Update file at path using diff'''
-    newdata = '\n'.join(difflib.restore(diff.split('\n'), 1))
-    print "writing file, %s" % path
-    open(path, 'w').write(newdata)
-
 class SSHbase(Bcfg2.Server.Plugin.Plugin,  Bcfg2.Server.Plugin.DirectoryBacked):
     '''The sshbase generator manages ssh host keys (both v1 and v2)
     for hosts.  It also manages the ssh_known_hosts file. It can
@@ -190,17 +184,14 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin,  Bcfg2.Server.Plugin.DirectoryBacked):
                 except OSError:
                     self.logger.error("Failed to unlink temporary ssh keys")
 
-    def AcceptEntry(self, meta, _, entry_name, diff, fulldata, metadata_updates={}):
-        '''per-plugin bcfg2-admin pull support'''
-        filename = "%s/%s.H_%s" % (self.data, entry_name.split('/')[-1],
-                                   meta.hostname)
-        print "This file will be installed as file %s" % filename
-        if raw_input("Should it be installed? (N/y): ") in ['Y', 'y']:
-            print "writing file, %s" % filename
-            if fulldata:
-                newdata = fulldata
-            else:
-                newdata = '\n'.join(difflib.restore(diff.split('\n'), 1))
-            open(filename, 'w').write(newdata)
+    def AcceptChoices(self, _, metadata):
+        return Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)
 
-        
+    def AcceptPullData(self, specific, entry, log):
+        '''per-plugin bcfg2-admin pull support'''
+        # specific will always be host specific
+        filename = "%s/%s.H_%s" % (self.data, entry['name'].split('/')[-1],
+                                   specific.hostname)
+        open(filename, 'w').write(entry['text'])
+        if log:
+            print "Wrote file %s" % filename