diff options
62 files changed, 2454 insertions, 2079 deletions
diff --git a/debian/bcfg2-server.init b/debian/bcfg2-server.init index 1baf44f08..e052a854f 100644 --- a/debian/bcfg2-server.init +++ b/debian/bcfg2-server.init @@ -6,28 +6,28 @@ # description: bcfg2 server for configuration requests # -PIDFILE=/var/tmp/bcfg2-xmlrpc.pid +PIDFILE=/var/tmp/bcfg2-server.pid case "$1" in start) - echo -n "Starting Bcfg2Server: " + echo -n "Starting bcfg2-server: " if [ -f "/etc/SuSE-release" ] ; then - /sbin/start_daemon -p "${PIDFILE}" /usr/sbin/Bcfg2Server + /sbin/start_daemon -p "${PIDFILE}" /usr/sbin/bcfg2-server elif [ -f "/etc/redhat-release" ]; then - /usr/sbin/Bcfg2Server -D "${PIDFILE}" + /usr/sbin/bcfg2-server -D "${PIDFILE}" else - /sbin/start-stop-daemon --pidfile "${PIDFILE}" --make-pidfile -b -S --startas /usr/sbin/Bcfg2Server + /sbin/start-stop-daemon --pidfile "${PIDFILE}" --make-pidfile -b -S --startas /usr/sbin/bcfg2-server fi - echo "Bcfg2Server" + echo "bcfg2-server" ;; stop) - echo -n "Stopping Bcfg2Server: " + echo -n "Stopping bcfg2-server: " if [ -f "/etc/SuSE-release" ] ; then - /sbin/killproc -p "${PIDFILE}" /usr/sbin/Bcfg2Server + /sbin/killproc -p "${PIDFILE}" /usr/sbin/bcfg2-server elif [ -f "/etc/redhat-release" ]; then kill -INT `cat ${PIDFILE}` else - /sbin/start-stop-daemon -p "${PIDFILE}" -K /usr/sbin/Bcfg2Server + /sbin/start-stop-daemon -p "${PIDFILE}" -K /usr/sbin/bcfg2-server fi echo done ;; diff --git a/debian/bcfg2-server.install b/debian/bcfg2-server.install index d1ab3e9ac..442ecedf5 100644 --- a/debian/bcfg2-server.install +++ b/debian/bcfg2-server.install @@ -1,7 +1,6 @@ -debian/bcfg2-install/usr/bin/Bcfg2Server* usr/sbin -debian/bcfg2-install/usr/bin/ValidateBcfg2Repo usr/sbin +debian/bcfg2-install/usr/bin/bcfg2-* usr/sbin debian/bcfg2-install/usr/bin/StatReports usr/sbin debian/bcfg2-install/usr/bin/GenerateHostInfo usr/sbin debian/bcfg2-install/usr/lib/python2.3/site-packages/Bcfg2/Server/* usr/lib/python2.3/site-packages/Bcfg2/Server debian/bcfg2-install/usr/share/bcfg2/* usr/share/bcfg2/ -debian/bcfg2-install/usr/share/man/man8/* usr/share/man/man8
\ No newline at end of file +debian/bcfg2-install/usr/share/man/man8/* usr/share/man/man8 diff --git a/debian/changelog b/debian/changelog index c331caf28..e7f23ba42 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,20 +1,39 @@ -bcfg2 (0.7.4-1) unstable; urgency=low +bcfg2 (0.8.0-1) unstable; urgency=low - * final release + * New upstream release - -- Narayan Desai <desai@mcs.anl.gov> Mon, 16 Jan 2006 10:52:13 -0600 + -- Narayan Desai <desai@mcs.anl.gov> Fri, 20 Jan 2006 15:53:30 -0600 -bcfg2 (0.7.4pre5-1) unstable; urgency=low +bcfg2 (0.8.0pre5-1) unstable; urgency=low - * new upstream + * New upstream release + + -- Narayan Desai <desai@mcs.anl.gov> Mon, 16 Jan 2006 12:11:09 -0600 + +bcfg2 (0.8.0pre4-1) unstable; urgency=low + + * New upstream release + + -- Narayan Desai <desai@mcs.anl.gov> Wed, 11 Jan 2006 16:04:53 -0600 + +bcfg2 (0.8.0pre3-1) unstable; urgency=low + + * New upstream release + + -- Narayan Desai <desai@mcs.anl.gov> Tue, 10 Jan 2006 15:05:48 -0600 + +bcfg2 (0.8.0pre2-1) unstable; urgency=low + + * New upstream release - -- Narayan Desai <desai@mcs.anl.gov> Tue, 10 Jan 2006 14:56:21 -0600 + -- Narayan Desai <desai@mcs.anl.gov> Fri, 6 Jan 2006 14:18:36 -0600 -bcfg2 (0.7.4pre4-1) unstable; urgency=low +bcfg2 (0.8.0pre1-1) unstable; urgency=low * new upstream + * major repository format change - -- Narayan Desai <desai@mcs.anl.gov> Thu, 5 Jan 2006 10:43:16 -0600 + -- Narayan Desai <desai@mcs.anl.gov> Thu, 5 Jan 2006 10:47:04 -0600 bcfg2 (0.7.4pre3-1) unstable; urgency=low diff --git a/debian/conffiles b/debian/conffiles deleted file mode 100644 index 8b1378917..000000000 --- a/debian/conffiles +++ /dev/null @@ -1 +0,0 @@ - diff --git a/debian/control b/debian/control index be50aba98..a4774e6d4 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Description: build configuration system Package: bcfg2-server Architecture: all -Depends: ${python:Depends}, bcfg2, python-lxml (>= 0.8), python-fam | python2.3-gamin, m2crypto, libxml2-utils +Depends: ${python:Depends}, bcfg2, python-lxml (>= 0.8), python2.3-fam | python2.3-gamin, m2crypto, libxml2-utils, fam | gamin Description: build configuration system, version two Bcfg2 is a symbolic configuration management system, produced at Argonne National Lab. diff --git a/doc/Makefile b/doc/Makefile index dcbb6f8ea..3f3c06e5b 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,4 +1,5 @@ SOURCES = *.xml manual.pdf: $(SOURCES) - docbook2pdf --dcl /usr/share/sgml/declaration/xml.dcl manual.xml
\ No newline at end of file + xsltproc --stringparam body.font.family Helvetica --stringparam fop.extensions 1 /usr/share/xml/docbook/stylesheet/nwalsh/fo/docbook.xsl manual.xml > manual.fo + fop -fo manual.fo -pdf manual.pdf diff --git a/doc/arch.txt b/doc/arch.txt deleted file mode 100644 index 73da91b53..000000000 --- a/doc/arch.txt +++ /dev/null @@ -1,25 +0,0 @@ -The BCFG2 Architecture consists of four main parts: - -The core is the component containing shared data: - - file alteration monitor - - metadata store - - generators - - structuring agents - - change notification mechanism - -Generators: configuration atom construction mechanism - - configuration repository - - package management - - service management - - ssh key management - - hostbase - -Structuring Agents - code that forms the configuration into - independant and dependant clauses - - Bundler - - Translator - - Literal - - Image data - -Client - state transition engine that runs on clients. -
\ No newline at end of file diff --git a/doc/architecture.xml b/doc/architecture.xml index abb90c640..7c54fca45 100644 --- a/doc/architecture.xml +++ b/doc/architecture.xml @@ -72,7 +72,7 @@ </listitem> <listitem> <para> - Generators and administrative applications. + Plugins and administrative applications. </para> </listitem> <listitem> @@ -104,6 +104,12 @@ simple as possible. The normal configuration process consists of four main steps:</para> +<!-- <procedure> --> +<!-- <title>Bcfg2 Client Execution Process</title> --> + +<!-- <step>foo</step> --> +<!-- </procedure> --> + <variablelist> <varlistentry> <term>Probe Execution</term> @@ -316,18 +322,18 @@ literal configuration information. After the abstract configuration is created, each configuration entry must be bound to a client-specific value. The Bcfg2 server - uses generators to provide these client-specific bindings. + uses plugins to provide these client-specific bindings. </para> <para> The Bcfg2 server core contains a dispatch table that - describes which generator can handle requests of a - particular type. The responsible generator is located + describes which plugins can handle requests of a + particular type. The responsible plugin is located for each entry. It is called, passing in the configuration entry and the client's metadata. The - behavior of generators is explicitly undefined, so as to - allow maximum flexibility. The behavior of the stock - generators is documented elsewhere in this manual. + behavior of plugins is explicitly undefined, so as to + allow maximum flexibility. The behaviors of the stock + plugins are documented elsewhere in this manual. </para> <para> diff --git a/doc/concepts.xml b/doc/concepts.xml deleted file mode 100644 index 1d9c3c498..000000000 --- a/doc/concepts.xml +++ /dev/null @@ -1,111 +0,0 @@ -<chapter> - <title>Design Goals & Concepts</title> - - <para> - Bcfg2 was designed with several goals in mind. This section will - describe those goals, and how they were manifested in the - design. This section will also define important concepts used in - Bcfg2. - </para> - - <section> - <title>Goals</title> - - <itemizedlist> - <listitem> - <para> - Model configurations using declarative - semantics. Declarative semantics maximize the utility of - configuration management tools; they provide the most - flexibility for the tool to determine the right course of - action in any given situation. This means that users can - focus on the task of describing the desired configuration, - while leaving the task of transitioning clients states to - the tool. - </para> - </listitem> - <listitem> - <para> - Configuration descriptions should be comprehensive. This - means that configurations served to the client should be - sufficent to reproduce all desired functionality. This - assumption allows the use of heuristics to detect extra - configuration, aiding in reliable, comprehensive - configuration definitions. - </para> - </listitem> - <listitem> - <para> - Provide a flexible approach to user interactions. Most - configuration management systems take a rigid approach to - user interactions; that is, either the client system is - always correct, or the central system is. This means that - users are forced into an overly proscribed model where the - system asserts where correct data is. Configuration data - modification is frequently undertaken on both the - configuration server and clients. Hence, the existance of a - single canonical data location can easily pose a problem - during normal tool use. - </para> - <para> - Bcfg2 takes a different approach. The default assumption is - that data on the server is correct, however, the client has - option to run in another mode where local changes are - catalogued for server-side integration. If the Bcfg2 client is - run in dry run mode, it can help to reconcile differences - between current client state and the configuration described - on the server. - </para> - <para> - The Bcfg2 client also searches for extra configuration; that - is, configuration that is not specified by the configuration - description. When extra configuration is found, either - configuration has been removed from the configuration - description on the server, or manual configuration has - occurred on the client. Options related to two-way - verification and removal are useful for configuration - reconciliation when interactive access is used. - </para> - </listitem> - <listitem> - <para> - Generators, and administrative applications. - </para> - </listitem> - <listitem> - <para> - Imcremental operations. - </para> - </listitem> - </itemizedlist> - </section> - - <section> - <title>Important Concepts</title> - <variablelist> - <varlistentry> - <term>Bundles</term> - <listitem> - <para> - Bundles are groups of interdependent configuration - elements. Service configurations including software, - configuration files, and service activations are a good - example of bundles. When any of these components are - modified, all should be re-checked, and any associated - services should be restarted. We refer to this process as - coherent reconfiguration; this guarentees that all - configuration changes are active before reconfiguration - has completed. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>Metadata</term> - <listitem> - <para/> - </listitem> - </varlistentry> - </variablelist> - </section> - -</chapter>
\ No newline at end of file diff --git a/doc/deployment.xml b/doc/deployment.xml index d687ea187..1114589d3 100644 --- a/doc/deployment.xml +++ b/doc/deployment.xml @@ -52,58 +52,6 @@ ports. </para> - <variablelist> - <varlistentry> - <term>Package</term> - <listitem> - <para> - A software package. This entity includes a package name - and version number. It may optionally include installation - information (such as a package source URL) if one is - needed. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>ConfigFile</term> - <listitem> - <para> - A configuration file. This entity includes a file path, - owner, group, permissions, and file contents. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>SymLink</term> - <listitem> - <para> - A symbolic link. This entity includes a source and - destination. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>Service</term> - <listitem> - <para> - A service (de)activation. This controls services, a la - chkconfig or update-rc.d. Services are restarted - whenever co-located configuration entities are - modified. This ensures that any configuration changes - are flushed out to all active processes. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>Directory</term> - <listitem> - <para> - A filesystem directory. This entity includes an owner, - group and permissions. - </para> - </listitem> - </varlistentry> - </variablelist> </section> <section> @@ -129,166 +77,12 @@ </section> <section> - <title>Object Oriented Configs</title> - <para> - One of the most powerful and useful parts about bcfg2 metadata - system is the ability to have truely Object Oriented configs. I - have found that this has made me understand the machines I am - managing in a whole new light. Instead of focusing on what is - installed, I now focus on how machines relate to each other, or - what pieces of the metadata are similar in their configs. To - illistrate this think about the following example machines: - </para> - - <itemizedlist> - <listitem><para>A users desktop machine</para></listitem> - <listitem><para>A multiuser compute machine</para></listitem> - <listitem><para>A users home machine. (telecommuter)</para></listitem> - </itemizedlist> - <para> - These 3 machines have 3 distinct focuses for usage, but in reality - they can have a very similar metadata, depending on how the config is - broken up or the view that is taken of the config. If you focus first - on where the machines are the same it will help to build the common - metadata. below is what all 3 machines have in common: - </para> - <itemizedlist> - <listitem><para>users need to have the software they need to perform - there work. </para></listitem> - </itemizedlist> - <para> - let just encode this into metadata by creating a Class called - common-software that contains all the common software between all 3 - machines. - </para> - <programlisting> - <![CDATA[ - <Class name='common-software'> - <Bundle ... /> - .... - </Class> - ]]> - </programlisting> - <para> - now we need to find where they differ: - </para> - <variablelist> - <varlistentry> - <term>Desktop machines</term> - <listitem> - <itemizedlist> - <listitem> - <para>need to have a GUI interface( Xwindows or what not )</para> - </listitem> - <listitem> - <para>use NIS for authentification</para> - </listitem> - <listitem> - <para>use Autofs to mount home directories</para> - </listitem> - </itemizedlist> - </listitem> - </varlistentry> - <varlistentry> - <term>Multiuser compute machines</term> - <listitem> - <itemizedlist> - <listitem> - <para>only accessible by SSH, No GUI interface</para> - </listitem> - <listitem> - <para>use NIS for authentification</para> - </listitem> - <listitem> - <para>use Autofs to mount home directories</para> - </listitem> - </itemizedlist> - </listitem> - </varlistentry> - <varlistentry> - <term>Home machine</term> - <listitem> - <itemizedlist> - <listitem> - <para>need to have a GUI interface( Xwindows or what not )</para> - </listitem> - <listitem> - <para>use static password file</para> - </listitem> - <listitem> - <para>use local disk for home directories</para> - </listitem> - </itemizedlist> - </listitem> - </varlistentry> - </variablelist> - <para> - As you can see that there are common things pairwise in these configs - that can be further exploited by the object oriented system of bcfg2 - </para> - <programlisting> - <![CDATA[ - <Class name="Gui-Interface"> - <Bundle ... /> - ... - </Class> - <Class name="NIS-Autofs"> - <Bundle ... /> - ... - </Class> - ]]> - </programlisting> - <para> - now all that is left is to ensure that all needs are met and we find - we need to make one more class: - </para> - <programlisting> - <![CDATA[ - <Class name="static-password-local-disk"> - <Bundle ... /> - ... - </Class> - ]]> - </programlisting> - <para> - Now we can mix and match these classes together to build the 3 - profiles or even build new profiles with these descrete entities. - </para> - <programlisting> - <![CDATA[ - <Profile name='desktop'> - <Class name='common-software'/> - <Class name="Gui-Interface"/> - <Class name="NIS-Autofs"> - </Profile> - <Profile name='computerserver'> - <Class name='common-software'/> - <Class name="NIS-Autofs"> - </Profile> - <Profile name='home-desktop'> - <Class name='common-software'/> - <Class name="Gui-Interface"/> - <Class name="static-password-local-disk"> - </Profile> - <Profile name='bare-system'> - <Class name='common-software'/> - </Profile> - ]]> - </programlisting> - <para> - The free form object oriented fashion in which metadata can be - constructed is truely a double edge sword. On one hand you can build - up a nice list of descrete entities that can compose even the most - complicated configs, but it also allows for the creation of entities - that could provide monolithic solutions to each machines config. It is - all in how one views the machines and how much they are willing to - harness the power of the OO based metadata system. - </para> - </section> - <section> - <title>Tips & Tricks</title> + <title>Bcfg2 Server Administration</title> + <para/> + </section> + <section> <title>An example application of bcfg2</title> <para> diff --git a/doc/development.xml b/doc/development.xml new file mode 100644 index 000000000..057d20786 --- /dev/null +++ b/doc/development.xml @@ -0,0 +1,165 @@ +<chapter> +<title>Developing for Bcfg2</title> + + <para> + While the Bcfg2 server provides a good interface for representing + general system configurations, its plugin interface offers the + ability to implement configuration interfaces and representation + tailored to problems encountered by a particular site. This + chapter describes what plugins are good for, what they can do, and + how to implement them. + </para> + + <section> + <title>Bcfg2 Plugins</title> + + <para> + Bcfg2 plugins are loadable python modules that the Bcfg2 server + loads at initialization time. These plugins can contribute to + the functions already offered by the Bcfg2 server or can extend + its functionality. In general, plugins will provide some portion + of the configuration for clients, with a data representation + that is tuned for a set of common tasks. Much of the core + functionality of Bcfg2 is implemented by several plugins, + however, they are not special in any way; new plugins could + easily supplant one or all of them. + </para> + + <table> + <title>Bcfg2 Plugin Functions</title> + <tgroup cols='2'> + <colspec colnum='1'/> + <colspec colnum='2'/> + <thead> + <row><entry>Name</entry><entry>Description</entry></row> + </thead> + <tbody> + <row><entry>Probes</entry><entry>Plugins can send executable + code to clients, where local contributions to configuration + state can be gathered.</entry></row> + <row><entry>Abstract Configuration Structures</entry> + <entry>A plugin can define new groups of interdependent + and independent configuration entities</entry></row> + <row><entry>Literal Configuration Entities</entry> + <entry>Plugins can provide literal configuration entity + information.</entry></row> + <row><entry>XML-RPC Functions</entry> + <entry>Plugins can expose a set of functions through the + Bcfg2 server's authenticated XML-RPC interface.</entry></row> + </tbody> + </tgroup> + </table> + + </section> + + <section> + <title>Writing Bcfg2 Plugins</title> + + <para> + Bcfg2 plugins are python classes that subclass from + Bcfg2.Server.Plugin.Plugin. Several plugin-specific values must + be set in the new plugin. These values dictate how the new + plugin will behave with respect to the above four functions. + </para> + + + <table> + <title>Bcfg2 Plugin Members</title> + <tgroup cols='3'> + <colspec colnum='1' colwidth='1*'/> + <colspec colnum='2' colwidth='3*'/> + <colspec colnum='3' colwidth='2*'/> + <thead> + <row><entry>Name</entry><entry>Description</entry><entry>Format</entry></row> + </thead> + <tbody> + <row><entry>__name__</entry><entry>The name of the + plugin</entry><entry>string</entry> + </row> + <row><entry>__version__</entry> + <entry>The plugin version (generally tied to revctl + keyword expansion).</entry><entry>string</entry></row> + <row><entry>__author__</entry> + <entry>The plugin author.</entry><entry>string</entry></row> + <row><entry>__rmi__</entry> + <entry>Set of functions to be exposed as XML-RPC + functions</entry> + <entry>List of function names (strings)</entry></row> + <row><entry>Entries</entry><entry>Multidimentional + dictionary of keys that point to the function used to bind + literal contents for a given configuration + entity.</entry><entry>Dictionary of + ConfigurationEntityType, Name keys and function reference + values</entry></row> + <row><entry>BuildStructures</entry><entry>Function that + returns a list of the structures for a given + client</entry><entry>Member function</entry></row> + <row><entry>GetProbes</entry><entry>Function that returns a + list of probes that a given client should + execute</entry><entry>Member function</entry></row> + <row><entry>ReceiveData</entry><entry>Function that accepts + the probe results for a given client.</entry><entry>Member + function</entry></row> + </tbody> + </tgroup> + </table> + + <section> + <title>An Example Plugin</title> + + <example> + <title>A Simple Plugin</title> + <programlisting>import socket, Bcfg2.Server.Plugin + +class Chiba(Bcfg2.Server.Plugin.Plugin): + '''the Chiba plugin builds the following files: + -> /etc/network/interfaces''' + + __name__ = 'Chiba' + __version__ = '$Id: chiba.py 1702 2006-01-19 20:20:51Z desai ' + __author__ = 'bcfg-dev@mcs.anl.gov' + Entries = {'ConfigFile':{}} + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + self.repo = Bcfg2.Server.Plugin.DirectoryBacked(self.data, + self.core.fam) + self.Entries['ConfigFile']['/etc/network/interfaces'] \ + = self.build_interfaces + + def build_interfaces(self, entry, metadata): + '''build network configs for clients''' + entry.attrib['owner'] = 'root' + entry.attrib['group'] = 'root' + entry.attrib['perms'] = '0644' + try: + myriaddr = socket.gethostbyname("%s-myr" % \ + metadata.hostname) + except socket.gaierror: + self.LogError("Failed to resolve %s-myr"% metadata.hostname) + raise Bcfg2.Server.Plugin.PluginExecutionError, ("%s-myr" \ + % metadata.hostname, 'lookup') + entry.text = self.repo.entries['interfaces-template'].data % \ + myriaddr + </programlisting> + </example> + + <para> + Bcfg2 server plugins must subclass the + Bcfg2.Server.Plugin.Plugin class. Plugin constructors must + take two arguments: an instance of a Bcfg2.Core object, and a + location for a datastore. __name__, __version__, __author__, + and Entries are used to describe what the plugin is and how it + works. Entries describes a set of configuration entries that + can be provided by the generator, and a set of handlers that + can bind in the proper data. build_interfaces is an example of + a handler. It gets client metadata and an configuration entry + passed in, and binds data into entry as appropriate. This + results in a <filename>/etc/network/interfaces</filename> file + that has static information derived from DNS for a given host. + </para> + + </section> + + </section> +</chapter>
\ No newline at end of file diff --git a/doc/generators.txt b/doc/generators.txt deleted file mode 100644 index 4ad6dc74e..000000000 --- a/doc/generators.txt +++ /dev/null @@ -1,74 +0,0 @@ -A generator is a module, loaded into bcfg2's address space, that can -be called to produce configuration data for clients. The code contains -can do anything, so any configuration pattern can be modeled. These -are useful for a variety of tasks; we currently use this interface to -define all of our sources of configuration logic. While this construct -was available in bcfg1, it suffered from a variety of issues, mainly -related to performance. The new API provides for high enough speed -execution that the generator API is the sole provider of configuration -data to the bcfg2 server. - -Several generators have been implemented in bcfg2: -Cfg: A basic configuration file repository -SSHBase: A ssh key management system -Pkgmgr: Provides information about available software -Svcmgr: Provides information about available services, and activations -Debconf: Generator debconf settings based on probed client data - -Any of these generators can be enabled in bcfg2.conf, and custom -generators can similarly be added. - -Generators are during the client configuration process at two -times. First, generators provide client-side probes. These can be used -to determine local client state where appropriate. Generators can -provide a list of probes for a client to execute. The data generated -by this client execution is routed back to the source generator, which -can then use this data to generate the client configuration. - -The second function is to provide explicit values for the elements of -a client configuration. This function is used in the following way: - -When a client requests its configuration, the bcfg2 daemon queries all -structuring agents for pertinent configurations. In general, this -means that all bundles and base configuration data for the appropriate -image are constructed. At this point, all of these are abstract, that -is, all entities contain name and type attributes, but don't contain -versions, or explicit file contents or status. Generators are queried -for individual configuration elements, of the form, which version of -package zsh should be installed on client X. - -Finding the generator responsible for a configuration entity has two -steps. First, each generator has a dispatch table (called -generator.__provides__) This dispatch table has a key for each type of -configuration element it handles, which in turn has keys for -particular instances of those types. For example, the generator that -handled configuration file /etc/ssh/sshd_config would have the -following dispatch table entry: -self.__provides__['ConfigFile']['/etc/ssh/sshd_config'] -The value of this key is a function that can be called to bind in -configuration elements. (like version, owner, permissions, or file -data) - -Generators can be activated in other ways, but i would just as soon -not admit it in public. - -Probes can be created in generators to discover information about -clients. These xml elements look like: -<Probe source='my-generator' name='test1' interpreter='/bin/sh'> -hostname</Probe> - -A generator can create as many of these as needed. Bcfg2 will query -for these by calling a generator's GetProbes method. By default, this -function returns an empty list, but can return an arbitrary number of -probe elements. - -These probes are forwarded to the client, where they are -executed. Output is collected and sent back to the bcfg2 server. the -source attribute is used to determine which data belongs to which -generator, and the AcceptProbeData method is called for each probe -result belonging to the generator. This data may then be used in -constructing configuration elements for the client. - -There is a generator base class that provides skeleton methods, which -custom generators should subclass. It is named -Bcfg2.Server.Generator.Generator.
\ No newline at end of file diff --git a/doc/generators.xml b/doc/generators.xml deleted file mode 100644 index 053c754e3..000000000 --- a/doc/generators.xml +++ /dev/null @@ -1,266 +0,0 @@ -<chapter> - <title>Generators</title> - - <para> - Generators are modules which are loaded by the Bcfg2 server, - based on directives in <filename>/etc/bcfg2.conf</filename>. They - provide concrete, fully-specified configuration entries for - clients. This chapter documents the function and usage of - generators bundled with Bcfg2 releases. It also describes the - interface used to communicate with generators; modeles - implementing this interface can provide configuration elements for - clients based on any representation or requirements that may - exist. - </para> - - <section> - <title>Bundled Generators</title> - - <para>This section describes the generators that come bundled with - Bcfg2. As a general rule, generators requiring more than one - configuration file will use a generator specific directory in the - configuration repository. - </para> - - <section> - <title>Cfg</title> - <para> - The Cfg generator provides a configuration file repository - that uses literal file contents to provide client-tailored - configuration file entries. The Cfg generator chooses which - data to provide for a given client based on the aspect-based - metadata system used for high-level client configuration. - </para> - <para> - The Cfg repository is structured much like the filesystem - hierarchy being configured. Each configuration file being - served has a corresponding directory in the configuration - repository. These directories have the same relative path as - the absolute path of the configuration file on the target - system. For example, if Cfg was serving data for the - configuration file <filename>/etc/services</filename>, then - its directory would be in the relative path - <filename>./etc/services</filename> inside of the Cfg - repository. - </para> - <para> - Inside of this file-specific directory, three types of files - may exist. Base files are complete instances of configuration - file. Deltas are differences between a base file and the - target file contents. Base files and deltas are tagged with - metadata specifiers, which describe which groups of clients - the fragment pertains to. Configuration files are constructed - by finding the most specific base file and applying any more - specific deltas. - </para> - <para> - Specifiers are embedded in fragment filenames. For example, in - the fragment <filename>services.C99_webserver</filename>, - "C99_webserver" is the specifier. This specifier applies to - the class (C) webserver with a priority of 99. Other metadata - categories which can be used include bundle (B), profile (P), - hostname (H), attribute (A), and image (I). These are ordered - from least to most specific: image, profile, class, bundle, - and hostname. Global files are the least specific. Priorities - are used as to break ties. - </para> - <para> - Info files, named <filename>:info</filename> are used to - specify target configuration file metadata, such as owner, - group and permissions. If no <filename>:info</filename> is - provided, targets are installed with default - information. Default metadata is root ownership, root group - memberships, and 0644 file permissions. - </para> - <example> - <title>Cfg generator :info files</title> - <programlisting> - owner:root - group:root - perms:0755 - </programlisting> - </example> - - <example> - <title>Cfg file repository example</title> - <programlisting> - $ ls - :info passwd passwd.C99_chiba-login - passwd.H_bio-debian passwd.H_cvstest passwd.H_foxtrot - passwd.H_reboot passwd.H_rudy2 passwd.C99_netserv - passwd.B99_tacacs-server.cat passwd.H_adenine - </programlisting> - </example> - - <para> - In the previous example, there exists files with each of the - characteristics mentioned above. All files ending in ".cat" - are deltas; ones with ".H_" are host specific files. There - exists a base file, a <filename>:info</filename> file, two - class-specified base files, and a bundle-specified base file. - </para> - </section> - - <section> - <title>The Scoped XML File Group: Servicemgr</title> - <para> - The generator Servicemgr uses files formatted similarly to the - metadata files. It - works based on a single file, which contains definitions - ordered by increasing specificity. - </para> - - <example> - <title><filename>services.xml</filename></title> - <programlisting> - <![CDATA[<Services> - <Class name='webserver'> - <Service status='on' name='httpd' port='80' protocol='tcp'> - <User address='0.0.0.0' mask='32'/> - </Service> - </Class> - <Host name='mailhost'> - <Service name='sendmail' status='on' protocol='tcp' port='80'> - <User address='0.0.0.0' mask='32'/> - </Service> - </Host> - <Host name='thai'> - <Service name='ssh' status='off'/> - </Host> - <Service name='ssh' status='on' protocol='tcp' port='22'/> - <Service name='ntp-server' status='on' /> -</Services>]]> - </programlisting> - </example> - - <para> - This set of service definitions is intrepreted in the - following way. Webservers run httpd, the host mailhost runs - sendmail, and all machines run ssh, and the ntp-server. - </para> - </section> - - </section> - - <section> - <title>The Generator API</title> - <para> - The Bcfg2 core has a well-formed API used to call - generators. This mechanism allows all stock generators to be - runtime selected; no stock generators are required. The - generator API has two main functions. The first is communication - to the Bcfg2 core: the list of entries a particular generator - can bind must be communicated to the core so that the proper - generator can be called. The second function is the actual - production of client-specific configuration element data; this - data is then included in client configurations. - </para> - - <para> - The inventory function is provided by a python dictionary, - called __provides__ in each generator object. This dictionary - has a key for each type of configuration entry (ConfigFile, - Package, Directory, SymLink, Service), whose value is a - dictionary indexed by configuration element name. For example, - the data path to information about the service "sshd" could be - reached at __provides__['Service']['sshd']. The value of each of - these keys is a function that can be called to bind - client-specific values to a configuration entry. This function - is used in the next section. - </para> - - <para> - The handler function located by the __provides__ dictionary is - called with a static API. The function prototype for each of - these handlers is: - </para> - - <example> - <title>The Generator handler API</title> - <programlisting> - def Handler(self, entry, metadata): - generator logic here - </programlisting> - </example> - - <para> - The data supplied upon handler invokation includes two - parts. The first is the entry. This is a ElementTree.Element - object, which already contains the configuration element type - (ie Service) and name. All other data is bound into this object - in this function. The range of data bound depends on the data - type. The other data provided to handlers is client metadata, - information about the current client, including hostname, image, - profile, classes and bundles. The metadata is typically used to - choose entry contents. - </para> - </section> - - <section> - <title>Writing a Generator</title> - <para> - Writing a generator is a fairly straightforward task. At a high - level, generators are instantiated by the Bcfg2 core, and then - used to provide configuration entry contents. This means that - the two points where control passes into a generator from Bcfg2 - are during initial object instantiation, and every time a - generator-provided configuration entry is bound. - </para> - - <para> - Currently, generators must be written in python. They can - perform arbitrary operations, hence, a generator could be - written that executed logic in another language, but this - functionality is currently not implemented. - </para> - - <example> - <title>Simple Generator</title> - <programlisting> - from socket import gethostbyname, gaierror - from syslog import syslog, LOG_ERR - from Bcfg2.Server.Generator import Generator, DirectoryBacked, SingleXMLFileBacked, GeneratorError - -class Chiba(Generator): - '''the Chiba generator builds the following files: - -> /etc/network/interfaces''' - - __name__ = 'Chiba' - __version__ = '$Id: Chiba.py 1.12 05/01/15 11:05:02-06:00 desai@topaz.mcs.anl.gov $' - __author__ = 'bcfg-dev@mcs.anl.gov' - __provides__ = {'ConfigFile':{}} - - def __init__(self, core, datastore): - Generator.__init__(self, core, datastore) - self.repo = DirectoryBacked(self.data, self.core.fam) - self.__provides__['ConfigFile']['/etc/network/interfaces'] = self.build_interfaces - - def build_interfaces(self, entry, metadata): - '''build network configs for clients''' - entry.attrib['owner'] = 'root' - entry.attrib['group'] = 'root' - entry.attrib['perms'] = '0644' - try: - myriaddr = gethostbyname("%s-myr" % metadata.hostname) - except gaierror: - syslog(LOG_ERR, "Failed to resolve %s-myr"% metadata.hostname) - raise GeneratorError, ("%s-myr" % metadata.hostname, 'lookup') - entry.text = self.repo.entries['interfaces-template'].data % myriaddr - </programlisting> - </example> - - <para> - Generators must subclass the Bcfg2.Server.Generator.Generator - class. Generator constructors must take two arguments: an - instance of a Bcfg2.Core object, and a location for a - datastore. __name__, __version__, __author__, and __provides__ - are used to describe what the generator is and how it - works. __provides__ describes a set of configuration entries - that can be provided by the generator, and a set of handlers - that can bind in the proper data. build_interfaces is an example - of a handler. It gets client metadata and an configuration entry - passed in, and binds data into entry as appropriate. - </para> - - </section> -</chapter>
\ No newline at end of file diff --git a/doc/install.xml b/doc/install.xml index 13cf2847f..27b88c636 100644 --- a/doc/install.xml +++ b/doc/install.xml @@ -5,8 +5,8 @@ <title>Pre-requisites</title> <para> Bcfg2 is written in python using several modules not included - with most distributions. Element Tree, available from - http://www.effbot.org provides convenient XML handling. + with most distributions. lxml provides convenient xml + handling. M2crypto wraps openssl calls for https support. </para> <para> @@ -16,125 +16,95 @@ use SSL functions. </para> - <para>ElementTree can be downloaded from - http://www.effbot.org/downloads. It can be installed by running - the setup script against the python installation. + <para>lxml is required for xml parsing. It can be downloaded from + http://www.codespeak.net/lxml. It, in turn, requires libxml2, + libxslt, and pyrex. </para> - - <programlisting>$ python setup.py build -running build -running build_py -creating build -creating build/lib -creating build/lib/elementtree -copying elementtree/ElementInclude.py -> build/lib/elementtree -copying elementtree/ElementPath.py -> build/lib/elementtree -copying elementtree/ElementTree.py -> build/lib/elementtree -copying elementtree/HTMLTreeBuilder.py -> build/lib/elementtree -copying elementtree/SgmlopXMLTreeBuilder.py -> build/lib/elementtree -copying elementtree/SimpleXMLTreeBuilder.py -> build/lib/elementtree -copying elementtree/SimpleXMLWriter.py -> build/lib/elementtree -copying elementtree/TidyHTMLTreeBuilder.py -> build/lib/elementtree -copying elementtree/TidyTools.py -> build/lib/elementtree -copying elementtree/XMLTreeBuilder.py -> build/lib/elementtree -copying elementtree/__init__.py -> build/lib/elementtree -$ python setup.py install -... - </programlisting> - + <para> The python fam binding can be downloaded from python-fam.sourceforge.net. FAM (on several linux distributions) has been depricated in favor of gamin. The Bcfg server will autodetect which modules are available, and use appropriate file - caching logic. - </para> - </sect1> - <sect1> - <title>Bcfg2 Installation</title> - <para> + caching logic. It can be installed by running the setup.py script. </para> + + <table> + <title>Bcfg2 Software Prerequisites</title> + <tgroup cols='3'> + <colspec colnum='1' colwidth='2*'/> + <colspec colnum='2' colwidth='5*'/> + <colspec colnum='3' colwidth='8*'/> + <thead> + <row><entry>Name</entry><entry>Description</entry><entry>URL</entry></row> + </thead> + <tbody> + <row><entry>lxml</entry><entry>XML + Processing</entry><entry><ulink + url="http://codespeak.net/lxml"/></entry></row> + <row><entry>pyrex</entry><entry>C to Python language + interoperability (needed for lxml)</entry><entry><ulink + url="http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex"/></entry></row> + <row><entry>M2Crypto</entry> + <entry>OpenSSL bindings for Python</entry><entry><ulink + url="http://wiki.osafoundation.org/bin/view/Projects/MeTooCrypto"/></entry></row> + <row><entry>Swig</entry> + <entry>C to Python language interoperability (needed for + M2Crypto)</entry><entry><ulink + url="http://www.swig.org"/></entry></row> + <row><entry>Fam</entry><entry>File Alteration + Monitor</entry><entry><ulink + url="http://oss.sgi.com"/></entry></row> + <row><entry>Gamin</entry><entry>Alternate File Alteration + Monitor</entry><entry><ulink + url="http://www.gnome.org/~veillard/gamin/"/></entry></row> + <row><entry>Python-fam</entry><entry>Python bindings for fam + (not needed with + gamin)</entry><entry><ulink url="http://python-fam.sourceforge.net"/></entry></row> + </tbody> + </tgroup> + </table> + </sect1> <sect1> <title>Bcfg2 Initial Setup and Testing</title> <para>Once the Bcfg2 software is installed, the configuration file and repository must be created. The example configuration file in <filename>bcfg2/examples/bcfg2.conf</filename> can be used, with - minor modifications. + minor modifications. This should be placed in + <filename>/etc/bcfg2.conf</filename>. If it is placed in another + location, each program takes a command line argument to specify + its alternate location. </para> + <example> - <title>bcfg2.conf</title> + <title>/etc/bcfg2.conf</title> <programlisting>[server] - repository = /disks/bcfg2 - structures = Bundler,Base - generators = SSHbase,Cfg,Pkgmgr,Svcmgr - metadata = /disks/bcfg2/etc - </programlisting> +repository = /disks/bcfg2 +structures = Bundler,Base +generators = SSHbase,Cfg,Pkgmgr,Svcmgr</programlisting> </example> - <para>This configuration file sets the location of the - configuration repository. It also activates two structures, and - four generators. Structures are components that generate - abstract configuration fragments. These are the form of the - configuration. Generators provide client-specific values for - each configuration settings contained in all abstract - configuration fragments. Both of these are described in Section - ???.</para> - </sect1> - <sect1> - <title>Daemon Configuration</title> - <para>Bcfg2 uses SSSlib, the - communication libraries from the Scalable Systems Software project - for communication abstraction. This library provides a unified - messaging interface on top of several wire protocols with - different authentication and encryption mechanisms. The default - protocol is "challenge" which is a challenge response protocol - with no data encryption. (SSL protection will be configured - later). SSSlib also includes service location functionality; - this allows software to locate components by name, regardless of - their respective network locations. This function is provided - with both static and dynamic implementations. Static component - location setup will be sufficient for most Bcfg2 deployments. - </para> <para> - Static component lookups depend on the file - <filename>/etc/sss.conf</filename>. This file contains - information about static service locations. This file must be - the same on the server and all clients for communication to work - properly. A location definition for the bcfg2 component will - allow all clients to find and connect to it. - </para> - <example> - <title>/etc/sss.conf</title> - <programlisting> - <![CDATA[ <locations> - <location component="bcfg2" host="bcfgserver" - port="8052" protocol="challenge" schema_version="1.0" tier="1"/> - </locations>]]> - </programlisting> - </example> - <para>This allows SSSlib to locate the bcfg2 component on the - machine bcfgserver, port 8052, with the wire protocol "challenge". + This configuration file sets the top level location of the + configuration repository. It also activates two structures, and + four generators. Both structures and generators are instances of + Bcfg2 server plugins. Structures generate abstract configuration + fragments. These form the inventory of the + configuration. Generators provide client-specific literal values + for each configuration entity contained in the abstract + configuration. </para> </sect1> - <sect1> - <title>New-Style XML-RPC Deployments</title> - <para> - A new version of the Bcfg2 software is in testing that will - provide simplified and standards compliant communications - facilities. Instead of the use of SSSlib for communication, the - server and clients can use HTTPS XML-RPC instead. This has - required reimplementing the server and providing XML-RPC support - for the client, but provides drastically simplified setup for - new installs. - </para> + <sect1> + <title>Daemon Configuration</title> <para> - The prerequisite list now includes ElementTree, M2Crypto (for - SSL functions) and Python 2.2 or newer. ElementTree and M2Crypto - are both python modules that can be easily installed and are - already packaged for many Linux distributions. + Bcfg2 uses XML-RPC over HTTPS for all communications. + All communications occur over this transport. HTTPS provides + data security, while an embedded username and password provide + authentication. </para> <sect2> @@ -146,33 +116,40 @@ $ python setup.py install </para> <programlisting> -openssl req -x509 -nodes -days 1000 -newkey rsa:1024 -out server.pem -keyout server.pem +openssl req -x509 -nodes -days 1000 -newkey rsa:1024 \ + -out bcfg2.key -keyout bcfg2.key </programlisting> - <para>This command will generate an SSL key including both an - RSA key and a certificate. This is suitable for use with the - Bcfg2 XML-RPC server.</para> + <para> + This command will generate an SSL key including both an + RSA key and a certificate. This is suitable for use with the + Bcfg2 server. The path to this key should be put in the + bcfg2 configuration file in section communication, setting + key. + </para> </sect2> <sect2> - <title>Communication Bootstrapping</title> + <title>Client Communication Setup</title> <para> The Bcfg2 client must be able to find the server's location. This is accomplished through the use of the communication settings in <filename>/etc/bcfg2.conf</filename> - Two settings for the this section are required: protocol and - server url. + Several settings must be included in this file: the server + url, a username and a password. </para> <example> - <title>Bcfg2 XML-RPC Communication Settings</title> - <programlisting> - [communication] - protocol = xmlrpc/ssl - url = https://localhost:9443 - </programlisting> + <title>/etc/bcfg2.conf</title> + <programlisting>[communication] +protocol = xmlrpc/ssl +password = pwd +user = root + +[components] +bcfg2 = https://bcfg2server:8765</programlisting> </example> </sect2> </sect1> diff --git a/doc/manual.xml b/doc/manual.xml index 88d7e0eb3..878a104fb 100644 --- a/doc/manual.xml +++ b/doc/manual.xml @@ -1,40 +1,51 @@ <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook V4.4//EN" "http://www.oasis-open.org/docbook/xml/4.4/docbookx.dtd" [ -<!ENTITY concepts SYSTEM "concepts.xml"> <!ENTITY architecture SYSTEM "architecture.xml"> <!ENTITY install SYSTEM "install.xml"> -<!ENTITY generators SYSTEM "generators.xml"> +<!ENTITY specification SYSTEM "specs.xml"> <!ENTITY deployment SYSTEM "deployment.xml"> +<!ENTITY development SYSTEM "development.xml"> <!ENTITY reports SYSTEM "reports.xml"> ]> <book id="bcfg2"> <bookinfo> <title>Bcfg2 Manual</title> - <author> - <firstname>Narayan</firstname> - <surname>Desai</surname> - <email>desai@mcs.anl.gov</email> - </author> - <author> - <firstname>Rick</firstname> - <surname>Bradshaw</surname> - <email>bradshaw@mcs.anl.gov</email> - </author> - <author> - <firstname>Joey</firstname> - <surname>Hagedorn</surname> - <email>hagedorn@mcs.anl.gov</email> - </author> - <affiliation> - <orgname>Argonne National Laboratory</orgname> - <orgdiv>MCS Division</orgdiv> - </affiliation> + <authorgroup> + <author> + <firstname>Narayan</firstname> + <surname>Desai</surname> + <email>desai@mcs.anl.gov</email> + <affiliation> + <orgname>Argonne National Laboratory</orgname> + <orgdiv>MCS Division</orgdiv> + </affiliation> + </author> + <author> + <firstname>Rick</firstname> + <surname>Bradshaw</surname> + <email>bradshaw@mcs.anl.gov</email> + <affiliation> + <orgname>Argonne National Laboratory</orgname> + <orgdiv>MCS Division</orgdiv> + </affiliation> + </author> + <author> + <firstname>Joey</firstname> + <surname>Hagedorn</surname> + <email>hagedorn@mcs.anl.gov</email> + <affiliation> + <orgname>Argonne National Laboratory</orgname> + <orgdiv>MCS Division</orgdiv> + </affiliation> + </author> + </authorgroup> <date>September 2, 2005</date> + <edition>Manual for version 0.8.0</edition> <releaseinfo>$Revision$</releaseinfo> - <pubdate>September 2005</pubdate> + <pubdate>January 2006</pubdate> <copyright> - <year>2005</year> + <year>2005-2006</year> <holder>Argonne National Laboratory</holder> </copyright> @@ -56,18 +67,17 @@ <revhistory> <revision> - <revnumber>0.7.1pre6</revnumber> + <revnumber>0.8.0</revnumber> <date>$Date$</date> <revremark>$Id$</revremark> </revision> </revhistory> </bookinfo> - <toc/> - &architecture; &install; -&generators; +&specification; &deployment; +&development; &reports; </book>
\ No newline at end of file diff --git a/doc/repo-quickstart.txt b/doc/repo-quickstart.txt deleted file mode 100644 index d52fc08a2..000000000 --- a/doc/repo-quickstart.txt +++ /dev/null @@ -1,61 +0,0 @@ -The Bcfg2 repository has a bunch of parts, each with different -functions. - -Cfg/ -> Configuration file repository -Pkgmgr/ -> Package indices for different images -SSHbase/ -> SSH Key data files -etc/ -> Single files for generators or system wide settings - -Adding to the repository: - -1. If the addition is a single package or configuration file (without - any associated service) it can be added to etc/base.xml. This file - contains single items which need to be installed on - clients. Entries can be nested, depending on where they should be - installed. Global entries go at the top level, as do containers - corresponding to particular Images. Items which should be on all - instances of an image should directly inside of the Image tag, and - containers for classes can also be added at this level. - -<Base> - <ConfigFile name="/etc/passwd" /> - <ConfigFile name="/etc/group" /> - <Image name="debian-sarge"> - <ConfigFile name="/etc/motd" /> - <Class name='workstation'> - <Package name='gdm'/> - </Class> - </Image> -</Base> - - In the above example, /etc/passwd and group belong on all - machines, while /etc/motd belongs on all debian-sarge - machines. the package gdm belongs only on debian-sarge machines in - class workstation. - -2. If a package is associated with a service, then it should be - installed as a part of a bundle. This means that the installation - mechanism will ensure that all components are installed properly - and that services are restarted when any underlying components - change. The bundle making process consist of the following steps: - - - Create the bundle file. (look at <REPO>/Bundler/ssh.xml for an - example) This file contains a list of all interrelated - configuration items. Abstraction based on system (a grouping of - images) is possible in this file. Note that explicit - configuration data, like file contents or package versions aren't - specified in this file. - - - Add all new configuration items into the generator - sources. The package list automatically contains all packages - available, so it shouldn't need to be changed. Config files must - be added to the repository in <REPO>/Cfg. Services must be - activated in <REPO>/etc/services.xml. - - - Activate the bundle for a class in <REPO>/etc/metadata.xml - -In all cases, run /usr/sbin/ValidateBcfg2Repo <repo> after changes are -made. This wil check that the files still parse (ie typos, or quote -errors) and check their values against xml schemas, which will ensure -that all element names are correct and that required attributes are -included.
\ No newline at end of file diff --git a/doc/reports.xml b/doc/reports.xml index 76f609fb8..e02e49cf4 100644 --- a/doc/reports.xml +++ b/doc/reports.xml @@ -1,109 +1,58 @@ <chapter> - <title>The BCFG2 Reporting System</title> - - <section> - <title>Reporting Quick Start</title> - - <example> - <title><filename>report-configuration.xml</filename></title> - <programlisting><![CDATA[<Reports> - <Report name='core_stats' good='Y' modified='Y'> - <Delivery mechanism='mail' type='nodes-digest'> - <Destination address='user@domain.tld'/> - <Destination address='user@otherdomain.tld'/> - </Delivery> - <Delivery mechanism='www' type='nodes-digest'> - <Destination address='/var/www/core_stats.html'/> - </Delivery> - <Machine name='.*'/> - </Report> - - <Report name='stats_for_a_machines' good='N' modified='Y'> - <Delivery mechanism='mail' type='nodes-digest'> - <Destination address='user@domain.tld'/> - </Delivery> - <Delivery mechanism='mail' type='overview-stats'> - <Destination address='user@otherdomain.tld'/> - </Delivery> - <Machine name='a.*'/> - <Machine name='x-aim'/> - </Report> - </Reports>]]> - </programlisting> - </example> - - <para>This configuration will generate two separate reports and deliver - them a number of different ways. For more information on exactly what - each section does, please refer to the Configuration section below</para> - - <para>In order to run reports, you need to run the following command - regularly via <command>Cron</command>: - <command>GenerateHostInfo</command></para> - <para>Once configured correctly, just wait for the e-mail or check - look at the output html files with a web browser. You can run - <command>GenerateHostInfo</command> by hand if you would like - in order to try it out immediately. If you are trying to view a - HTML file and find that it doesn't work as expected, make sure - that the required source directory is in the same directory. They - are static, but might need to be moved from <![CDATA[prefix]]>/share/bcfg2/ - in to the directory from which you are viewing your reports.</para> -</section> - - <section> - <title>Concepts; Why to report</title> - - <para>Reports play an important role in effectively managing systems - with BCFG. There are two primary functions they fulfill; providing - otherwise unobtainable information, and presenting additional - helpful information to allow for easier admistration. Reports - can contain information including system statistics, discrepancies - between specified and actual configuration, invalid configuration, - and auditing information among other things.</para> - - <para>The flexible XML configuration file allows reports to be configured - to deliver only the information that is important. Additional reports - can easily be created, providing site-specific capability to manage - at record effiency. The capability to harvest information regarding - statistics, configuration, and problems in a single location should - prove to be powerful.</para> - </section> + <title>BCFG2 Reports</title> + + <para> + Reports play an important role in effectively managing systems + with BCFG. There are two primary functions they fulfill; providing + otherwise unobtainable information, and presenting additional + helpful information to allow for easier admistration. Reports + can contain information including system statistics, discrepancies + between specified and actual configuration, invalid configuration, + and auditing information among other things. + </para> + + <para> + The flexible XML configuration file allows reports to be configured + to deliver only the information that is important. Additional reports + can easily be created, providing site-specific capability to manage + at record effiency. The capability to harvest information regarding + statistics, configuration, and problems in a single location should + prove to be powerful. + </para> <section> <title>How it all works</title> - <para>The BCFG2 Reporting System consists of a number of - elements. The core is the <command>StatReports</command> - Executable. <command>StatReports</command> reads - a default configuration file (or a config file specified on the - command line) then prepares and delivers the reports specified in - the configuration file. It is expected that this executable will be + <para> + The BCFG2 Reporting System consists of a number of elements. The + core is the <command>StatReports</command> + Executable. <command>StatReports</command> reads a default + configuration file (or a config file specified on the command + line) then prepares and delivers the reports specified in the + configuration file. It is expected that this executable will be run by the adminstrator periodically via <command>cron</command> - or similar - facilty. The executable can also be run manually on demand for a - special sort of report that needs to be generated immediately. + or similar facilty. The executable can also be run manually on + demand for a special sort of report that needs to be generated + immediately. </para> <para> - <command>StatReports</command> gets the data it reports from a number of - sources. The <filename>hostinfo.xml</filename> - file is specific to the reporting system - and generated by the executable python script <command>GenerateHostInfo - </command>. - <filename>Hostinfo.xml</filename> contains information about if a - host is currently - pingable or not, and a mapping of short hostnames to - FQDNs. <command>GenerateHostInfo</command> will be run automatically - by <command>StatReports</command> if the <filename>hostinfo.xml</filename> - file is older than 23.5 hours. This number is - chosen, because it is likely an administrator will recieve reports - daily about the status of hosts and if they had run BCFG the - previous night. It is possible to execute - <command>GenerateHostInfo</command> to update the - <filename>hostinfo.xml</filename> file at any interval via cron, - but it does take - some time to complete, so be sure to give it a few minutes. This - will only likely be of use if your BCFG clients are set to update - more often than nightly and you would like reports after each run. + <command>StatReports</command> gets the data it reports from a + number of sources. <filename>Metadata/clients.xml</filename> + contains information about if a host is currently pingable or + not, and a mapping of short hostnames to + FQDNs. <command>GenerateHostInfo</command> will be run + automatically by <command>StatReports</command> if the + <filename>Metadata/clients.xml</filename> file is older than + 23.5 hours. This number is chosen, because it is likely an + administrator will recieve reports daily about the status of + hosts and if they had run BCFG the previous night. It is + possible to execute <command>GenerateHostInfo</command> to + update the <filename>Metadata/clients.xml</filename> file at any + interval via cron, but it does take some time to complete, so be + sure to give it a few minutes. This will only likely be of use + if your BCFG clients are set to update more often than nightly + and you would like reports after each run. </para> <para> @@ -115,11 +64,9 @@ </para> <para> - Finally <command>StatReports</command> is able to get any information - out of the <filename>metadata.xml</filename> file. - This will likely be most useful in the future - where the range of reports will include auditing style reports, - that must include configuration information. + Finally <command>StatReports</command> is able to get information + out of the <filename>Metadata/groups.xml</filename> file as well. + This allows reports to describe the configured profile for each client. </para> </section> @@ -134,91 +81,73 @@ options. For each of those reports, each can be delivered by Mail, WWW, or via RSS (or any combination of the three.) In the future, additional report types will be added, and if necessary, - additional types of deliveries will be created. + additional types of deliveries will be created. Tables + describing report types and report delivery mechanisms follow. </para> - <para>Here is a list of the report types currently defined.</para> - - <variablelist> - <varlistentry> - <term>Overview-Stats</term> - <listitem> - <para> - This report provides informatoin about a large number of - machines and their states. It is often found to be useful - when the group of machines it is connected with is simply - All Nodes, which gives an overall outlook on your - network's health. It makes sense to get this report via - any mechanism - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>Nodes-Digest</term> - <listitem> - <para> - This is a report that includes details about each node, - specifically what packages, files, etc are broken, and - other node specific info. It makes sense to recieve this - via any mechanism - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>Nodes-Individual</term> - <listitem> - <para> - This report includes details about each node, but - information is separated in to separate sections (such as - separate e-mails or RSS articles) before sending. This - works well with e-mail filters and error - detection. Currently WWW is not a supported delivery - mechanism for this type of report, because it is not - completely clear how such a report could be used. - </para> - </listitem> - </varlistentry> - </variablelist> + <table> + <title>Bcfg2 Report Types</title> + <tgroup cols='2'> + <colspec colnum='1' colwidth='1*'/> + <colspec colnum='2' colwidth='4*'/> + <thead> + <row><entry>Report Type</entry><entry>Description</entry></row> + </thead> + <tbody> + <row><entry>Overview-Stats</entry> + <entry> + <para> + This report provides informatoin about a large number of + machines and their states. It is often found to be useful + when the group of machines it is connected with is simply + All Nodes, which gives an overall outlook on your + network's health. It makes sense to get this report via + any mechanism + </para> + </entry></row> + <row><entry>Nodes-Digest</entry> + <entry> + <para> + This report includes details about each node, + specifically what packages, files, etc are broken, and + other node specific info. It makes sense to recieve + this via any mechanism + </para> + </entry></row> + <row><entry>Nodes-Individual</entry> + <entry> + <para> + This report includes details about each node, but + information is separated in to separate sections (such + as separate e-mails or RSS articles) before + sending. This works well with e-mail filters and error + detection. Currently WWW is not a supported delivery + mechanism for this type of report, because it is not + completely clear how such a report could be used. + </para> + </entry></row> + </tbody> + </tgroup> + </table> + + <table> + <title>Bcfg2 Report Delivery Mechanisms</title> + <tgroup cols='2'> + <colspec colnum='1' colwidth='1*'/> + <colspec colnum='2' colwidth='3*'/> + <thead> + <row><entry>Name</entry><entry>Description</entry></row> + </thead> + <tbody> + <row><entry>www</entry><entry>XHTML file</entry></row> + <row><entry>rss</entry><entry>An RSS file <comment>(links do + not point at real web links, since they may not exist)</comment></entry></row> + <row><entry>mail</entry><entry>A plaintext email + message</entry></row> + </tbody> + </tgroup> + </table> - <para>A list of delivery mechanisms follows:</para> - - <variablelist> - <varlistentry> - <term>www</term> - <listitem> - <para> - This delivery produces HTML reports which can be delivered - via the WWW. it is important that you copy the directory: - <![CDATA[<prefix>]]>/share/bcfg2/reports/web-rprt-srcs in to the - same directory as you will be serving web pages from. It - includes CSS and JavaScript files necessary to properly - view the WWW files. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>rss</term> - <listitem> - <para> - This delivery type also should be written to some area - that is likely web accessable. It is plain RSS-- One - Caveat-- the "Article Link" does not actually point to any - web information, because it is not clear that a - corresponding web page exists for any given RSS article. - </para> - </listitem> - </varlistentry> - <varlistentry> - <term>mail</term> - <listitem> - <para> - Mail is simple plaintext e-mail that is sent to the - specified recipients directly from the machine on which - BCFG is running by opening a pipe to sendmail. - </para> - </listitem> - </varlistentry> - </variablelist> </section> <section> @@ -244,7 +173,7 @@ around a group of machines. <![CDATA[<Machine/>]]> tags may individually reference a machine by hostname (not FQDN), or also by a Python Regular Expression. More information can be found about such Regexes at: - http://docs.python.org/lib/re-syntax.html.</para> + <ulink url="http://docs.python.org/lib/re-syntax.html"/>.</para> <para>Any number of <![CDATA[<Delivery/>]]> elements can be made for a given @@ -261,4 +190,52 @@ a complete e-mail address.</para> </section> + + <section> + <title>Reporting Quick Start</title> + + <para> + This configuration will generate two separate reports and + deliver them a number of different ways. For more information on + exactly what each section does, please refer to the + Configuration section above. + </para> + + <example> + <title><filename>etc/report-configuration.xml</filename></title> + <programlisting><![CDATA[<Reports> + <Report name='core_stats' good='Y' modified='Y'> + <Delivery mechanism='mail' type='nodes-digest'> + <Destination address='user@domain.tld'/> + <Destination address='user@otherdomain.tld'/> + </Delivery> + <Delivery mechanism='www' type='nodes-digest'> + <Destination address='/var/www/core_stats.html'/> + </Delivery> + <Machine name='.*'/> + </Report> + + <Report name='stats_for_a_machines' good='N' modified='Y'> + <Delivery mechanism='mail' type='nodes-digest'> + <Destination address='user@domain.tld'/> + </Delivery> + <Delivery mechanism='mail' type='overview-stats'> + <Destination address='user@otherdomain.tld'/> + </Delivery> + <Machine name='a.*'/> + <Machine name='x-aim'/> + </Report> + </Reports>]]> + </programlisting> + </example> + + <para> + Once configured correctly, just wait for the e-mail or check + look at the output html files with a web browser. You can run + <command>StatReports</command> by hand if you would like + in order to try it out immediately. + </para> + +</section> + </chapter>
\ No newline at end of file diff --git a/doc/specs.xml b/doc/specs.xml new file mode 100644 index 000000000..9b51a3309 --- /dev/null +++ b/doc/specs.xml @@ -0,0 +1,501 @@ +<chapter> + <title>Writing Bcfg2 Specifications</title> + + <para> + The Bcfg2 specification is a set of directives that describe how + hosts should be configured. This information is used to generate + client configurations. This section describes the steps taken in + during the composition of bcfg2 specifications. + </para> + + <orderedlist> + <listitem> + <para> + All parts of the specification will correspond to some + subset of the clients that bcfg2 has record of. Find or create a + group corresponding to the target of this specification. + </para> + </listitem> + <listitem> + <para> + Add each new configuration entry to the "abstract + configuration" for the target group. This can be done in one + of two ways. If the new configuration has to do with a + service, or some other piece of inter-dependent configuration, + write a bundle. If not, add the extra configuration entries to + the base. + </para> + </listitem> + <listitem> + <para> + Add configuration data for the above entries that apply only + to the target group. + </para> + </listitem> + <listitem> + <para> + Verify that clients not in the target group remain unaffected + by these changes. + </para> + </listitem> + <listitem> + <para> + Re-validate the bcfg2 repository by running + <command>bcfg2-repo-validate</command>. + </para> + </listitem> + </orderedlist> + + <section> + <title>Interacting with Client Groups in Bcfg2</title> + + <para> + Bcfg2 uses an aspect based classing mechanism to describe + configuration patterns in its specifications. The Bcfg2 metadata + mechanism has two types of information, client metadata and + group metadata. Client metadata describes what top level group a + client is associated. Its configuration is derived from a + combination of its host information and this group. Group + definitions describe groups in terms of what bundles they + include and other groups they include. Groups have a set of + properties that describe how they can be used. + </para> + + <table> + <title>Bcfg2 Group Parameters</title> + <tgroup cols='3'> + <colspec colnum='1' colwidth='1*'/> + <colspec colnum='2' colwidth='7*'/> + <colspec colnum='3' colwidth='2*'/> + <thead> + <row><entry>Name</entry><entry>Description</entry> + <entry>Values</entry></row> + </thead> + <tbody> + <row><entry>profile</entry><entry>If a client can be + directly associated with this group</entry><entry>(True|False*)</entry></row> + <row><entry>public</entry><entry>If a client can freely + associate itself with this group</entry><entry>(True|False*)</entry></row> + <row><entry>toolset</entry> + <entry>Describes which client-side logic should be used to + make configuration + changes</entry><entry>(rh|debian|solaris)</entry></row> + <row><entry>category</entry> + <entry>A group can only contain one instance of a group of + any category. This provides the basis for representing + groups which are conjugates of one another in a rigorous + way. It also provides the basis for negation.</entry><entry>string</entry></row> + </tbody> + </tgroup> + </table> + </section> + + <para> + When a client's configuration is generated, its metadata is the + fetched. This includes a list of all groups recursively + dereferenced, and all bundles included by those groups. This + collection has already been processed using the group category + rules, so only one instance from each group category is + included. This metadata is used throughout the rest of the + configuration generation process; it defines the client's abstract + configuration and specifies all literal contents of all + configuration entities. + </para> + + <section> + <title>Adding to the Abstract Configuration</title> + + <para> + When writing bcfg2 specification, administrators primarily + perform one of two operations: addition of new configuration + entities or the modification of existing entries. If new + entities need to be added, then they must be added to the + abstract configuration. This is the inventory of configuration + entities that should be installed on a client. Two plugins + provide the basis for the abstract configuration, the bundler + and base. The bundler builds descriptions of interrelated + configuration entities. These are typically used for the + representation of services, or other complex groups of + entities. Base provides a laundry list of configuration entities + that need to be installed on hosts. These entities are + independent from one another, and can be installed individually + without worrying about the impact on other entities. + </para> + + <para> + Entities in the abstract configuration (and correspondingly in + the literal configuration) can have one of several types. In the + abstract configuration, each of these entities only has a tag + and the name attribute set. + </para> + + <table> + <title>Bcfg2 Configuration Entity Types</title> + <tgroup cols='2'> + <colspec colnum='1'/> + <colspec colnum='2'/> + <thead> + <row><entry>Name</entry><entry>Description</entry></row> + </thead> + <tbody> + <row><entry>Package</entry><entry>Software Package</entry></row> + <row><entry>ConfigFile</entry><entry>Configuration File</entry></row> + <row><entry>Service</entry> + <entry>Persistent system services and daemons</entry></row> + <row><entry>Directory</entry><entry>Filesystem Directories</entry></row> + <row><entry>SymLink</entry><entry>Symbolic links</entry></row> + <row><entry>Permissions</entry> + <entry>The permissions (not contents) of a POSIX path</entry></row> + </tbody> + </tgroup> + </table> + + <section> + <title>Writing Bundles</title> + + <para> + Bundles consist of a set of configuration entities. These + entities are grouped together due to a configuration-time + interdependency. Basic services tend to be the simplest + example of these: they consist of some software package(s) + some configuration files and an indication that some service + should be activated. If any of these pieces are installed or + updated, all should be re-checked and any associated services + should be restarted. + </para> + + <para> + Bundles can also contain conditonal entries that are only used + for hosts in some particular groups. This is useful when a + service name varies from platform to platform. A group is + defined for each platform, hence a different service can be + associated with the bundle for clients in the different + groups. Conditional additions can also add extra functionality + to services as needed. This has proven useful in the case of + configuring webservers; php and ssl can be configured as extra + features that some webservers have and others do not. At the + same time, all configuration-time interdependencies are + maintained. + </para> + </section> + + <section> + <title>Using Base</title> + + <para> + The Base plugin provides a mechanism to add independent + configuration entities to a client's abstract + configuration. All files in the Base/ subdirectory of the + respository are processed, and all entries that fall within + the scope of the client metadata are included in its abstract + configuration. These files are similar to those used by the + Bundler, Svcmgr, and Pkgmgr, without the need for + prioritization used by the later two. + </para> + </section> + </section> + + <section> + <title>Adding to the Literal Configuration</title> + + <para> + During the construction of the literal configuration, first the + abstract configuration is built, and then explicit data is bound + in to each entity. The previous section describes how the first + stage works, and this section describes the second stage. Each + entity will be served by one plugin. This plugin will decide + what explicit data should be bound in to a particular entity for + a given client. Each of these plugins has a specific area of the + configuration repository, corresponding to its name. This + section describes how several of the basic plugins works. + </para> + + <section> + <title>Cfg</title> + <para> + The Cfg plugin provides a configuration file repository that + uses literal file contents to provide client-tailored + configuration file entries. It chooses which data to provide + for a given client based on the aspect-based metadata system + used for high-level client configuration. + </para> + <para> + The Cfg repository is structured much like the filesystem + hierarchy being configured. Each configuration file being + served has a corresponding directory in the configuration + repository. These directories have the same relative path as + the absolute path of the configuration file on the target + system. For example, if Cfg was serving data for the + configuration file <filename>/etc/services</filename>, then + its directory would be in the relative path + <filename>./etc/services</filename> inside of the Cfg + repository. + </para> + <para> + Inside of this file-specific directory, three types of files + may exist. Base files are complete instances of configuration + file. Deltas are differences between a base file and the + target file contents. Base files and deltas are tagged with + metadata specifiers, which describe which groups of clients + the fragment pertains to. Configuration files are constructed + by finding the most specific base file and applying any more + specific deltas. + </para> + <para> + Specifiers are embedded in fragment filenames. For example, in + the fragment <filename>services.G99_webserver</filename>, + "G99_webserver" is the specifier. This specifier applies to + the group (G) webserver with a priority of 99. Files can also + be tagged with a host-specific (H) specifier.Global files are + the least specific. Priorities are used as to break ties. + </para> + <para> + Info files, named <filename>:info</filename> are used to + specify target configuration file metadata, such as owner, + group and permissions. If no <filename>:info</filename> is + provided, targets are installed with default + information. Default metadata is root ownership, root group + memberships, and 0644 file permissions. + </para> + <example> + <title>Cfg/filepath/:info</title> + <programlisting> owner:root + group:root + perms:0755</programlisting> + </example> + + <example> + <title>Cfg/etc/passwd/</title> + <programlisting> $ ls + :info passwd passwd.G99_chiba-login + passwd.H_bio-debian passwd.H_cvstest passwd.H_foxtrot + passwd.H_reboot passwd.H_rudy2 passwd.G98_netserv + passwd.G99_tacacs-server.cat passwd.H_adenine</programlisting> + </example> + + <para> + In the previous example, there exists files with each of the + characteristics mentioned above. All files ending in ".cat" + are deltas; ones with ".H_" are host specific files. There + exists a base file, a <filename>:info</filename> file, two + class-specified base files, and a bundle-specified base file. + </para> + </section> + + <section> + <title>Pkgmgr</title> + + <para> + The Pkgmgr plugin is responsible for providing package version + and installation information. In the case of each "Package" + entity in the configuration, it binds in information needed to + detect, verify and install the package. It has a similar + format to the files used by Base and Bundler, but with a few + differences. First, each file has a priority. This allows the + same entity to be served by multiple files. The priorities can + be used to disambiguate in the case that multiple files serve + data for the same package. The other difference is that + automatic deriviation of package information from the file + attribute. The Pkgmgr has a set of regexes that can split + package names for several formats. The filenames are used to + construct installation URLs, and set several important fields + like package name and version. + </para> + </section> + <section> + <title>Svcmgr</title> + + <para> + The Svcmgr plugin describes where services should be active + and inactive. Its files have a similar form to those used by + the Pkgmgr. Several files in the Svcmgr repository can contain + overlapping definitions, and a per-file priority is used to + determine precedence, the highest priority file serving data + for a particular service wins, on a service by service basis. + </para> + + <para> + These files also have a similar set of semantics to those used + by the Pkgmgr. Entries in the top level element (Services) are + global definitions. Group elements describe additional + conditions that must be matched for that definition to + supercede less specific ones. Deeply nested definitions must + have their parent condition matched, plus all parent + conditions as well. For example, the following declaration + turns ssh on by default, disables it if the client is a part + of group a, and reenables it if the client is a part of both + groups a and b. + </para> + + <example> + <title>Svcmgr/ssh.xml</title> + <programlisting><![CDATA[<Services priority='0'> + <Service name='ssh' status='on'/> + <Group name='a'> + <Service name='ssh' status='off'/> + <Group name='b'> + <Service name='ssh' status='on'/> + </Group> + </Group> +</Services>]]></programlisting> + </example> + + <para> + The files used by this plugin can be structured in a number of + ways. The most common method is to use one large file, but + this can be inconvenience due to the large size of the + file. The data can also be split up using any convenient + mechanism: per-service, per-administrator, etc. + </para> + </section> + <section> + <title>SSHbase</title> + + <para> + The SSHbase plugin implements ssh public and private key + management functionality. This means that a central record of + ssh host keys is maintained. Also, a correct ssh_known_hosts + file is maintained. This means that the keys for new hosts are + added to this configuration, and also that a correct line for + localhost is created. SSHbase will generate a new key for any + hosts that doesn't already have a key stored in the + repository, so it should be pre-seeded with the keys (public + and private) of pre-existing clients. + </para> + </section> + </section> + + <section> + <title>Checking Group-External Clients for Unintended + Changes</title> + + <para> + Any configuration change will apply to some set of clients. + Often, repository changes can have unintended consequences to + clients not included in the target group. To address this issue, + consider the changes performed, and if they can affect clients + in unexpected ways. + </para> + + </section> + + <section> + <title>Validating the Bcfg2 Repository</title> + + <para> + Bcfg2 includes a repository validation tool that will check all + XML files in the repository against included XML schemas. It is + critical to run this command, + <command>bcfg2-repo-valdate</command> after any modifications to + XML files. If all files validate properly, then no output will + be returned. It takes a "-v" option that prints out a line for + each file that is validated. This can be used to ensure that all + files are checked. + </para> + </section> + + <section> + <title>Annotated Configuration Examples</title> + + <para> + In addition to the description of the abstract process + above, we present several examples of the thought process and + actions taken to achieve a particular configuration goal. These + will start simple, but become more complex. + </para> + + <section> + <title>Configuring /etc/motd on all hosts</title> + + <para>The goal for this example is to install a uniform copy of + a specified <filename>/etc/motd</filename> on all hosts.</para> + + <orderedlist> + <listitem><para> + In this case, the target group is global, since we want + this version of <filename>/etc/motd</filename>. As + mentioned earlier, the global group is handled specially, + so that all new clients, even newly created ones, are in it. + </para></listitem> + <listitem><para> + Since <filename>/etc/motd</filename> is not interdependent + with any other configuration entities, it can be installed + using Base instead of using Bundler. The ConfigFile + entity should be placed in the globally scoped section of + a base file. This adds the configuration file to the + abstract configuration. + </para></listitem> + <listitem><para> + A file with the correct contents for + <filename>/etc/motd</filename> should be installed in the + Cfg repository area as a global file. This will provide + the right literal configuration specification for each + client. + </para></listitem> + <listitem><para> + Since this change is globally scoped, there are not any + clients that shoud not be affected. + </para></listitem> + <listitem><para> + Finally, <command>bcfg2-repo-validate</command> should be + run to catch typos. + </para></listitem> + </orderedlist> + </section> + + <section> + <title>Configuring NTP for a network</title> + + <para> + The goal for this example is to configure NTP for an entire + network. This means several things. All clients should run NTP + as clients. One client should run NTP as a server, and + ntp clients should use it instead of an external server. + </para> + + <orderedlist> + <listitem><para> + Two discrete groups are used for the different parts of + the desired configuration state. The first is the global + group, handling the "all clients run ntp" part of the + configuration specification. The other part corresponds to + a new group "ntp-server", which contains only clients that + should function as an ntp server. + </para></listitem> + <listitem><para> + This configuration specification describes configuration + entities that have interdependencies, so a bundle should + be used. This bundle should contain the ntp software, the + ntp configuration file, and the ntpd service. + </para></listitem> + <listitem><para> + The literal portions of three different configuration + entities need to be represented. First, the Pkgmgr needs + to be configured to bind a package entity named ntp. Next, + the Svcmgr needs to have a global declaration that the + service ntpd should be on. Finally, two different + configuration files should be added to Cfg. The global + version of <filename>/etc/ntp.conf</filename> should have + an ntp configuration that points hosts at the local ntp + server. The group "ntp-server" specific version of this + file should have the proper configuration for local ntp + servers. + </para></listitem> + <listitem><para> + A quick inspection of these configuration changes show + only minor possibilities for bad interactions. All + machines should run ntp, and the only scope where bad + interactions can occur is on ntp servers. + </para></listitem> + <listitem><para> + Finally, run <command>bcfg2-repo-validate</command>. It + will validate the new bundle that has been added, and + changes to the Svcmgr and Pkgmgr indices. + </para></listitem> + </orderedlist> + </section> + + <!-- <para>Need more examples here</para> --> + + </section> +</chapter>
\ No newline at end of file diff --git a/man/Bcfg2debug.8 b/man/Bcfg2debug.8 deleted file mode 100644 index 6c4947d2d..000000000 --- a/man/Bcfg2debug.8 +++ /dev/null @@ -1,15 +0,0 @@ -.TH "Bcfg2debug" 8 -.SH NAME -Bcfg2debug \- Debugging environment for Bcfg2 -.SH SYNOPSIS -.B Bcfg2debug -.SH DESCRIPTION -.PP -.B Bcfg2debug -Instantiate an instance of the Bcfg2 core for data examination and -debugging purposes. -.SH "SEE ALSO" -.BR bcfg(1), -.BR Bcfg2Server(8) -.SH "BUGS" -None currently known
\ No newline at end of file diff --git a/man/bcfg2-info.8 b/man/bcfg2-info.8 new file mode 100644 index 000000000..a71bc21a4 --- /dev/null +++ b/man/bcfg2-info.8 @@ -0,0 +1,16 @@ +.TH "bcfg2-info" 8 +.SH NAME +bcfg2-info \- Creates a local version of the bcfg2 server core for +state observation +.SH SYNOPSIS +.B bcfg2-info +.SH DESCRIPTION +.PP +.B bcfg2-info +Instantiate an instance of the Bcfg2 core for data examination and +debugging purposes. +.SH "SEE ALSO" +.BR bcfg2(1), +.BR bcfg2-server(8) +.SH "BUGS" +None currently known diff --git a/man/ValidateBcfg2Repo.8 b/man/bcfg2-repo-validate.8 index fe0551fdd..cebcbd301 100644 --- a/man/ValidateBcfg2Repo.8 +++ b/man/bcfg2-repo-validate.8 @@ -1,12 +1,12 @@ -.TH "ValidateBcfg2Repo" 8 +.TH "bcfg2-repo-validate" 8 .SH NAME -ValidateBcfg2Repo \- Check Bcfg2 repository data against data schemas +bcfg2-repo-validate \- Check Bcfg2 repository data against data schemas .SH SYNOPSIS -.B ValidateBcfg2Repo +.B bcfg2-repo-validate .I [ Repository Location ] [ Schema Location ] .SH DESCRIPTION .PP -.B ValidateBcfg2Repo +.B bcfg2-repo-validate This script checks data against schemas, and it quite helpful in finding typos or malformed data. .SH ARGUMENTS @@ -15,7 +15,7 @@ finding typos or malformed data. .TP .B Schema Location - where schemas are. (Optional). .SH "SEE ALSO" -.BR bcfg(1), -.BR Bcfg2Server(8) +.BR bcfg2(1), +.BR bcfg2-server(8) .SH "BUGS" -None currently known
\ No newline at end of file +None currently known diff --git a/man/Bcfg2Server.8 b/man/bcfg2-server.8 index dc7b80daa..3274638c6 100644 --- a/man/Bcfg2Server.8 +++ b/man/bcfg2-server.8 @@ -1,12 +1,12 @@ -.TH "Bcfg2Server" 8 +.TH "bcfg2-server" 8 .SH NAME -Bcfg2Server \- Server for client configuration specifications +bcfg2-server \- Server for client configuration specifications .SH SYNOPSIS -.B Bcfg2Server +.B bcfg2-server .I [-D <pidfile>] [-d] [-v] [-C <Client>] .SH DESCRIPTION .PP -.B Bcfg2Server +.B bcfg2-server This daemon serves configurations to clients based on the data in its repository. .SH OPTIONS @@ -20,7 +20,7 @@ particular client's configs. .TP .B \-D Daemonize, placing the program pid in the specified pidfile .SH "SEE ALSO" -.BR bcfg(1), -.BR ValidateBcfg2Repo(8) +.BR bcfg2(1), +.BR bcfg2-repo-validate(8) .SH "BUGS" -None currently known
\ No newline at end of file +None currently known diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 41f49663f..f049f6994 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -14,15 +14,13 @@ structures=list of enabled structures .TP bundles=list of enabled bundles .TP -metadata=/path/to/metadata -.TP .B [communication] .TP password=communication password .TP key=/path/to/ssl/key .TP -protocol=(sss|xmlrpc/ssl) +protocol=xmlrpc/ssl .TP .B [components] list of component urls @@ -31,4 +29,4 @@ componentname=url .SH "SEE ALSO" .BR bcfg2(1), -.BR Bcfg2Server(8)
\ No newline at end of file +.BR bcfg2-server(8)
\ No newline at end of file diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index af83abe01..48a65d0dd 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -1,5 +1,5 @@ %define name bcfg2 -%define version 0.7.4 +%define version 0.8.0 %define release 1 %define pythonversion 2.3 @@ -38,9 +38,7 @@ python%{pythonversion} setup.py build python%{pythonversion} setup.py install --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES mkdir -p ${RPM_BUILD_ROOT}/usr/sbin mkdir -p ${RPM_BUILD_ROOT}/etc/init.d/ -mv ${RPM_BUILD_ROOT}/usr/bin/Bcfg2Server ${RPM_BUILD_ROOT}/usr/sbin -mv ${RPM_BUILD_ROOT}/usr/bin/ValidateBcfg2Repo ${RPM_BUILD_ROOT}/usr/sbin -mv ${RPM_BUILD_ROOT}/usr/bin/bcfg2 ${RPM_BUILD_ROOT}/usr/sbin +mv ${RPM_BUILD_ROOT}/usr/bin/bcfg2* ${RPM_BUILD_ROOT}/usr/sbin mv ${RPM_BUILD_ROOT}/usr/bin/StatReports ${RPM_BUILD_ROOT}/usr/sbin install -m 755 debian/bcfg2.init ${RPM_BUILD_ROOT}/etc/init.d/bcfg2 install -m 755 debian/bcfg2-server.init ${RPM_BUILD_ROOT}/etc/init.d/bcfg2-server @@ -50,8 +48,9 @@ rm -rf $RPM_BUILD_ROOT %files -n bcfg2-server %defattr(-,root,root) -/usr/sbin/Bcfg2Server* -/usr/sbin/ValidateBcfg2Repo +/usr/sbin/bcfg2-server +/usr/sbin/bcfg2-repo-validate +/usr/sbin/bcfg2-info /usr/sbin/StatReports /usr/bin/GenerateHostInfo /usr/lib/python%{pythonversion}/site-packages/Bcfg2/Server/* diff --git a/reports/xsl-transforms/nodes-digest-www.xsl b/reports/xsl-transforms/nodes-digest-www.xsl index 9585738c9..95026120f 100644 --- a/reports/xsl-transforms/nodes-digest-www.xsl +++ b/reports/xsl-transforms/nodes-digest-www.xsl @@ -10,8 +10,8 @@ <xsl:variable name="dirtynodes" select="/Report/Node[count(Statistics/Bad)>0]"/> <xsl:variable name="modifiednodes" select="/Report/Node[count(Statistics/Modified)>0]"/> <xsl:variable name="stalenodes" select="/Report/Node[count(Statistics/Stale)>0]"/> - <xsl:variable name="unpingablenodes" select="/Report/Node[HostInfo/@pingable='N']"/> - <xsl:variable name="pingablenodes" select="/Report/Node[HostInfo/@pingable='Y']"/> + <xsl:variable name="unpingablenodes" select="/Report/Node[Client/@pingable='N']"/> + <xsl:variable name="pingablenodes" select="/Report/Node[Client/@pingable='Y']"/> <html> <head> @@ -48,10 +48,10 @@ <div class="items" id="goodsummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Good) > 0"> <li><b>Node: </b> - <tt><a href="#{HostInfo/@fqdn}"><xsl:value-of select="HostInfo/@fqdn" /></a></tt></li> + <tt><a href="#{Client/@name}"><xsl:value-of select="Client/@name" /></a></tt></li> </xsl:if> </xsl:for-each> </ul></div> @@ -64,10 +64,10 @@ <div class="items" id="badsummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Bad) > 0"> <li><b>Node: </b> - <tt><a href="#{HostInfo/@fqdn}"><xsl:value-of select="HostInfo/@fqdn" /></a></tt></li> + <tt><a href="#{Client/@name}"><xsl:value-of select="Client/@name" /></a></tt></li> </xsl:if> </xsl:for-each> </ul></div> @@ -79,10 +79,10 @@ <span class="nodelisttitle"><a href="javascript:toggleLayer('extrasummary');" title="Click to Expand" class="commentLink"><xsl:value-of select="count(/Report/Node/Statistics/Extra)" /></a> nodes have extra configuration. (includes both good and bad nodes)<br /></span> <div class="items" id="extrasummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Extra) > 0"> <li><b>Node: </b> - <tt><a href="#{HostInfo/@fqdn}"><xsl:value-of select="HostInfo/@fqdn" /></a></tt></li> + <tt><a href="#{Client/@name}"><xsl:value-of select="Client/@name" /></a></tt></li> </xsl:if> </xsl:for-each> </ul></div> @@ -96,10 +96,10 @@ <div class="items" id="modifiedsummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Modified) > 0"> <li><b>Node: </b> - <tt><a href="#{HostInfo/@fqdn}"><xsl:value-of select="HostInfo/@fqdn" /></a></tt></li> + <tt><a href="#{Client/@name}"><xsl:value-of select="Client/@name" /></a></tt></li> </xsl:if> </xsl:for-each> </ul></div> @@ -111,10 +111,10 @@ <span class="nodelisttitle"><a href="javascript:toggleLayer('vstalesummary');" title="Click to Expand" class="commentLink"><xsl:value-of select="count($stalenodes[count(.|$pingablenodes)= count($pingablenodes)])" /></a> nodes did not run within the last 24 hours but were pingable.<br /></span> <div class="items" id="vstalesummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> - <xsl:if test="count(Statistics/Stale)-count(HostInfo[@pingable='N']) > 0"> + <xsl:sort select="Client/@name"/> + <xsl:if test="count(Statistics/Stale)-count(Client[@pingable='N']) > 0"> <li><b>Node: </b> - <tt><a href="#{HostInfo/@fqdn}"><xsl:value-of select="HostInfo/@fqdn" /></a></tt></li> + <tt><a href="#{Client/@name}"><xsl:value-of select="Client/@name" /></a></tt></li> </xsl:if> </xsl:for-each> </ul></div> @@ -127,10 +127,10 @@ <div class="items" id="stalesummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Stale) > 0"> <li><b>Node: </b> - <tt><a href="#{HostInfo/@fqdn}"><xsl:value-of select="HostInfo/@fqdn" /></a></tt></li> + <tt><a href="#{Client/@name}"><xsl:value-of select="Client/@name" /></a></tt></li> </xsl:if> </xsl:for-each> </ul></div> @@ -138,16 +138,16 @@ </div> </xsl:if> - <xsl:if test="count(/Report/Node[HostInfo/@pingable='N']) > 0"> + <xsl:if test="count(/Report/Node[Client/@pingable='N']) > 0"> <div class="down"> - <span class="nodelisttitle"><a href="javascript:toggleLayer('unpingablesummary');" title="Click to Expand" class="commentLink"><xsl:value-of select="count(/Report/Node/HostInfo[@pingable='N'])" /></a> nodes were down.<br /></span> + <span class="nodelisttitle"><a href="javascript:toggleLayer('unpingablesummary');" title="Click to Expand" class="commentLink"><xsl:value-of select="count(/Report/Node/Client[@pingable='N'])" /></a> nodes were down.<br /></span> <div class="items" id="unpingablesummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> - <xsl:if test="count(HostInfo[@pingable='N']) > 0"> + <xsl:sort select="Client/@name"/> + <xsl:if test="count(Client[@pingable='N']) > 0"> <li><b>Node: </b> - <tt><a href="#{HostInfo/@fqdn}"><xsl:value-of select="HostInfo/@fqdn" /></a></tt></li> + <tt><a href="#{Client/@name}"><xsl:value-of select="Client/@name" /></a></tt></li> </xsl:if> </xsl:for-each> </ul></div> diff --git a/reports/xsl-transforms/overview-stats-mail.xsl b/reports/xsl-transforms/overview-stats-mail.xsl index a134d8048..8527defe5 100644 --- a/reports/xsl-transforms/overview-stats-mail.xsl +++ b/reports/xsl-transforms/overview-stats-mail.xsl @@ -8,8 +8,8 @@ <xsl:variable name="dirtynodes" select="/Report/Node[count(Statistics/Bad)>0]"/> <xsl:variable name="modifiednodes" select="/Report/Node[count(Statistics/Modified)>0]"/> <xsl:variable name="stalenodes" select="/Report/Node[count(Statistics/Stale)>0]"/> -<xsl:variable name="unpingablenodes" select="/Report/Node[HostInfo/@pingable='N']"/> -<xsl:variable name="pingablenodes" select="/Report/Node[HostInfo/@pingable='Y']"/> +<xsl:variable name="unpingablenodes" select="/Report/Node[Client/@pingable='N']"/> +<xsl:variable name="pingablenodes" select="/Report/Node[Client/@pingable='Y']"/> Summary: <xsl:text> </xsl:text><xsl:value-of select="count($cleannodes)" /> nodes are clean. @@ -26,9 +26,9 @@ Summary: <xsl:if test="count($cleannodes) > 0"> CLEAN: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> +<xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Good) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -38,9 +38,9 @@ CLEAN: <xsl:if test="count($dirtynodes) > 0"> DIRTY: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> +<xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Bad) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -50,9 +50,9 @@ DIRTY: <xsl:if test="count($modifiednodes) > 0"> MODIFIED: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> +<xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Modified) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -63,9 +63,9 @@ MODIFIED: <xsl:if test="count($stalenodes[count(.|$pingablenodes)= count($pingablenodes)]) > 0"> STALE: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> -<xsl:if test="count(Statistics/Stale)-count(HostInfo[@pingable='N']) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:sort select="Client/@name"/> +<xsl:if test="count(Statistics/Stale)-count(Client[@pingable='N']) > 0"> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -76,9 +76,9 @@ STALE: <xsl:if test="count($unpingablenodes) > 0"> UNPINGABLE: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> -<xsl:if test="count(HostInfo[@pingable='N']) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:sort select="Client/@name"/> +<xsl:if test="count(Client[@pingable='N']) > 0"> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> diff --git a/reports/xsl-transforms/overview-stats-rss.xsl b/reports/xsl-transforms/overview-stats-rss.xsl index c4c689a99..1df5e29cf 100644 --- a/reports/xsl-transforms/overview-stats-rss.xsl +++ b/reports/xsl-transforms/overview-stats-rss.xsl @@ -12,8 +12,8 @@ Report Run @ <xsl:value-of select="@time" /> <xsl:variable name="dirtynodes" select="/Report/Node[count(Statistics/Bad)>0]"/> <xsl:variable name="modifiednodes" select="/Report/Node[count(Statistics/Modified)>0]"/> <xsl:variable name="stalenodes" select="/Report/Node[count(Statistics/Stale)>0]"/> -<xsl:variable name="unpingablenodes" select="/Report/Node[HostInfo/@pingable='N']"/> -<xsl:variable name="pingablenodes" select="/Report/Node[HostInfo/@pingable='Y']"/> +<xsl:variable name="unpingablenodes" select="/Report/Node[Client/@pingable='N']"/> +<xsl:variable name="pingablenodes" select="/Report/Node[Client/@pingable='Y']"/> Summary: <xsl:text> </xsl:text><xsl:value-of select="count($cleannodes)" /> nodes are clean. @@ -30,9 +30,9 @@ Summary: <xsl:if test="count($cleannodes) > 0"> CLEAN: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> +<xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Good) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -42,9 +42,9 @@ CLEAN: <xsl:if test="count($dirtynodes) > 0"> DIRTY: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> +<xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Bad) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -54,9 +54,9 @@ DIRTY: <xsl:if test="count($modifiednodes) > 0"> MODIFIED: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> +<xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Modified) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -67,9 +67,9 @@ MODIFIED: <xsl:if test="count($stalenodes[count(.|$pingablenodes)= count($pingablenodes)]) > 0"> STALE: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> -<xsl:if test="count(Statistics/Stale)-count(HostInfo[@pingable='N']) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:sort select="Client/@name"/> +<xsl:if test="count(Statistics/Stale)-count(Client[@pingable='N']) > 0"> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> @@ -80,9 +80,9 @@ STALE: <xsl:if test="count($unpingablenodes) > 0"> UNPINGABLE: <xsl:for-each select="Node"> -<xsl:sort select="HostInfo/@fqdn"/> -<xsl:if test="count(HostInfo[@pingable='N']) > 0"> -<xsl:text> </xsl:text><xsl:value-of select="HostInfo/@fqdn" /><xsl:text> +<xsl:sort select="Client/@name"/> +<xsl:if test="count(Client[@pingable='N']) > 0"> +<xsl:text> </xsl:text><xsl:value-of select="Client/@name" /><xsl:text> </xsl:text> </xsl:if> </xsl:for-each><xsl:text> diff --git a/reports/xsl-transforms/overview-stats-www.xsl b/reports/xsl-transforms/overview-stats-www.xsl index afcd06de4..3b1e252ac 100644 --- a/reports/xsl-transforms/overview-stats-www.xsl +++ b/reports/xsl-transforms/overview-stats-www.xsl @@ -8,8 +8,8 @@ <xsl:variable name="dirtynodes" select="/Report/Node[count(Statistics/Bad)>0]"/> <xsl:variable name="modifiednodes" select="/Report/Node[count(Statistics/Modified)>0]"/> <xsl:variable name="stalenodes" select="/Report/Node[count(Statistics/Stale)>0]"/> - <xsl:variable name="unpingablenodes" select="/Report/Node[HostInfo/@pingable='N']"/> - <xsl:variable name="pingablenodes" select="/Report/Node[HostInfo/@pingable='Y']"/> + <xsl:variable name="unpingablenodes" select="/Report/Node[Client/@pingable='N']"/> + <xsl:variable name="pingablenodes" select="/Report/Node[Client/@pingable='Y']"/> <html> <head> @@ -32,9 +32,9 @@ <div class="items" id="goodsummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Good) > 0"> - <tt><xsl:value-of select="HostInfo/@fqdn" /></tt><br/> + <tt><xsl:value-of select="Client/@name" /></tt><br/> </xsl:if> </xsl:for-each> </ul></div> @@ -47,9 +47,9 @@ <div class="items" id="badsummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Bad) > 0"> - <tt><xsl:value-of select="HostInfo/@fqdn" /></tt><br/> + <tt><xsl:value-of select="Client/@name" /></tt><br/> </xsl:if> </xsl:for-each> </ul></div> @@ -62,9 +62,9 @@ <div class="items" id="modifiedsummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> + <xsl:sort select="Client/@name"/> <xsl:if test="count(Statistics/Modified) > 0"> - <tt><xsl:value-of select="HostInfo/@fqdn" /></tt><br/> + <tt><xsl:value-of select="Client/@name" /></tt><br/> </xsl:if> </xsl:for-each> </ul></div> @@ -77,9 +77,9 @@ <div class="items" id="stalesummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> - <xsl:if test="count(Statistics/Stale)-count(HostInfo[@pingable='N']) > 0"> - <tt><xsl:value-of select="HostInfo/@fqdn" /></tt><br/> + <xsl:sort select="Client/@name"/> + <xsl:if test="count(Statistics/Stale)-count(Client[@pingable='N']) > 0"> + <tt><xsl:value-of select="Client/@name" /></tt><br/> </xsl:if> </xsl:for-each> </ul></div> @@ -90,13 +90,13 @@ <xsl:if test="count($unpingablenodes) > 0"> <div class="down"> - <span class="nodelisttitle"><a href="javascript:toggleLayer('unpingablesummary');" title="Click to Expand" class="commentLink"><xsl:value-of select="count(/Report/Node/HostInfo[@pingable='N'])" /></a> nodes were down.<br /></span> + <span class="nodelisttitle"><a href="javascript:toggleLayer('unpingablesummary');" title="Click to Expand" class="commentLink"><xsl:value-of select="count(/Report/Node/Client[@pingable='N'])" /></a> nodes were down.<br /></span> <div class="items" id="unpingablesummary"><ul class="plain"> <xsl:for-each select="Node"> - <xsl:sort select="HostInfo/@fqdn"/> - <xsl:if test="count(HostInfo[@pingable='N']) > 0"> - <tt><xsl:value-of select="HostInfo/@fqdn" /></tt><br/> + <xsl:sort select="Client/@name"/> + <xsl:if test="count(Client[@pingable='N']) > 0"> + <tt><xsl:value-of select="Client/@name" /></tt><br/> </xsl:if> </xsl:for-each> </ul></div> diff --git a/reports/xsl-transforms/timing-summary-www.xsl b/reports/xsl-transforms/timing-summary-www.xsl index 838995243..2b8b94699 100644 --- a/reports/xsl-transforms/timing-summary-www.xsl +++ b/reports/xsl-transforms/timing-summary-www.xsl @@ -36,7 +36,7 @@ <th class="sortable">Total</th> </tr> <xsl:apply-templates select="Node"> - <xsl:sort select="HostInfo/@fqdn" order="ascending"/> + <xsl:sort select="Client/@name" order="ascending"/> <xsl:sort select="@name"/> </xsl:apply-templates> </table> @@ -54,7 +54,7 @@ <xsl:template match="Node"> <xsl:if test="count(Statistics/OpStamps) > 0"> <tr> - <td class="sortable"><xsl:value-of select="HostInfo/@fqdn" /></td><!--node name--> + <td class="sortable"><xsl:value-of select="Client/@name" /></td><!--node name--> <td class="sortable"><xsl:value-of select="format-number(number(Statistics/OpStamps/@config_parse - Statistics/OpStamps/@config_download),'#.##')"/></td><!--parse--> <td class="sortable"><xsl:value-of select="format-number(number(Statistics/OpStamps/@probe_upload - Statistics/OpStamps/@start),'#.##')"/></td><!--probe download--> <td class="sortable"><xsl:value-of select="format-number(number(Statistics/OpStamps/@inventory - Statistics/OpStamps/@initialization),'#.##')"/></td><!--inventory--> diff --git a/reports/xsl-transforms/xsl-transform-includes/boxypastel-css.xsl b/reports/xsl-transforms/xsl-transform-includes/boxypastel-css.xsl index 1ac508ea6..2fdd7aee4 100644 --- a/reports/xsl-transforms/xsl-transform-includes/boxypastel-css.xsl +++ b/reports/xsl-transforms/xsl-transform-includes/boxypastel-css.xsl @@ -144,8 +144,8 @@ ul.plain { .configbox { position: absolute; - top: 1.6em; - right: 1px; + bottom: 0px; + right: 0px; padding: 1px; text-indent:0px; border: 1px solid #999; diff --git a/reports/xsl-transforms/xsl-transform-includes/html-templates.xsl b/reports/xsl-transforms/xsl-transform-includes/html-templates.xsl index cfe494b05..7e9e08055 100644 --- a/reports/xsl-transforms/xsl-transform-includes/html-templates.xsl +++ b/reports/xsl-transforms/xsl-transform-includes/html-templates.xsl @@ -3,10 +3,10 @@ <xsl:template match="Node"> <xsl:if test="count(Statistics/Good)+count(Statistics/Bad)+count(Statistics/Extra)+count(Statistics/Modified)+count(Statistics/Stale) > 0"> - <div class="nodebox" name="{HostInfo/@fqdn}"> + <div class="nodebox" name="{Client/@name}"> <span class="notebox">Time Ran: <xsl:value-of select="Statistics/@time" /></span> - <span class="configbox">(<xsl:value-of select="Client/@image" />/<xsl:value-of select="Client/@profile" />)</span> - <h2>Node: <span class="nodename"><xsl:value-of select="HostInfo/@fqdn" /></span></h2> + <span class="configbox">(<xsl:value-of select="Client/@profile" />)</span> + <h2>Node: <span class="nodename"><xsl:value-of select="Client/@name" /></span></h2> <xsl:apply-templates select="Statistics" /> </div> </xsl:if> diff --git a/reports/xsl-transforms/xsl-transform-includes/text-templates.xsl b/reports/xsl-transforms/xsl-transform-includes/text-templates.xsl index c393692a4..8b820415b 100644 --- a/reports/xsl-transforms/xsl-transform-includes/text-templates.xsl +++ b/reports/xsl-transforms/xsl-transform-includes/text-templates.xsl @@ -5,9 +5,9 @@ <xsl:text> - </xsl:text>Node:<xsl:value-of select="HostInfo/@fqdn" /><xsl:text> + </xsl:text>Node:<xsl:value-of select="Client/@name" /><xsl:text> </xsl:text>Time Ran: <xsl:value-of select="Statistics/@time" />.<xsl:text> - </xsl:text>(<xsl:value-of select="Client/@image" />/<xsl:value-of select="Client/@profile" />) + </xsl:text>(<xsl:value-of select="Client/@profile" />) <xsl:apply-templates select="Statistics" /> </xsl:if> </xsl:template> diff --git a/schemas/base.xsd b/schemas/base.xsd index 14009cf4e..61e065ecc 100644 --- a/schemas/base.xsd +++ b/schemas/base.xsd @@ -4,7 +4,7 @@ <xsd:documentation> base schema for bcfg2 Narayan Desai, Argonne National Laboratory - $Id: $ + $Id$ </xsd:documentation> </xsd:annotation> @@ -18,8 +18,7 @@ <xsd:element name='SymLink' type='SymLinkType'/> <xsd:element name='Directory' type='DirectoryType'/> <xsd:element name='Permissions' type='PermissionType'/> - <xsd:element name='Class' type='ContainerType'/> - <xsd:element name='Image' type='ContainerType'/> + <xsd:element name='Group' type='ContainerType'/> </xsd:choice> <xsd:attribute name='name' type='xsd:string'/> </xsd:complexType> @@ -27,8 +26,7 @@ <xsd:element name='Base'> <xsd:complexType> <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='Image' type='ContainerType'/> - <xsd:element name='Class' type='ContainerType'/> + <xsd:element name='Group' type='ContainerType'/> </xsd:choice> </xsd:complexType> </xsd:element> diff --git a/schemas/bundle.xsd b/schemas/bundle.xsd index 22e8336ba..391a5a27e 100644 --- a/schemas/bundle.xsd +++ b/schemas/bundle.xsd @@ -25,8 +25,7 @@ <xsd:element name='Bundle'> <xsd:complexType> <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element type='GroupType' name='System'/> - <xsd:element type='GroupType' name='Attribute'/> + <xsd:element type='GroupType' name='Group'/> <xsd:element name='Package' type='PackageType'/> <xsd:element name='Service' type='ServiceType'/> <xsd:element name='ConfigFile' type='ConfigFileType'/> @@ -34,6 +33,7 @@ <xsd:element name='SymLink' type='SymLinkType'/> <xsd:element name='Permission' type='PermissionType'/> <xsd:element name='PostInstall' type='PostInstallType'/> + <xsd:element name='Group' type='GroupType'/> </xsd:choice> <xsd:attribute type='xsd:string' name='name'/> <xsd:attribute type='xsd:string' name='version'/> diff --git a/schemas/clients.xsd b/schemas/clients.xsd new file mode 100644 index 000000000..f1a3ec27e --- /dev/null +++ b/schemas/clients.xsd @@ -0,0 +1,26 @@ +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> + + <xsd:annotation> + <xsd:documentation> + client schema for bcfg2 + Narayan Desai, Argonne National Laboratory + $Id$ + </xsd:documentation> + </xsd:annotation> + + <xsd:complexType name='ClientType'> + <xsd:attribute type='xsd:string' name='name' use='required'/> + <xsd:attribute type='xsd:string' name='profile' use='required'/> + <xsd:attribute type='xsd:string' name='pingable' use='required'/> + <xsd:attribute type='xsd:string' name='pingtime' use='required'/> + </xsd:complexType> + + <xsd:element name='Clients'> + <xsd:complexType> + <xsd:choice minOccurs='0' maxOccurs='unbounded'> + <xsd:element name='Client' type='ClientType'/> + </xsd:choice> + <xsd:attribute name='version' type='xsd:string'/> + </xsd:complexType> + </xsd:element> +</xsd:schema>
\ No newline at end of file diff --git a/schemas/metadata.xsd b/schemas/metadata.xsd index 3636bb2b8..d200694cc 100644 --- a/schemas/metadata.xsd +++ b/schemas/metadata.xsd @@ -4,69 +4,44 @@ <xsd:documentation> metadata schema for bcfg2 Narayan Desai, Argonne National Laboratory - $Id: $ + $Id$ </xsd:documentation> </xsd:annotation> - <xsd:complexType name='ClientType'> - <xsd:attribute type='xsd:string' name='name' use='required'/> - <xsd:attribute type='xsd:string' name='image' use='required'/> - <xsd:attribute type='xsd:string' name='profile' use='required'/> - </xsd:complexType> + <xsd:simpleType name='toolsetType'> + <xsd:restriction base="xsd:string"> + <xsd:pattern value="rh|debian|solaris"/> + </xsd:restriction> + </xsd:simpleType> - <xsd:complexType name='ProfileType'> - <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='Class'> - <xsd:complexType> - <xsd:attribute type='xsd:string' name='name' use='required'/> - </xsd:complexType> - </xsd:element> - <xsd:element name='Attribute'> - <xsd:complexType> - <xsd:attribute name='name' type='xsd:string' use='required'/> - <xsd:attribute name='scope' type='xsd:string' use='required'/> - </xsd:complexType> - </xsd:element> - </xsd:choice> - <xsd:attribute type='xsd:string' name='name' use='required'/> - <xsd:attribute type='xsd:string' name='public' use='required'/> - </xsd:complexType> + <xsd:simpleType name='booleanType'> + <xsd:restriction base="xsd:string"> + <xsd:pattern value="true|false"/> + </xsd:restriction> + </xsd:simpleType> - <xsd:complexType name='ClassType'> + <xsd:complexType name='groupType'> <xsd:choice minOccurs='0' maxOccurs='unbounded'> <xsd:element name='Bundle'> <xsd:complexType> <xsd:attribute type='xsd:string' name='name' use='required'/> </xsd:complexType> </xsd:element> + <xsd:element name='Group' type='groupType'/> </xsd:choice> + <xsd:attribute type='booleanType' name='profile' use='optional'/> + <xsd:attribute type='booleanType' name='public' use='optional'/> <xsd:attribute type='xsd:string' name='name' use='required'/> - </xsd:complexType> - - <xsd:simpleType name='ToolSetType'> - <xsd:restriction base="xsd:string"> - <xsd:pattern value="rh|debian|solaris"/> - </xsd:restriction> - </xsd:simpleType> - - <xsd:complexType name='ImageType'> - <xsd:attribute type='xsd:string' name='name' use='required'/> - <xsd:attribute type='ToolSetType' name='toolset' use='required'/> + <xsd:attribute type='toolsetType' name='toolset' use='optional'/> + <xsd:attribute type='xsd:string' name='category' use='optional'/> </xsd:complexType> - - <xsd:element name='Metadata'> + <xsd:element name='Groups'> <xsd:complexType> <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='Client' type='ClientType'/> - <xsd:element name='Class' type='ClassType'/> - <xsd:element name='Profile' type='ProfileType'/> - <xsd:element name='Image' type='ImageType'/> + <xsd:element name='Group' type='groupType'/> </xsd:choice> <xsd:attribute name='version' type='xsd:string'/> - <xsd:attribute type='xsd:string' name='default_image' use='required'/> - <xsd:attribute type='xsd:string' name='default_profile' use='required'/> </xsd:complexType> </xsd:element> - </xsd:schema> diff --git a/schemas/pkglist.xsd b/schemas/pkglist.xsd index 31fa9959d..d262963ee 100644 --- a/schemas/pkglist.xsd +++ b/schemas/pkglist.xsd @@ -4,7 +4,7 @@ <xsd:documentation> package list schema for bcfg2 Narayan Desai, Argonne National Laboratory - $Id: $ + $Id$ </xsd:documentation> </xsd:annotation> @@ -16,20 +16,21 @@ <xsd:attribute type='xsd:string' name='simplefile'/> </xsd:complexType> - <xsd:complexType name='LocationType'> + <xsd:complexType name='GroupType'> <xsd:choice minOccurs='0' maxOccurs='unbounded'> <xsd:element name='Package' type='PackageType'/> </xsd:choice> - <xsd:attribute type='xsd:string' name='uri' use='required'/> - <xsd:attribute type='xsd:string' name='type' use='required'/> + <xsd:attribute type='xsd:string' name='name' use='required'/> </xsd:complexType> <xsd:element name='PackageList'> <xsd:complexType> <xsd:choice minOccurs='0' maxOccurs='unbounded'> - <xsd:element name='Location' type='LocationType'/> + <xsd:element name='Group' type='GroupType'/> </xsd:choice> - <xsd:attribute name='image' type='xsd:string' use='required'/> + <xsd:attribute name='priority' type='xsd:integer' use='required'/> + <xsd:attribute name='type' type='xsd:string' use='required'/> + <xsd:attribute name='uri' type='xsd:string' use='optional'/> </xsd:complexType> </xsd:element> diff --git a/schemas/services.xsd b/schemas/services.xsd index 012bb2876..286c925aa 100644 --- a/schemas/services.xsd +++ b/schemas/services.xsd @@ -4,7 +4,7 @@ <xsd:documentation> services schema for bcfg2 Narayan Desai, Argonne National Laboratory - $Id: $ + $Id$ </xsd:documentation> </xsd:annotation> @@ -35,11 +35,10 @@ <xsd:complexType> <xsd:choice minOccurs='0' maxOccurs='unbounded'> <xsd:element name='Service' type='ServiceType'/> - <xsd:element name='Class' type='ContainerType'/> - <xsd:element name='Image' type='ContainerType'/> + <xsd:element name='Group' type='ContainerType'/> <xsd:element name='Host' type='ContainerType'/> - <xsd:element name='Global' type='ContainerType'/> </xsd:choice> + <xsd:attribute name='priority' type='xsd:integer'/> </xsd:complexType> </xsd:element> diff --git a/schemas/translation.xsd b/schemas/translation.xsd index cdf117a8a..9a58ab640 100644 --- a/schemas/translation.xsd +++ b/schemas/translation.xsd @@ -4,7 +4,7 @@ <xsd:documentation> translation image info schema for bcfg2 Narayan Desai, Argonne National Laboratory - $Id: $ + $Id$ </xsd:documentation> </xsd:annotation> @@ -4,13 +4,13 @@ from distutils.core import setup from glob import glob setup(name="Bcfg2.Server", - version="0.7.3", + version="0.8.0", description="Bcfg2 Server", author="Narayan Desai", author_email="desai@mcs.anl.gov", packages=["Bcfg2", 'Bcfg2.Server', "Bcfg2.Server.Plugins", "Bcfg2.Client"], package_dir = {'Bcfg2':'src/lib'}, - scripts = ['src/sbin/Bcfg2Server', 'src/sbin/bcfg2', 'src/sbin/ValidateBcfg2Repo', 'src/sbin/StatReports', 'src/sbin/GenerateHostInfo'], + scripts = glob('src/sbin/*'), data_files = [('share/bcfg2/schemas', glob('schemas/*.xsd')), ('share/bcfg2/xsl-transforms', diff --git a/src/lib/Client/Redhat.py b/src/lib/Client/Redhat.py index d68288b79..9ddd0f142 100644 --- a/src/lib/Client/Redhat.py +++ b/src/lib/Client/Redhat.py @@ -117,7 +117,7 @@ class ToolsetImpl(Toolset): if self.setup['remove'] in ['all', 'services']: self.CondDisplayList('verbose', 'Removing services:', self.extra_services) for service in self.extra_services: - if not system("/sbin/chkconfig %s off" % service): + if not system("/sbin/chkconfig --level 123456 %s off" % service): self.extra_services.remove(service) else: self.CondDisplayList('verbose', 'Need to remove services:', self.extra_services) diff --git a/src/lib/Client/Solaris.py b/src/lib/Client/Solaris.py index 69d6ee69b..48f26c1f8 100644 --- a/src/lib/Client/Solaris.py +++ b/src/lib/Client/Solaris.py @@ -83,7 +83,7 @@ class ToolsetImpl(Toolset): def VerifyService(self, entry): '''Verify Service status for entry''' if not entry.attrib.has_key('FMRI'): - (rc, name) = self.saferun("/usr/bin/svcs -H -o FMRI %s 2>/dev/null" % entry.get('name')) + name = self.saferun("/usr/bin/svcs -H -o FMRI %s 2>/dev/null" % entry.get('name'))[1] if name: entry.set('FMRI', name[0]) else: diff --git a/src/lib/Client/Toolset.py b/src/lib/Client/Toolset.py index b358ac92c..9cf9ad4b1 100644 --- a/src/lib/Client/Toolset.py +++ b/src/lib/Client/Toolset.py @@ -4,7 +4,7 @@ __revision__ = '$Revision$' from binascii import a2b_base64 from copy import deepcopy from grp import getgrgid, getgrnam -from os import chown, chmod, lstat, mkdir, stat, system, unlink, rename, readlink, symlink +from os import chown, chmod, lstat, mkdir, stat, unlink, rename, readlink, symlink from pwd import getpwuid, getpwnam from stat import S_ISVTX, S_ISGID, S_ISUID, S_IXUSR, S_IWUSR, S_IRUSR, S_IXGRP from stat import S_IWGRP, S_IRGRP, S_IXOTH, S_IWOTH, S_IROTH, ST_MODE, S_ISDIR @@ -87,6 +87,7 @@ class Toolset(object): pass def get_height_width(self): + '''Get Terminal information''' try: import termios, struct, fcntl height, width = struct.unpack('hhhh', @@ -99,6 +100,7 @@ class Toolset(object): return 25, 80 def FormattedCondPrint(self, state, items): + '''Formatted conditional print''' items.sort() screenWidth = self.width - len("%s[%s]:" % (self.__name__, state)) columnWidth = 1 @@ -121,11 +123,13 @@ class Toolset(object): self.CondPrint(state, lineText.rstrip()) def CondDisplayList(self, state, title, items): + '''Conditionally print a list of data''' self.CondPrint(state, title) self.FormattedCondPrint(state, items) self.CondPrint(state, '') def CondDisplayState(self, state, phase): + '''Conditionally print tracing information''' self.CondPrint(state, 'Phase: %s' % phase) self.CondPrint(state, 'Correct entries:\t%d' % self.states.values().count(True)) @@ -625,17 +629,19 @@ class Toolset(object): self.CondPrint("debug", "Installing packages: :%s:" % pkgargs) self.CondPrint("debug", "Running command ::%s::" % (pkgtool[0] % pkgargs)) - (cmdrc, cmdoutput) = self.saferun(pkgtool[0] % pkgargs) + cmdrc = self.saferun(pkgtool[0] % pkgargs)[0] if cmdrc == 0: self.CondPrint('verbose', "Single Pass Succeded") # set all package states to true and flush workqueues pkgnames = [pkg.get('name') for pkg in pkglist] for entry in [entry for entry in self.states.keys() - if entry.tag == 'Package' and entry.get('type') == pkgtype and entry.get('name') in pkgnames]: + if entry.tag == 'Package' and entry.get('type') == pkgtype + and entry.get('name') in pkgnames]: self.CondPrint('debug', 'Setting state to true for pkg %s' % (entry.get('name'))) self.states[entry] = True - [self.pkgwork[listname].remove(entry) for listname in ['add', 'update'] if self.pkgwork[listname].count(entry)] + [self.pkgwork[listname].remove(entry) for listname in ['add', 'update'] + if self.pkgwork[listname].count(entry)] self.Refresh() else: self.CondPrint("verbose", "Single Pass Failed") @@ -649,10 +655,10 @@ class Toolset(object): else: self.CondPrint("verbose", "Installing pkg %s version %s" % (pkg.get('name'), pkg.get('version'))) - (cmdrc, cmdoutput) = self.saferun(pkgtool[0] % - (pkgtool[1][0] % - tuple([pkg.get(field) for field in pkgtool[1][1]]))) - if cmdrc == 0: + cmdrc = self.saferun(pkgtool[0] % + (pkgtool[1][0] % + tuple([pkg.get(field) for field in pkgtool[1][1]]))) + if cmdrc[0] == 0: self.states[pkg] = True else: self.CondPrint('verbose', "Failed to install package %s" % (pkg.get('name'))) diff --git a/src/lib/Server/Component.py b/src/lib/Server/Component.py index 97444bb10..5c19c3bdd 100644 --- a/src/lib/Server/Component.py +++ b/src/lib/Server/Component.py @@ -5,7 +5,6 @@ from ConfigParser import ConfigParser, NoOptionError from cPickle import loads, dumps from M2Crypto import SSL from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler -from select import select from socket import gethostname from sys import exc_info import sys diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py index a2100ea53..91da366b8 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Server/Core.py @@ -11,9 +11,11 @@ from ConfigParser import ConfigParser from lxml.etree import Element from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError -from Bcfg2.Server.Metadata import MetadataStore, MetadataConsistencyError + from Bcfg2.Server.Statistics import Statistics +import Bcfg2.Server.Metadata + def log_failure(msg): syslog(LOG_ERR, "Unexpected failure in %s" % (msg)) (trace, val, trb) = exc_info() @@ -210,7 +212,7 @@ class Core(object): mpath = cfile.get('server','repository') try: - self.metadata = MetadataStore("%s/etc/metadata.xml" % mpath, self.fam) + self.metadata = Bcfg2.Server.Metadata.Metadata(self.fam, mpath) except OSError: raise CoreInitError, "metadata path incorrect" @@ -269,21 +271,15 @@ class Core(object): generators = ", ".join([gen.__name__ for gen in glist]) syslog(LOG_ERR, "%s %s served by multiple generators: %s" % (entry.tag, entry.get('name'), generators)) - raise PluginExecutionError, (entry.tag, entry.get('name')) - else: - for gen in self.generators: - if hasattr(gen, "FindHandler"): - return gen.FindHandler(entry)(entry, metadata) - syslog(LOG_ERR, "Failed to find handler for %s:%s" % (entry.tag, entry.get('name'))) - raise PluginExecutionError, (entry.tag, entry.get('name')) + raise PluginExecutionError, (entry.tag, entry.get('name')) def BuildConfiguration(self, client): '''Build Configuration for client''' start = time() config = Element("Configuration", version='2.0') try: - meta = self.metadata.FetchMetadata(client) - except MetadataConsistencyError: + meta = self.metadata.get_metadata(client) + except Bcfg2.Server.Metadata.MetadataConsistencyError: syslog(LOG_ERR, "Metadata consistency error for client %s" % client) return Element("error", type='metadata error') diff --git a/src/lib/Server/Metadata.py b/src/lib/Server/Metadata.py index 47bbb3ecb..ecf636476 100644 --- a/src/lib/Server/Metadata.py +++ b/src/lib/Server/Metadata.py @@ -1,130 +1,190 @@ '''This file stores persistent metadata for the BCFG Configuration Repository''' __revision__ = '$Revision$' -from lxml.etree import XML, SubElement, Element, _Comment, tostring from syslog import syslog, LOG_ERR, LOG_INFO -from Bcfg2.Server.Plugin import SingleXMLFileBacked +import lxml.etree, os, time, threading class MetadataConsistencyError(Exception): '''This error gets raised when metadata is internally inconsistent''' pass -class Metadata(object): - '''The Metadata class is a container for all classes of metadata used by Bcfg2''' - def __init__(self, all, image, classes, bundles, attributes, hostname, toolset): - self.all = all - self.image = image - self.classes = classes +class MetadataRuntimeError(Exception): + '''This error is raised when the metadata engine is called prior to reading enough data''' + pass + +class ClientMetadata(object): + '''This object contains client metadata''' + def __init__(self, client, groups, bundles, toolset): + self.hostname = client self.bundles = bundles - self.attributes = attributes - self.hostname = hostname + self.groups = groups self.toolset = toolset - def Applies(self, other): - '''Check if metadata styled object applies to current metadata''' - if (other.all or (other.image and (self.image == other.image)) or - (other.classes and (other.classes in self.classes)) or - (other.attributes and (other.attributes in self.attributes)) or - (other.bundles and (other.bundles in self.bundles)) or - (other.hostname and (self.hostname == other.hostname)) or - (other.hostname and (self.hostname.split('.')[0] == other.hostname))): - return True - else: - return False - -class Profile(object): - '''Profiles are configuration containers for sets of classes and attributes''' - def __init__(self, xml): - object.__init__(self) - self.classes = [cls.attrib['name'] for cls in xml.findall("Class")] - self.attributes = ["%s.%s" % (attr.attrib['scope'], attr.attrib['name']) for - attr in xml.findall("Attribute")] - -class MetadataStore(SingleXMLFileBacked): - '''The MetadataStore is a filebacked xml repository that contains all setup info for all clients''' +class Metadata: + '''This class contains data for bcfg2 server metadata''' + __name__ = 'Metadata' + __version__ = '$Id$' + __author__ = 'bcfg-dev@mcs.anl.gov' - def __init__(self, filename, fam): - # initialize Index data to avoid race - self.defaults = {} + def __init__(self, fam, datastore): + self.data = "%s/%s" % (datastore, self.__name__) + fam.AddMonitor("%s/%s" % (self.data, "groups.xml"), self) + fam.AddMonitor("%s/%s" % (self.data, "clients.xml"), self) + self.states = {'groups.xml':False, 'clients.xml':False} self.clients = {} - self.profiles = {} - self.classes = {} - self.images = {} - self.element = Element("dummy") - SingleXMLFileBacked.__init__(self, filename, fam) - - def Index(self): - '''Build data structures for XML data''' - self.element = XML(self.data) - self.defaults = {} - self.clients = {} - self.profiles = {} - self.classes = {} - self.images = {} - for prof in self.element.findall("Profile"): - self.profiles[prof.attrib['name']] = Profile(prof) - for cli in self.element.findall("Client"): - self.clients[cli.attrib['name']] = (cli.attrib['image'], cli.attrib['profile']) - for cls in self.element.findall("Class"): - self.classes[cls.attrib['name']] = [bundle.attrib['name'] for bundle in cls.findall("Bundle")] - for img in self.element.findall("Image"): - self.images[img.attrib['name']] = img.attrib['toolset'] - for key in [key[8:] for key in self.element.attrib if key[:8] == 'default_']: - self.defaults[key] = self.element.get("default_%s" % key) + self.aliases = {} + self.groups = {} + self.public = [] + self.profiles = [] + self.toolsets = {} + self.categories = {} + self.clientdata = None + self.default = None - def FetchMetadata(self, client, image=None, profile=None): - '''Get metadata for client''' - if ((image != None) and (profile != None)): - # Client asserted profile/image - self.clients[client] = (image, profile) - syslog(LOG_INFO, "Metadata: Asserted metadata for %s: %s, %s" % (client, image, profile)) - [self.element.remove(cli) for cli in self.element.findall("Client") if cli.get('name') == client] - SubElement(self.element, "Client", name=client, image=image, profile=profile) - self.WriteBack() + def HandleEvent(self, event): + '''Handle update events for data files''' + filename = event.filename.split('/')[-1] + if filename not in ['groups.xml', 'clients.xml']: + return + if event.code2str() == 'endExist': + return + try: + xdata = lxml.etree.parse("%s/%s" % (self.data, filename)) + except lxml.etree.XMLSyntaxError: + syslog(LOG_ERR, 'Metadata: Failed to parse %s' % (filename)) + return + if filename == 'clients.xml': + self.clients = {} + self.aliases = {} + self.clientdata = xdata + for client in xdata.findall('./Client'): + self.clients.update({client.get('name'): client.get('profile')}) + [self.aliases.update({alias.get('name'): client.get('name')}) for alias in client.findall('Alias')] else: - # no asserted metadata - if self.clients.has_key(client): - (image, profile) = self.clients[client] - else: - # default profile stuff goes here - (image, profile) = (self.defaults['image'], self.defaults['profile']) - SubElement(self.element, "Client", name=client, profile=profile, image=image) - self.WriteBack() + self.public = [] + self.profiles = [] + self.toolsets = {} + self.groups = {} + grouptmp = {} + self.categories = {} + for group in xdata.findall('./Group'): + grouptmp[group.get('name')] = tuple([[item.get('name') for item in group.findall(spec)] + for spec in ['./Bundle', './Group']]) + grouptmp[group.get('name')][1].append(group.get('name')) + if group.get('default', 'false') == 'true': + self.default = group.get('name') + if group.get('profile', 'false') == 'true': + self.profiles.append(group.get('name')) + if group.get('public', 'false') == 'true': + self.public.append(group.get('name')) + if group.attrib.has_key('toolset'): + self.toolsets[group.get('name')] = group.get('toolset') + if group.attrib.has_key('category'): + self.categories[group.get('name')] = group.get('category') + for group in grouptmp: + self.groups[group] = ([], []) + gcategories = [] + tocheck = [group] + while tocheck: + now = tocheck.pop() + if now not in self.groups[group][1]: + self.groups[group][1].append(now) + if grouptmp.has_key(now): + (bundles, groups) = grouptmp[now] + for ggg in [ggg for ggg in groups if ggg not in self.groups[group][1]]: + if not self.categories.has_key(ggg) or (self.categories[ggg] not in gcategories): + self.groups[group][1].append(ggg) + tocheck.append(ggg) + if self.categories.has_key(ggg): + gcategories.append(self.categories[ggg]) + [self.groups[group][0].append(bund) for bund in bundles + if bund not in self.groups[group][0]] + self.states[filename] = True + if False not in self.states.values(): + # check that all client groups are real and complete + real = self.groups.keys() + for client in self.clients.keys(): + if self.clients[client] not in real or self.clients[client] not in self.profiles: + syslog(LOG_ERR, "Metadata: Client %s set as nonexistant or incomplete group %s" \ + % (client, self.clients[client])) + syslog(LOG_ERR, "Metadata: Removing client mapping for %s" % (client)) + del self.clients[client] - if not self.profiles.has_key(profile): - syslog(LOG_ERR, "Metadata: profile %s not defined" % profile) - raise MetadataConsistencyError - prof = self.profiles[profile] - # should we uniq here? V - bundles = reduce(lambda x, y:x + y, [self.classes.get(cls, []) for cls in prof.classes]) - if not self.images.has_key(image): - syslog(LOG_ERR, "Metadata: Image %s not defined" % image) + def set_group(self, client, group): + '''Set group parameter for provided client''' + if False in self.states.values(): + raise MetadataRuntimeError + if group not in self.public: + syslog(LOG_ERR, "Metadata: Failed to set client %s to private group %s" % (client, + group)) raise MetadataConsistencyError - toolset = self.images[image] - return Metadata(False, image, prof.classes, bundles, prof.attributes, client, toolset) + if self.clients.has_key(client): + syslog(LOG_INFO, "Metadata: Changing %s group from %s to %s" % (client, + self.clients[client], group)) + cli = self.clientdata.xpath('/Clients/Client[@name="%s"]' % (client)) + cli[0].set('group', group) + else: + lxml.etree.SubElement(self.clientdata.getroot(), 'Client', name=client, group=group) + self.clients[client] = group + self.write_back_clients() + + def write_back_clients(self): + '''Write changes to client.xml back to disk''' + try: + datafile = open("%s/%s" % (self.data, 'clients.xml'), 'w') + except IOError: + syslog(LOG_ERR, "Metadata: Failed to write clients.xml") + raise MetadataRuntimeError + datafile.write(lxml.etree.tostring(self.clientdata)) + datafile.close() - def pretty_print(self, element, level=0): - '''Produce a pretty-printed text representation of element''' - if isinstance(element, _Comment): - return (level * " ") + tostring(element) - if element.text: - fmt = "%s<%%s %%s>%%s</%%s>" % (level*" ") - data = (element.tag, (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])), - element.text, element.tag) - numchild = len(element.getchildren()) - if numchild: - fmt = "%s<%%s %%s>\n" % (level*" ",) + (numchild * "%s") + "%s</%%s>\n" % (level*" ") - data = (element.tag, ) + (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib]),) - data += tuple([self.pretty_print(entry, level+2) for entry in element.getchildren()]) + (element.tag, ) + def find_toolset(self, client): + '''Find the toolset for a given client''' + tgroups = [self.toolsets[group] for group in self.groups[client][1] if self.toolsets.has_key(group)] + if len(tgroups) == 1: + return tgroups[0] + elif len(tgroups) == 0: + syslog(LOG_ERR, "Metadata: Couldn't find toolset for client %s" % (client)) + raise MetadataConsistencyError else: - fmt = "%s<%%s %%s/>\n" % (level * " ") - data = (element.tag, " ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])) - return fmt % data + syslog(LOG_ERR, "Metadata: Got goofy toolset result for client %s" % (client)) + raise MetadataConsistencyError - def WriteBack(self): - '''Write metadata changes back to persistent store''' - fout = open(self.name, 'w') - fout.write(self.pretty_print(self.element)) - fout.close() + def get_config_template(self, client): + '''Build the configuration header for a client configuration''' + return lxml.etree.Element("Configuration", version='2.0', toolset=self.find_toolset(client)) + def get_metadata(self, client): + '''Return the metadata for a given client''' + if self.aliases.has_key(client): + client = self.aliases[client] + if self.clients.has_key(client): + [bundles, groups] = self.groups[self.clients[client]] + else: + if self.default == None: + syslog(LOG_ERR, "Cannot set group for client %s; no default group set" % (client)) + raise MetadataConsistencyError + [bundles, groups] = self.groups[self.default] + toolinfo = [self.toolsets[group] for group in groups if self.toolsets.has_key(group)] + if len(toolinfo) > 1: + syslog(LOG_ERR, "Metadata: Found multiple toolsets for client %s; choosing one" % (client)) + elif len(toolinfo) == 0: + syslog(LOG_ERR, "Metadata: Cannot determine toolset for client %s" % (client)) + raise MetadataConsistencyError + toolset = toolinfo[0] + return ClientMetadata(client, groups, bundles, toolset) + + def ping_sweep_clients(self): + '''Find live and dead clients''' + live = {} + dead = {} + work = self.clients.keys() + while work: + client = work.pop() + rc = os.system("/bin/ping -w 5 -c 1 %s > /dev/null 2>&1" % client) + if not rc: + live[client] = time.time() + else: + dead[client] = time.time() + diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py index 262d6092d..890084c98 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Server/Plugin.py @@ -1,10 +1,12 @@ '''This module provides the baseclass for Bcfg2 Server Plugins''' -__revision__ = '$Revision:$' +__revision__ = '$Revision$' -from lxml.etree import XML, XMLSyntaxError, _Comment, tostring -from os import stat -from stat import ST_MTIME -from syslog import syslog, LOG_ERR, LOG_INFO +import lxml.etree +import os +import stat +import syslog + +from lxml.etree import XML, XMLSyntaxError class PluginInitError(Exception): '''Error raised in cases of Plugin initialization errors''' @@ -39,7 +41,7 @@ class Plugin(object): def LogError(self, msg): '''Log error message tagged with Plugin name''' - syslog(LOG_ERR, "%s: %s" % (self.__name__, msg)) + syslog.syslog(syslog.LOG_ERR, "%s: %s" % (self.__name__, msg)) def BuildStructures(self, metadata): '''Build a set of structures tailored to the client metadata''' @@ -73,15 +75,15 @@ class FileBacked(object): '''Read file upon update''' oldmtime = self.mtime try: - self.mtime = stat(self.name)[ST_MTIME] + self.mtime = os.stat(self.name)[stat.ST_MTIME] except OSError: - syslog(LOG_ERR, "Failed to stat file %s" % (self.name)) + syslog.syslog(syslog.LOG_ERR, "Failed to stat file %s" % (self.name)) try: self.data = file(self.name).read() self.Index() except IOError: - syslog(LOG_ERR, "Failed to read file %s" % (self.name)) + syslog.syslog(syslog.LOG_ERR, "Failed to read file %s" % (self.name)) def Index(self): '''Update local data structures based on current file state''' @@ -108,9 +110,9 @@ class DirectoryBacked(object): def AddEntry(self, name): '''Add new entry to data structures upon file creation''' if name == '': - syslog(LOG_INFO, "got add for empty name") + syslog.syslog(syslog.LOG_INFO, "got add for empty name") elif self.entries.has_key(name): - syslog(LOG_INFO, "got multiple adds for %s" % name) + syslog.syslog(syslog.LOG_INFO, "got multiple adds for %s" % name) else: if ((name[-1] == '~') or (name[:2] == '.#') or (name[-4:] == '.swp') or (name in ['SCCS', '.svn'])): return @@ -121,7 +123,7 @@ class DirectoryBacked(object): '''Propagate fam events to underlying objects''' action = event.code2str() if event.filename == '': - syslog(LOG_INFO, "Got event for blank filename") + syslog.syslog(syslog.LOG_INFO, "Got event for blank filename") return if action == 'exists': if event.filename != self.name: @@ -153,7 +155,7 @@ class XMLFileBacked(FileBacked): try: xdata = XML(self.data) except XMLSyntaxError: - syslog(LOG_ERR, "Failed to parse %s"%(self.name)) + syslog.syslog(syslog.LOG_ERR, "Failed to parse %s"%(self.name)) return self.label = xdata.attrib[self.__identifier__] self.entries = xdata.getchildren() @@ -167,83 +169,141 @@ class SingleXMLFileBacked(XMLFileBacked): XMLFileBacked.__init__(self, filename) fam.AddMonitor(filename, self) -class ScopedXMLFile(SingleXMLFileBacked): - '''Scoped XML files are coherent files with Metadata structured data''' - __containers__ = ['Class', 'Host', 'Image'] - - def __init__(self, filename, fam): - self.store = {} - self.__provides__ = {} - SingleXMLFileBacked.__init__(self, filename, fam) +class StructFile(XMLFileBacked): + '''This file contains a set of structure file formatting logic''' + def __init__(self, name): + XMLFileBacked.__init__(self, name) + self.fragments = {} - def StoreRecord(self, metadata, entry): - '''Store scoped record based on metadata''' - if isinstance(entry, _Comment): - return - elif not entry.attrib.has_key('name'): - syslog(LOG_ERR, "Got malformed record %s" % (tostring(entry))) - if not self.store.has_key(entry.tag): - self.store[entry.tag] = {} - if not self.store[entry.tag].has_key(entry.attrib['name']): - self.store[entry.tag][entry.attrib['name']] = [] - self.store[entry.tag][entry.attrib['name']].append((metadata, entry)) - def Index(self): '''Build internal data structures''' try: - xdata = XML(self.data) - except XMLSyntaxError, msg: - syslog(LOG_ERR, "Failed to parse %s"%(self.name)) - # need to add in lxml error messages, once they are supported + xdata = lxml.etree.XML(self.data) + except lxml.etree.XMLSyntaxError: + syslog.syslog(syslog.LOG_ERR, "Failed to parse file %s" % self.name) return - self.store = {} - for entry in [ent for ent in xdata.getchildren() if not isinstance(ent, _Comment)]: - if entry.tag not in self.__containers__: - self.StoreRecord(('Global','all'), entry) + self.fragments = {} + work = {lambda x:True: xdata.getchildren()} + while work: + (predicate, worklist) = work.popitem() + self.fragments[predicate] = [item for item in worklist if item.tag != 'Group' + and not isinstance(item, lxml.etree._Comment)] + for group in [item for item in worklist if item.tag == 'Group']: + # if only python had forceable early-binding + newpred = eval("lambda x:'%s' in x.groups and predicate(x)" % (group.get('name')), + {'predicate':predicate}) + work[newpred] = group.getchildren() + + def Match(self, metadata): + '''Return matching fragments of independant''' + return reduce(lambda x, y:x+y, [frag for (pred, frag) in self.fragments.iteritems() + if pred(metadata)]) + +class LNode: + '''LNodes provide lists of things available at a particular group intersection''' + raw = {'Client':"lambda x:'%s' == x.hostname and predicate(x)", + 'Group':"lambda x:'%s' in x.groups and predicate(x)"} + __leaf__ = './Child' + + def __init__(self, data, plist, parent=None): + self.data = data + self.contents = {} + if parent == None: + self.predicate = lambda x:True + else: + predicate = parent.predicate + if data.tag in self.raw.keys(): + self.predicate = eval(self.raw[data.tag] % (data.get('name')), {'predicate':predicate}) else: - name = (entry.tag, entry.get('name')) - [self.StoreRecord(name, child) - for child in entry.getchildren() if not isinstance(entry, _Comment)] - # now to build the __provides__ table - for key in self.__provides__.keys(): - del self.__provides__[key] - for key in self.store.keys(): - self.__provides__[key] = {} - for name in self.store[key].keys(): - self.__provides__[key][name] = self.FetchRecord - # also need to sort all leaf node lists - self.store[key][name].sort(self.Sort) - - def Sort(self, meta1, meta2): - '''Sort based on specificity''' - order = ['Global', 'Image', 'Profile', 'Class', 'Host'] - return order.index(meta1[0][0]) - order.index(meta2[0][0]) - - def MatchMetadata(self, mdata, metadata): - '''Match internal metadata representation against metadata''' - (mtype, mvalue) = mdata - if mtype == 'Global': - return True - elif mtype == 'Profile': - if mvalue == metadata.profile: - return True - elif mtype == 'Image': - if mvalue == metadata.image: - return True - elif mtype == 'Class': - if mvalue in metadata.classes: - return True - elif mtype == 'Host': - if mvalue == metadata.hostname: - return True - return False - - def FetchRecord(self, entry, metadata): - '''Build a data for specified metadata''' - dlist = self.store[entry.tag][entry.get('name')] - useful = [ent for ent in dlist if self.MatchMetadata(ent[0], metadata)] - if not useful: - syslog(LOG_ERR, "Failed to FetchRecord %s:%s"%(entry.tag, entry.get('name'))) + print data.tag + raise Exception + mytype = self.__class__ + self.children = [mytype(child, plist, self) for child in data.getchildren() + if child.tag in ['Group', 'Client']] + for leaf in data.findall(self.__leaf__): + self.contents[leaf.get('name')] = leaf.attrib + if leaf.get('name') not in plist: + plist.append(leaf.get('name')) + + def Match(self, metadata, data): + '''Return a dictionary of package mappings''' + if self.predicate(metadata): + data.update(self.contents) + for child in self.children: + child.Match(metadata, data) + +class XMLSrc(XMLFileBacked): + '''XMLSrc files contain a LNode hierarchy that returns matching entries''' + __node__ = LNode + + def __init__(self, filename): + XMLFileBacked.__init__(self, filename) + self.names = [] + self.cache = None + self.pnode = None + self.priority = '1000' + + def Index(self): + self.names = [] + xdata = XML(self.data) + self.pnode = self.__node__(xdata, self.names) + self.cache = None + self.priority = xdata.attrib['priority'] + + def Cache(self, metadata): + '''Build a package dict for a given host''' + if self.cache == None or self.cache[0] != metadata: + cache = (metadata, {}) + if self.pnode == None: + syslog.syslog(syslog.LOG_ERR, + "Cache method called early for %s; forcing data load" % (self.name)) + self.HandleEvent() + return + self.pnode.Match(metadata, cache[1]) + self.cache = cache + +class XMLPrioDir(Plugin, DirectoryBacked): + '''This is a generator that handles package assignments''' + __name__ = 'XMLPrioDir' + __child__ = XMLSrc + __element__ = 'Dummy' + + def __init__(self, core, datastore): + Plugin.__init__(self, core, datastore) + self.Entries[self.__element__] = {} + try: + DirectoryBacked.__init__(self, self.data, self.core.fam) + except OSError: + self.LogError("Failed to load %s indices" % (self.__element__.lower())) + raise PluginInitError + + def HandleEvent(self, event): + '''Handle events and update dispatch table''' + DirectoryBacked.HandleEvent(self, event) + for src in self.entries.values(): + for child in src.names: + self.Entries[self.__element__][child] = self.BindEntry + + def BindEntry(self, entry, metadata): + '''Check package lists of package entries''' + [src.Cache(metadata) for src in self.entries.values()] + name = entry.get('name') + if not src.cache: + self.LogError("Called before data loaded") + raise PluginExecutionError + matching = [src for src in self.entries.values() + if src.cache[1].has_key(name)] + if len(matching) == 0: + raise PluginExecutionError + elif len(matching) == 1: + index = 0 else: - data = useful[-1][-1] - [entry.attrib.__setitem__(x, data.attrib[x]) for x in data.attrib] + prio = [int(src.priority) for src in matching] + if prio.count(max(prio)) > 1: + self.LogError("Found multiple %s sources with same priority for %s, pkg %s" % + (self.__element__.lower(), metadata.hostname, entry.get('name'))) + raise PluginExecutionError + index = prio.index(max(prio)) + + data = matching[index].cache[1][name] + [entry.attrib.__setitem__(key, data[key]) for key in data.keys()] diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py index 1cdd7599c..3be30bc6a 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Server/Plugins/Base.py @@ -1,62 +1,31 @@ '''This module sets up a base list of configuration entries''' __revision__ = '$Revision$' -from copy import deepcopy -from lxml.etree import Element, XML, XMLSyntaxError, _Comment +import Bcfg2.Server.Plugin +import copy +import lxml.etree -from Bcfg2.Server.Plugin import Plugin, PluginInitError, SingleXMLFileBacked - -class Base(Plugin, SingleXMLFileBacked): +class Base(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): '''This Structure is good for the pile of independent configs needed for most actual systems''' __name__ = 'Base' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' + __child__ = Bcfg2.Server.Plugin.StructFile '''base creates independent clauses based on client metadata''' def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - self.store = {'all':[], 'Class':{'all':[]}, 'Image':{'all':[]}, 'all':[]} + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + self.fragements = {} try: - SingleXMLFileBacked.__init__(self, "%s/etc/base.xml"%(datastore), self.core.fam) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, self.core.fam) except OSError: - self.LogError("Failed to load base.xml") - raise PluginInitError + self.LogError("Failed to load Base repository") + raise Bcfg2.Server.Plugin.PluginInitError - def Index(self): - '''Store XML data in reasonable structures''' - try: - xdata = XML(self.data) - except XMLSyntaxError: - self.LogError("Failed to parse base.xml") - return - self.store = {'all':[], 'Class':{'all':[]}, 'Image':{'all':[]}, 'all':[]} - for entry in [ent for ent in xdata.getchildren() if not isinstance(ent, _Comment)]: - if entry.tag in ['Image', 'Class']: - if not self.store[entry.tag].has_key(entry.get('name')): - self.store[entry.tag][entry.get('name')] = {'all':[], 'Class':{}, 'Image':{}} - for child in [ent for ent in entry.getchildren() if not isinstance(ent, _Comment)]: - if child.tag in ['Image', 'Class']: - self.store[entry.tag][entry.get('name')][child.tag][child.get('name')] = \ - [ent for ent in child.getchildren() if \ - not isinstance(ent, _Comment)] - else: - self.store[entry.tag][entry.get('name')]['all'].append(child) - else: - self.store['all'].append(child) - def BuildStructures(self, metadata): '''Build structures for client described by metadata''' - ret = Element("Independant", version='2.0') - [ret.append(deepcopy(entry)) for entry in self.store['all']] - idata = self.store['Image'].get(metadata.image, {'all':[], 'Class':{}}) - for entry in idata['all']: - ret.append(deepcopy(entry)) - for cls in metadata.classes: - for entry in idata['Class'].get(cls, []): - ret.append(deepcopy(entry)) - cdata = self.store['Class'].get(cls, {'all':[], 'Image':{}}) - for entry in cdata['all']: - ret.append(deepcopy(entry)) - for entry in cdata['Image'].get(metadata.image, []): - ret.append(deepcopy(entry)) + ret = lxml.etree.Element("Independant", version='2.0') + fragments = reduce(lambda x, y: x+y, + [base.Match(metadata) for base in self.entries.values()]) + [ret.append(copy.deepcopy(frag)) for frag in fragments] return [ret] diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py index 4b357f121..cbbb6c671 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Server/Plugins/Bundler.py @@ -1,122 +1,36 @@ '''This provides bundle clauses with translation functionality''' __revision__ = '$Revision$' -from copy import deepcopy -from syslog import LOG_ERR, syslog -from lxml.etree import Element, XML, XMLSyntaxError, _Comment +import Bcfg2.Server.Plugin +import copy +import lxml.etree -from Bcfg2.Server.Plugin import Plugin, SingleXMLFileBacked, XMLFileBacked, DirectoryBacked - - -class ImageFile(SingleXMLFileBacked): - '''This file contains image -> system mappings''' - def __init__(self, filename, fam): - self.images = {} - SingleXMLFileBacked.__init__(self, filename, fam) - - def Index(self): - '''Build data structures out of the data''' - try: - xdata = XML(self.data) - except XMLSyntaxError, err: - syslog(LOG_ERR, "Failed to parse file %s" % (self.name)) - syslog(LOG_ERR, err) - del self.data - return - self.images = {} - for child in xdata.getchildren(): - [name, pkg, service] = [child.get(field) for field in ['name', 'package', 'service']] - for grandchild in child.getchildren(): - self.images[grandchild.get('name')] = (name, pkg, service) - -class Bundle(XMLFileBacked): - '''Bundles are configuration specifications (with image/translation abstraction)''' - - def __init__(self, filename): - self.all = [] - self.attributes = {} - self.systems = {} - XMLFileBacked.__init__(self, filename) - - def Index(self): - '''Build data structures from the source data''' - try: - xdata = XML(self.data) - except XMLSyntaxError, err: - syslog(LOG_ERR, "Failed to parse file %s" % (self.name)) - syslog(LOG_ERR, str(err)) - del self.data - return - self.all = [] - self.systems = {} - self.attributes = {} - for entry in [ent for ent in xdata.getchildren() if not isinstance(ent, _Comment)]: - if entry.tag == 'System': - self.systems[entry.attrib['name']] = [ent for ent in entry.getchildren() \ - if not isinstance(ent, _Comment)] - elif entry.tag == 'Attribute': - self.attributes[entry.get('name')] = [ent for ent in entry.getchildren() \ - if not isinstance(ent, _Comment)] - else: - self.all.append(entry) - del self.data - - def BuildBundle(self, metadata, system): - '''Build a bundle for a particular client''' - bundlename = self.name.split('/')[-1] - bundle = Element('Bundle', name=bundlename) - for entry in self.all + self.systems.get(system, []): - bundle.append(deepcopy(entry)) - for attribute in [aname for (scope, aname) in [item.split('.') for item in metadata.attributes] - if scope == bundlename[:-4]]: - for entry in self.attributes.get(attribute, []): - bundle.append(deepcopy(entry)) - return bundle - -class BundleSet(DirectoryBacked): - '''The Bundler handles creation of dependent clauses based on bundle definitions''' - __child__ = Bundle - -class Bundler(Plugin): +class Bundler(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.DirectoryBacked): '''The bundler creates dependent clauses based on the bundle/translation scheme from bcfg1''' __name__ = 'Bundler' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' + __child__ = Bcfg2.Server.Plugin.StructFile def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - self.imageinfo = ImageFile("%s/etc/imageinfo.xml"%(datastore), self.core.fam) - self.bundles = BundleSet(self.data, self.core.fam) + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + try: + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, self.core.fam) + except OSError: + self.LogError("Failed to load Bundle repository") + raise Bcfg2.Server.Plugin.PluginInitError def BuildStructures(self, metadata): '''Build all structures for client (metadata)''' - try: - (system, package, service) = self.GetTransInfo(metadata) - except KeyError: - syslog(LOG_ERR, "Failed to find translation information for image %s" % metadata.image) - return [] bundleset = [] for bundlename in metadata.bundles: - if not self.bundles.entries.has_key("%s.xml"%(bundlename)): - syslog(LOG_ERR, "Client %s requested nonexistent bundle %s"%(metadata.hostname, bundlename)) + if not self.entries.has_key("%s.xml"%(bundlename)): + self.LogError("Client %s requested nonexistent bundle %s" % \ + (metadata.hostname, bundlename)) continue - - bundle = self.bundles.entries["%s.xml" % (bundlename)].BuildBundle(metadata, system) - # now we need to populate service/package types - for entry in bundle.getchildren(): - if entry.tag == 'Package': - entry.attrib['type'] = package - elif entry.tag == 'Service': - entry.attrib['type'] = service + bundle = lxml.etree.Element('Bundle', name=bundlename) + [bundle.append(copy.deepcopy(item)) + for item in self.entries["%s.xml" % (bundlename)].Match(metadata)] bundleset.append(bundle) return bundleset - def GetTransInfo(self, metadata): - '''Get Translation info for metadata.image''' - if self.imageinfo.images.has_key(metadata.image): - return self.imageinfo.images[metadata.image] - else: - raise KeyError, metadata.image - - - diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py index 953401e7a..b325144e5 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Server/Plugins/Cfg.py @@ -1,7 +1,6 @@ '''This module implements a config file repository''' __revision__ = '$Revision$' -from binascii import b2a_base64 from os import stat from re import compile as regcompile from stat import S_ISDIR, ST_MODE @@ -9,54 +8,73 @@ from syslog import syslog, LOG_INFO, LOG_ERR from Bcfg2.Server.Plugin import Plugin, PluginExecutionError, FileBacked +import binascii +import exceptions + +specific = regcompile('(.*/)(?P<filename>[\S\-.]+)\.((H_(?P<hostname>\S+))|' + + '(G(?P<prio>\d+)_(?P<group>\S+)))$') + +class SpecificityError(Exception): + '''Thrown in case of filename parse failure''' + pass + class FileEntry(FileBacked): '''The File Entry class pertains to the config files contained in a particular directory. This includes :info, all base files and deltas''' - - def __init__(self, name, all, image, classes, bundles, attribs, hostname): + + def __init__(self, myid, name): FileBacked.__init__(self, name) - self.all = all - self.image = image - self.bundles = bundles - self.classes = classes - self.attributes = attribs - self.hostname = hostname + self.name = name + self.identity = myid + self.all = False + self.hostname = False + self.group = False + self.op = False + self.prio = False + if name.split('.')[-1] in ['cat', 'diff']: + self.op = name.split('.')[-1] + name = name[:-(len(self.op) + 1)] + if self.name.split('/')[-1] == myid.split('/')[-1]: + self.all = True + else: + data = specific.match(name) + if not data: + syslog(LOG_ERR, "Cfg: Failed to match %s" % name) + raise SpecificityError + if data.group('hostname') != None: + self.hostname = data.group('hostname') + else: + self.group = data.group('group') + self.prio = int(data.group('prio')) def __cmp__(self, other): - fields = ['all', 'image', 'classes', 'bundles', 'attributes', 'hostname'] - try: - most1 = [index for index in range(len(fields)) if getattr(self, fields[index])][0] - except IndexError: - most1 = 0 - try: - most2 = [index for index in range(len(fields)) if getattr(other, fields[index])][0] - except IndexError: - most2 = 0 - if most1 == most2: - if self.name.split('.')[-1] in ['cat', 'diff']: - meta1 = self.name.split('.')[-2] - else: - meta1 = self.name.split('.')[-1] - if other.name.split('.')[-1] in ['cat', 'diff']: - meta2 = other.name.split('.')[-2] + data = [[getattr(self, field) for field in ['all', 'group', 'hostname']], + [getattr(other, field) for field in ['all', 'group', 'hostname']]] + for index in range(3): + if data[0][index] and not data[1][index]: + return -1 + elif data[1][index] and not data[0][index]: + return 1 + elif data[0][index] and data[1][index]: + if hasattr(self, 'prio') and hasattr(other, 'prio'): + return self.prio - other.prio + else: + return 0 else: - meta2 = other.name.split('.')[-1] + pass + syslog(LOG_ERR, "Cfg: Critical: Ran off of the end of the world sorting %s" % (self.name)) - if meta1[0] not in ['C', 'B']: - return 0 - # need to tiebreak with numeric prio - prio1 = int(meta1[1:3]) - prio2 = int(meta2[1:3]) - return prio1 - prio2 + def applies(self, metadata): + '''Predicate if fragment matches client metadata''' + if self.all or (self.hostname == metadata.hostname) or \ + (self.group in metadata.groups): + return True else: - return most1 - most2 + return False class ConfigFileEntry(object): '''ConfigFileEntry is a repository entry for a single file, containing all data for all clients.''' - specific = regcompile('(.*/)(?P<filename>[\S\-.]+)\.((H_(?P<hostname>\S+))|' + - '(B(?P<bprio>\d+)_(?P<bundle>\S+))|(A(?P<aprio>\d+)_(?P<attr>\S+))|' + - '(I_(?P<image>\S+))|(C(?P<cprio>\d+)_(?P<class>\S+)))$') info = regcompile('^owner:(\s)*(?P<owner>\w+)|group:(\s)*(?P<group>\w+)|' + 'perms:(\s)*(?P<perms>\w+)|encoding:(\s)*(?P<encoding>\w+)|' + '(?P<paranoid>paranoid(\s)*)$') @@ -65,8 +83,7 @@ class ConfigFileEntry(object): object.__init__(self) self.path = path self.repopath = repopath - self.basefiles = [] - self.deltas = [] + self.fragments = [] self.metadata = {'encoding': 'ascii', 'owner':'root', 'group':'root', 'perms':'0644'} self.paranoid = False @@ -94,41 +111,15 @@ class ConfigFileEntry(object): def AddEntry(self, name): '''add new file additions for a single cf file''' - delta = False - oldname = name if name[-5:] == ':info': return self.read_info() - if name.split('/')[-1] == self.path.split('/')[-1]: - self.basefiles.append(FileEntry(name, True, None, [], [], [], None)) - self.basefiles.sort() - return - - if name.split('/')[-1].split('.')[-1] in ['cat']: - delta = True - oldname = name - name = name[:-4] - - specmatch = self.specific.match(name) - if specmatch == None: - syslog(LOG_ERR, "Cfg: Failed to match file %s" % (name)) + try: + self.fragments.append(FileEntry(self.path, name)) + self.fragments.sort() + except SpecificityError: return - data = {} - for item, value in specmatch.groupdict().iteritems(): - if value != None: - data[item] = value - - cfile = FileEntry(oldname, False, data.get('image', None), data.get('class', []), - data.get('bundle', []), data.get('attr', []), data.get('hostname', None)) - - if delta: - self.deltas.append(cfile) - self.deltas.sort() - else: - self.basefiles.append(cfile) - self.basefiles.sort() - def HandleEvent(self, event): '''Handle FAM updates''' action = event.code2str() @@ -136,11 +127,11 @@ class ConfigFileEntry(object): if action in ['changed', 'exists', 'created']: return self.read_info() if event.filename != self.path.split('/')[-1]: - if not self.specific.match('/' + event.filename): + if not specific.match('/' + event.filename): syslog(LOG_INFO, 'Cfg: Suppressing event for bogus file %s' % event.filename) return - entries = [entry for entry in self.basefiles + self.deltas if + entries = [entry for entry in self.fragments if entry.name.split('/')[-1] == event.filename] if len(entries) == 0: @@ -152,10 +143,8 @@ class ConfigFileEntry(object): syslog(LOG_INFO, "Cfg: Removing entry %s" % event.filename) for entry in entries: syslog(LOG_INFO, "Cfg: Removing entry %s" % (entry.name)) - if entry in self.basefiles: - self.basefiles.remove(entry) - if entry in self.deltas: - self.deltas.remove(entry) + self.fragments.remove(entry) + self.fragments.sort() syslog(LOG_INFO, "Cfg: Entry deletion completed") elif action in ['changed', 'exists', 'created']: [entry.HandleEvent(event) for entry in entries] @@ -168,13 +157,13 @@ class ConfigFileEntry(object): filedata = "" # first find basefile try: - basefile = [bfile for bfile in self.basefiles if metadata.Applies(bfile)][-1] + basefile = [bfile for bfile in self.fragments if bfile.applies(metadata) and not bfile.op][-1] except IndexError: syslog(LOG_ERR, "Cfg: Failed to locate basefile for %s" % name) raise PluginExecutionError, ('basefile', name) filedata += basefile.data - for delta in [x for x in self.deltas if metadata.Applies(x)]: + for delta in [delta for delta in self.fragments if delta.applies(metadata) and delta.op]: # find applicable deltas lines = filedata.split('\n') if not lines[-1]: @@ -188,15 +177,15 @@ class ConfigFileEntry(object): lines.append(line[1:]) filedata = "\n".join(lines) + "\n" - [entry.attrib.__setitem__(x,y) for (x,y) in self.metadata.iteritems()] + [entry.attrib.__setitem__(key, value) for (key, value) in self.metadata.iteritems()] if self.paranoid: entry.attrib['paranoid'] = 'true' if entry.attrib['encoding'] == 'base64': - entry.text = b2a_base64(filedata) + entry.text = binascii.b2a_base64(filedata) else: try: entry.text = filedata - except: + except exceptions.AttributeError: syslog(LOG_ERR, "Failed to marshall file %s. Mark it as base64" % (entry.get('name'))) class Cfg(Plugin): diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Server/Plugins/Hostbase.py index 9a488fc74..e729030fb 100644 --- a/src/lib/Server/Plugins/Hostbase.py +++ b/src/lib/Server/Plugins/Hostbase.py @@ -2,9 +2,12 @@ __revision__ = '$Revision$' from syslog import syslog, LOG_INFO -from lxml.etree import XML +from lxml.etree import XML, SubElement from Cheetah.Template import Template from Bcfg2.Server.Plugin import Plugin, PluginExecutionError, PluginInitError, DirectoryBacked +from time import strftime +from sets import Set +import re class DataNexus(DirectoryBacked): '''DataNexus is an object that watches multiple files and @@ -39,7 +42,7 @@ class Hostbase(Plugin, DataNexus): def __init__(self, core, datastore): self.ready = False - files = ['dnsdata.xml', 'hostbase.xml', 'networks.xml'] + files = ['zones.xml', 'hostbase.xml', 'hostbase-dns.xml', 'hostbase-dhcp.xml'] Plugin.__init__(self, core, datastore) try: DataNexus.__init__(self, datastore + '/Hostbase/data', @@ -49,11 +52,15 @@ class Hostbase(Plugin, DataNexus): raise PluginInitError self.xdata = {} self.filedata = {} + self.dnsservers = ['scotty.mcs.anl.gov'] + self.dhcpservers = ['thwap.mcs.anl.gov', 'squeak.mcs.anl.gov'] self.templates = {'zone':Template(open(self.data + '/templates/' + 'zonetemplate.tmpl').read()), 'reversesoa':Template(open(self.data + '/templates/' + 'reversesoa.tmpl').read()), 'named':Template(open(self.data + '/templates/' + 'namedtemplate.tmpl').read()), 'reverseapp':Template(open(self.data + '/templates/' + 'reverseappend.tmpl').read()), - 'dhcp':Template(open(self.data + '/templates/' + 'dhcpd_template.tmpl').read())} + 'dhcp':Template(open(self.data + '/templates/' + 'dhcpd_template.tmpl').read()), + 'hosts':Template(open(self.data + '/templates/' + 'hosts.tmpl').read()), + 'hostsapp':Template(open(self.data + '/templates/' + 'hostsappend.tmpl').read())} self.Entries['ConfigFile'] = {} def FetchFile(self, entry, metadata): @@ -65,8 +72,38 @@ class Hostbase(Plugin, DataNexus): [entry.attrib.__setitem__(key, value) for (key, value) in perms.iteritems()] entry.text = self.filedata[fname] + def BuildStructures(self, metadata): + '''Build hostbase bundle''' + if metadata.hostname in self.dnsservers or metadata.hostname in self.dhcpservers: + output = [] + if metadata.hostname in self.dnsservers: + dnsbundle = XML(self.entries['hostbase-dns.xml'].data) + for configfile in self.Entries['ConfigFile']: + if re.search('/etc/bind/', configfile): + SubElement(dnsbundle, "ConfigFile", name=configfile) + output.append(dnsbundle) + if metadata.hostname in self.dhcpservers: + dhcpbundle = XML(self.entries['hostbase-dhcp.xml'].data) + output.append(dhcpbundle) + return output + else: + return [] + def rebuildState(self, event): '''Pre-cache all state information for hostbase config files''' + def get_serial(zone): + '''I think this does the zone file serial number hack but whatever''' + todaydate = (strftime('%Y%m%d')) + try: + if todaydate == zone.get('serial')[:8]: + serial = atoi(zone.get('serial')) + 1 + else: + serial = atoi(todaydate) * 100 + return str(serial) + except (KeyError): + serial = atoi(todaydate) * 100 + return str(serial) + if self.entries.has_key(event.filename) and not self.xdata.has_key(event.filename): self.xdata[event.filename] = XML(self.entries[event.filename].data) if [item for item in self.files if not self.entries.has_key(item)]: @@ -74,163 +111,293 @@ class Hostbase(Plugin, DataNexus): # we might be able to rebuild data more sparsely, # but hostbase.xml is the only one that will really change often # rebuild zoneinfo - iplist = [] - for zone in self.xdata['dnsdata.xml']: + hosts = {} + zones = self.xdata['zones.xml'] + hostbase = self.xdata['hostbase.xml'] + ## this now gets all hosts associated with the zone file being initialized + ## all ip addresses and cnames are grabbed from each host and passed to the appropriate template + for zone in zones: + hosts[zone.get('domain')] = [] + for host in hostbase: + if host.get('domain') in hosts: + hosts[host.get('domain')].append(host) + for zone in zones: zonehosts = [] - for host in [host for host in self.xdata['hostbase.xml'] - if host.get('domain') == zone.get('domain')]: - hostname = host.get('hostname') - if zone.get('domain') == 'mcs.anl.gov': - ## special cases for the mcs.anl.gov domain - ## all machines have a "-eth" entry as well as an entry identifying their subnet - ## they also have their mail exchangers after every address - ipnodes = host.findall("interface/ip") - zonehosts.append((hostname, ipnodes[0].attrib['ip'], ipnodes[0].findall("name/mx"), None)) - [zonehosts.append(("-".join([hostname, ipnode.attrib['dnssuffix']]), \ - ipnode.attrib['ip'], ipnode.findall("name/mx"), None)) - for ipnode in ipnodes] - [zonehosts.append(("-".join([hostname, namenode.attrib['name']]), \ - ipnode.attrib['ip'], namenode.findall("mx"), None)) - for ipnode in ipnodes - for namenode in ipnode - if namenode.attrib['name'] != ""] - else: - ipnodes = host.findall("interface/ip") - zonehosts.append((host.attrib['hostname'], ipnodes[0].attrib['ip'], None, None)) - [zonehosts.append(("-".join([host.attrib['hostname'], namenode.attrib['name']]), - ipnode.attrib['ip'], None, None)) - for ipnode in ipnodes - for namenode in ipnode - if namenode.attrib['name'] != ""] - - [zonehosts.append((host.attrib['hostname'], None, None, cnamenode.attrib['cname'])) - for cnamenode in host.findall("interface/ip/name/cname") - if cnamenode.attrib['cname'] != ""] - - [iplist.append(ipnode.attrib['ip']) for ipnode in host.findall("interface/ip")] + for host in hosts[zone.get('domain')]: + hostname = host.attrib['hostname'] + ipnodes = host.findall("interface/ip") + #gets all the forward look up stuff + [zonehosts.append((namenode.get('name').split(".")[0], ipnode.get('ip'), + namenode.findall('mx'))) + for ipnode in ipnodes + for namenode in ipnode] + #gets cname stuff + [zonehosts.append((cnamenode.get('cname') + '.', namenode.get('name').split('.')[0], None)) + for namenode in host.findall("interface/ip/name") + for cnamenode in namenode.findall("cname") + if (cnamenode.get('cname').split(".")[0], namenode.get('name').split('.')[0], None) not in zonehosts + and cnamenode.get('cname') is not None] + zonehosts.sort() self.templates['zone'].zone = zone - self.templates['zone'].root = self.xdata['dnsdata.xml'] + self.templates['zone'].root = zones self.templates['zone'].hosts = zonehosts self.filedata[zone.get('domain')] = str(self.templates['zone']) + self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone.get('domain'))] = self.FetchFile # now all zone forward files are built - iplist.sort() filelist = [] - temp = None - for x in range(len(iplist)-1): - addressparts = iplist[x].split(".") - if addressparts[:3] != iplist[x+1].split(".")[:3] and addressparts[:2] == iplist[x+1].split(".")[:2] \ - and ".".join([addressparts[1], addressparts[0]]) not in filelist: - filelist.append(".".join([addressparts[1], addressparts[0]])) - elif addressparts[:3] != iplist[x+1].split(".")[:3] and \ - addressparts[:2] != iplist[x+1].split(".")[:2] and \ - ".".join([addressparts[1], addressparts[0]]) not in filelist: - filelist.append(".".join([addressparts[2], addressparts[1], addressparts[0]])) - if x+1 == len(iplist) - 1: - temp = iplist[x+1].split(".") - if ".".join([temp[2], temp[1], temp[0]]) not in filelist \ - and ".".join([temp[1], temp[0]]) not in filelist: - filelist.append(".".join([temp[2], temp[1], temp[0]])) - + three_subnet = [ip.get('ip').rstrip('0123456789').rstrip('.') + for ip in hostbase.findall('host/interface/ip')] + three_subnet_set = Set(three_subnet) + two_subnet = [subnet.rstrip('0123456789').rstrip('.') + for subnet in three_subnet_set] + two_subnet_set = Set(two_subnet) + filelist = [each for each in two_subnet_set + if two_subnet.count(each) > 1] + [filelist.append(each) for each in three_subnet_set + if each.rstrip('0123456789').rstrip('.') not in filelist] + + reversenames = [] for filename in filelist: - self.templates['reversesoa'].inaddr = filename + towrite = filename.split('.') + towrite.reverse() + reversename = '.'.join(towrite) + self.templates['reversesoa'].inaddr = reversename self.templates['reversesoa'].zone = zone - self.templates['reversesoa'].root = self.xdata['dnsdata.xml'] - self.filedata["%s.rev" % filename] = str(self.templates['reversesoa']) + self.templates['reversesoa'].root = self.xdata['zones.xml'] + self.filedata['%s.rev' % reversename] = str(self.templates['reversesoa']) + reversenames.append(reversename) - self.templates['named'].zones = self.xdata['dnsdata.xml'] - self.templates['named'].reverses = filelist + self.templates['named'].zones = self.xdata['zones.xml'] + self.templates['named'].reverses = reversenames self.filedata["named.conf"] = str(self.templates['named']) + self.Entries['ConfigFile']["%s/%s" % (self.filepath, 'named.conf')] = self.FetchFile - for filename in filelist: + reversenames.sort() + for filename in reversenames: originlist = [] + reversehosts = [] towrite = filename.split(".") towrite.reverse() if len(towrite) > 2: - self.templates['reverseapp'].hosts = [(ipnode.get('ip').split('.'), host.get('hostname'), - host.get('domain'), ipnode.get('num'), ipnode.get('dnssuffix')) - for host in self.xdata['hostbase.xml'] - for ipnode in host.findall('interface/ip') - if ipnode.get('ip').split('.')[:3] == towrite] - + [reversehosts.append((ipnode.attrib['ip'].split("."), host.attrib['hostname'], + host.attrib['domain'], ipnode.get('num'), None)) + for host in self.xdata['hostbase.xml'] + for ipnode in host.findall("interface/ip") + if ipnode.attrib['ip'].split(".")[:3] == towrite] + self.templates['reverseapp'].hosts = reversehosts self.templates['reverseapp'].inaddr = filename self.templates['reverseapp'].fileorigin = None self.filedata["%s.rev" % filename] += str(self.templates['reverseapp']) else: - revhosts = [(ipnode.get('ip').split('.'), host.get('hostname'), host.get('domain'), - ipnode.get('num'), ipnode.get('dnssuffix')) - for host in self.xdata['hostbase.xml'] - for ipnode in host.findall("interface/ip") - if ipnode.get('ip').split(".")[:2] == towrite] + [reversehosts.append((ipnode.attrib['ip'].split("."), host.attrib['hostname'], + host.attrib['domain'], ipnode.get('num'), None)) + for host in self.xdata['hostbase.xml'] + for ipnode in host.findall("interface/ip") + if ipnode.attrib['ip'].split(".")[:2] == towrite] [originlist.append(".".join([reversehost[0][2], reversehost[0][1], reversehost[0][0]])) - for reversehost in revhosts + for reversehost in reversehosts if ".".join([reversehost[0][2], reversehost[0][1], reversehost[0][0]]) not in originlist] - revhosts.sort() + reversehosts.sort() originlist.sort() for origin in originlist: - outputlist = [rhost for rhost in revhosts - if ".".join([rhost[0][2], rhost[0][1], rhost[0][0]]) == origin] + outputlist = [] + [outputlist.append(reversehost) + for reversehost in reversehosts + if ".".join([reversehost[0][2], reversehost[0][1], reversehost[0][0]]) == origin] self.templates['reverseapp'].fileorigin = filename self.templates['reverseapp'].hosts = outputlist self.templates['reverseapp'].inaddr = origin self.filedata["%s.rev" % filename] += str(self.templates['reverseapp']) + self.Entries['ConfigFile']["%s/%s.rev" % (self.filepath, filename)] = self.FetchFile self.buildDHCP() - for key in self.filedata: - self.Entries['ConfigFile']["%s/%s" % (self.filepath, key)] = self.FetchFile + self.buildHosts() + self.buildHostsLPD() + self.buildPrinters() + self.buildNetgroups() def buildDHCP(self): '''Pre-build dhcpd.conf and stash in the filedata table''' - if 'networks.xml' not in self.xdata.keys(): - print "not running before networks is cached" - return - networkroot = self.xdata['networks.xml'] if 'hostbase.xml' not in self.xdata.keys(): print "not running before hostbase is cached" return hostbase = self.xdata['hostbase.xml'] - vlanandsublist = [] - subnets = networkroot.findall("subnet") - for vlan in networkroot.findall("vlan"): - vlansubs = vlan.findall("subnet") - vlansubs.sort(lambda x, y: cmp(x.get("address"), y.get("address"))) - vlanandsublist.append((vlan, vlansubs)) - - subnets140 = [subnet for subnet in subnets if subnet.attrib['address'].split(".")[0] == "140"] - privatesubnets = [subnet for subnet in subnets if subnet.attrib['address'].split(".")[0] != "140"] - subnets140.sort(lambda x, y: cmp(x.get("address"), y.get("address"))) - privatesubnets.sort(lambda x, y: cmp(x.get("address"), y.get("address"))) - - dhcphosts = [host for host in hostbase if host.get('dhcp') == 'y' \ - and host.find("interface").get('mac') != 'float' \ - and host.find("interface").get('mac') != ""] + dhcphosts = [host for host in hostbase if host.find('dhcp').get('dhcp') == 'y' + and host.find("interface").attrib['mac'] != 'float' + and host.find("interface").attrib['mac'] != "" + and host.find("interface").attrib['mac'] != "unknown"] + numips = 0 hosts = [] for host in dhcphosts: if len(host.findall("interface")) == 1 and len(host.findall("interface/ip")) == 1: - hosts.append([host.get('hostname'), host.get('domain'), \ - host.find("interface").get('mac'), \ - host.find("interface/ip").get('ip')]) - elif len(host.findall("interface")) > 1: + hosts.append([host.attrib['hostname'], host.attrib['domain'], \ + host.find("interface").attrib['mac'], \ + host.find("interface/ip").attrib['ip']]) + else: count = 0 - for interface in host.findall("interface"): + for interface in host.findall('interface'): if count == 0 and interface.find("ip") is not None: - hostdata = [host.get('hostname'), host.get('domain'), \ - interface.get('mac'), interface.find("ip").get('ip')] + hostdata = [host.attrib['hostname'], host.attrib['domain'], + interface.attrib['mac'], interface.find("ip").attrib['ip']] elif count != 0 and interface.find("ip") is not None: - hostdata = [host.get('hostname'), "-".join([host.get('domain'), str(count)]), \ - interface.get('mac'), interface.find("ip").get('ip')] + hostdata = [host.attrib['hostname'], "-".join([host.attrib['domain'], str(count)]), + interface.attrib['mac'], interface.find("ip").attrib['ip']] if len(interface.findall("ip")) > 1: - for ipnode in interface.findall("ip")[1:]: - hostdata[3] = ", ".join([hostdata[3], ipnode.get('ip')]) + for ip in interface.findall("ip")[1:]: + hostdata[3] = ", ".join([hostdata[3], ip.attrib['ip']]) count += 1 hosts.append(hostdata) + + numips += len(host.findall("interface/ip")) hosts.sort(lambda x, y: cmp(x[0], y[0])) self.templates['dhcp'].hosts = hosts - self.templates['dhcp'].privatesubnets = privatesubnets - self.templates['dhcp'].subnets140 = subnets140 - self.templates['dhcp'].vlans = vlanandsublist - self.templates['dhcp'].networkroot = networkroot - self.filedata['/etc/dhcpd.conf'] = str(self.templates['dhcp']) + self.templates['dhcp'].numips = numips + self.templates['dhcp'].timecreated = strftime("%a %b %d %H:%M:%S %Z %Y") + self.filedata['dhcpd.conf'] = str(self.templates['dhcp']) + self.Entries['ConfigFile']['/etc/dhcpd.conf'] = self.FetchFile + + def buildHosts(self): + '''This will rebuild the hosts file to include all important machines''' + hostbase = self.xdata['hostbase.xml'] + domains = [host.get('domain') for host in hostbase] + domains_set = Set(domains) + domain_data = [(domain, domains.count(domain)) for domain in domains_set] + domain_data.sort() + ips = [(ip, host) for host in hostbase.findall('host') + for ip in host.findall("interface/ip")] + three_octets = [ip[0].get('ip').rstrip('0123456789').rstrip('.') + for ip in ips] + three_octets_set = list(Set(three_octets)) + three_sort = [tuple([int(num) for num in each.split('.')]) for each in three_octets_set] + three_sort.sort() + three_octets_set = ['.'.join([str(num) for num in each]) for each in three_sort] + three_octets_data = [(octet, three_octets.count(octet)) + for octet in three_octets_set] + append_data = [(subnet, [ip for ip in ips \ + if ip[0].get('ip').rstrip("0123456789").rstrip('.') + == subnet[0]]) for subnet in three_octets_data] + for each in append_data: + each[1].sort(lambda x, y: cmp(int(x[0].get('ip').split('.')[-1]), int(y[0].get('ip').split('.')[-1]))) + two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] + two_octets_set = list(Set(two_octets)) + two_sort = [tuple([int(num) for num in each.split('.')]) for each in two_octets_set] + two_sort.sort() + two_octets_set = ['.'.join([str(num) for num in each]) for each in two_sort] + two_octets_data = [(octet, two_octets.count(octet)) for octet in two_octets_set] + self.templates['hosts'].domain_data = domain_data + self.templates['hosts'].three_octets_data = three_octets_data + self.templates['hosts'].two_octets_data = two_octets_data + self.templates['hosts'].three_octets = len(three_octets) + self.templates['hosts'].timecreated = strftime("%a %b %d %H:%M:%S %Z %Y") + self.filedata['hosts'] = str(self.templates['hosts']) + for subnet in append_data: + self.templates['hostsapp'].ips = subnet[1] + self.templates['hostsapp'].subnet = subnet[0] + self.filedata['hosts'] += str(self.templates['hostsapp']) + self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile + + + def buildPrinters(self): + '''this will rebuild the printers.data file used in + our local printers script''' + header = """# This file is automatically generated. DO NOT EDIT IT! +# This datafile is for use with /mcs/bin/printers. +# +Name Room User Type Notes +============== ========== ============ ======================== ==================== +""" + + printers = [host for host in self.xdata['hostbase.xml'] + if host.find('whatami').get('whatami') == "printer" + and host.get('domain') == 'mcs.anl.gov'] + self.filedata['printers.data'] = header + output_list = [] + for printer in printers: + if printer.find('printq').get('printq'): + for printq in re.split(',[ ]*', printer.find('printq').get('printq')): + output_list.append((printq, printer.find('room').get('room'), printer.find('user').get('user'), + printer.find('model').get('model'), printer.find('note').get('note'))) + output_list.sort() + for printer in output_list: + self.filedata['printers.data'] += ("%-16s%-12s%-14s%-26s%s\n" % printer) + self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile + + def buildHostsLPD(self): + '''this rebuilds the hosts.lpd file''' + header = """+@machines ++@all-machines +achilles.ctd.anl.gov +raven.ops.anl.gov +seagull.hr.anl.gov +parrot.ops.anl.gov +condor.ops.anl.gov +delphi.esh.anl.gov +anlcv1.ctd.anl.gov +anlvms.ctd.anl.gov +olivia.ctd.anl.gov\n\n""" + + hostbase = self.xdata['hostbase.xml'] + redmachines = [".".join([host.get('hostname'), host.get('domain')]) + for host in hostbase if host.find('netgroup').get('netgroup') == 'red'] + winmachines = [".".join([host.get('hostname'), host.get('domain')]) + for host in hostbase if host.find('netgroup').get('netgroup') == 'win'] + redmachines += [name.get('name') for host in hostbase + for name in host.findall('interface/ip/name') + if host.find('netgroup').get('netgroup') == 'red' and name.get('only') != 'no'] + winmachines += [name.get('name') for host in hostbase + for name in host.findall('interface/ip/name') + if host.find('netgroup').get('netgroup') == 'win' and name.get('only') != 'no'] + redmachines.sort() + winmachines.sort() + self.filedata['hosts.lpd'] = header + for machine in redmachines: + self.filedata['hosts.lpd'] += machine + "\n" + self.filedata['hosts.lpd'] += "\n" + for machine in winmachines: + self.filedata['hosts.lpd'] += machine + "\n" + self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile + + def buildNetgroups(self): + '''this rebuilds the many different files that will eventually + get post processed and converted into a ypmap for netgroups''' + header = """################################################################### +# This file lists hosts in the '%s' machine netgroup, it is +# automatically generated. DO NOT EDIT THIS FILE! To update +# the hosts in this file, edit hostbase and do a 'make nets' +# in /mcs/adm/hostbase. +# +# Number of hosts in '%s' machine netgroup: %i +#\n\n""" + + netgroups = {} + for host in self.xdata['hostbase.xml']: + if host.find('netgroup').get('netgroup') == "" or host.find('netgroup').get('netgroup')== 'none': + continue + if host.find('netgroup').get('netgroup') not in netgroups: + netgroups.update({host.find('netgroup').get('netgroup') : + [".".join([host.get('hostname'), host.get('domain')])]}) + else: + netgroups[host.find('netgroup').get('netgroup')].append(".".join([host.get('hostname'), + host.get('domain')])) + + for name in host.findall('interface/ip/name'): + if name.get('only') != 'no': + netgroups[host.find('netgroup').get('netgroup')].append(name.get('name')) + + for netgroup in netgroups: + self.filedata["%s-machines" % netgroup] = header % (netgroup, netgroup, len(netgroups[netgroup])) + netgroups[netgroup].sort() + for each in netgroups[netgroup]: + self.filedata["%s-machines" % netgroup] += each + "\n" + self.Entries['ConfigFile']["/var/yp/netgroups/%s-machines" % netgroup] = self.FetchFile + + def dumpXML(self): + '''this just dumps the info in the hostbase.xml file to be used + with external programs''' + self.filedata['hostbase.xml'] = self.xdata['hostbase.xml'] + self.Entries['ConfigFile']['/etc/hostbase.xml'] = self.FetchFile + diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py index 8521994e0..e77dd99e5 100644 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ b/src/lib/Server/Plugins/Pkgmgr.py @@ -1,100 +1,54 @@ '''This module implements a package management scheme for all images''' __revision__ = '$Revision$' -from copy import deepcopy -from re import compile as regcompile +import re from syslog import syslog, LOG_ERR +import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugin import Plugin, PluginInitError, PluginExecutionError, DirectoryBacked, XMLFileBacked - -class PackageEntry(XMLFileBacked): - '''PackageEntry is a set of packages and locations for a single image''' - __identifier__ = 'image' - splitters = {'rpm':regcompile('^(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ +class PNode(Bcfg2.Server.Plugin.LNode): + '''PNode has a list of packages available at a particular group intersection''' + splitters = {'rpm':re.compile('^(?P<name>[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ '(?P<version>[\w\d\.]+-([\w\d\.]+))\.(?P<arch>\w+)\.rpm$'), - 'encap':regcompile('^(?P<name>\w+)-(?P<version>[\w\d\.-]+).encap.*$')} - - def __init__(self, filename): - XMLFileBacked.__init__(self, filename) - self.packages = {} - - def Index(self): - '''Build internal data structures''' - XMLFileBacked.Index(self) - self.packages = {} - for location in self.entries: - for pkg in location.getchildren(): - if location.attrib.has_key('type'): - pkg.set('type', location.get('type')) - if pkg.attrib.has_key("simplefile"): - self.packages[pkg.get('name')] = {} - for key in pkg.attrib: - self.packages[pkg.get('name')][key] = pkg.attrib[key] - # most attribs will be set from pkg - self.packages[pkg.get('name')]['url'] = "%s/%s" % (location.get('uri'), pkg.get('simplefile')) - elif pkg.attrib.has_key("file"): - if self.splitters.has_key(pkg.get('type')): - mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) - if not mdata: - syslog(LOG_ERR, "Failed to match pkg %s" % pkg.get('file')) - continue - pkgname = mdata.group('name') - self.packages[pkgname] = mdata.groupdict() - self.packages[pkgname]['url'] = location.get('uri') + '/' + pkg.get('file') - self.packages[pkgname]['type'] = pkg.get('type') - else: - derived = [(ptype, self.splitters[ptype].match(pkg.get('file')).groupdict()) - for ptype in self.splitters if self.splitters[ptype].match(pkg.get('file'))] - if not derived: - syslog("Failed to match pkg %s" % pkg.get('file')) - else: - (ptype, mdata) = derived[0] - pkgname = mdata['name'] - self.packages[pkgname] = mdata - self.packages[pkgname]['url'] = location.get('uri') + '/' + pkg.get('file') - self.packages[pkgname]['type'] = ptype + 'encap':re.compile('^(?P<name>\w+)-(?P<version>[\w\d\.-]+).encap.*$')} + + def __init__(self, data, plist, parent=None): + # copy local attributes to all child nodes if no local attribute exists + for child in data.getchildren(): + for attr in [key for key in data.attrib.keys() if key != 'name' and not child.attrib.has_key(key)]: + child.set(attr, data.get(attr)) + Bcfg2.Server.Plugin.LNode.__init__(self, data, plist, parent) + for pkg in data.findall('./Package'): + if pkg.attrib.has_key('name') and pkg.get('name') not in plist: + plist.append(pkg.get('name')) + if pkg.attrib.has_key('simplefile'): + pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) + self.contents[pkg.get('name')] = pkg.attrib + else: + if pkg.attrib.has_key('file'): + pkg.set('url', '%s/%s' % (pkg.get('uri'), pkg.get('file'))) + if self.splitters.has_key(pkg.get('type')): + mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) + if not mdata: + syslog(LOG_ERR, "Pkgmgr: Failed to match pkg %s" % pkg.get('file')) + continue + pkgname = mdata.group('name') + self.contents[pkgname] = mdata.groupdict() + if pkg.attrib.get('file'): + self.contents[pkgname]['url'] = pkg.get('url') + self.contents[pkgname]['type'] = pkg.get('type') + if pkgname not in plist: + plist.append(pkgname) else: - self.packages[pkg.get('name')] = pkg.attrib + self.contents[pkg.get('name')] = pkg.attrib -class PackageDir(DirectoryBacked): - '''A directory of package files''' - __child__ = PackageEntry +class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): + '''PkgSrc files contain a PNode hierarchy that returns matching package entries''' + __node__ = PNode -class Pkgmgr(Plugin): +class Pkgmgr(Bcfg2.Server.Plugin.XMLPrioDir): '''This is a generator that handles package assignments''' __name__ = 'Pkgmgr' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - try: - self.pkgdir = PackageDir(self.data, self.core.fam) - except OSError: - self.LogError("Pkgmgr: Failed to load package indices") - raise PluginInitError - - def FindHandler(self, entry): - '''Non static mechanism of determining entry provisioning''' - if entry.tag != 'Package': - raise PluginExecutionError, (entry.tag, entry.get('name')) - return self.LocatePackage - - def LocatePackage(self, entry, metadata): - '''Locates a package entry for particular metadata''' - pkgname = entry.get('name') - if self.pkgdir.entries.has_key("%s.xml" % metadata.hostname): - pkglist = self.pkgdir["%s.xml" % metadata.hostname] - if pkglist.packages.has_key(pkgname): - pkginfo = pkglist.packages[pkgname] - [entry.attrib.__setitem__(field, pkginfo[field]) for field in pkginfo] - return - elif not self.pkgdir.entries.has_key("%s.xml" % metadata.image): - self.LogError("Pkgmgr: no package index for image %s" % metadata.image) - raise PluginExecutionError, ("Image", metadata.image) - pkglist = self.pkgdir["%s.xml" % (metadata.image)] - if pkglist.packages.has_key(pkgname): - pkginfo = pkglist.packages[pkgname] - [entry.attrib.__setitem__(x, pkginfo[x]) for x in pkginfo] - else: - raise PluginExecutionError, ("Package", pkgname) + __child__ = PkgSrc + __element__ = 'Package' diff --git a/src/lib/Server/Plugins/Svcmgr.py b/src/lib/Server/Plugins/Svcmgr.py index 2f2c7e5eb..da5ab341c 100644 --- a/src/lib/Server/Plugins/Svcmgr.py +++ b/src/lib/Server/Plugins/Svcmgr.py @@ -1,23 +1,20 @@ '''This generator provides service mappings''' __revision__ = '$Revision$' -from Bcfg2.Server.Plugin import Plugin, ScopedXMLFile, PluginInitError +import Bcfg2.Server.Plugin -class Svcmgr(Plugin): +class SNode(Bcfg2.Server.Plugin.LNode): + '''SNode has a list of services available at a particular group intersection''' + __leaf__ = './Service' + +class SvcSrc(Bcfg2.Server.Plugin.XMLSrc): + '''SvcSrc files contain prioritized service definitions''' + __node__ = SNode + +class Svcmgr(Bcfg2.Server.Plugin.XMLPrioDir): '''This is a generator that handles service assignments''' __name__ = 'Svcmgr' __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - try: - self.svc = ScopedXMLFile("%s/etc/services.xml"%(datastore), self.core.fam) - except OSError: - self.LogError("Failed to load service definition file") - raise PluginInitError - self.Entries = self.svc.__provides__ - - - - + __child__ = SvcSrc + __element__ = 'Service' diff --git a/src/sbin/Bcfg2debug b/src/sbin/Bcfg2debug deleted file mode 100644 index 95508a5b4..000000000 --- a/src/sbin/Bcfg2debug +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python -'''This tool loads the Bcfg2 core into an interactive debugger''' -__revision__ = '$Revision$' - - -from sys import argv -from time import sleep -from Bcfg2.Server.Core import Core, CoreInitError - -def get_input(): - '''read commands from stdin''' - try: - return raw_input('> ').split(" ") - except: - return [''] - -if __name__ == '__main__': - settings = {} - if '-c' in argv: - cfile = argv[-1] - else: - cfile = '/etc/bcfg2.conf' - try: - core = Core({}, cfile) - except CoreInitError, msg: - print "Core load failed because %s" % msg - raise SystemExit, 1 - for i in range(25): - core.fam.Service() - sleep(0.5) - cmd = get_input() - while cmd != ['']: - if cmd[0] in ['exit', 'quit']: - raise SystemExit, 0 - elif cmd[0] == 'generators': - for generator in core.generators: - print generator.__version__ - elif cmd[0] == 'help': - print 'Commands:' - print 'exit - quit' - print 'generators - list current versions of generators' - print 'help - print this text' - print 'mappings <type*> - print generator mappings for optional type' - print 'set <variable> <value> - set variable for later use' - print 'settings - display all settings' - print 'shell - shell out to native python interpreter' - print 'update - process pending fam events' - print 'version - print version of this tool' - elif cmd[0] == 'set': - settings[cmd[1]] = cmd[2] - elif cmd[0] == 'settings': - for (key, value) in settings.iteritems(): - print "%s --> %s" % (key, value) - elif cmd[0] == 'shell': - cmd = [''] - continue - elif cmd[0] == 'version': - print 'Bcfg2debug v. %s' % __revision__ - elif cmd[0] == 'mappings': - # dump all mappings unless type specified - for generator in core.generators: - print "Generator -> ", generator.__name__ - for key, value in generator.__provides__.iteritems(): - for instance in generator.__provides__[key].keys(): - print " ", key, instance - elif cmd[0] == 'update': - core.fam.Service() - else: - print "Unknown command %s" % cmd[0] - cmd = get_input() diff --git a/src/sbin/GenerateHostInfo b/src/sbin/GenerateHostInfo index 04d257e8a..67521d628 100644..100755 --- a/src/sbin/GenerateHostInfo +++ b/src/sbin/GenerateHostInfo @@ -6,42 +6,30 @@ __revision__ = '$Revision$' from ConfigParser import ConfigParser -from lxml.etree import Element, SubElement, parse -from os import fork, execl, dup2, wait +from lxml.etree import Element, SubElement, parse, tostring +from os import fork, execl, dup2, wait, uname import sys -def pretty_print(element, level=0): - '''Produce a pretty-printed text representation of element''' - if element.text: - fmt = "%s<%%s %%s>%%s</%%s>" % (level*" ") - data = (element.tag, (" ".join(["%s='%s'" % keyval for keyval in element.attrib.iteritems()])), - element.text, element.tag) - children = element.getchildren() - if children: - fmt = "%s<%%s %%s>\n" % (level*" ",) + (len(children) * "%s") + "%s</%%s>\n" % (level*" ") - data = (element.tag, ) + (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib]),) - data += tuple([pretty_print(entry, level+2) for entry in children]) + (element.tag, ) - else: - fmt = "%s<%%s %%s/>\n" % (level * " ") - data = (element.tag, " ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])) - return fmt % data - - if __name__ == '__main__': c = ConfigParser() c.read(['/etc/bcfg2.conf']) configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') - hostinfopath = "%s/etc/hostinfo.xml" % c.get('server', 'repository') - metadatapath = "%s/etc/metadata.xml" % c.get('server', 'repository') + clientdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') sendmailpath = c.get('statistics','sendmailpath') - metaElement = parse(metadatapath) - hostlist = [client.get('name') for client in metaElement.findall("Client")] + clientElement = parse(clientdatapath) + hostlist = [client.get('name') for client in clientElement.findall("Client")] - HostInfo = Element("HostInformation") pids = {} fullnames = {} null = open('/dev/null', 'w+') + + + #use uname to detect OS and use -t for darwin and -w for linux + #/bin/ping on linux /sbin/ping on os x + osname = uname()[0] + + while hostlist or pids: if hostlist and len(pids.keys()) < 15: host = hostlist.pop() @@ -51,26 +39,39 @@ if __name__ == '__main__': dup2(null.fileno(), sys.__stdin__.fileno()) dup2(null.fileno(), sys.__stdout__.fileno()) dup2(null.fileno(), sys.__stderr__.fileno()) - execl('/bin/ping', 'ping', '-w', '5', '-c', '1', host) + if osname == 'Linux': + execl('/bin/ping', 'ping', '-w', '5', '-c', '1', host) + elif osname == 'Darwin': + execl('/sbin/ping', 'ping', '-t', '5', '-c', '1', host) + elif osname == 'SunOS': + execl('/usr/sbin/ping', 'ping', host, '56', '1') + else: #default + execl('/bin/ping', 'ping', '-w', '5', '-c', '1', host) else: pids[pid] = host else: try: (cpid, status) = wait() - chost = pids[cpid] - del pids[cpid] - if status == 0: - SubElement(HostInfo, "HostInfo", name=chost, fqdn=chost, pingable='Y') + except OSError: + continue + chost = pids[cpid] + del pids[cpid] + if status == 0: + try: + clientElement.xpath("//Client[@name='%s']"%chost)[0].set("pingable",'Y') + except:#i think this is for a problem with aliases? + clientElement.xpath("//Client[@name='%s']"%fullnames[chost])[0].set("pingable",'Y') + #also set pingtime, if you can get it + + else: + if chost.count('.') > 0: + fullnames[chost.split('.')[0]] = chost + hostlist.append(chost.split('.')[0]) else: - if chost.count('.') > 0: - fullnames[chost.split('.')[0]] = chost - hostlist.append(chost.split('.')[0]) - else: - SubElement(HostInfo, "HostInfo", name=fullnames[chost], fqdn=fullnames[chost], pingable='N') - except: - pass + clientElement.xpath("//Client[@name='%s']"%(fullnames[chost]))[0].set("pingable",'N') + #also set pingtime if you can get it - fout = open(hostinfopath, 'w') - fout.write(pretty_print(HostInfo)) + fout = open(clientdatapath, 'w') + fout.write(tostring(clientElement.getroot())) fout.close() diff --git a/src/sbin/StatReports b/src/sbin/StatReports index ef6a8df66..cb74eb4bb 100644..100755 --- a/src/sbin/StatReports +++ b/src/sbin/StatReports @@ -30,8 +30,7 @@ def generatereport(rspec, nrpt): pattern = re.compile( '|'.join([item.get("name") for item in reportspec.findall('Machine')])) for node in nodereprt.findall('Node'): - if not (node.findall("HostInfo") and node.findall("Statistics") and - node.find("HostInfo").get("fqdn") and pattern.match(node.get('name'))): + if not (node.findall("Statistics") and pattern.match(node.get('name'))): # don't know enough about node nodereprt.remove(node) continue @@ -151,8 +150,7 @@ if __name__ == '__main__': c.read(['/etc/bcfg2.conf']) configpath = "%s/etc/report-configuration.xml" % c.get('server', 'repository') statpath = "%s/etc/statistics.xml" % c.get('server', 'repository') - hostinfopath = "%s/etc/hostinfo.xml" % c.get('server', 'repository') - metadatapath = "%s/etc/metadata.xml" % c.get('server', 'repository') + clientsdatapath = "%s/Metadata/clients.xml" % c.get('server', 'repository') transformpath = "/usr/share/bcfg2/xsl-transforms/" #websrcspath = "/usr/share/bcfg2/web-rprt-srcs/" @@ -173,12 +171,12 @@ if __name__ == '__main__': #See if hostinfo.xml exists, and is less than 23.5 hours old - try: - hostinstat = os.stat(hostinfopath) - if (time() - hostinstat[9])/(60*60) > 23.5: - os.system('GenerateHostInfo')#Generate HostInfo needs to be in path - except OSError: - os.system('GenerateHostInfo')#Generate HostInfo needs to be in path + #try: + #hostinstat = os.stat(hostinfopath) + #if (time() - hostinstat[9])/(60*60) > 23.5: + os.system('GenerateHostInfo')#Generate HostInfo needs to be in path + #except OSError: + # os.system('GenerateHostInfo')#Generate HostInfo needs to be in path '''Reads Data & Config files''' @@ -193,16 +191,10 @@ if __name__ == '__main__': print("StatReports: Failed to parse %s"%(configpath)) raise SystemExit, 1 try: - metadata = XML(open(metadatapath).read()) + clientsdata = XML(open(clientsdatapath).read()) except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s"%(metadatapath)) + print("StatReports: Failed to parse %s"%(clientsdatapath)) raise SystemExit, 1 - try: - hostinfodata = XML(open(hostinfopath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s. Is GenerateHostInfo in your path?"%(hostinfopath)) - raise SystemExit, 1 - #Merge data from three sources nodereport = Element("Report", attrib={"time" : asctime()}) @@ -210,12 +202,9 @@ if __name__ == '__main__': #should all of the other info in Metadata be appended? #What about all of the package stuff for other types of reports? - for client in metadata.findall("Client"): + for client in clientsdata.findall("Client"): nodel = Element("Node", attrib={"name" : client.get("name")}) nodel.append(client) - for hostinfo in hostinfodata.findall("HostInfo"): - if hostinfo.get("name") == client.get("name"): - nodel.append(hostinfo) for nod in statsdata.findall("Node"): if client.get('name').find(nod.get('name')) == 0: diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index fcc3a757c..fcc3a757c 100644..100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info new file mode 100755 index 000000000..e325b2fd0 --- /dev/null +++ b/src/sbin/bcfg2-info @@ -0,0 +1,154 @@ +#!/usr/bin/python -i +'''This tool loads the Bcfg2 core into an interactive debugger''' +__revision__ = '$Revision$' + +from sys import argv +from time import sleep +from Bcfg2.Server.Core import Core, CoreInitError +from lxml.etree import tostring + +def print_tabular(rows): + '''print data in tabular format''' + cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 for index in xrange(len(rows[0]))]) + fstring = (" %%-%ss |" * len(cmax)) % cmax + fstring = ('|'.join([" %%-%ss "] * len(cmax))) % cmax + print fstring % rows[0] + print (sum(cmax) + (len(cmax) * 2) + (len(cmax) - 1)) * '=' + for row in rows[1:]: + print fstring % row + +def get_input(): + '''read commands from stdin''' + try: + return raw_input('> ').split(" ") + except: + return [''] + +def do_build(cmd, core): + '''build client configuration''' + if len(cmd) == 3: + output = open(cmd[2], 'w') + output.write(tostring(core.BuildConfiguration(cmd[1]))) + output.close() + else: + print 'Usage: build <hostname> <output file>' + +def do_bundles(cmd, core): + '''print out group/bundle info''' + data = [('Group', 'Bundles')] + groups = core.metadata.groups.keys() + groups.sort() + for group in groups: + data.append((group, ','.join(core.metadata.groups[group][0]))) + print_tabular(data) + +def do_clients(cmd, core): + '''print out client info''' + data = [('Client', 'Profile')] + clist = core.metadata.clients.keys() + clist.sort() + for client in clist: + data.append((client, core.metadata.clients[client])) + print_tabular(data) + +def do_help(cmd, core): + '''print out usage info''' + print 'Commands:' + print 'build <hostname> <filename> - build config for hostname, writing to filename' + print 'bundles - print out group/bundle information' + print 'clients - print out client/profile information' + print 'debug - shell out to native python interpreter' + print 'generators - list current versions of generators' + print 'groups - list groups' + print 'help - print this text' + print 'mappings <type*> <name*>- print generator mappings for optional type and name' + print 'quit' + print 'update - process pending file events' + print 'version - print version of this tool' + +def do_generators(cmd, core): + '''print out generator info''' + for generator in core.generators: + print generator.__version__ + +def do_groups(cmd, core): + '''print out group info''' + data = [("Groups", "Profile", "Category", "Contains")] + grouplist = core.metadata.groups.keys() + grouplist.sort() + for group in grouplist: + if group in core.metadata.profiles: + prof = 'yes' + else: + prof = 'no' + if core.metadata.categories.has_key(group): + cat = core.metadata.categories[group] + else: + cat = '' + gdata = [grp for grp in core.metadata.groups[group][1]] + gdata.remove(group) + data.append((group, prof, cat, ','.join(gdata))) + print_tabular(data) + +def do_mappings(cmd, core): + '''print out mapping info''' + # dump all mappings unless type specified + data = [('Plugin', 'Type', 'Name')] + for generator in core.generators: + if len(cmd) == 1: + etypes = generator.Entries.keys() + elif len(cmd) > 1: + etypes = [cmd[1]] + if len(cmd) == 3: + interested = [(etype, [cmd[2]]) for etype in etypes] + else: + interested = [(etype, generator.Entries[etype].keys()) for etype in etypes + if generator.Entries.has_key(etype)] + if [etype for (etype, names) in interested + if generator.Entries.has_key(etype) and [name for name in names + if generator.Entries[etype].has_key(name)]]: + for (etype, names) in interested: + for name in names: + if generator.Entries.has_key(etype) and generator.Entries[etype].has_key(name): + data.append((generator.__name__, etype, name)) + print_tabular(data) + +def do_quit(cmd, core): + '''exit program''' + raise SystemExit, 0 + +def do_update(cmd, core): + '''Process pending fs events''' + core.fam.Service() + +def do_version(cmd, core): + '''print out code version''' + print __revision__ + +if __name__ == '__main__': + dispatch = {'build': do_build, 'bundles': do_bundles, 'clients': do_clients, + 'generators': do_generators, 'groups': do_groups, + 'help': do_help, 'mappings': do_mappings, 'quit': do_quit, + 'update': do_update, 'version': do_version} + if '-c' in argv: + cfile = argv[-1] + else: + cfile = '/etc/bcfg2.conf' + try: + bcore = Core({}, cfile) + except CoreInitError, msg: + print "Core load failed because %s" % msg + raise SystemExit, 1 + for i in range(25): + bcore.fam.Service() + sleep(0.5) + ucmd = get_input() + while True: + if ucmd[0] == 'debug': + break + else: + if dispatch.has_key(ucmd[0]): + dispatch[ucmd[0]](ucmd, bcore) + else: + print "Unknown command %s" % ucmd[0] + ucmd = get_input() diff --git a/src/sbin/ValidateBcfg2Repo b/src/sbin/bcfg2-repo-validate index a1b9552ce..a87b31cc6 100644 --- a/src/sbin/ValidateBcfg2Repo +++ b/src/sbin/bcfg2-repo-validate @@ -1,7 +1,7 @@ #!/usr/bin/env python -'''ValidateBcfg2Repo checks all xml files in Bcfg2 repos against their respective XML schemas''' -__revision__ = '0.7.3' +'''bcfg2-repo-validate checks all xml files in Bcfg2 repos against their respective XML schemas''' +__revision__ = '$Revision$' from glob import glob from lxml.etree import parse, XMLSchema @@ -12,27 +12,32 @@ from ConfigParser import ConfigParser, NoSectionError, NoOptionError if __name__ == '__main__': cf = ConfigParser() schemadir = '/usr/share/bcfg2/schemas' - cf.read(['/etc/bcfg2.conf']) - try: - repo = cf.get('server', 'repository') - except (NoSectionError, NoOptionError): - if len(argv) == 1: + if len(argv) > 1: + repo = argv[1] + else: + cf.read(['/etc/bcfg2.conf']) + try: + repo = cf.get('server', 'repository') + except (NoSectionError, NoOptionError): print "Repository location not specified in config file or on command line" - print "Usage: validate_repo <repo directory>" + print "Usage: bcfg2-repo-validate <repo directory>" raise SystemExit, 1 - repo = argv[1] # add more validation as more schemas get written - filesets = {'metadata':("%s/etc/metadata.xml", "%s/metadata.xsd"), + filesets = {'metadata':("%s/Metadata/groups.xml", "%s/metadata.xsd"), + 'clients':("%s/Metadata/clients.xml", "%s/clients.xsd"), 'bundle':("%s/Bundler/*.xml", "%s/bundle.xsd"), 'pkglist':("%s/Pkgmgr/*.xml", "%s/pkglist.xsd"), - 'base':("%s/etc/base.xml", "%s/base.xsd"), - 'imageinfo':("%s/etc/imageinfo.xml", "%s/translation.xsd"), + 'base':("%s/Base/*.xml", "%s/base.xsd"), 'imageinfo':("%s/etc/reports.xml", "%s/report-configuration.xsd"), - 'services':("%s/etc/services.xml", "%s/services.xsd")} + 'services':("%s/Svcmgr/*.xml", "%s/services.xsd")} for k, (spec, schemaname) in filesets.iteritems(): - schema = XMLSchema(parse(open(schemaname%(schemadir)))) + try: + schema = XMLSchema(parse(open(schemaname%(schemadir)))) + except: + print "Failed to process schema %s" % (schemaname%(schemadir)) + continue for filename in glob(spec%(repo)): try: datafile = parse(open(filename)) diff --git a/src/sbin/Bcfg2Server b/src/sbin/bcfg2-server index c1b93644b..2da6ff325 100644..100755 --- a/src/sbin/Bcfg2Server +++ b/src/sbin/bcfg2-server @@ -1,7 +1,7 @@ #!/usr/bin/env python '''The XML-RPC Bcfg2 Server''' -__revision__ = '$Revision:$' +__revision__ = '$Revision$' from getopt import getopt, GetoptError from sys import argv, exc_info @@ -77,7 +77,7 @@ def verbose(message): syslog(LOG_INFO, "%s" % (message)) def print_usage(opt, vopt, descs, argDescs): - print "Bcfg2Server usage:" + print "bcfg2-server usage:" for arg in opt.iteritems(): print " -%s\t\t\t%s" % (arg[0], descs[arg[0]]) for arg in vopt.iteritems(): @@ -176,6 +176,8 @@ class Bcfg2(Component): return False def resolve_client(self, client): + if self.setup['client']: + return self.setup['client'] try: return gethostbyaddr(client)[0] except herror: @@ -189,7 +191,7 @@ class Bcfg2(Component): resp = Element('probes') try: - meta = self.Core.metadata.FetchMetadata(client) + meta = self.Core.metadata.get_metadata(client) for generator in self.Core.generators: for probe in generator.GetProbes(meta): @@ -241,8 +243,8 @@ class Bcfg2(Component): # Update statistics self.Core.stats.updateStats(sdata, client) - verbose("Client %s reported state %s" % - (client, state.attrib['state'])) + syslog(LOG_INFO, "Client %s reported state %s" % + (client, state.attrib['state'])) return "<ok/>" if __name__ == '__main__': diff --git a/tools/convert-metadata.py b/tools/convert-metadata.py new file mode 100755 index 000000000..012262ea3 --- /dev/null +++ b/tools/convert-metadata.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python + +import glob +import os +import stat +import sys +import lxml.etree + +def create_if_noent(name): + if name[0] != '/': + name = os.getcwd() + '/' + name + for count in range(len(name.split('/')))[1:]: + partial = '/'.join(name.split('/')[:count+1]) + try: + os.stat(partial) + except: + os.mkdir(partial) + +def pretty_print(element, level=0): + '''Produce a pretty-printed text representation of element''' + if isinstance(element, lxml.etree._Comment): + return (level * " ") + lxml.etree.tostring(element) + if element.text: + fmt = "%s<%%s %%s>%%s</%%s>" % (level*" ") + data = (element.tag, (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])), + element.text, element.tag) + numchild = len(element.getchildren()) + if numchild: + fmt = "%s<%%s %%s>\n" % (level*" ",) + (numchild * "%s") + "%s</%%s>\n" % (level*" ") + data = (element.tag, ) + (" ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib]),) + data += tuple([pretty_print(entry, level+2) for entry in element.getchildren()]) + (element.tag, ) + else: + fmt = "%s<%%s %%s/>\n" % (level * " ") + data = (element.tag, " ".join(["%s='%s'" % (key, element.attrib[key]) for key in element.attrib])) + return fmt % data + +if __name__ == '__main__': + try: + [source, dest] = sys.argv[1:3] + except ValueError: + print "Usage: convert-metadata.py <oldpath> <newpath>" + raise SystemExit, 1 + contained = {} + create_if_noent(dest) + mdata = lxml.etree.parse(source + '/etc/metadata.xml') + newm = lxml.etree.Element("Groups", version='3.0') + newc = lxml.etree.Element("Clients", version='3.0') + newi = lxml.etree.Element("Images", version='3.0') + names = [block.get('name') for block in mdata.getroot().findall('.//Profile')] + for block in mdata.getroot().getchildren(): + if block.tag == 'Profile': + newp = lxml.etree.SubElement(newm, 'Group', profile='true', name=block.get('name'), + public=block.get('public', 'false')) + elif block.tag == 'Client': + lxml.etree.SubElement(newc, 'Client', name=block.get('name'), profile=block.get('profile'), + pingable='N', pingtime='0') + continue + elif block.tag == 'Class': + if block.get('name') in names: + print "%s is a dup profile/class; collapsing" % (block.get('name')) + newp = newm.xpath('./Group[@name="%s"]' % block.get('name'))[0] + try: + oldincl = newp.xpath('./Group[@name="%s"]' % block.get('name'))[0] + newp.remove(oldincl) + except: + print "failed to locate old class inclusion for %s" % (block.get('name')) + else: + newp = lxml.etree.SubElement(newm, 'Group', profile='false', name=block.get('name')) + elif block.tag == 'Image': + newi.append(block) + continue + elif block.tag == None: + continue + else: + print "Unknown block tag %s" % block.tag + continue + for child in block.getchildren(): + if child.tag == 'Class': + lxml.etree.SubElement(newp, 'Group', name=child.get('name')) + if contained.has_key(child.get('name')): + contained[child.get('name')].append(block.get('name')) + else: + contained[child.get('name')] = [block.get('name')] + elif child.tag == 'Attribute': + lxml.etree.SubElement(newp, 'Group', name=child.get('name'), scope=child.get('scope')) + elif child.tag == 'Bundle': + lxml.etree.SubElement(newp, "Bundle", name=child.get('name')) + else: + print "Unknown child tag %s" % child.tag + + iinfo = lxml.etree.parse(source + '/etc/imageinfo.xml') + for system in iinfo.findall('./System'): + if system.get('name') == 'debian': + lxml.etree.SubElement(newm, 'Group', name=system.get('name'), toolset='debian') + elif system.get('name') == 'redhat': + lxml.etree.SubElement(newm, 'Group', name=system.get('name'), toolset='rh') + elif system.get('name') == 'solaris': + lxml.etree.SubElement(newm, 'Group', name=system.get('name'), toolset='solaris') + else: + lxml.etree.SubElement(newm, 'Group', name=system.get('name')) + for image in system.findall('./Image'): + newi = lxml.etree.SubElement(newm, 'Group', name=image.get('name')) + lxml.etree.SubElement(newi, 'Group', name=system.get('name')) + + create_if_noent(dest) + create_if_noent(dest + '/Metadata') + open(dest + '/Metadata/groups.xml', 'w').write(pretty_print(newm)) + open(dest + '/Metadata/clients.xml', 'w').write(pretty_print(newc)) + + print "*** Image interpolation is not performed automatically" + + bsrc = source + '/Bundler/' + bdst = dest + '/Bundler/' + create_if_noent(bdst) + for bundle in glob.glob(bsrc + '*.xml'): + bname = bundle.split('/')[-1] + bdata = lxml.etree.parse(bundle).getroot() + for sys in bdata.findall('./System'): + sys.tag = 'Group' + open(bdst + bname, 'w').write(pretty_print(bdata)) + + os.system("rsync -a %s/SSHbase %s > /dev/null" % (source, dest)) + create_if_noent("%s/etc" % ( dest)) + os.system("cp %s/etc/*report*xml %s/etc" % (source, dest)) + os.system("cp %s/etc/statistics.xml %s/etc" % (source, dest)) + + # handle base + basedata = lxml.etree.parse("%s/etc/base.xml" % source) + left = basedata.getroot().getchildren() + while left: + next = left.pop() + if next.tag in ['Image', 'Class']: + next.tag = 'Group' + left += next.getchildren() + + create_if_noent("%s/Base" % dest) + open(dest + '/Base/converted.xml', 'w').write(pretty_print(basedata.getroot())) + + # handle packages + create_if_noent("%s/Pkgmgr" % dest) + for pkgsrc in glob.glob("%s/Pkgmgr/*.xml" % source): + pname = pkgsrc.split('/')[-1] + pdata = lxml.etree.parse(pkgsrc).getroot() + image = pdata.get('image') + del pdata.attrib['image'] + for loc in pdata.findall('./Location'): + loc.tag = 'Group' + loc.set('name', image) + if loc.attrib.has_key('uri'): + pdata.set('uri', loc.get('uri')) + del loc.attrib['uri'] + if loc.attrib.has_key('type'): + pdata.set('type', loc.get('type')) + del loc.attrib['type'] + pdata.set('priority', '0') + open(dest + '/Pkgmgr/' + pname, 'w').write(pretty_print(pdata)) + + # handle services.xml + svcdata = lxml.etree.parse("%s/etc/services.xml" % source) + left = svcdata.getroot().getchildren() + while left: + next = left.pop() + if next.tag in ['Image', 'Class']: + next.tag = 'Group' + left += next.getchildren() + + svcdata.getroot().set('priority', '0') + create_if_noent("%s/Svcmgr" % dest) + open(dest + '/Svcmgr/converted.xml', 'w').write(pretty_print(svcdata.getroot())) + + # handle Cfg + os.chdir("%s/Cfg" % source) + p = os.popen("find . -depth -type d -print") + path = p.readline().strip() + paths = [] + while path: + paths.append(path) + create_if_noent("%s/Cfg/%s" % (dest, path)) + path = p.readline().strip() + # paths are now created + for path in paths: + for filename in os.listdir(path): + if stat.S_ISDIR(os.stat(path + '/' + filename)[stat.ST_MODE]): + continue + if filename in [':info', path.split('/')[-1]]: + os.system("cp -f %s/%s %s/Cfg/%s/%s" % (path, filename, dest, path, filename)) + else: + meta = filename[len(path.split('/')[-1])+1:] + if meta[:2] == 'H_': + os.system("cp -f %s/%s %s/Cfg/%s/%s" % (path, filename, dest, path, filename)) + elif meta[:1] in ['C', 'B', 'I']: + print filename, "moved" + meta = 'G' + meta[1:] + os.system("cp -f %s/%s %s/Cfg/%s/%s.%s" % (path, filename, dest, + path, path.split('/')[-1], meta)) + else: + print "=========> don't know what to do with %s/%s" % (path, filename) + diff --git a/tools/crosscheck.py b/tools/crosscheck.py new file mode 100644 index 000000000..034fb1a90 --- /dev/null +++ b/tools/crosscheck.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +import lxml.etree +import sys + +important = {'Package':['name', 'version'], + 'Service':['name', 'status'], + 'Directory':['name', 'owner', 'group', 'perms'], + 'SymLink':['name', 'owner', 'from'], + 'ConfigFile':['name', 'owner', 'group', 'perms']} + +def compare(new, old): + for child in new.getchildren(): + equiv = old.xpath('%s[@name="%s"]' % (child.tag, child.get('name'))) + if not important.has_key(child.tag): + print "tag type %s not handled" % (child.tag) + continue + if len(equiv) == 0: + print "didn't find matching %s %s" % (child.tag, child.get('name')) + continue + elif len(equiv) == 1: + if child.tag == 'ConfigFile': + if child.text != equiv[0].text: + continue + if [child.get(field) for field in important[child.tag]] == \ + [equiv[0].get(field) for field in important[child.tag]]: + new.remove(child) + old.remove(equiv[0]) + else: + print "+", lxml.etree.tostring(child), + print "-", lxml.etree.tostring(equiv[0]), + else: + print "tag %s.%s listed > 1?" % (child.tag, child.get('name')) + if len(old.getchildren()) == 0 and len(new.getchildren()) == 0: + return True + if new.tag == 'Independant': + name = 'Indep' + else: + name = new.get('name') + print name, ["%s.%s" % (child.tag, child.get('name')) for child in old.getchildren()], + print ["%s.%s" % (child.tag, child.get('name')) for child in new.getchildren()] + return False + + +if __name__ == '__main__': + try: + (new, old) = sys.argv[1:3] + except IndexError: + print "Usage: crosscheck.py <new> <old>" + raise SystemExit + + new = lxml.etree.parse(new).getroot() + old = lxml.etree.parse(old).getroot() + for src in [new, old]: + for bundle in src.findall('./Bundle'): + if bundle.get('name')[-4:] == '.xml': + bundle.set('name', bundle.get('name')[:-4]) + + for bundle in new.findall('./Bundle'): + equiv = old.xpath('Bundle[@name="%s"]' % (bundle.get('name'))) + if len(equiv) == 0: + print "couldnt find matching bundle for %s" % bundle.get('name') + continue + if len(equiv) == 1: + if compare(bundle, equiv[0]): + new.remove(bundle) + old.remove(equiv[0]) + else: + print "dunno what is going on for bundle %s" % (bundle.get('name')) + i1 = new.find('./Independant') + i2 = old.find('./Independant') + if compare(i1, i2): + new.remove(i1) + old.remove(i2) + + #print lxml.etree.tostring(new) + #print lxml.etree.tostring(old) |