diff options
122 files changed, 3926 insertions, 1450 deletions
diff --git a/debian/bcfg2-server.postinst b/debian/bcfg2-server.postinst index 2f65fe847..77dea5f22 100644 --- a/debian/bcfg2-server.postinst +++ b/debian/bcfg2-server.postinst @@ -40,21 +40,4 @@ esac #DEBHELPER# -# We do a restart manually here because with autogenerated code -# we get this traceback (eg something isn't done yet): -# This happens due to debhelper bug #546293, fixed in version 7.4.2. -## Setting up bcfg2-server (1.0.0~rc3+r5542-0.1+dctest8) ... -## Starting Configuration Management Server: Traceback (most recent call last): -## File "/usr/sbin/bcfg2-server", line 12, in <module> -## import Bcfg2.Server.Plugins.Metadata -## ImportError: No module named Server.Plugins.Metadata -## * bcfg2-server -if [ -x "/etc/init.d/bcfg2-server" ]; then - if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then - invoke-rc.d bcfg2-server start || exit $? - else - /etc/init.d/bcfg2-server start || exit $? - fi -fi - exit 0 diff --git a/debian/bcfg2.default b/debian/bcfg2.default index 0164e5531..8ed0da74a 100644 --- a/debian/bcfg2.default +++ b/debian/bcfg2.default @@ -20,7 +20,7 @@ #BCFG2_INIT=1 # BCFG2_AGENT: -# Bcfg2 no longer supports agent mode please use the Agent+SSH method +# Bcfg2 no longer supports agent mode, please see NEWS.Debian # BCFG2_CRON: # Set the frequency of cron runs. diff --git a/debian/changelog b/debian/changelog index 298e695c5..5da9d27aa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +bcfg2 (1.3.2-0.0) unstable; urgency=low + + * New upstream release + + -- Sol Jerome <sol.jerome@gmail.com> Mon, 01 Jul 2013 16:24:46 -0500 + bcfg2 (1.3.1-0.0) unstable; urgency=low * New upstream release diff --git a/debian/control b/debian/control index 7b27b27ed..aee6f1c24 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,8 @@ Build-Depends: debhelper (>= 7.0.50~), python-sphinx (>= 1.0.7+dfsg) | python3-sphinx, python-lxml, python-daemon, - python-cherrypy, + python-boto, + python-cherrypy3, python-gamin, python-genshi, python-pyinotify, @@ -43,7 +44,7 @@ Description: Configuration management server Package: bcfg2-web Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django, +Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django, python-django-south (>= 0.7.5) Suggests: python-mysqldb, python-psycopg2, python-sqlite, libapache2-mod-wsgi Description: Configuration management web interface Bcfg2 is a configuration management system that generates configuration sets diff --git a/doc/appendix/files/mysql.txt b/doc/appendix/files/mysql.txt index 81104ec17..5adf2e27f 100644 --- a/doc/appendix/files/mysql.txt +++ b/doc/appendix/files/mysql.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _appendix-files-mysql: @@ -17,7 +18,7 @@ I added a new bundle: <Bundle name="mysql-server" version="3.0"> <Path name="/root/bcfg2-install/mysql/users.sh"/> <Path name="/root/bcfg2-install/mysql/users.sql"/> - <PostInstall name="/root/bcfg2-install/mysql/users.sh"/> + <Action name="users.sh"/> <Package name="mysql-server-4.1"/> <Service name="mysql"/> </Bundle> diff --git a/doc/appendix/guides/ubuntu.txt b/doc/appendix/guides/ubuntu.txt index 5a67d0a37..f68c8b9ad 100644 --- a/doc/appendix/guides/ubuntu.txt +++ b/doc/appendix/guides/ubuntu.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _appendix-guides-ubuntu: @@ -8,7 +9,7 @@ Ubuntu .. note:: - This particular how to was done on lucid, but should apply to any + This particular how to was done on saucy, but should apply to any other `stable`__ version of Ubuntu. __ ubuntu-releases_ @@ -23,11 +24,6 @@ version available in the ubuntu archives, but it is not as up to date). .. _PPA: https://launchpad.net/~bcfg2/+archive/ppa -Add the Ubuntu PPA listing to your APT sources ----------------------------------------------- - -See http://trac.mcs.anl.gov/projects/bcfg2/wiki/PrecompiledPackages#UbuntuLucid - Install bcfg2-server -------------------- :: @@ -36,7 +32,7 @@ Install bcfg2-server Remove the default configuration preseeded by the ubuntu package:: - root@lucid:~# rm -rf /etc/bcfg2* /var/lib/bcfg2 + root@saucy:~# rm -rf /etc/bcfg2* /etc/ssl/bcfg2* /var/lib/bcfg2 Initialize your repository ========================== @@ -45,63 +41,95 @@ Now that you're done with the install, you need to intialize your repository and setup your bcfg2.conf. bcfg2-admin init is a tool which allows you to automate this process.:: - root@lucid:~# bcfg2-admin init - Store bcfg2 configuration in [/etc/bcfg2.conf]: - Location of bcfg2 repository [/var/lib/bcfg2]: + root@saucy:~# bcfg2-admin init + Store Bcfg2 configuration in [/etc/bcfg2.conf]: + Location of Bcfg2 repository [/var/lib/bcfg2]: Input password used for communication verification (without echoing; leave blank for a random): - What is the server's hostname: [lucid] - Input the server location [https://lucid:6789]: + What is the server's hostname: [saucy] + Input the server location (the server listens on a single interface by default) [https://saucy:6789]: Input base Operating System for clients: - 1: Redhat/Fedora/RHEL/RHAS/Centos + 1: Redhat/Fedora/RHEL/RHAS/CentOS 2: SUSE/SLES 3: Mandrake 4: Debian 5: Ubuntu 6: Gentoo 7: FreeBSD + 8: Arch : 5 + Path where Bcfg2 server private key will be created [/etc/ssl/bcfg2.key]: + Path where Bcfg2 server cert will be created [/etc/ssl/bcfg2.crt]: + The following questions affect SSL certificate generation. + If no data is provided, the default values are used. + Country name (2 letter code) for certificate: US + State or Province Name (full name) for certificate: Illinois + Locality Name (eg, city) for certificate: Argonne + Repository created successfuly in /var/lib/bcfg2 Generating a 2048 bit RSA private key - ......................................................................................+++ - ...+++ - writing new private key to '/etc/bcfg2.key' + ....................................................................................................................+++ + ..............................+++ + writing new private key to '/etc/ssl/bcfg2.key' ----- Signature ok - subject=/C=US/ST=Illinois/L=Argonne/CN=lucid + subject=/C=US/ST=Illinois/L=Argonne/CN=saucy Getting Private key - Repository created successfuly in /var/lib/bcfg2 - Of course, change responses as necessary. Start the server ================ +Before you start the server, you need to fix your network resolution for +this host. The short and easy way is to remove the 127.0.1.1 line in +``/etc/hosts`` and move your hostname to the 127.0.0.1 line. + +:: + + 127.0.0.1 saucy localhost + + # The following lines are desirable for IPv6 capable hosts + ... + +.. _Debian Manual: http://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution + +.. note:: + + This configuration is not recommended except as a quick hack to get + you through this guide. Ideally you'd add a line containing the + host's actual IP address. More information on why this is broken + can be found in the `Debian Manual`_. + You are now ready to start your bcfg2 server for the first time.:: - root@lucid:~# /etc/init.d/bcfg2-server start - root@lucid:~# tail /var/log/syslog - Dec 17 22:07:02 lucid bcfg2-server[17523]: serving bcfg2-server at https://lucid:6789 - Dec 17 22:07:02 lucid bcfg2-server[17523]: serve_forever() [start] - Dec 17 22:07:02 lucid bcfg2-server[17523]: Processed 16 fam events in 0.502 seconds. 0 coalesced + root@saucy:~# /etc/init.d/bcfg2-server start + Starting Configuration Management Server: * bcfg2-server + root@saucy:~# tail /var/log/syslog + Jul 18 17:50:48 saucy bcfg2-server[5872]: Reconnected to syslog + Jul 18 17:50:48 saucy bcfg2-server[5872]: bcfg2-server daemonized + Jul 18 17:50:48 saucy bcfg2-server[5872]: service available at https://saucy:6789 + Jul 18 17:50:48 saucy bcfg2-server[5872]: serving bcfg2-server at https://saucy:6789 + Jul 18 17:50:48 saucy bcfg2-server[5872]: serve_forever() [start] + Jul 18 17:50:48 saucy bcfg2-server[5872]: Handled 13 events in 0.006s Run bcfg2 to be sure you are able to communicate with the server:: - root@lucid:~# bcfg2 -vqn + root@saucy:~# bcfg2 -vqn + Starting Bcfg2 client run at 1374188552.53 Loaded tool drivers: - APT Action DebInit POSIX - + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers Phase: initial Correct entries: 0 Incorrect entries: 0 Total managed entries: 0 - Unmanaged entries: 382 - - + Unmanaged entries: 590 Phase: final Correct entries: 0 Incorrect entries: 0 Total managed entries: 0 - Unmanaged entries: 382 + Unmanaged entries: 590 + Finished Bcfg2 client run at 1374188563.26 Bring your first machine under Bcfg2 control ============================================ @@ -114,92 +142,101 @@ Setup the :ref:`server-plugins-generators-packages` plugin Replace Pkgmgr with Packages in the plugins line of ``bcfg2.conf``:: - root@lucid:~# cat /etc/bcfg2.conf + root@saucy:~# cat /etc/bcfg2.conf [server] repository = /var/lib/bcfg2 - plugins = SSHbase,Cfg,Packages,Rules,Metadata,Base,Bundler + plugins = Bundler,Cfg,Metadata,Packages,Rules,SSHbase + # Uncomment the following to listen on all interfaces + #listen_all = true [statistics] sendmailpath = /usr/lib/sendmail + #web_debug = False + #time_zone = [database] - engine = sqlite3 + #engine = sqlite3 # 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. - name = + #name = # Or path to database file if using sqlite3. - #<repository>/etc/brpt.sqlite is default path if left empty - user = + #<repository>/bcfg2.sqlite is default path if left empty + #user = # Not used with sqlite3. - password = + #password = # Not used with sqlite3. - host = + #host = # Not used with sqlite3. - port = + #port = + + [reporting] + transport = LocalFilesystem [communication] protocol = xmlrpc/ssl password = secret - certificate = /etc/bcfg2.crt - key = /etc/bcfg2.key - ca = /etc/bcfg2.crt + certificate = /etc/ssl/bcfg2.crt + key = /etc/ssl/bcfg2.key + ca = /etc/ssl/bcfg2.crt [components] - bcfg2 = https://lucid:6789 + bcfg2 = https://saucy:6789 Create Packages layout (as per :ref:`packages-exampleusage`) in ``/var/lib/bcfg2`` .. code-block:: xml - root@lucid:~# mkdir /var/lib/bcfg2/Packages - root@lucid:~# cat /var/lib/bcfg2/Packages/packages.conf + root@saucy:~# mkdir /var/lib/bcfg2/Packages + root@saucy:~# cat /var/lib/bcfg2/Packages/packages.conf [global] - root@lucid:~# cat /var/lib/bcfg2/Packages/sources.xml + root@saucy:~# cat /var/lib/bcfg2/Packages/sources.xml <Sources> - <Group name="ubuntu-lucid"> - <Source type="apt" url="http://archive.ubuntu.com/ubuntu" version="lucid"> + <Group name="ubuntu-saucy"> + <Source type="apt" debsrc="true" recommended="true" url="http://archive.ubuntu.com/ubuntu" version="saucy"> <Component>main</Component> <Component>multiverse</Component> <Component>restricted</Component> <Component>universe</Component> <Arch>amd64</Arch> + <Blacklist>bcfg2</Blacklist> + <Blacklist>bcfg2-server</Blacklist> </Source> - <Source type="apt" url="http://archive.ubuntu.com/ubuntu" version="lucid-updates"> + <Source type="apt" debsrc="true" recommended="true" url="http://archive.ubuntu.com/ubuntu" version="saucy-updates"> <Component>main</Component> <Component>multiverse</Component> <Component>restricted</Component> <Component>universe</Component> <Arch>amd64</Arch> + <Blacklist>bcfg2</Blacklist> + <Blacklist>bcfg2-server</Blacklist> </Source> - <Source type="apt" url="http://security.ubuntu.com/ubuntu" version="lucid-security"> + <Source type="apt" debsrc="true" recommended="true" url="http://security.ubuntu.com/ubuntu" version="saucy-security"> <Component>main</Component> <Component>multiverse</Component> <Component>restricted</Component> <Component>universe</Component> <Arch>amd64</Arch> + <Blacklist>bcfg2</Blacklist> + <Blacklist>bcfg2-server</Blacklist> + </Source> + <Source type="apt" debsrc="true" recommended="true" url="http://ppa.launchpad.net/bcfg2/ppa/ubuntu" version="saucy"> + <Component>main</Component> + <Arch>amd64</Arch> </Source> </Group> </Sources> -Due to the :ref:`server-plugins-generators-packages-magic-groups`, -we need to modify our Metadata. Let's add an **ubuntu-lucid** -group which inherits the **ubuntu** group already present in -``/var/lib/bcfg2/Metadata/groups.xml``. The resulting file should look -something like this - -.. note:: - - The reason we are creating a release-specific group in this case is - that the APTSource above is specific to the lucid release of ubuntu. - That is, it should not apply to other releases (hardy, maverick, etc). +Above, we have grouped our package sources under **ubuntu-saucy**. We +need to add this group to our ``/var/lib/bcfg2/Metadata/groups.xml`` so +that our client is able to obtain these sources. .. code-block:: xml <Groups version='3.0'> <Group profile='true' public='true' default='true' name='basic'> - <Group name='ubuntu-lucid'/> + <Group name='ubuntu-saucy'/> </Group> - <Group name='ubuntu-lucid'> + <Group name='ubuntu-saucy'> <Group name='ubuntu'/> </Group> <Group name='ubuntu'/> @@ -214,7 +251,7 @@ something like this .. note:: When editing your xml files by hand, it is useful to occasionally run - `bcfg2-lint` to ensure that your xml validates properly. + ``bcfg2-lint -v`` to ensure that your xml validates properly. The last thing we need is for the client to have the proper arch group membership. For this, we will make use of the @@ -223,13 +260,13 @@ Probes to your plugins line in ``bcfg2.conf`` and create the Probe. .. code-block:: sh - root@lucid:~# grep plugins /etc/bcfg2.conf - plugins = Base,Bundler,Cfg,...,Probes - root@lucid:~# mkdir /var/lib/bcfg2/Probes - root@lucid:~# cat /var/lib/bcfg2/Probes/groups + root@saucy:~# grep plugins /etc/bcfg2.conf + plugins = Bundler,Cfg,Metadata,...,Probes + root@saucy:~# mkdir /var/lib/bcfg2/Probes + root@saucy:~# cat /var/lib/bcfg2/Probes/groups #!/bin/sh - ARCH=`uname -m` + ARCH=$(uname -m) case "$ARCH" in "x86_64") echo "group:amd64" @@ -241,33 +278,36 @@ Probes to your plugins line in ``bcfg2.conf`` and create the Probe. Now we restart the bcfg2-server:: - root@lucid:~# /etc/init.d/bcfg2-server restart + root@saucy:~# /etc/init.d/bcfg2-server restart Stopping Configuration Management Server: * bcfg2-server Starting Configuration Management Server: * bcfg2-server - root@lucid:~# tail /var/log/syslog - Dec 17 22:36:47 lucid bcfg2-server[17937]: Packages: File read failed; falling back to file download - Dec 17 22:36:47 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/main/binary-amd64/Packages.gz - Dec 17 22:36:54 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/multiverse/binary-amd64/Packages.gz - Dec 17 22:36:55 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/restricted/binary-amd64/Packages.gz - Dec 17 22:36:56 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/universe/binary-amd64/Packages.gz - Dec 17 22:37:27 lucid bcfg2-server[17937]: Failed to read file probed.xml - Dec 17 22:37:27 lucid bcfg2-server[17937]: Loading experimental plugin(s): Packages - Dec 17 22:37:27 lucid bcfg2-server[17937]: NOTE: Interfaces subject to change - Dec 17 22:37:27 lucid bcfg2-server[17937]: service available at https://lucid:6789 - Dec 17 22:37:27 lucid bcfg2-server[17937]: serving bcfg2-server at https://lucid:6789 - Dec 17 22:37:27 lucid bcfg2-server[17937]: serve_forever() [start] - Dec 17 22:37:28 lucid bcfg2-server[17937]: Processed 17 fam events in 0.502 seconds. 0 coalesced + root@saucy:~# tail /var/log/syslog + Jul 18 18:43:22 saucy bcfg2-server[6215]: Reconnected to syslog + Jul 18 18:43:22 saucy bcfg2-server[6215]: bcfg2-server daemonized + Jul 18 18:43:22 saucy bcfg2-server[6215]: service available at https://saucy:6789 + Jul 18 18:43:22 saucy bcfg2-server[6215]: Failed to read file probed.xml: Error reading file '/var/lib/bcfg2/Probes/probed.xml': failed to load external entity "/var/lib/bcfg2/Probes/probed.xml" + Jul 18 18:43:22 saucy bcfg2-server[6215]: serving bcfg2-server at https://saucy:6789 + Jul 18 18:43:22 saucy bcfg2-server[6215]: serve_forever() [start] + Jul 18 18:43:22 saucy bcfg2-server[6215]: Reloading Packages plugin + Jul 18 18:43:22 saucy bcfg2-server[6215]: Handled 15 events in 0.205s + +.. note:: + + The error regarding *probed.xml* is non-fatal and just telling you + that the file doesn't yet exist. It will be populated once you have + run a client with the Probes plugin enabled. Start managing packages ----------------------- -Add a base-packages bundle. Let's see what happens when we just populate -it with the ubuntu-standard package. +Add a base-saucy (or whatever release you happen to be using) +bundle. Let's see what happens when we just populate it with the +ubuntu-standard package. .. code-block:: xml - root@lucid:~# cat /var/lib/bcfg2/Bundler/base-packages.xml - <Bundle name='base-packages'> + root@saucy:~# cat /var/lib/bcfg2/Bundler/base-saucy.xml + <Bundle name='base-saucy'> <Package name='ubuntu-standard'/> </Bundle> @@ -277,218 +317,473 @@ profile group might look something like this .. code-block:: xml <Group profile='true' public='true' default='true' name='basic'> - <Bundle name='base-packages'/> - <Group name='ubuntu-lucid'/> + <Bundle name='base-saucy'/> + <Group name='ubuntu-saucy'/> </Group> Now if we run the client in debug mode (-d), we can see what this has done for us.:: - root@lucid:~# bcfg2 -vqdn + root@saucy:/var/lib/bcfg2# bcfg2 -vqdn + Configured logging: DEBUG to console; DEBUG to syslog + {'help': False, 'extra': False, 'ppath': '/var/cache/bcfg2', 'ca': '/etc/ssl/bcfg2.crt', 'rpm_version_fail_action': 'upgrade', 'yum_version_fail_action': 'upgrade', 'retry_delay': '1', 'posix_uid_whitelist': [], 'rpm_erase_flags': ['allmatches'], 'verbose': True, 'certificate': '/etc/ssl/bcfg2.crt', 'paranoid': False, 'rpm_installonly': ['kernel', 'kernel-bigmem', 'kernel-enterprise', 'kernel-smp', 'kernel-modules', 'kernel-debug', 'kernel-unsupported', 'kernel-devel', 'kernel-source', 'kernel-default', 'kernel-largesmp-devel', 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], 'cache': None, 'yum24_autodep': True, 'yum_pkg_verify': True, 'probe_timeout': None, 'yum_installed_action': 'install', 'rpm_verify_fail_action': 'reinstall', 'dryrun': True, 'retries': '3', 'apt_install_path': '/usr', 'quick': True, 'password': 'secret', 'yum24_installed_action': 'install', 'kevlar': False, 'max_copies': 1, 'syslog': True, 'decision_list': False, 'configfile': '/etc/bcfg2.conf', 'remove': None, 'server': 'https://saucy:6789', 'encoding': 'UTF-8', 'timeout': 90, 'debug': True, 'yum24_installonly': ['kernel', 'kernel-bigmem', 'kernel-enterprise', 'kernel-smp', 'kernel-modules', 'kernel-debug', 'kernel-unsupported', 'kernel-devel', 'kernel-source', 'kernel-default', 'kernel-largesmp-devel', 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], 'yum24_erase_flags': ['allmatches'], 'yum24_pkg_checks': True, 'interactive': False, 'apt_etc_path': '/etc', 'rpm_installed_action': 'install', 'yum24_verify_fail_action': 'reinstall', 'omit_lock_check': False, 'yum24_pkg_verify': True, 'serverCN': None, 'file': None, 'apt_var_path': '/var', 'posix_gid_whitelist': [], 'posix_gid_blacklist': [], 'indep': False, 'decision': 'none', 'servicemode': 'default', 'version': False, 'rpm_pkg_checks': True, 'profile': None, 'yum_pkg_checks': True, 'args': [], 'bundle': [], 'posix_uid_blacklist': [], 'user': 'root', 'key': '/etc/ssl/bcfg2.key', 'command_timeout': None, 'probe_exit': True, 'lockfile': '/var/lock/bcfg2.run', 'yum_verify_fail_action': 'reinstall', 'yum24_version_fail_action': 'upgrade', 'yum_verify_flags': [], 'logging': None, 'rpm_pkg_verify': True, 'bundle_quick': False, 'rpm_verify_flags': [], 'yum24_verify_flags': [], 'skipindep': False, 'skipbundle': [], 'portage_binpkgonly': False, 'drivers': ['APK', 'APT', 'Action', 'Blast', 'Chkconfig', 'DebInit', 'Encap', 'FreeBSDInit', 'FreeBSDPackage', 'IPS', 'MacPorts', 'OpenCSW', 'POSIX', 'POSIXUsers', 'Pacman', 'Portage', 'RPM', 'RPMng', 'RcUpdate', 'SELinux', 'SMF', 'SYSV', 'Systemd', 'Upstart', 'VCS', 'YUM', 'YUM24', 'YUMng', 'launchd']} + Starting Bcfg2 client run at 1374191628.88 Running probe groups + Running: /tmp/tmpEtgdwo + < group:amd64 Probe groups has result: - amd64 + group:amd64 + + POSIX: Handlers loaded: nonexistent, directory, hardlink, symlink, file, device, permissions Loaded tool drivers: - APT Action DebInit POSIX + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers The following packages are specified in bcfg2: ubuntu-standard The following packages are prereqs added by Packages: - adduser debconf hdparm libdevmapper1.02.1 libk5crypto3 libparted1.8-12 libxml2 passwd upstart - apt debianutils info libdns53 libkeyutils1 libpci3 logrotate pciutils usbutils - aptitude dmidecode install-info libelf1 libkrb5-3 libpopt0 lsb-base perl-base wget - at dnsutils iptables libept0 libkrb5support0 libreadline5 lshw popularity-contest zlib1g - base-files dosfstools libacl1 libgcc1 liblwres50 libreadline6 lsof psmisc - base-passwd dpkg libattr1 libgdbm3 libmagic1 libselinux1 ltrace readline-common - bsdmainutils ed libbind9-50 libgeoip1 libmpfr1ldbl libsigc++-2.0-0c2a man-db rsync - bsdutils file libc-bin libgmp3c2 libncurses5 libssl0.9.8 memtest86+ sed - cpio findutils libc6 libgssapi-krb5-2 libncursesw5 libstdc++6 mime-support sensible-utils - cpp ftp libcap2 libisc50 libpam-modules libusb-0.1-4 ncurses-bin strace - cpp-4.4 gcc-4.4-base libcomerr2 libisccc50 libpam-runtime libuuid1 netbase time - cron groff-base libcwidget3 libisccfg50 libpam0g libxapian15 parted tzdata - + accountsservice libdrm2 libusb-1.0-0 + adduser libedit2 libustr-1.0-1 + apparmor libelf1 libuuid1 + apt libexpat1 libwind0-heimdal + apt-transport-https libffi6 libx11-6 + apt-utils libfribidi0 libx11-data + base-files libfuse2 libxau6 + base-passwd libgcc1 libxcb1 + bash libgck-1-0 libxdmcp6 + bash-completion libgcr-3-common libxext6 + bsdmainutils libgcr-base-3-1 libxml2 + bsdutils libgcrypt11 libxmuu1 + busybox-initramfs libgdbm3 libxtables10 + busybox-static libgeoip1 locales + ca-certificates libglib2.0-0 login + command-not-found libglib2.0-data logrotate + command-not-found-data libgnutls26 lsb-base + coreutils libgpg-error0 lsb-release + cpio libgpm2 lshw + cron libgssapi-krb5-2 lsof + dash libgssapi3-heimdal ltrace + dbus libhcrypto4-heimdal makedev + debconf libheimbase1-heimdal man-db + debconf-i18n libheimntlm0-heimdal manpages + debianutils libhx509-5-heimdal memtest86+ + diffutils libidn11 mime-support + dmidecode libisc92 mlocate + dmsetup libisccc90 module-init-tools + dnsutils libisccfg90 mount + dosfstools libjson-c2 mountall + dpkg libjson0 mtr-tiny + e2fslibs libk5crypto3 multiarch-support + e2fsprogs libkeyutils1 nano + ed libklibc ncurses-base + file libkmod2 ncurses-bin + findutils libkrb5-26-heimdal netbase + friendly-recovery libkrb5-3 ntfs-3g + ftp libkrb5support0 openssh-client + fuse libldap-2.4-2 openssl + gcc-4.8-base liblocale-gettext-perl parted + geoip-database liblwres90 passwd + gettext-base liblzma5 pciutils + gnupg libmagic1 perl-base + gpgv libmount1 plymouth + grep libncurses5 plymouth-theme-ubuntu-text + groff-base libncursesw5 popularity-contest + gzip libnewt0.52 powermgmt-base + hdparm libnfnetlink0 ppp + hostname libnih-dbus1 pppconfig + ifupdown libnih1 pppoeconf + info libnuma1 procps + initramfs-tools libp11-kit0 psmisc + initramfs-tools-bin libpam-modules python-apt-common + initscripts libpam-modules-bin python3 + insserv libpam-runtime python3-apt + install-info libpam-systemd python3-commandnotfound + iproute libpam0g python3-dbus + iproute2 libparted0debian1 python3-distupgrade + iptables libpcap0.8 python3-gdbm + iputils-tracepath libpci3 python3-minimal + irqbalance libpcre3 python3-update-manager + iso-codes libpipeline1 python3.3 + klibc-utils libplymouth2 python3.3-minimal + kmod libpng12-0 readline-common + krb5-locales libpolkit-gobject-1-0 rsync + language-selector-common libpopt0 sed + libaccountsservice0 libprocps0 sensible-utils + libacl1 libpython3-stdlib sgml-base + libapparmor-perl libpython3.3-minimal shared-mime-info + libapparmor1 libpython3.3-stdlib strace + libapt-inst1.5 libreadline6 systemd-services + libapt-pkg4.12 libroken18-heimdal sysv-rc + libasn1-8-heimdal librtmp0 sysvinit-utils + libasprintf0c2 libsasl2-2 tar + libatm1 libsasl2-modules tcpdump + libattr1 libselinux1 telnet + libaudit-common libsemanage-common time + libaudit1 libsemanage1 tzdata + libbind9-90 libsepol1 ubuntu-keyring + libblkid1 libslang2 ubuntu-release-upgrader-core + libbsd0 libsqlite3-0 ucf + libbz2-1.0 libss2 udev + libc-bin libssl1.0.0 ufw + libc6 libstdc++6 update-manager-core + libcap-ng0 libsystemd-daemon0 upstart + libcap2 libsystemd-login0 usbutils + libcomerr2 libtasn1-3 util-linux + libcurl3-gnutls libtext-charwidth-perl uuid-runtime + libdb5.1 libtext-iconv-perl wget + libdbus-1-3 libtext-wrapi18n-perl whiptail + libdbus-glib-1-2 libtinfo5 xauth + libdevmapper1.02.1 libudev1 xml-core + libdns95 libusb-0.1-4 zlib1g Phase: initial - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 - - + Total managed entries: 280 + Unmanaged entries: 313 + Installing entries in the following bundle(s): + base-saucy + Bundle base-saucy was not modified Phase: final - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 + Total managed entries: 280 + Unmanaged entries: 313 + Finished Bcfg2 client run at 1374191642.69 As you can see, the Packages plugin has generated the dependencies required for the ubuntu-standard package for us automatically. The ultimate goal should be to move all the packages from the **Unmanaged** entries section to the **Managed** entries section. So, what exactly *are* -those Unmanaged entries?:: +those Unmanaged entries? + +:: - root@lucid:~# bcfg2 -vqen + Starting Bcfg2 client run at 1374192077.76 Running probe groups Probe groups has result: - amd64 - Loaded tool drivers: - APT Action DebInit POSIX + group:amd64 + Loaded tool drivers: + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers Phase: initial - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 - - + Total managed entries: 280 + Unmanaged entries: 313 Phase: final - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 - Package:apparmor - Package:apparmor-utils - Package:apport - ... - -Now you can go through these and continue adding the packages you want to -your Bundle. Note that ``aptitude why`` is useful when trying to figure -out the reason for a package being installed. Also, deborphan is helpful -for removing leftover dependencies which are no longer needed. After a -while, I ended up with a minimal bundle that looks like this + Total managed entries: 280 + Unmanaged entries: 313 + POSIXGroup:adm + POSIXGroup:audio + POSIXGroup:backup + ... + Package:deb:apt-xapian-index + Package:deb:aptitude + Package:deb:aptitude-common + ... + +Now you can go through these and continue adding the packages you want +to your Bundle. Note that ``aptitude why`` is useful when trying to +figure out the reason for a package being installed. Also, ``deborphan`` +is helpful for removing leftover dependencies which are no longer +needed. After a while, I ended up with a minimal bundle that looks +like this: .. code-block:: xml - <Bundle name='base-packages'> - <Package name='bash-completion'/> + <Bundle name='base-saucy'> + <!-- packages --> <Package name='bcfg2-server'/> - <Package name='debconf-i18n'/> + <!-- or dependencies --> + <Package name='python-pyinotify'/> + <Package name='ttf-dejavu-core'/> + <Package name='bind9-host'/> + <Package name='crda'/> <Package name='deborphan'/> - <Package name='diffutils'/> - <Package name='e2fsprogs'/> - <Package name='fam'/> - <Package name='grep'/> <Package name='grub-pc'/> - <Package name='gzip'/> - <Package name='hostname'/> - <Package name='krb5-config'/> - <Package name='krb5-user'/> - <Package name='language-pack-en-base'/> + <Package name='language-pack-en'/> <Package name='linux-generic'/> <Package name='linux-headers-generic'/> - <Package name='login'/> - <Package name='manpages'/> - <Package name='mlocate'/> - <Package name='ncurses-base'/> - <Package name='openssh-server'/> - <Package name='python-fam'/> - <Package name='tar'/> + <Package name='systemd-shim'/> + <Package name='tasksel'/> <Package name='ubuntu-minimal'/> <Package name='ubuntu-standard'/> + <!-- or dependencies --> + <Package name='python3-gi'/> + <Package name='wamerican'/> + <Package name='wbritish'/> <Package name='vim'/> - <Package name='vim-runtime'/> - - <!-- PreDepends --> - <Package name='dash'/> - <Package name='initscripts'/> - <Package name='libdbus-1-3'/> - <Package name='libnih-dbus1'/> - <Package name='lzma'/> - <Package name='mountall'/> - <Package name='sysvinit-utils'/> - <Package name='sysv-rc'/> - - <!-- vim dependencies --> - <Package name='libgpm2'/> - <Package name='libpython2.6'/> </Bundle> -As you can see below, I no longer have any unmanaged packages. :: +Once your ``bcfg2 -vqen`` output no longer shows Package entries, you +can move on to the next section. - root@lucid:~# bcfg2 -vqen - Running probe groups - Probe groups has result: - amd64 - Loaded tool drivers: - APT Action DebInit POSIX +Manage users +------------ - Phase: initial - Correct entries: 247 - Incorrect entries: 0 - Total managed entries: 247 - Unmanaged entries: 10 +The default setting in ``login.defs`` is for system accounts to be UIDs +< 1000. We will ignore those accounts for now (you can manage them if +you like at a later time). +To ignore system UID/GIDs, add the following lines to ``bcfg2.conf`` +(we will also ignore the nobody uid and nogroup gid--65534). - Phase: final - Correct entries: 247 - Incorrect entries: 0 - Total managed entries: 247 - Unmanaged entries: 10 - Service:bcfg2 Service:fam Service:killprocs Service:rc.local Service:single - Service:bcfg2-server Service:grub-common Service:ondemand Service:rsync Service:ssh +:: + + [POSIXUsers] + uid_blacklist = 0-999,65534 + gid_blacklist = 0-999,65534 + +If you run the client again with ``bcfg2 -vqen``, you should now see a +:ref:`POSIXUser <server-plugins-generators-rules-posixuser-tag>` entry +and :ref:`POSIXGroup <server-plugins-generators-rules-posixgroup-tag>` +entry for your user account (assuming this is a fresh install with a +regular user). + +You can manage this user by adding the following to your bundle. + +.. code-block:: xml + + <BoundPOSIXUser name='username' uid='1000' gecos="Your Name"> + <MemberOf>adm</MemberOf> + <MemberOf>cdrom</MemberOf> + <MemberOf>dip</MemberOf> + <MemberOf>lpadmin</MemberOf> + <MemberOf>plugdev</MemberOf> + <MemberOf>sambashare</MemberOf> + <MemberOf>sudo</MemberOf> + </BoundPOSIXUser> Manage services --------------- -Now let's clear up the unmanaged service entries by adding the following -entries to our bundle... +To clear up the unmanaged service entries, you will need to add the +entries to your bundle. Here's an example of what that might look like. .. code-block:: xml - <!-- basic services --> + <!-- services --> <Service name='bcfg2'/> + <Service name='bcfg2-report-collector'/> <Service name='bcfg2-server'/> - <Service name='fam'/> + <Service name='bootmisc.sh'/> + <Service name='checkfs.sh'/> + <Service name='checkroot-bootclean.sh'/> + <Service name='checkroot.sh'/> + <Service name='console'/> + <Service name='console-font'/> + <Service name='console-setup'/> + <Service name='container-detect'/> + <Service name='control-alt-delete'/> + <Service name='cron'/> + <Service name='dbus'/> + <Service name='dmesg'/> + <Service name='dns-clean'/> + <Service name='failsafe'/> + <Service name='flush-early-job-log'/> + <Service name='friendly-recovery'/> <Service name='grub-common'/> + <Service name='hostname'/> + <Service name='hwclock'/> + <Service name='hwclock-save'/> + <Service name='irqbalance'/> <Service name='killprocs'/> + <Service name='kmod'/> + <Service name='mountall'/> + <Service name='mountall.sh'/> + <Service name='mountall-bootclean.sh'/> + <Service name='mountall-net'/> + <Service name='mountall-reboot'/> + <Service name='mountall-shell'/> + <Service name='mountdevsubfs.sh'/> + <Service name='mounted-debugfs'/> + <Service name='mounted-dev'/> + <Service name='mounted-proc'/> + <Service name='mounted-run'/> + <Service name='mounted-tmp'/> + <Service name='mounted-var'/> + <Service name='mountkernfs.sh'/> + <Service name='mountnfs-bootclean.sh'/> + <Service name='mountnfs.sh'/> + <Service name='mtab.sh'/> + <Service name='network-interface'/> + <Service name='network-interface-container'/> + <Service name='network-interface-security'/> + <Service name='networking'/> <Service name='ondemand'/> + <Service name='passwd'/> + <Service name='plymouth'/> + <Service name='plymouth-log'/> + <Service name='plymouth-ready'/> + <Service name='plymouth-splash'/> + <Service name='plymouth-stop'/> + <Service name='plymouth-upstart-bridge'/> + <Service name='pppd-dns'/> + <Service name='procps'/> + <Service name='rc'/> <Service name='rc.local'/> + <Service name='rc-sysinit'/> + <Service name='rcS'/> + <Service name='resolvconf'/> <Service name='rsync'/> + <Service name='rsyslog'/> + <Service name='setvtrgb'/> + <Service name='shutdown'/> <Service name='single'/> - <Service name='ssh'/> - - -...and bind them in Rules + <Service name='startpar-bridge'/> + <Service name='sudo'/> + <Service name='systemd-logind'/> + <Service name='tty1'/> + <Service name='tty2'/> + <Service name='tty3'/> + <Service name='tty4'/> + <Service name='tty5'/> + <Service name='tty6'/> + <Service name='udev'/> + <Service name='udev-fallback-graphics'/> + <Service name='udev-finish'/> + <Service name='udevmonitor'/> + <Service name='udevtrigger'/> + <Service name='ufw'/> + <Service name='upstart-file-bridge'/> + <Service name='upstart-socket-bridge'/> + <Service name='upstart-udev-bridge'/> + <Service name='ureadahead'/> + <Service name='ureadahead-other'/> + <Service name='wait-for-state'/> + +Add the literal entries in Rules to bind the Service entries from above. .. code-block:: xml - root@lucid:~# cat /var/lib/bcfg2/Rules/services.xml + root@saucy:~# cat /var/lib/bcfg2/Rules/services.xml <Rules priority='1'> - <!-- basic services --> - <Service type='deb' status='on' name='bcfg2'/> - <Service type='deb' status='on' name='bcfg2-server'/> - <Service type='deb' status='on' name='fam'/> - <Service type='deb' status='on' name='grub-common'/> - <Service type='deb' status='on' name='killprocs'/> - <Service type='deb' status='on' name='ondemand'/> - <Service type='deb' status='on' name='rc.local'/> - <Service type='deb' status='on' name='rsync'/> - <Service type='deb' status='on' name='single'/> - <Service type='deb' status='on' name='ssh'/> + <!-- sysv services --> + <Service name='bcfg2' type='deb' status='on'/> + <Service name='bcfg2-server' type='deb' status='on'/> + <Service name='dns-clean' type='deb' status='on'/> + <Service name='grub-common' type='deb' status='on'/> + <Service name='sudo' type='deb' status='on'/> + + <Service name='killprocs' type='deb' bootstatus='on' status='ignore'/> + <Service name='ondemand' type='deb' bootstatus='on' status='ignore'/> + <Service name='pppd-dns' type='deb' bootstatus='on' status='ignore'/> + <Service name='rc.local' type='deb' bootstatus='on' status='ignore'/> + <Service name='rsync' type='deb' bootstatus='on' status='ignore'/> + <Service name='single' type='deb' bootstatus='on' status='ignore'/> + + <Service name='bcfg2-report-collector' type='deb' status='off'/> + + <!-- upstart services --> + <Service name='bootmisc.sh' type='upstart' status='on'/> + <Service name='checkfs.sh' type='upstart' status='on'/> + <Service name='checkroot-bootclean.sh' type='upstart' status='on'/> + <Service name='checkroot.sh' type='upstart' status='on'/> + <Service name='cron' type='upstart' status='on'/> + <Service name='dbus' type='upstart' status='on'/> + <Service name='mountall.sh' type='upstart' status='on'/> + <Service name='mountall-bootclean.sh' type='upstart' status='on'/> + <Service name='mountdevsubfs.sh' type='upstart' status='on'/> + <Service name='mountkernfs.sh' type='upstart' status='on'/> + <Service name='mountnfs-bootclean.sh' type='upstart' status='on'/> + <Service name='mountnfs.sh' type='upstart' status='on'/> + <Service name='mtab.sh' type='upstart' status='on'/> + <Service name='network-interface' type='upstart' status='on' parameters='INTERFACE=eth0'/> + <Service name='network-interface-security' type='upstart' status='on' parameters='JOB=network-interface/eth0'/> + <Service name='networking' type='upstart' status='on'/> + <Service name='plymouth-ready' type='upstart' status='ignore'/> + <Service name='resolvconf' type='upstart' status='on'/> + <Service name='rsyslog' type='upstart' status='on'/> + <Service name='startpar-bridge' type='upstart' status='ignore'/> + <Service name='systemd-logind' type='upstart' status='on'/> + <Service name='tty1' type='upstart' status='on'/> + <Service name='tty2' type='upstart' status='on'/> + <Service name='tty3' type='upstart' status='on'/> + <Service name='tty4' type='upstart' status='on'/> + <Service name='tty5' type='upstart' status='on'/> + <Service name='tty6' type='upstart' status='on'/> + <Service name='udev' type='upstart' status='on'/> + <Service name='ufw' type='upstart' status='on'/> + <Service name='upstart-file-bridge' type='upstart' status='on'/> + <Service name='upstart-socket-bridge' type='upstart' status='on'/> + <Service name='upstart-udev-bridge' type='upstart' status='on'/> + <Service name='wait-for-state' type='upstart' status='ignore'/> + + <Service name='console' type='upstart' status='off'/> + <Service name='console-font' type='upstart' status='off'/> + <Service name='console-setup' type='upstart' status='off'/> + <Service name='container-detect' type='upstart' status='off'/> + <Service name='control-alt-delete' type='upstart' status='off'/> + <Service name='dmesg' type='upstart' status='off'/> + <Service name='failsafe' type='upstart' status='off'/> + <Service name='flush-early-job-log' type='upstart' status='off'/> + <Service name='friendly-recovery' type='upstart' status='off'/> + <Service name='hostname' type='upstart' status='off'/> + <Service name='hwclock' type='upstart' status='off'/> + <Service name='hwclock-save' type='upstart' status='off'/> + <Service name='irqbalance' type='upstart' status='off'/> + <Service name='kmod' type='upstart' status='off'/> + <Service name='mountall' type='upstart' status='off'/> + <Service name='mountall-net' type='upstart' status='off'/> + <Service name='mountall-reboot' type='upstart' status='off'/> + <Service name='mountall-shell' type='upstart' status='off'/> + <Service name='mounted-debugfs' type='upstart' status='off'/> + <Service name='mounted-dev' type='upstart' status='off'/> + <Service name='mounted-proc' type='upstart' status='off'/> + <Service name='mounted-run' type='upstart' status='off'/> + <Service name='mounted-tmp' type='upstart' status='off'/> + <Service name='mounted-var' type='upstart' status='off'/> + <Service name='network-interface-container' type='upstart' status='off'/> + <Service name='passwd' type='upstart' status='off'/> + <Service name='plymouth' type='upstart' status='off'/> + <Service name='plymouth-log' type='upstart' status='off'/> + <Service name='plymouth-splash' type='upstart' status='off'/> + <Service name='plymouth-stop' type='upstart' status='off'/> + <Service name='plymouth-upstart-bridge' type='upstart' status='off'/> + <Service name='procps' type='upstart' status='off'/> + <Service name='rc' type='upstart' status='off'/> + <Service name='rc-sysinit' type='upstart' status='off'/> + <Service name='rcS' type='upstart' status='off'/> + <Service name='setvtrgb' type='upstart' status='off'/> + <Service name='shutdown' type='upstart' status='off'/> + <Service name='udev-fallback-graphics' type='upstart' status='off'/> + <Service name='udev-finish' type='upstart' status='off'/> + <Service name='udevmonitor' type='upstart' status='off'/> + <Service name='udevtrigger' type='upstart' status='off'/> + <Service name='ureadahead' type='upstart' status='off'/> + <Service name='ureadahead-other' type='upstart' status='off'/> </Rules> -Now we run the client and see there are no more unmanaged entries! :: +Now we run the client and see there are no more unmanaged entries! - root@lucid:~# bcfg2 -vqn +:: + + root@saucy:~# bcfg2 -vqn + Starting Bcfg2 client run at 1374271524.83 Running probe groups Probe groups has result: - amd64 - Loaded tool drivers: - APT Action DebInit POSIX + group:amd64 + Loaded tool drivers: + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers Phase: initial - Correct entries: 257 + Correct entries: 519 Incorrect entries: 0 - Total managed entries: 257 + Total managed entries: 519 Unmanaged entries: 0 - - All entries correct. - Phase: final - Correct entries: 257 + Correct entries: 519 Incorrect entries: 0 - Total managed entries: 257 + Total managed entries: 519 Unmanaged entries: 0 - All entries correct. + Finished Bcfg2 client run at 1374271541.56 .. warning:: diff --git a/doc/appendix/tools.txt b/doc/appendix/tools.txt index 1d7a8dd90..92bde683b 100644 --- a/doc/appendix/tools.txt +++ b/doc/appendix/tools.txt @@ -11,4 +11,4 @@ can help you to maintain your Bcfg2 configuration, to make the initial setup easier, or to do some other tasks. -http://trac.mcs.anl.gov/projects/bcfg2/browser/tools +https://github.com/Bcfg2/bcfg2/tree/maint/tools diff --git a/doc/client/tools/actions.txt b/doc/client/tools/actions.txt index 81486ecd1..e5fdb1f39 100644 --- a/doc/client/tools/actions.txt +++ b/doc/client/tools/actions.txt @@ -31,10 +31,11 @@ central reporting of action failure is desired, set this attribute to 'check'. Also note that Action entries included in Base will not be executed. -Actions cannot be completely defined inside of a bundle; they are a bound -entry, much like Packages, Services or Paths. The Rules plugin can bind -these entries. For example to include the above action in a bundle, -first the Action entry must be included in the bundle: +Actions may be completely defined inside of a bundle with the use of +:ref:`server-configurationentries`, much like Packages, Services or Paths. +The Rules plugin can also bind these entries. For example to include the +above action in a bundle, first the Action entry must be included in the +bundle: .. code-block:: xml @@ -70,3 +71,18 @@ requires this key. <Action timing='post' name='apt-key-update' command='apt-key adv --recv-keys --keyserver hkp://pgp.mit.edu 0C5A2783' when='modified' status='check'/> </Group> </Rules> + +Example BoundAction (add RPM GPG keys) +====================================== + +This example will add the RPM-GPG-KEY-redhat-release key to the RPM +GPG keyring **before** Package entries are handled on the client run. + +.. code-block:: xml + + <Bundle name="rpm-gpg-keys"> + <Group name='rhel'> + <Path name="/etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release"/> + <BoundAction timing="pre" name="install rpm key" command="rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release" when="modified" status="check"/> + </Group> + </Bundle> diff --git a/doc/conf.py b/doc/conf.py index d3d30687b..d1bb029d2 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -66,7 +66,7 @@ else: # The short X.Y version. version = '1.3' # The full version, including alpha/beta/rc tags. -release = '1.3.1' +release = '1.3.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -254,6 +254,8 @@ man_pages = [ [], 8), ('man/bcfg2-lint.conf', 'bcfg2-lint.conf', 'Configuration parameters for bcfg2-lint', [], 5), + ('man/bcfg2-report-collector', 'bcfg2-report-collector', + 'Reports collection daemon', [], 8), ('man/bcfg2-reports', 'bcfg2-reports', 'Query reporting system for client status', [], 8), ('man/bcfg2-server', 'bcfg2-server', diff --git a/doc/development/lint.txt b/doc/development/lint.txt index 6a4651f92..6c0be960d 100644 --- a/doc/development/lint.txt +++ b/doc/development/lint.txt @@ -106,6 +106,11 @@ Basics Existing ``bcfg2-lint`` Plugins =============================== +AWSTagsLint +----------- + +.. autoclass:: Bcfg2.Server.Plugins.AWSTags.AWSTagsLint + BundlerLint ----------- diff --git a/doc/getting_started/index.txt b/doc/getting_started/index.txt index a9e91e6b8..a9b1b847f 100644 --- a/doc/getting_started/index.txt +++ b/doc/getting_started/index.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _getting_started-index: @@ -115,7 +116,7 @@ files: ``clients.xml`` and ``groups.xml``. Your current .. code-block:: xml - <Clients version="3.0"> + <Clients> <Client profile="basic" pingable="Y" pingtime="0" name="bcfg-server.example.com"/> </Clients> @@ -132,7 +133,7 @@ Our simple ``groups.xml`` file looks like: .. code-block:: xml - <Groups version='3.0'> + <Groups> <Group profile='true' public='false' name='basic'> <Group name='suse'/> </Group> @@ -168,14 +169,14 @@ Next, we create a motd.xml file in the Bundler directory: .. code-block:: xml - <Bundle name='motd' version='2.0'> + <Bundle name='motd'> <Path name='/etc/motd' /> </Bundle> Now when we run the client, we get slightly different output:: Loaded tool drivers: - Chkconfig POSIX YUMng + Chkconfig POSIX YUM Incomplete information for entry Path:/etc/motd; cannot verify Phase: initial @@ -205,7 +206,7 @@ real ``/etc/motd`` file to that location, run the client again, and you will find that we now have a correct entry:: Loaded tool drivers: - Chkconfig POSIX PostInstall RPM + Chkconfig POSIX YUM Phase: initial Correct entries: 1 diff --git a/doc/man/bcfg2-report-collector.txt b/doc/man/bcfg2-report-collector.txt new file mode 100644 index 000000000..07c618537 --- /dev/null +++ b/doc/man/bcfg2-report-collector.txt @@ -0,0 +1,40 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + + +bcfg2-report-collector +====================== + +.. program:: bcfg2-report-collector + +Synopsis +-------- + +**bcfg2-report-collector** [*options*] + +Description +----------- + +:program:`bcfg2-report-collector` runs a daemon to collect logs from the +LocalFilesystem :ref:`Bcfg2 Reports <reports-dynamic>` transport object +and add them to the Reporting storage backend. + +Options +------- + +-C configfile Specify alternate bcfg2.conf location. +-D pidfile Daemonize, placing the program pid in *pidfile*. +-E encoding Specify the encoding of config files. +-Q path Specify the path to the server repository. +-W configfile Specify the path to the web interface + configuration file. +-d Enable debugging output. +-h Print usage information. +-o path Set path of file log +-v Run in verbose mode. +--version Print the version and exit + +See Also +-------- + +:manpage:`bcfg2-server(8)`, :manpage:`bcfg2-reports(8)` diff --git a/doc/man/bcfg2-server.txt b/doc/man/bcfg2-server.txt index d5945cad6..3f8f3ea21 100644 --- a/doc/man/bcfg2-server.txt +++ b/doc/man/bcfg2-server.txt @@ -23,8 +23,7 @@ Options ------- -C configfile Specify alternate bcfg2.conf location. --D pidfile Daemonize, placing the program pid in the specified - pidfile. +-D pidfile Daemonize, placing the program pid in *pidfile*. -E encoding Specify the encoding of config files. -Q path Specify the path to the server repository. -S server Manually specify the server location (as opposed to diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 3a0217aef..6faf48a1a 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -46,6 +46,12 @@ filemonitor fam pseudo +fam_blocking + Whether the server should block at startup until the file monitor + backend has processed all events. This can cause a slower startup, + but ensure that all files are recognized before the first client + is handled. + ignore_files A comma-separated list of globs that should be ignored by the file monitor. Default values are:: @@ -729,6 +735,11 @@ control the database connection of the server. port Port for database connections. Not used for sqlite3. + options + Various options for the database connection. The value is + expected as multiple key=value pairs, separated with commas. + The concrete value depends on the database engine. + Reporting options ----------------- diff --git a/doc/server/admin/index.txt b/doc/server/admin/index.txt index ee03cedda..c563ead9c 100644 --- a/doc/server/admin/index.txt +++ b/doc/server/admin/index.txt @@ -23,7 +23,6 @@ functionality. Available modes are listed below. minestruct perf pull - query snapshots tidy viz diff --git a/doc/server/admin/query.txt b/doc/server/admin/query.txt deleted file mode 100644 index 65851a43d..000000000 --- a/doc/server/admin/query.txt +++ /dev/null @@ -1,15 +0,0 @@ -.. -*- mode: rst -*- - -.. _server-admin-query: - -query -===== - -Query clients. - -The default result format is suitable for consumption by `pdsh`_. -This example queries the server for all clients in the *ubuntu* group:: - - bcfg2-admin query g=ubuntu - -.. _pdsh: http://sourceforge.net/projects/pdsh/ diff --git a/doc/server/configuration.txt b/doc/server/configuration.txt index 7892c2612..bb4983367 100644 --- a/doc/server/configuration.txt +++ b/doc/server/configuration.txt @@ -149,7 +149,7 @@ Consequently, you simply have to run: .. code-block:: bash chown bcfg2:bcfg2 /var/run/bcfg2-server - chmod 0644 /var/run/bcfg2-server + chmod 0755 /var/run/bcfg2-server Additionally, the server daemon itself supports dropping privileges natively in 1.3. Simply add the following lines to ``bcfg2.conf``:: diff --git a/doc/server/database.txt b/doc/server/database.txt index b0ec7b571..3c8970f68 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -49,6 +49,12 @@ of ``/etc/bcfg2.conf``. +-------------+------------------------------------------------------------+-------------------------------+ | port | The port to connect to | None | +-------------+------------------------------------------------------------+-------------------------------+ +| options | Extra parameters to use when connecting to the database. | None | +| | Available parameters vary depending on your database | | +| | backend. The parameters are supplied as comma separated | | +| | key=value pairs. | | ++-------------+------------------------------------------------------------+-------------------------------+ + Database Schema Sync ==================== diff --git a/doc/server/plugins/connectors/awstags.txt b/doc/server/plugins/connectors/awstags.txt new file mode 100644 index 000000000..b884ca065 --- /dev/null +++ b/doc/server/plugins/connectors/awstags.txt @@ -0,0 +1,124 @@ +.. -*- mode: rst -*- + +.. _server-plugins-connectors-awstags: + +========= + AWSTags +========= + +The AWSTags plugin is a connector that retrieves tags from instances +in EC2, and can assign optionally assign +group membership pased on patterns in the tags. See `Using Tags +<http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html>`_ +for details on using tags in EC2. + +AWSTags queries EC2 for instances whose ``private-dns-name`` property +matches the hostname of the client. + +Setup +===== + +#. Add ``AWSTags`` to the ``plugins`` option in ``/etc/bcfg2.conf`` +#. Configure AWS credentials in ``/etc/bcfg2.conf`` (See + `Configuration`_ below for details.) +#. Optionally, create ``AWSTags/config.xml`` (See `Assigning Groups`_ + below for details.) +#. Restart the Bcfg2 server. + +Using Tag Data +============== + +AWSTags exposes the data in templates as a dict available as +``metadata.AWSTags``. E.g., in a :ref:`Genshi template +<server-plugins-generators-cfg-genshi>`, you could do: + +.. code-block:: genshitext + + Known tags on ${metadata.hostname}: + {% for key, val in metadata.AWSTags.items() %}\ + ${key} ${val} + {% end %}\ + +This would produce something like:: + + Known tags on foo.example.com: + Name foo.example.com + some random tag the value + +Assigning Groups +================ + +AWSTags can assign groups based on the tag data. This functionality +is configured in ``AWSTags/config.xml``. + +Example +------- + +.. code-block:: xml + + <AWSTags> + <Tag name="^foo$"> + <Group>foo</Group> + </Tag> + <Tag name="^bar$" value="^bar$"> + <Group>bar</Group> + </Tag> + <Tag name="^bcfg2 group$" value="(.*)"> + <Group>$1</Group> + </Tag> + </AWSTags> + +In this example, any machine with a tag named ``foo`` would be added +to the ``foo`` group. Any machine with a tag named ``bar`` whose +value was also ``bar`` would be added to the ``bar`` group. Finally, +any machine with a tag named ``bcfg2 group`` would be added to the +group named in the value of that tag. + +Note that both the ``name`` and ``value`` attributes are *always* +regular expressions. + +If a ``<Tag/>`` element has only a ``name`` attribute, then it only +checks for existence of a matching tag. If it has both ``name`` and +``value``, then it checks for a matching tag with a matching value. + +You can use backreferences (``$1``, ``$2``, etc.) in the group names. +If only ``name`` is specified, then the backreferences will refer to +groups in the ``name`` regex. If ``name`` and ``value`` are both +specified, then backreferences will refer to groups in the ``value`` +regex. If you specify both ``name`` and ``value``, it is not possible +to refer to groups in the ``name`` regex. + +Schema Reference +---------------- + +.. xml:schema:: awstags.xsd + +Configuration +============= + +AWSTags recognizes several options in ``/etc/bcfg2.conf``; at a +minimum, you must configure an AWS access key ID and secret key. All +of the following options are in the ``[awstags]`` section: + ++-----------------------+-----------------------------------------------------+ +| Option | Description | ++=======================+=====================================================+ +| ``access_key_id`` | The AWS access key ID | ++-----------------------+-----------------------------------------------------+ +| ``secret_access_key`` | The AWS secret access key | ++-----------------------+-----------------------------------------------------+ +| ``cache`` | Whether or not to cache tag lookups. See `Caching`_ | +| | for details. Default is to cache. | ++-----------------------+-----------------------------------------------------+ + +Caching +======= + +Since the AWS API isn't always very quick to respond, AWSTags caches +its results by default. The cache is fairly short-lived: the cache +for each host is expired when it starts a client run, so it will start +the run with fresh data. + +If you frequently update tags on your instances, you may wish to +disable caching. That's probably a bad idea, and would tend to +suggest that updating tags frequently is perhaps the Wrong Thing. diff --git a/doc/server/plugins/connectors/properties.txt b/doc/server/plugins/connectors/properties.txt index da511736d..47e82fdbf 100644 --- a/doc/server/plugins/connectors/properties.txt +++ b/doc/server/plugins/connectors/properties.txt @@ -181,6 +181,8 @@ XML tag should be ``<Properties>``. JSON Property Files ------------------- +.. versionadded:: 1.3.0 + The data in a JSON property file can be accessed with the ``json`` attribute, which is the loaded JSON data. The JSON properties interface does not provide any additional functionality beyond the @@ -189,6 +191,8 @@ interface does not provide any additional functionality beyond the YAML Property Files ------------------- +.. versionadded:: 1.3.0 + The data in a YAML property file can be accessed with the ``yaml`` attribute, which is the loaded YAML data. Only a single YAML document may be included in a file. @@ -229,10 +233,10 @@ simply:: %} You can also enable automatch for individual Property files by setting -the attribute ``automatch="true"`` on the top-level ``<Property>`` +the attribute ``automatch="true"`` on the top-level ``<Properties>`` tag. Conversely, if automatch is enabled by default in ``bcfg2.conf``, you can disable it for an individual Property file by -setting ``automatch="false"`` on the top-level ``<Property>`` tag. +setting ``automatch="false"`` on the top-level ``<Properties>`` tag. If you want to see what ``XMLMatch()``/automatch would produce for a given client on a given Properties file, you can use :ref:`bcfg2-info diff --git a/doc/server/plugins/generators/cfg.txt b/doc/server/plugins/generators/cfg.txt index e3768a3ba..0f0601105 100644 --- a/doc/server/plugins/generators/cfg.txt +++ b/doc/server/plugins/generators/cfg.txt @@ -541,7 +541,8 @@ Example </Group> <Allow from="/root/.ssh/id_rsa.pub" host="foo.example.com"/> <Allow from="/home/foo_user/.ssh/id_rsa.pub"> - <Params command="/home/foo_user/.ssh/ssh_command_filter"/> + <Option name="command" value="/home/foo_user/.ssh/ssh_command_filter"/> + <Option name="no-X11-forwarding"/> </Allow> <Allow> ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDw/rgKQeARRAHK5bQQhAAe1b+gzdtqBXWrZIQ6cIaLgxqj76TwZ3DY4A6aW9RgC4zzd0p4a9MfsScUIB4+UeZsx9GopUj4U6H8Vz7S3pXxrr4E9logVLuSfOLFbI/wMWNRuOANqquLYQ+JYWKeP4kagkVp0aAWp7mH5IOI0rp0A6qE2you4ep9N/nKvHDrtypwhYBWprsgTUXXMHnAWGmyuHGYWxNYBV9AARPdAvZfb8ggtuwibcOULlyK4DdVNbDTAN1/BDBE1ve6WZDcrc386KhqUGj/yoRyPjNZ46uZiOjRr3cdY6yUZoCwzzxvm5vle6mEbLjHgjGEMQMArzM9 vendor@example.com diff --git a/doc/server/plugins/generators/nagiosgen.txt b/doc/server/plugins/generators/nagiosgen.txt index ee99b2dc1..0ae922fa3 100644 --- a/doc/server/plugins/generators/nagiosgen.txt +++ b/doc/server/plugins/generators/nagiosgen.txt @@ -8,7 +8,7 @@ NagiosGen This page describes the installation and use of the `NagiosGen`_ plugin. -.. _NagiosGen: http://trac.mcs.anl.gov/projects/bcfg2/browser/src/lib/Server/Plugins/NagiosGen.py +.. _NagiosGen: https://github.com/Bcfg2/bcfg2/blob/maint/src/lib/Bcfg2/Server/Plugins/NagiosGen.py Update ``/etc/bcfg2.conf``, adding NagiosGen to plugins:: diff --git a/doc/server/plugins/generators/packages.txt b/doc/server/plugins/generators/packages.txt index cdc4f7282..31f3ccf22 100644 --- a/doc/server/plugins/generators/packages.txt +++ b/doc/server/plugins/generators/packages.txt @@ -490,17 +490,18 @@ Benefits to this include: * Much lower memory usage by the ``bcfg2-server`` process. * Much faster ``Packages.Refresh`` behavior. * More accurate dependency resolution. +* Better use of multiple processors/cores. Drawbacks include: -* More disk I/O. In some cases, you may have to raise the open file +* Resolution of package dependencies is slower and more + resource-intensive. At times it can be much slower, particularly + after running ``Packages.Refresh``. +* More disk I/O. This can be alleviated by putting + ``/var/lib/bcfg2/Packages/cache`` on tmpfs, but that offsets the + lower memory usage. In some cases, you may have to raise the open file limit for the user who runs your Bcfg2 server process, particularly if you have a lot of repositories. -* Resolution of package dependencies is slower in some cases, - particularly after running ``Packages.Refresh``. -* If you have a very large number of clients using a very small number - of repositories, using native yum libraries may actually increase - memory usage. Configuring the Yum Helper -------------------------- diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 2493be53f..9ba70238d 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-plugins-generators-rules: @@ -59,7 +60,7 @@ Rules Tag .. xml:element:: Rules :linktotype: :noautodep: - :inlinetypes: PostInstall,RContainerType + :inlinetypes: RContainerType Package Tag ----------- @@ -376,6 +377,8 @@ SEModule Tag See also :ref:`server-plugins-generators-semodules`. +.. _server-plugins-generators-rules-posixuser-tag: + POSIXUser Tag ------------- @@ -411,6 +414,8 @@ Defaults plugin <server-plugins-structures-defaults>`. See :ref:`client-tools-posixusers` for more information on managing users and groups. +.. _server-plugins-generators-rules-posixgroup-tag: + POSIXGroup Tag -------------- diff --git a/doc/server/plugins/index.txt b/doc/server/plugins/index.txt index 4f2b484ac..f3d6daa73 100644 --- a/doc/server/plugins/index.txt +++ b/doc/server/plugins/index.txt @@ -31,7 +31,7 @@ Default Plugins The `Bcfg2 repository`_ contains the all plugins currently distributed with Bcfg2. -.. _Bcfg2 repository: http://trac.mcs.anl.gov/projects/bcfg2/browser/src/lib/Server/Plugins +.. _Bcfg2 repository: https://github.com/Bcfg2/bcfg2/tree/maint/src/lib/Bcfg2/Server/Plugins Metadata (Grouping) ------------------- diff --git a/doc/server/plugins/structures/bundler/kernel.txt b/doc/server/plugins/structures/bundler/kernel.txt index 2e3d84e93..d83679683 100644 --- a/doc/server/plugins/structures/bundler/kernel.txt +++ b/doc/server/plugins/structures/bundler/kernel.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _server-plugins-structures-bundler-kernel: @@ -21,7 +22,7 @@ some of which might be better than this one. Feel free to hack as needed. .. code-block:: xml - <Bundle name='kernel' version='2.0'> + <Bundle name='kernel'> <Group name='sles8'> <!-- =================== ia32 ==================== --> <Group name='ia32'> @@ -30,7 +31,7 @@ some of which might be better than this one. Feel free to hack as needed. <Path name='/boot/initrd'/> <Path name='/boot/vmlinuz.old'/> <Path name='/boot/initrd.old'/> - <PostInstall name='/sbin/lilo'/> + <BoundAction name='lilo' command='/sbin/lilo' timing='post' when='modified'/> <!-- Current kernel --> <Package name='linux-2.4.21-314.tg1'/> <Package name='linux-2.4.21-314.tg1-source'/> diff --git a/doc/unsorted/python-ssl_1.14-1_amd64.deb b/doc/unsorted/python-ssl_1.14-1_amd64.deb Binary files differdeleted file mode 100644 index e6c8ad137..000000000 --- a/doc/unsorted/python-ssl_1.14-1_amd64.deb +++ /dev/null diff --git a/doc/unsorted/python-stdeb_0.3-1_all.deb b/doc/unsorted/python-stdeb_0.3-1_all.deb Binary files differdeleted file mode 100644 index 5cee96e3c..000000000 --- a/doc/unsorted/python-stdeb_0.3-1_all.deb +++ /dev/null diff --git a/doc/unsorted/ssl.txt b/doc/unsorted/ssl.txt deleted file mode 100644 index 91b62ca59..000000000 --- a/doc/unsorted/ssl.txt +++ /dev/null @@ -1,68 +0,0 @@ -.. -*- mode: rst -*- - -.. _unsorted-ssl: - -========== -Python SSL -========== - -The ssl module can be found `here <http://pypi.python.org/pypi/ssl>`_. - -With this change, SSL certificate based client authentication is -supported. In order to use this, based CA-type capabilities are -required. A central CA needs to be created, with each server and all -clients getting a signed cert. See [wiki:Authentication] for details. - -Setting up keys is accomplished with three settings, each in the -"`[communication]`" section of ``bcfg2.conf``:: - - key = /path/to/ssl private key - certificate = /path/to/signed cert for that key - ca = /path/to/cacert.pem - - -Python SSL Backport Packaging -============================= - -Both the Bcfg2 server and client are able to use the in-tree ssl module -included with python 2.6. The client is also able to still use M2Crypto. A -python ssl backport exists for 2.3, 2.4, and 2.5. With this, M2Crypto -is not needed, and tlslite is no longer included with Bcfg2 sources. See -[wiki:Authentication] for details. - -To build a package of the ssl backport for .deb based distributions -that don't ship with python 2.6, you can follow these instructions, -which use `stdeb`_. Alternatively if you happen to have .deb packaging -skills, it would be great to get policy-complaint .debs into the major -deb-based distributions. - -.. _stdeb: http://github.com/astraw/stdeb/tree/master - -The following commands were used to generate :download:`this -<python-ssl_1.14-1_amd64.deb>` debian package The ``easy_install`` command -can be found in the `python-setuptools` package.:: - - sudo aptitude install python-all-dev fakeroot - sudo easy_install stdeb - wget http://pypi.python.org/packages/source/s/ssl/ssl-1.14.tar.gz#md5=4e08aae0cd2c7388d1b4bbb7f374b14a - tar xvfz ssl-1.14.tar.gz - cd ssl-1.14 - stdeb_run_setup - cd deb_dist/ssl-1.14 - dpkg-buildpackage -rfakeroot -uc -us - sudo dpkg -i ../python-ssl_1.14-1_amd64.deb - -.. note:: Version numbers for the SSL module have changed. - -For complete Bcfg2 goodness, you'll also want to package stdeb using stdeb. -The completed debian package can be grabbed from :download:`here -<python-stdeb_0.3-1_all.deb>`, which was generated using the following:: - - sudo aptitude install apt-file - wget http://pypi.python.org/packages/source/s/stdeb/stdeb-0.3.tar.gz#md5=e692f745597dcdd9343ce133e3b910d0 - tar xvfz stdeb-0.3.tar.gz - cd stdeb-0.3 - stdeb_run_setup - cd deb_dist/stdeb-0.3 - dpkg-buildpackage -rfakeroot -uc -us - sudo dpkg -i ../python-stdeb_0.3-1_all.deb diff --git a/man/bcfg2-report-collector.8 b/man/bcfg2-report-collector.8 new file mode 100644 index 000000000..195b15ec8 --- /dev/null +++ b/man/bcfg2-report-collector.8 @@ -0,0 +1,79 @@ +.TH "BCFG2-REPORT-COLLECTOR" "8" "July 27, 2013" "1.3" "Bcfg2" +.SH NAME +bcfg2-report-collector \- Reports collection daemon +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.\" Man page generated from reStructuredText. +. +.SH SYNOPSIS +.sp +\fBbcfg2\-report\-collector\fP [\fIoptions\fP] +.SH DESCRIPTION +.sp +\fBbcfg2\-report\-collector\fP runs a daemon to collect logs from the +LocalFilesystem \fIBcfg2 Reports\fP transport object +and add them to the Reporting storage backend. +.SH OPTIONS +.INDENT 0.0 +.TP +.BI \-C \ configfile +Specify alternate bcfg2.conf location. +.TP +.BI \-D \ pidfile +Daemonize, placing the program pid in \fIpidfile\fP. +.TP +.BI \-E \ encoding +Specify the encoding of config files. +.TP +.BI \-Q \ path +Specify the path to the server repository. +.TP +.BI \-W \ configfile +Specify the path to the web interface +configuration file. +.TP +.B \-d +Enable debugging output. +.TP +.B \-h +Print usage information. +.TP +.BI \-o \ path +Set path of file log +.TP +.B \-v +Run in verbose mode. +.TP +.B \-\-version +Print the version and exit +.UNINDENT +.SH SEE ALSO +.sp +\fIbcfg2\-server(8)\fP, \fIbcfg2\-reports(8)\fP +.\" Generated by docutils manpage writer. +. diff --git a/man/bcfg2-server.8 b/man/bcfg2-server.8 index 27f6a7b01..dcec03252 100644 --- a/man/bcfg2-server.8 +++ b/man/bcfg2-server.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-SERVER" "8" "March 18, 2013" "1.3" "Bcfg2" +.TH "BCFG2-SERVER" "8" "July 27, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-server \- Server for client configuration specifications . @@ -46,8 +46,7 @@ configurations to clients based on the data in its repository. Specify alternate bcfg2.conf location. .TP .BI \-D \ pidfile -Daemonize, placing the program pid in the specified -pidfile. +Daemonize, placing the program pid in \fIpidfile\fP. .TP .BI \-E \ encoding Specify the encoding of config files. diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index b0db91a5b..5e64caae9 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "March 18, 2013" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "July 19, 2013" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -76,6 +76,13 @@ pseudo .UNINDENT .UNINDENT .TP +.B fam_blocking +. +Whether the server should block at startup until the file monitor +backend has processed all events. This can cause a slower startup, +but ensure that all files are recognized before the first client +is handled. +.TP .B ignore_files A comma\-separated list of globs that should be ignored by the file monitor. Default values are: @@ -771,6 +778,11 @@ Host for database connections. Not used for sqlite3. .TP .B port Port for database connections. Not used for sqlite3. +.TP +.B options +Various options for the database connection. The value is +expected as multiple key=value pairs, separated with commas. +The concrete value depends on the database engine. .UNINDENT .UNINDENT .UNINDENT diff --git a/misc/bcfg2-selinux.spec b/misc/bcfg2-selinux.spec index 9c5262dfd..fa70d2c42 100644 --- a/misc/bcfg2-selinux.spec +++ b/misc/bcfg2-selinux.spec @@ -8,7 +8,7 @@ %global selinux_variants %([ -z "%{selinux_types}" ] && echo mls strict targeted || echo %{selinux_types}) Name: bcfg2-selinux -Version: 1.3.1 +Version: 1.3.2 Release: 1 Summary: Bcfg2 Client and Server SELinux policy @@ -120,6 +120,9 @@ if [ $1 -eq 0 ] ; then fi %changelog +* Mon Jul 01 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.2-1 +- New upstream release + * Thu Mar 21 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.1-1 - New upstream release diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 518ebd245..ddd694d51 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -1,13 +1,31 @@ -%global __python python -%{!?py_ver: %global py_ver %(%{__python} -c 'import sys;print(sys.version[0:3])')} -%global pythonversion %{py_ver} -%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} +# Fedora 13+ and EL6 contain these macros already; only needed for EL5 +%if 0%{?rhel} && 0%{?rhel} <= 5 +%global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") +%define python_version %(%{__python} -c 'import sys;print(sys.version[0:3])') +%endif + +# openSUSE macro translation +%if 0%{?suse_version} +%global python_version %{py_ver} %{!?_initrddir: %global _initrddir %{_sysconfdir}/rc.d/init.d} +# openSUSE < 11.2 +%if %{suse_version} < 1120 +%global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()") +%endif +%endif + +# For -pre or -rc releases, remove the initial <hash><percent> +# characters from the appropriate line below. +# +# Don't forget to change the Release: tag below to something like 0.1 +#%%global _rc 1 +#%%global _pre 2 +%global _pre_rc %{?_pre:.pre%{_pre}}%{?_rc:.rc%{_rc}} Name: bcfg2 -Version: 1.3.1 -Release: 1 -Summary: Configuration management system +Version: 1.3.2 +Release: 2%{?_pre_rc}%{?dist} +Summary: A configuration management system %if 0%{?suse_version} # http://en.opensuse.org/openSUSE:Package_group_guidelines @@ -18,11 +36,54 @@ Group: Applications/System License: BSD URL: http://bcfg2.org Source0: ftp://ftp.mcs.anl.gov/pub/bcfg/%{name}-%{version}.tar.gz +# Used in %%check +Source1: http://www.w3.org/2001/XMLSchema.xsd +%if %{?rhel}%{!?rhel:10} <= 5 || 0%{?suse_version} +# EL5 and OpenSUSE require the BuildRoot tag BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +%endif BuildArch: noarch +BuildRequires: python BuildRequires: python-devel BuildRequires: python-lxml +BuildRequires: python-boto +%if 0%{?suse_version} +BuildRequires: python-M2Crypto +BuildRequires: python-Genshi +BuildRequires: python-gamin +BuildRequires: python-pyinotify +BuildRequires: python-python-daemon +BuildRequires: python-CherryPy >= 3 +%else # ! suse_version +BuildRequires: python-daemon +BuildRequires: python-inotify +%if "%{_vendor}" == "redhat" && 0%{!?rhel:1} && 0%{!?fedora:1} +# by default, el5 doesn't have the %%rhel macro, provided by this +# package; EPEL build servers install buildsys-macros by default, but +# explicitly requiring this may help builds in other environments +BuildRequires: buildsys-macros +%else # vendor != redhat || rhel defined +%if 0%{?rhel} && 0%{?rhel} < 6 +BuildRequires: python-ssl +%else # rhel > 5 +# EL5 lacks python-mock, so test suite is disabled +BuildRequires: python-sqlalchemy +BuildRequires: python-nose +BuildRequires: mock +BuildRequires: m2crypto +BuildRequires: Django +BuildRequires: python-genshi +BuildRequires: python-cheetah +BuildRequires: pylibacl +BuildRequires: libselinux-python +BuildRequires: python-pep8 +BuildRequires: python-cherrypy >= 3 +BuildRequires: python-mock +BuildRequires: pylint +%endif # rhel > 5 +%endif # vendor != redhat || rhel defined +%endif # ! suse_version %if 0%{?mandriva_version} # mandriva seems to behave differently than other distros and needs @@ -40,23 +101,40 @@ BuildRequires: libsane1 # a different package name in EPEL. %if "%{_vendor}" == "redhat" && 0%{?rhel} <= 6 && 0%{?fedora} == 0 BuildRequires: python-sphinx10 -# the python-sphinx10 package doesn't set sys.path correctly, so we -# have to do it for them +# python-sphinx10 doesn't set sys.path correctly; do it for them %global pythonpath %(find %{python_sitelib} -name Sphinx*.egg) %else BuildRequires: python-sphinx >= 1.0 %endif +BuildRequires: python-docutils + +%if 0%{?fedora} >= 16 +BuildRequires: systemd-units +%endif -%if 0%{?rhel_version} -# the debian init script needs redhat-lsb. -# iff we switch to the redhat one, this might not be needed anymore. -Requires: redhat-lsb +Requires: python-lxml +%if 0%{?rhel} && 0%{?rhel} < 6 +Requires: python-ssl %endif +Requires: libselinux-python + +%if 0%{?fedora} >= 16 +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units +%else +Requires(post): /sbin/chkconfig +Requires(preun): /sbin/chkconfig +Requires(preun): /sbin/service +Requires(postun): /sbin/service +%endif + %if "%{_vendor}" != "redhat" # fedora and rhel (and possibly other distros) do not know this tag. Recommends: cron %endif + %description Bcfg2 helps system administrators produce a consistent, reproducible, and verifiable description of their environment, and offers @@ -86,17 +164,13 @@ deployment strategies. This package includes the Bcfg2 client software. %package server -Version: 1.3.1 Summary: Bcfg2 Server %if 0%{?suse_version} Group: System/Management %else -Group: System Tools -%endif -Requires: bcfg2 = %{version} -%if "%{py_ver}" < "2.6" -Requires: python-ssl +Group: System Environment/Daemons %endif +Requires: bcfg2 = %{version}-%{release} Requires: python-lxml >= 1.2.1 %if 0%{?suse_version} Requires: python-pyinotify @@ -107,8 +181,24 @@ Requires: python-daemon %endif Requires: /usr/sbin/sendmail Requires: /usr/bin/openssl +Requires: graphviz Requires: python-nose +%if %{_vendor} == redhat +%if 0%{?fedora} >= 16 +Requires(post): systemd-units +Requires(preun): systemd-units +Requires(postun): systemd-units +Requires(post): systemd-sysv +%else +Requires(post): /sbin/chkconfig +Requires(preun): /sbin/chkconfig +Requires(preun): /sbin/service +Requires(postun): /sbin/service +%endif +%endif + + %description server Bcfg2 helps system administrators produce a consistent, reproducible, and verifiable description of their environment, and offers @@ -138,15 +228,14 @@ deployment strategies. This package includes the Bcfg2 server software. %package server-cherrypy -Version: 1.3.1 Summary: Bcfg2 Server - CherryPy backend %if 0%{?suse_version} Group: System/Management %else -Group: System Tools +Group: System Environment/Daemons %endif -Requires: bcfg2 = %{version} -Requires: bcfg2-server = %{version} +Requires: bcfg2 = %{version}-%{release} +Requires: bcfg2-server = %{version}-%{release} # cherrypy 3.3 actually doesn't exist yet, but 3.2 has bugs that # prevent it from working: @@ -181,35 +270,67 @@ deployment strategies. This package includes the Bcfg2 CherryPy server backend. -%package doc -Summary: Configuration management system documentation +%package web +Summary: Bcfg2 Web Reporting Interface + %if 0%{?suse_version} -Group: Documentation/HTML +Group: System/Management +Requires: python-django >= 1.2 +Requires: python-django-south >= 0.7 %else -Group: Documentation +Group: System Tools +Requires: Django >= 1.2 +Requires: Django-south >= 0.7 +Requires: bcfg2-server %endif -%if 0%{?suse_version} -BuildRequires: python-M2Crypto -BuildRequires: python-Genshi -BuildRequires: python-gamin -BuildRequires: python-pyinotify -BuildRequires: python-python-daemon -BuildRequires: python-CherryPy >= 3 +Requires: httpd +%if "%{_vendor}" == "redhat" +Requires: mod_wsgi +%global apache_conf %{_sysconfdir}/httpd %else -BuildRequires: m2crypto -BuildRequires: python-genshi -BuildRequires: gamin-python -BuildRequires: python-inotify -BuildRequires: python-daemon +Requires: apache2-mod_wsgi +%global apache_conf %{_sysconfdir}/apache2 %endif -%if "%{_vendor}" == "redhat" && 0%{?rhel} < 6 && 0%{?fedora} == 0 -BuildRequires: python-ssl + +%description web +Bcfg2 helps system administrators produce a consistent, reproducible, +and verifiable description of their environment, and offers +visualization and reporting tools to aid in day-to-day administrative +tasks. It is the fifth generation of configuration management tools +developed in the Mathematics and Computer Science Division of Argonne +National Laboratory. + +It is based on an operational model in which the specification can be +used to validate and optionally change the state of clients, but in a +feature unique to bcfg2 the client's response to the specification can +also be used to assess the completeness of the specification. Using +this feature, bcfg2 provides an objective measure of how good a job an +administrator has done in specifying the configuration of client +systems. Bcfg2 is therefore built to help administrators construct an +accurate, comprehensive specification. + +Bcfg2 has been designed from the ground up to support gentle +reconciliation between the specification and current client states. It +is designed to gracefully cope with manual system modifications. + +Finally, due to the rapid pace of updates on modern networks, client +systems are constantly changing; if required in your environment, +Bcfg2 can enable the construction of complex change management and +deployment strategies. + +This package includes the Bcfg2 reports web frontend. + + +%package doc +Summary: Documentation for Bcfg2 +%if 0%{?suse_version} +Group: Documentation/HTML %else -BuildRequires: python-cherrypy >= 3 -BuildRequires: python-mock +Group: Documentation %endif + %description doc Bcfg2 helps system administrators produce a consistent, reproducible, and verifiable description of their environment, and offers @@ -238,26 +359,13 @@ deployment strategies. This package includes the Bcfg2 documentation. -%package web -Version: 1.3.1 -Summary: Bcfg2 Web Reporting Interface -%if 0%{?suse_version} -Group: System/Management -Requires: httpd,python-django >= 1.2,python-django-south >= 0.7 -%else -Group: System Tools -Requires: httpd,Django >= 1.2,Django-south >= 0.7 -%endif -Requires: bcfg2-server -%if "%{_vendor}" == "redhat" -Requires: mod_wsgi -%global apache_conf %{_sysconfdir}/httpd -%else -Requires: apache2-mod_wsgi -%global apache_conf %{_sysconfdir}/apache2 -%endif -%description web +%package examples +Summary: Examples for Bcfg2 +Group: Documentation + + +%description examples Bcfg2 helps system administrators produce a consistent, reproducible, and verifiable description of their environment, and offers visualization and reporting tools to aid in day-to-day administrative @@ -283,199 +391,387 @@ systems are constantly changing; if required in your environment, Bcfg2 can enable the construction of complex change management and deployment strategies. -This package includes the Bcfg2 reports web frontend. +This package includes the examples files for Bcfg2. + %prep -%setup -q -n %{name}-%{version} +%setup -q -n %{name}-%{version}%{?_pre_rc} -%build -%{__python}%{pythonversion} setup.py build +# The pylint and pep8 unit tests fail on RH-derivative distros +%if "%{_vendor}" == "redhat" +mv testsuite/Testsrc/test_code_checks.py \ + testsuite/Testsrc/test_code_checks.py.disable_unit_tests +awk ' + BEGIN {line=0} + /class Test(Pylint|PEP8)/ {line=FNR+1} + FNR==line {sub("True","False")} + {print $0} + ' testsuite/Testsrc/test_code_checks.py.disable_unit_tests \ + > testsuite/Testsrc/test_code_checks.py +%endif + +# Fixup some paths +%{__perl} -pi -e 's@/etc/default@%{_sysconfdir}/sysconfig@g' tools/bcfg2-cron + +%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.daily +%{__perl} -pi -e 's@/usr/lib/bcfg2@%{_libexecdir}@g' debian/bcfg2.cron.hourly -%{?pythonpath: export PYTHONPATH="%{pythonpath}"} -%{__python}%{pythonversion} setup.py build_sphinx +# Get rid of extraneous shebangs +for f in `find src/lib -name \*.py` +do + sed -i -e '/^#!/,1d' $f +done sed -i "s/apache2/httpd/g" misc/apache/bcfg2.conf + +%build +%{__python} setup.py build +%{?pythonpath: PYTHONPATH="%{pythonpath}"} \ + %{__python} setup.py build_sphinx + + %install +%if 0%{?rhel} == 5 || 0%{?suse_version} +# EL5 and OpenSUSE require the buildroot to be cleaned manually rm -rf %{buildroot} -%{__python}%{pythonversion} setup.py install --root=%{buildroot} --record=INSTALLED_FILES --prefix=/usr -%{__install} -d %{buildroot}%{_bindir} -%{__install} -d %{buildroot}%{_sbindir} -%{__install} -d %{buildroot}%{_initrddir} -%{__install} -d %{buildroot}%{_sysconfdir}/default -%{__install} -d %{buildroot}%{_sysconfdir}/cron.daily -%{__install} -d %{buildroot}%{_sysconfdir}/cron.hourly -%{__install} -d %{buildroot}%{_prefix}/lib/bcfg2 -mkdir -p %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version} -mkdir -p %{buildroot}%{_defaultdocdir}/bcfg2-server-%{version} +%endif + +%{__python} setup.py install -O1 --skip-build --root=%{buildroot} --prefix=/usr +install -d %{buildroot}%{_bindir} +install -d %{buildroot}%{_sbindir} +install -d %{buildroot}%{_initrddir} +install -d %{buildroot}%{_sysconfdir}/cron.daily +install -d %{buildroot}%{_sysconfdir}/cron.hourly +install -d %{buildroot}%{_sysconfdir}/sysconfig +install -d %{buildroot}%{_libexecdir} +install -d %{buildroot}%{_localstatedir}/cache/%{name} +install -d %{buildroot}%{_localstatedir}/lib/%{name} %if 0%{?suse_version} -%{__install} -d %{buildroot}/var/adm/fillup-templates -%endif - -%{__mv} %{buildroot}%{_bindir}/bcfg2* %{buildroot}%{_sbindir} -%{__install} -m 755 debian/bcfg2.init %{buildroot}%{_initrddir}/bcfg2 -%{__install} -m 755 debian/bcfg2-server.init %{buildroot}%{_initrddir}/bcfg2-server -%{__install} -m 755 debian/bcfg2-server.bcfg2-report-collector.init %{buildroot}%{_initrddir}/bcfg2-report-collector -%{__install} -m 755 debian/bcfg2.default %{buildroot}%{_sysconfdir}/default/bcfg2 -%{__install} -m 755 debian/bcfg2-server.default %{buildroot}%{_sysconfdir}/default/bcfg2-server -%{__install} -m 755 debian/bcfg2.cron.daily %{buildroot}%{_sysconfdir}/cron.daily/bcfg2 -%{__install} -m 755 debian/bcfg2.cron.hourly %{buildroot}%{_sysconfdir}/cron.hourly/bcfg2 -%{__install} -m 755 tools/bcfg2-cron %{buildroot}%{_prefix}/lib/bcfg2/bcfg2-cron +install -d %{buildroot}/var/adm/fillup-templates +%endif + +mv %{buildroot}%{_bindir}/bcfg2* %{buildroot}%{_sbindir} + +%if 0%{?fedora} < 16 +# Install SysV init scripts for everyone but new Fedoras +install -m 755 redhat/scripts/bcfg2.init \ + %{buildroot}%{_initrddir}/bcfg2 +install -m 755 redhat/scripts/bcfg2-server.init \ + %{buildroot}%{_initrddir}/bcfg2-server +install -m 755 redhat/scripts/bcfg2-report-collector.init \ + %{buildroot}%{_initrddir}/bcfg2-report-collector +%endif +install -m 755 debian/bcfg2.cron.daily \ + %{buildroot}%{_sysconfdir}/cron.daily/bcfg2 +install -m 755 debian/bcfg2.cron.hourly \ + %{buildroot}%{_sysconfdir}/cron.hourly/bcfg2 +install -m 755 tools/bcfg2-cron \ + %{buildroot}%{_libexecdir}/bcfg2-cron + +install -m 644 debian/bcfg2.default \ + %{buildroot}%{_sysconfdir}/sysconfig/bcfg2 +install -m 644 debian/bcfg2-server.default \ + %{buildroot}%{_sysconfdir}/sysconfig/bcfg2-server %if 0%{?suse_version} -%{__install} -m 755 debian/bcfg2.default %{buildroot}/var/adm/fillup-templates/sysconfig.bcfg2 -%{__install} -m 755 debian/bcfg2-server.default %{buildroot}/var/adm/fillup-templates/sysconfig.bcfg2-server +install -m 755 debian/bcfg2.default \ + %{buildroot}/var/adm/fillup-templates/sysconfig.bcfg2 +install -m 755 debian/bcfg2-server.default \ + %{buildroot}/var/adm/fillup-templates/sysconfig.bcfg2-server ln -s %{_initrddir}/bcfg2 %{buildroot}%{_sbindir}/rcbcfg2 ln -s %{_initrddir}/bcfg2-server %{buildroot}%{_sbindir}/rcbcfg2-server %endif -cp -r tools/* %{buildroot}%{_defaultdocdir}/bcfg2-server-%{version} -cp -r build/sphinx/html/* %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version} +touch %{buildroot}%{_sysconfdir}/%{name}.{cert,conf,key} -%{__install} -d %{buildroot}%{apache_conf}/conf.d -%{__install} -m 644 misc/apache/bcfg2.conf %{buildroot}%{apache_conf}/conf.d/wsgi_bcfg2.conf +# systemd +install -d %{buildroot}%{_unitdir} +install -p -m 644 redhat/systemd/%{name}.service \ + %{buildroot}%{_unitdir}/%{name}.service +install -p -m 644 redhat/systemd/%{name}-server.service \ + %{buildroot}%{_unitdir}/%{name}-server.service -%{__mkdir_p} %{buildroot}%{_localstatedir}/cache/%{name} -%{__mkdir_p} %{buildroot}%{_localstatedir}/lib/%{name} +# Webserver +install -d %{buildroot}%{apache_conf}/conf.d +install -p -m 644 misc/apache/bcfg2.conf \ + %{buildroot}%{apache_conf}/conf.d/wsgi_bcfg2.conf -# mandriva and RHEL 5 cannot handle %ghost without the file existing, +# mandriva cannot handle %ghost without the file existing, # so let's touch a bunch of empty config files -touch %{buildroot}%{_sysconfdir}/bcfg2.conf \ - %{buildroot}%{_sysconfdir}/bcfg2-web.conf +touch %{buildroot}%{_sysconfdir}/bcfg2.conf +%if 0%{?rhel} == 5 +# Required for EL5 %clean -[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} || exit 2 +rm -rf %{buildroot} +%endif + + +%if 0%{?rhel} != 5 +# EL5 lacks python-mock, so test suite is disabled +%check +# Downloads not allowed in koji; fix .xsd urls to point to local files +sed -i "s@schema_url = .*\$@schema_url = 'file://`pwd`/`basename %{SOURCE1}`'@" \ + testsuite/Testschema/test_schema.py +sed "s@http://www.w3.org/2001/xml.xsd@file://$(pwd)/schemas/xml.xsd@" \ + %{SOURCE1} > `basename %{SOURCE1}` +%{__python} setup.py test +%endif + + +%post +%if 0%{?fedora} >= 18 + %systemd_post bcfg2.service +%else + if [ $1 -eq 1 ] ; then + # Initial installation + %if 0%{?suse_version} + %fillup_and_insserv -f bcfg2 + %else %if 0%{?fedora} >= 16 + /bin/systemctl daemon-reload >/dev/null 2>&1 || : + %else + /sbin/chkconfig --add bcfg2 + %endif %endif + fi +%endif + +%post server +%if 0%{?fedora} >= 18 + %systemd_post bcfg2-server.service +%else + if [ $1 -eq 1 ] ; then + # Initial installation + %if 0%{?suse_version} + %fillup_and_insserv -f bcfg2-server + %else %if 0%{?fedora} >= 16 + /bin/systemctl daemon-reload >/dev/null 2>&1 || : + %else + /sbin/chkconfig --add bcfg2-server + %endif %endif + fi +%endif + +%preun +%if 0%{?fedora} >= 18 + %systemd_preun bcfg2.service +%else + if [ $1 -eq 0 ]; then + # Package removal, not upgrade + %if 0%{?suse_version} + %stop_on_removal bcfg2 + %else %if 0%{?fedora} >= 16 + /bin/systemctl --no-reload disable bcfg2.service > /dev/null 2>&1 || : + /bin/systemctl stop bcfg2.service > /dev/null 2>&1 || : + %else + /sbin/service bcfg2 stop &>/dev/null || : + /sbin/chkconfig --del bcfg2 + %endif %endif + fi +%endif + +%preun server +%if 0%{?fedora} >= 18 + %systemd_preun bcfg2-server.service +%else + if [ $1 -eq 0 ]; then + # Package removal, not upgrade + %if 0%{?suse_version} + %stop_on_removal bcfg2-server + %stop_on_removal bcfg2-report-collector + %else %if 0%{?fedora} >= 16 + /bin/systemctl --no-reload disable bcfg2-server.service > /dev/null 2>&1 || : + /bin/systemctl stop bcfg2-server.service > /dev/null 2>&1 || : + %else + /sbin/service bcfg2-server stop &>/dev/null || : + /sbin/chkconfig --del bcfg2-server + %endif %endif + fi +%endif + +%postun +%if 0%{?fedora} >= 18 + %systemd_postun bcfg2.service +%else + %if 0%{?fedora} >= 16 + /bin/systemctl daemon-reload >/dev/null 2>&1 || : + %endif + if [ $1 -ge 1 ] ; then + # Package upgrade, not uninstall + %if 0%{?suse_version} + %insserv_cleanup + %else %if 0%{?fedora} >= 16 + /bin/systemctl try-restart bcfg2.service >/dev/null 2>&1 || : + %else + /sbin/service bcfg2 condrestart &>/dev/null || : + %endif %endif + fi +%endif + +%postun server +%if 0%{?fedora} >= 18 + %systemd_postun bcfg2-server.service +%else + %if 0%{?fedora} >= 16 + /bin/systemctl daemon-reload >/dev/null 2>&1 || : + %endif + if [ $1 -ge 1 ] ; then + # Package upgrade, not uninstall + %if 0%{?fedora} >= 16 + /bin/systemctl try-restart bcfg2-server.service >/dev/null 2>&1 || : + %else + /sbin/service bcfg2-server condrestart &>/dev/null || : + %endif + fi + %if 0%{?suse_version} + if [ $1 -eq 0 ]; then + # clean up on removal. + %insserv_cleanup + fi + %endif +%endif + +%if 0%{?fedora} || 0%{?rhel} +%triggerun -- bcfg2 < 1.2.1-1 +/usr/bin/systemd-sysv-convert --save bcfg2 >/dev/null 2>&1 || : +/bin/systemctl --no-reload enable bcfg2.service >/dev/null 2>&1 || : +/sbin/chkconfig --del bcfg2 >/dev/null 2>&1 || : +/bin/systemctl try-restart bcfg2.service >/dev/null 2>&1 || : + +%triggerun server -- bcfg2-server < 1.2.1-1 +/usr/bin/systemd-sysv-convert --save bcfg2-server >/dev/null 2>&1 || : +/bin/systemctl --no-reload enable bcfg2-server.service >/dev/null 2>&1 || : +/sbin/chkconfig --del bcfg2-server >/dev/null 2>&1 || : +/bin/systemctl try-restart bcfg2-server.service >/dev/null 2>&1 || : +%endif + %files +%if 0%{?rhel} == 5 || 0%{?suse_version} +# Required for EL5 and OpenSUSE %defattr(-,root,root,-) +%endif +%doc COPYRIGHT LICENSE README +%{_mandir}/man1/bcfg2.1* +%{_mandir}/man5/bcfg2.conf.5* +%ghost %attr(600,root,root) %config(noreplace,missingok) %{_sysconfdir}/bcfg2.cert +%ghost %attr(0600,root,root) %config(noreplace,missingok) %{_sysconfdir}/bcfg2.conf +%if 0%{?fedora} >= 16 + %config(noreplace) %{_unitdir}/%{name}.service +%else + %{_initrddir}/bcfg2 +%endif +%if 0%{?fedora} || 0%{?rhel} +%config(noreplace) %{_sysconfdir}/sysconfig/bcfg2 +%else +%config(noreplace) %{_sysconfdir}/default/bcfg2 +%endif +%{_sysconfdir}/cron.daily/bcfg2 +%{_sysconfdir}/cron.hourly/bcfg2 %{_sbindir}/bcfg2 +%{_libexecdir}/bcfg2-cron +%dir %{_localstatedir}/cache/%{name} +%{python_sitelib}/Bcfg2*.egg-info %dir %{python_sitelib}/Bcfg2 -%{python_sitelib}/Bcfg2/Compat.py* %{python_sitelib}/Bcfg2/__init__.py* +%{python_sitelib}/Bcfg2/Client +%{python_sitelib}/Bcfg2/Compat.py* %{python_sitelib}/Bcfg2/Logger.py* %{python_sitelib}/Bcfg2/Options.py* %{python_sitelib}/Bcfg2/Proxy.py* %{python_sitelib}/Bcfg2/Utils.py* %{python_sitelib}/Bcfg2/version.py* -%{python_sitelib}/Bcfg2/Client -%{_mandir}/man1/bcfg2.1* -%{_mandir}/man5/bcfg2.conf.5* -%{_initrddir}/bcfg2 -%config(noreplace) %{_sysconfdir}/default/bcfg2 -%{_sysconfdir}/cron.hourly/bcfg2 -%{_sysconfdir}/cron.daily/bcfg2 -%{_prefix}/lib/bcfg2/bcfg2-cron -%{_localstatedir}/cache/%{name} -%{_localstatedir}/lib/%{name} %if 0%{?suse_version} %{_sbindir}/rcbcfg2 %config(noreplace) /var/adm/fillup-templates/sysconfig.bcfg2 %endif -%ghost %config(noreplace,missingok) %attr(0600,root,root) %{_sysconfdir}/bcfg2.conf %files server +%if 0%{?rhel} == 5 || 0%{?suse_version} %defattr(-,root,root,-) -%{_initrddir}/bcfg2-server -%{_initrddir}/bcfg2-report-collector -%dir %{python_sitelib}/Bcfg2 +%endif +%ghost %attr(600,root,root) %config(noreplace) %{_sysconfdir}/bcfg2.key +%if 0%{?fedora} >= 16 + %config(noreplace) %{_unitdir}/%{name}-server.service +%else + %{_initrddir}/bcfg2-server + %{_initrddir}/bcfg2-report-collector +%endif +%config(noreplace) %{_sysconfdir}/sysconfig/bcfg2-server +%{_sbindir}/bcfg2-* +%dir %{_localstatedir}/lib/%{name} %{python_sitelib}/Bcfg2/Cache.py* %{python_sitelib}/Bcfg2/Encryption.py* %{python_sitelib}/Bcfg2/SSLServer.py* %{python_sitelib}/Bcfg2/Statistics.py* -%{python_sitelib}/Bcfg2/manage.py* %{python_sitelib}/Bcfg2/settings.py* %{python_sitelib}/Bcfg2/Server %{python_sitelib}/Bcfg2/Reporting +%{python_sitelib}/Bcfg2/manage.py* %exclude %{python_sitelib}/Bcfg2/Server/CherryPyCore.py -%{python_sitelib}/*egg-info - %dir %{_datadir}/bcfg2 -%{_datadir}/bcfg2/Hostbase %{_datadir}/bcfg2/schemas %{_datadir}/bcfg2/xsl-transforms -%config(noreplace) %{_sysconfdir}/default/bcfg2-server -%{_sbindir}/bcfg2-admin -%{_sbindir}/bcfg2-build-reports -%{_sbindir}/bcfg2-crypt -%{_sbindir}/bcfg2-info -%{_sbindir}/bcfg2-lint -%{_sbindir}/bcfg2-repo-validate -%{_sbindir}/bcfg2-reports -%{_sbindir}/bcfg2-report-collector -%{_sbindir}/bcfg2-server -%{_sbindir}/bcfg2-yum-helper -%{_sbindir}/bcfg2-test +%{_datadir}/bcfg2/Hostbase %if 0%{?suse_version} %{_sbindir}/rcbcfg2-server %config(noreplace) /var/adm/fillup-templates/sysconfig.bcfg2-server %endif %{_mandir}/man5/bcfg2-lint.conf.5* -%{_mandir}/man8/*.8* -%dir %{_prefix}/lib/bcfg2 -%ghost %config(noreplace,missingok) %attr(0600,root,root) %{_sysconfdir}/bcfg2.conf +%{_mandir}/man8/bcfg2*.8* -%doc %{_defaultdocdir}/bcfg2-server-%{version} +%doc tools/* %files server-cherrypy +%if 0%{?rhel} == 5 || 0%{?suse_version} %defattr(-,root,root,-) +%endif %{python_sitelib}/Bcfg2/Server/CherryPyCore.py -%files doc -%defattr(-,root,root,-) -%doc %{_defaultdocdir}/bcfg2-doc-%{version} - %files web +%if 0%{?rhel} == 5 || 0%{?suse_version} %defattr(-,root,root,-) +%endif %{_datadir}/bcfg2/reports.wsgi %{_datadir}/bcfg2/site_media -%dir %{apache_conf} -%dir %{apache_conf}/conf.d %config(noreplace) %{apache_conf}/conf.d/wsgi_bcfg2.conf -%ghost %config(noreplace,missingok) %attr(0640,root,apache) %{_sysconfdir}/bcfg2-web.conf - -%post server -# enable daemon on first install only (not on update). -if [ $1 -eq 1 ]; then -%if 0%{?suse_version} - %fillup_and_insserv -f bcfg2-server -%else - /sbin/chkconfig --add bcfg2-server -%endif -fi -%preun -%if 0%{?suse_version} -# stop on removal (not on update). -if [ $1 -eq 0 ]; then - %stop_on_removal bcfg2 -fi -%endif - -%preun server -%if 0%{?suse_version} -if [ $1 -eq 0 ]; then - %stop_on_removal bcfg2-server - %stop_on_removal bcfg2-report-collector -fi +%files doc +%if 0%{?rhel} == 5 || 0%{?suse_version} +%defattr(-,root,root,-) %endif +%doc build/sphinx/html/* -%postun -%if 0%{?suse_version} -if [ $1 -eq 0 ]; then - %insserv_cleanup -fi +%files examples +%if 0%{?rhel} == 5 || 0%{?suse_version} +%defattr(-,root,root,-) %endif +%doc examples/* -%postun server -%if 0%{?suse_version} -if [ $1 -eq 0 ]; then - # clean up on removal. - %insserv_cleanup -fi -%endif %changelog +* Sun Aug 04 2013 John Morris <john@zultron.com> - 1.3.2-2 +- Reconcile divergences with Fedora specfile, as requested by upstream + (equally large changes made in Fedora version to reconcile with + this file) +- Python macro cleanups +- Accommodations for OpenSUSE +- Macros for pre and rc releases +- %%check section +- Move BRs to top of file +- Rearrange lines to match Fedora +- Group: tag tweaks +- Startup/shutdown changes +- Separate examples package +- Remove %%{__install} macros; RH has backed away from those +- Add fedora systemd units, both f16 and f18 variants :P + - Changes to %%post* scripts +- Rearrange %%files sections + +* Mon Jul 01 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.2-1 +- New upstream release + * Thu Mar 21 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.1-1 - New upstream release diff --git a/osx/Makefile b/osx/Makefile index f25e71927..0fc47aa7b 100644 --- a/osx/Makefile +++ b/osx/Makefile @@ -29,9 +29,9 @@ SITELIBDIR = /Library/Python/${PYVERSION}/site-packages # an Info.plist file for packagemaker to look at for package creation # and substitute the version strings. Major/Minor versions can only be # integers (e.g. "1" and "00" for bcfg2 version 1.0.0. -BCFGVER = 1.3.1 +BCFGVER = 1.3.2 MAJOR = 1 -MINOR = 31 +MINOR = 32 default: clean client diff --git a/osx/macports/Portfile b/osx/macports/Portfile index 45cf3dd2b..11c1d1908 100644 --- a/osx/macports/Portfile +++ b/osx/macports/Portfile @@ -5,7 +5,7 @@ PortSystem 1.0 PortGroup python26 1.0 name bcfg2 -version 1.3.1 +version 1.3.2 categories sysutils python maintainers gmail.com:sol.jerome license BSD diff --git a/redhat/VERSION b/redhat/VERSION index 3a3cd8cc8..1892b9267 100644 --- a/redhat/VERSION +++ b/redhat/VERSION @@ -1 +1 @@ -1.3.1 +1.3.2 diff --git a/redhat/bcfg2.spec.in b/redhat/bcfg2.spec.in index b1cd0d097..62097be93 100644 --- a/redhat/bcfg2.spec.in +++ b/redhat/bcfg2.spec.in @@ -262,6 +262,9 @@ fi %doc %{_defaultdocdir}/bcfg2-doc-%{version} %changelog +* Mon Jul 01 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.2-1 +- New upstream release + * Thu Mar 21 2013 Sol Jerome <sol.jerome@gmail.com> 1.3.1-1 - New upstream release diff --git a/redhat/scripts/bcfg2-report-collector.init b/redhat/scripts/bcfg2-report-collector.init index 43e875a6b..3c112006d 100755 --- a/redhat/scripts/bcfg2-report-collector.init +++ b/redhat/scripts/bcfg2-report-collector.init @@ -17,7 +17,7 @@ ### END INIT INFO # Include lsb functions -. /etc//init.d/functions +. /etc/init.d/functions # Commonly used stuff DAEMON=/usr/sbin/bcfg2-report-collector @@ -25,7 +25,7 @@ PIDFILE=/var/run/bcfg2-server/bcfg2-report-collector.pid PARAMS="-D $PIDFILE" # Include default startup configuration if exists -test -f "/etc/sysconfig/bcfg2-server" && . /etc/sysconfig/bcfg2-server +test -f "/etc/sysconfig/bcfg2-report-collector" && . /etc/sysconfig/bcfg2-report-collector # Exit if $DAEMON doesn't exist and is not executable test -x $DAEMON || exit 5 diff --git a/schemas/authorizedkeys.xsd b/schemas/authorizedkeys.xsd index 848f99bae..e59c964f6 100644 --- a/schemas/authorizedkeys.xsd +++ b/schemas/authorizedkeys.xsd @@ -42,6 +42,43 @@ </xsd:attribute> </xsd:complexType> + <xsd:complexType name="OptionContainerType"> + <xsd:annotation> + <xsd:documentation> + An **OptionContainerType** is a tag used to provide logic. + Child entries of an OptionContainerType tag only apply to + machines that match the condition specified -- either + membership in a group, or a matching client name. + :xml:attribute:`OptionContainerType:negate` can be set to + negate the sense of the match. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="0" maxOccurs="unbounded"> + <xsd:element name="Group" type="OptionContainerType"/> + <xsd:element name="Client" type="OptionContainerType"/> + <xsd:element name="Option" type="AuthorizedKeysOptionType"/> + <xsd:element name="Params" type="AuthorizedKeysParamsType"/> + </xsd:choice> + <xsd:attribute name='name' type='xsd:string'> + <xsd:annotation> + <xsd:documentation> + The name of the client or group to match on. Child entries + will only apply to this client or group (unless + :xml:attribute:`OptionContainerType:negate` is set). + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name='negate' type='xsd:boolean'> + <xsd:annotation> + <xsd:documentation> + Negate the sense of the match, so that child entries only + apply to a client if it is not a member of the given group + or does not have the given name. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + <xsd:complexType name="AllowType" mixed="true"> <xsd:annotation> <xsd:documentation> @@ -50,6 +87,9 @@ </xsd:documentation> </xsd:annotation> <xsd:choice minOccurs="0" maxOccurs="unbounded"> + <xsd:element name="Group" type="OptionContainerType"/> + <xsd:element name="Client" type="OptionContainerType"/> + <xsd:element name="Option" type="AuthorizedKeysOptionType"/> <xsd:element name="Params" type="AuthorizedKeysParamsType"/> </xsd:choice> <xsd:attribute name="from" type="xsd:string"> @@ -68,6 +108,15 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> + <xsd:attribute name="category" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + Use a public key specific to the group in the given + category, instead of the category specified in + ``bcfg2.conf``. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> <xsd:attribute name="host" type="xsd:string"> <xsd:annotation> <xsd:documentation> @@ -77,12 +126,36 @@ </xsd:attribute> </xsd:complexType> + <xsd:complexType name="AuthorizedKeysOptionType"> + <xsd:annotation> + <xsd:documentation> + Specify options for public key authentication and connection. + See :manpage:`sshd(8)` for details on allowable options. + </xsd:documentation> + </xsd:annotation> + <xsd:attribute name="name" type="xsd:string" use="required"> + <xsd:annotation> + <xsd:documentation> + The name of the sshd option. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="value" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + The value of the sshd option. This can be omitted for + options that take no value. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + <xsd:complexType name="AuthorizedKeysParamsType"> <xsd:annotation> <xsd:documentation> - Specify parameters for public key authentication and - connection. See :manpage:`sshd(8)` for details on allowable - parameters. + **Deprecated** way to specify options for public key + authentication and connection. See :manpage:`sshd(8)` for + details on allowable parameters. </xsd:documentation> </xsd:annotation> <xsd:anyAttribute processContents="lax"/> diff --git a/schemas/awstags.xsd b/schemas/awstags.xsd new file mode 100644 index 000000000..72be0366f --- /dev/null +++ b/schemas/awstags.xsd @@ -0,0 +1,73 @@ +<?xml version="1.0" encoding="utf-8"?> +<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema" xml:lang="en"> + + <xsd:annotation> + <xsd:documentation> + :ref:`AWSTags <server-plugins-connectors-awstags>` config + schema for bcfg2 + </xsd:documentation> + </xsd:annotation> + + <xsd:import namespace="http://www.w3.org/XML/1998/namespace" + schemaLocation="xml.xsd"/> + + <xsd:complexType name="TagType"> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Group" type="xsd:string" minOccurs="1" + maxOccurs="unbounded"> + <xsd:annotation> + <xsd:documentation> + The group to assign to machines with tags that match the + enclosing Tag expression. More than one group can be + specified. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + </xsd:choice> + <xsd:attribute name="name" type="xsd:string" use="required"> + <xsd:annotation> + <xsd:documentation> + The name pattern to match against. This is a regular + expression. It is not anchored. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute name="value" type="xsd:string"> + <xsd:annotation> + <xsd:documentation> + The value pattern to match against. This is a regular + expression. It is not anchored. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + </xsd:complexType> + + <xsd:complexType name="AWSTagsType"> + <xsd:annotation> + <xsd:documentation> + Top-level tag for ``AWSTags/config.xml``. + </xsd:documentation> + </xsd:annotation> + <xsd:choice minOccurs="1" maxOccurs="unbounded"> + <xsd:element name="Tag" type="TagType"> + <xsd:annotation> + <xsd:documentation> + Representation of a pattern that matches AWS tags. Tags can be + matched in one of two ways: + + * If only :xml:attribute:`TagType:name` is specified, then + AWSTags will only look for a tag with a matching name, and + the value of tags is ignored. + * If both :xml:attribute:`TagType:name` and + :xml:attribute:`TagType:value` are specified, a tag must + have a matching name *and* a matching value. + </xsd:documentation> + </xsd:annotation> + </xsd:element> + <xsd:element name="AWSTags" type="AWSTagsType"/> + </xsd:choice> + <xsd:attribute ref="xml:base"/> + </xsd:complexType> + + <xsd:element name="AWSTags" type="AWSTagsType"/> +</xsd:schema> diff --git a/schemas/types.xsd b/schemas/types.xsd index 4e3dfd70f..17b7f05f0 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -122,7 +122,10 @@ <xsd:attribute type='ActionTimingEnum' name='timing'> <xsd:annotation> <xsd:documentation> - When the action is run. + When the action is run. Actions with "pre" timing are run + after important entries have been installed and before + bundle entries are installed. Actions with "post" timing + are run after bundle entries are installed. </xsd:documentation> </xsd:annotation> </xsd:attribute> @@ -130,9 +133,7 @@ <xsd:annotation> <xsd:documentation> If the action is always run, or is only run when a bundle - has been modified. Actions that run before bundle - installation ("pre" and "both") ignore the setting of - ``when`` and are always run regardless. + has been modified. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/solaris-ips/MANIFEST.bcfg2-server.header b/solaris-ips/MANIFEST.bcfg2-server.header index efa11181f..382595338 100644 --- a/solaris-ips/MANIFEST.bcfg2-server.header +++ b/solaris-ips/MANIFEST.bcfg2-server.header @@ -1,5 +1,4 @@ license ../../LICENSE license=simplified_bsd set name=description value="Configuration management server" set name=pkg.summary value="Configuration management server" -set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.3.1" - +set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.3.2" diff --git a/solaris-ips/MANIFEST.bcfg2.header b/solaris-ips/MANIFEST.bcfg2.header index 8358aafca..2896b94ed 100644 --- a/solaris-ips/MANIFEST.bcfg2.header +++ b/solaris-ips/MANIFEST.bcfg2.header @@ -1,6 +1,5 @@ license ../../LICENSE license=simplified_bsd set name=description value="Configuration management client" set name=pkg.summary value="Configuration management client" -set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.3.1" - +set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.3.2" file usr/bin/bcfg2 group=bin mode=0755 owner=root path=usr/bin/bcfg2 diff --git a/solaris-ips/Makefile b/solaris-ips/Makefile index 343150dc5..6d55881dc 100644 --- a/solaris-ips/Makefile +++ b/solaris-ips/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/gmake -VERS=1.2.4-1 +VERS=1.3.2-1 PYVERSION := $(shell python -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris-ips/pkginfo.bcfg2 b/solaris-ips/pkginfo.bcfg2 index 90c628c53..47fc96244 100644 --- a/solaris-ips/pkginfo.bcfg2 +++ b/solaris-ips/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.2.4" +VERSION="1.3.2" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris-ips/pkginfo.bcfg2-server b/solaris-ips/pkginfo.bcfg2-server index 0e865522c..c9dd0c45b 100644 --- a/solaris-ips/pkginfo.bcfg2-server +++ b/solaris-ips/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.2.4" +VERSION="1.3.2" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/Makefile b/solaris/Makefile index fd2c254bb..e0c005f88 100644 --- a/solaris/Makefile +++ b/solaris/Makefile @@ -1,7 +1,7 @@ #!/usr/sfw/bin/gmake PYTHON="/usr/local/bin/python" -VERS=1.3.1-1 +VERS=1.3.2-1 PYVERSION := $(shell $(PYTHON) -c "import sys; print sys.version[0:3]") default: clean package diff --git a/solaris/pkginfo.bcfg2 b/solaris/pkginfo.bcfg2 index 2bf3abaf5..47fc96244 100644 --- a/solaris/pkginfo.bcfg2 +++ b/solaris/pkginfo.bcfg2 @@ -1,7 +1,7 @@ PKG="SCbcfg2" NAME="bcfg2" ARCH="sparc" -VERSION="1.3.1" +VERSION="1.3.2" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/solaris/pkginfo.bcfg2-server b/solaris/pkginfo.bcfg2-server index 4425220c2..c9dd0c45b 100644 --- a/solaris/pkginfo.bcfg2-server +++ b/solaris/pkginfo.bcfg2-server @@ -1,7 +1,7 @@ PKG="SCbcfg2-server" NAME="bcfg2-server" ARCH="sparc" -VERSION="1.3.1" +VERSION="1.3.2" CATEGORY="application" VENDOR="Argonne National Labratory" EMAIL="bcfg-dev@mcs.anl.gov" diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index 3254da9e9..ad718749e 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -1,6 +1,7 @@ """ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ +import copy import time import fnmatch import logging @@ -328,11 +329,13 @@ class Frame(object): if bundle.tag != 'Bundle': continue bmodified = len([item for item in bundle - if item in self.whitelist]) + if item in self.whitelist or + item in self.modified]) actions = [a for a in bundle.findall('./Action') if (a.get('timing') != 'post' and (bmodified or a.get('when') == 'always'))] - # now we process all "always actions" + # now we process all "pre" and "both" actions that are either + # always or the bundle has been modified if self.setup['interactive']: self.promptFilter(iprompt, actions) self.DispatchInstallCalls(actions) @@ -522,7 +525,7 @@ class Frame(object): container = Bcfg2.Client.XML.SubElement(stats, ename) for item in data: item.set('qtext', '') - container.append(item) + container.append(copy.deepcopy(item)) item.text = None timeinfo = Bcfg2.Client.XML.Element("OpStamps") diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index ac874c94c..4833f3f68 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -89,7 +89,7 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): if bootstatus is not None: if bootstatus == 'on': # make sure service is enabled on boot - bootcmd = '/sbin/chkconfig %s %s --level 0123456' % \ + bootcmd = '/sbin/chkconfig %s %s' % \ (entry.get('name'), bootstatus) elif bootstatus == 'off': # make sure service is disabled on boot @@ -116,8 +116,8 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): def FindExtra(self): """Locate extra chkconfig Services.""" allsrv = [line.split()[0] - for line in self.cmd.run("/sbin/chkconfig", - "--list").stdout.splitlines() + for line in + self.cmd.run("/sbin/chkconfig --list").stdout.splitlines() if ":on" in line] self.logger.debug('Found active services:') self.logger.debug(allsrv) diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 761c51db7..b544e44d4 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -108,7 +108,7 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service entry.""" self.logger.info("Installing Service %s" % (entry.get('name'))) - bootstatus = entry.get('bootstatus') + bootstatus = self.get_bootstatus(entry) # check if init script exists try: diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 9f47fb53a..b1bde1057 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -53,6 +53,10 @@ class POSIXFile(POSIXTool): def verify(self, entry, modlist): ondisk = self._exists(entry) tempdata, is_binary = self._get_data(entry) + if isinstance(tempdata, str) and str != unicode: + tempdatasize = len(tempdata) + else: + tempdatasize = len(tempdata.encode(self.setup['encoding'])) different = False content = None @@ -61,7 +65,7 @@ class POSIXFile(POSIXTool): # they're clearly different different = True content = "" - elif len(tempdata) != ondisk[stat.ST_SIZE]: + elif tempdatasize != ondisk[stat.ST_SIZE]: # next, see if the size of the target file is different # from the size of the desired content different = True @@ -72,6 +76,9 @@ class POSIXFile(POSIXTool): # for everything else try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), + encoding=self.setup['encoding']).read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) @@ -89,7 +96,7 @@ class POSIXFile(POSIXTool): def _write_tmpfile(self, entry): """ Write the file data to a temp file """ - filedata, _ = self._get_data(entry) + filedata = self._get_data(entry)[0] # get a temp file to write to that is in the same directory as # the existing file in order to preserve any permissions # protections on that directory, and also to avoid issues with @@ -105,7 +112,11 @@ class POSIXFile(POSIXTool): (os.path.dirname(entry.get('name')), err)) return False try: - os.fdopen(newfd, 'w').write(filedata) + if isinstance(filedata, str) and str != unicode: + os.fdopen(newfd, 'w').write(filedata) + else: + os.fdopen(newfd, 'wb').write( + filedata.encode(self.setup['encoding'])) except (OSError, IOError): err = sys.exc_info()[1] self.logger.error("POSIX: Failed to open temp file %s for writing " diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/base.py b/src/lib/Bcfg2/Client/Tools/POSIX/base.py index 16fe0acb5..85da3576b 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/base.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/base.py @@ -232,6 +232,11 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): else: defacl = None + if not acls: + self.logger.debug("POSIX: Removed ACLs from %s" % + entry.get("name")) + return True + for aclkey, perms in acls.items(): atype, scope, qualifier = aclkey if atype == "default": @@ -525,7 +530,8 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): if entry.get("secontext") == "__default__": try: wanted_secontext = \ - selinux.matchpathcon(path, 0)[1].split(":")[2] + selinux.matchpathcon( + path, ondisk[stat.ST_MODE])[1].split(":")[2] except OSError: errors.append("%s has no default SELinux context" % entry.get("name")) @@ -686,7 +692,7 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): """ os.makedirs helpfully creates all parent directories for us, but it sets permissions according to umask, which is probably wrong. we need to find out which directories were - created and set permissions on those + created and try to set permissions on those (http://trac.mcs.anl.gov/projects/bcfg2/ticket/1125 and http://trac.mcs.anl.gov/projects/bcfg2/ticket/1134) """ created = [] @@ -706,22 +712,17 @@ class POSIXTool(Bcfg2.Client.Tools.Tool): (path, err)) rv = False - # we need to make sure that we give +x to everyone who needs - # it. E.g., if the file that's been distributed is 0600, we - # can't make the parent directories 0600 also; that'd be - # pretty useless. They need to be 0700. + # set auto-created directories to mode 755 and use best effort for + # permissions. If you need something else, you should specify it in + # your config. tmpentry = copy.deepcopy(entry) - newmode = int(entry.get('mode'), 8) - for i in range(0, 3): - if newmode & (6 * pow(8, i)): - newmode |= 1 * pow(8, i) - tmpentry.set('mode', oct_mode(newmode)) + tmpentry.set('mode', '0755') for acl in tmpentry.findall('ACL'): acl.set('perms', oct_mode(self._norm_acl_perms(acl.get('perms')) | ACL_MAP['x'])) for cpath in created: - rv &= self._set_perms(tmpentry, path=cpath) + self._set_perms(tmpentry, path=cpath) return rv diff --git a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py index 8226392f9..bf23aca6b 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIXUsers.py +++ b/src/lib/Bcfg2/Client/Tools/POSIXUsers.py @@ -189,14 +189,18 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): else: for attr, idx in self.attr_mapping[entry.tag].items(): val = str(self.existing[entry.tag][entry.get("name")][idx]) - entry.set("current_%s" % attr, val) + entry.set("current_%s" % + attr, val.decode(self.setup['encoding'])) if attr in ["uid", "gid"]: if entry.get(attr) is None: # no uid/gid specified, so we let the tool # automatically determine one -- i.e., it always # verifies continue - if val != entry.get(attr): + entval = entry.get(attr) + if not isinstance(entval, str): + entval = entval.encode('utf-8') + if val != entval: errors.append("%s for %s %s is incorrect. Current %s is " "%s, but should be %s" % (attr.title(), entry.tag, entry.get("name"), @@ -249,7 +253,6 @@ class POSIXUsers(Bcfg2.Client.Tools.Tool): if entry.get('gid'): cmd.extend(['-g', entry.get('gid')]) elif entry.tag == 'POSIXUser': - cmd.append('-m') if entry.get('uid'): cmd.extend(['-u', entry.get('uid')]) cmd.extend(['-g', entry.get('group')]) diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index 8e9626521..e0c913dcd 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -89,7 +89,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service entry.""" self.logger.info('Installing Service %s' % entry.get('name')) - bootstatus = entry.get('bootstatus') + bootstatus = self.get_bootstatus(entry) if bootstatus is not None: if bootstatus == 'on': # make sure service is enabled on boot diff --git a/src/lib/Bcfg2/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 11fe55bd6..703b8ff57 100644 --- a/src/lib/Bcfg2/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -594,13 +594,14 @@ class SvcTool(Tool): if not self.handlesEntry(entry): continue + estatus = entry.get('status') restart = entry.get("restart", "true").lower() - if (restart == "false" or + if (restart == "false" or estatus == 'ignore' or (restart == "interactive" and not self.setup['interactive'])): continue success = False - if entry.get('status') == 'on': + if estatus == 'on': if self.setup['servicemode'] == 'build': success = self.stop_service(entry) elif entry.get('name') not in self.restarted: diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 25603186e..6d1cb9d40 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -21,6 +21,9 @@ def prompt(msg): try: ans = input(msg) return ans in ['y', 'Y'] + except UnicodeEncodeError: + ans = input(msg.encode('utf-8')) + return ans in ['y', 'Y'] except EOFError: # handle ^C on rhel-based platforms raise SystemExit(1) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 243c4ed2a..673fb125c 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -319,6 +319,28 @@ def colon_split(c_string): return [] +def dict_split(c_string): + """ split an option string on commas, optionally surrounded by + whitespace and split the resulting items again on equals signs, + returning a dict """ + result = dict() + if c_string: + items = re.split(r'\s*,\s*', c_string) + for item in items: + if r'=' in item: + key, value = item.split(r'=', 1) + try: + result[key] = get_bool(value) + except ValueError: + try: + result[key] = get_int(value) + except ValueError: + result[key] = value + else: + result[item] = True + return result + + def get_bool(val): """ given a string value of a boolean configuration option, return an actual bool (True or False) """ @@ -651,6 +673,15 @@ DB_PORT = \ default='', cf=('database', 'port'), deprecated_cf=('statistics', 'database_port')) +DB_OPTIONS = \ + Option('Database options', + default=dict(), + cf=('database', 'options'), + cook=dict_split) +DB_SCHEMA = \ + Option('Database schema', + default='', + cf=('database', 'schema')) # Django options WEB_CFILE = \ @@ -1193,7 +1224,8 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, authentication=SERVER_AUTHENTICATION, perflog=LOG_PERFORMANCE, perflog_interval=PERFLOG_INTERVAL, - children=SERVER_CHILDREN) + children=SERVER_CHILDREN, + client_timeout=CLIENT_TIMEOUT) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, @@ -1233,9 +1265,9 @@ DRIVER_OPTIONS = \ yum_verify_fail_action=CLIENT_YUM_VERIFY_FAIL_ACTION, yum_verify_flags=CLIENT_YUM_VERIFY_FLAGS, posix_uid_whitelist=CLIENT_POSIX_UID_WHITELIST, - posix_gid_whitelist=CLIENT_POSIX_UID_WHITELIST, + posix_gid_whitelist=CLIENT_POSIX_GID_WHITELIST, posix_uid_blacklist=CLIENT_POSIX_UID_BLACKLIST, - posix_gid_blacklist=CLIENT_POSIX_UID_BLACKLIST) + posix_gid_blacklist=CLIENT_POSIX_GID_BLACKLIST) CLIENT_COMMON_OPTIONS = \ dict(extra=CLIENT_EXTRA_DISPLAY, @@ -1285,6 +1317,8 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, db_password=DB_PASSWORD, db_host=DB_HOST, db_port=DB_PORT, + db_options=DB_OPTIONS, + db_schema=DB_SCHEMA, time_zone=DJANGO_TIME_ZONE, django_debug=DJANGO_DEBUG, web_prefix=DJANGO_WEB_PREFIX) diff --git a/src/lib/Bcfg2/Proxy.py b/src/lib/Bcfg2/Proxy.py index f6db66a93..34080da6b 100644 --- a/src/lib/Bcfg2/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -104,7 +104,6 @@ class RetryMethod(xmlrpclib._Method): err = sys.exc_info()[1] msg = err except: - raise etype, err = sys.exc_info()[:2] msg = "Unknown failure: %s (%s)" % (err, etype.__name__) if msg: diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index 3d224432e..b42364d8d 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -20,10 +20,38 @@ from Bcfg2.Reporting.Transport.DirectStore import DirectStore from Bcfg2.Reporting.Storage import load_storage_from_config, \ StorageError, StorageImportError + class ReportingError(Exception): """Generic reporting exception""" pass + +class ReportingStoreThread(threading.Thread): + """Thread for calling the storage backend""" + def __init__(self, interaction, storage, group=None, target=None, + name=None, args=(), kwargs=None): + """Initialize the thread with a reference to the interaction + as well as the storage engine to use""" + threading.Thread.__init__(self, group, target, name, args, + kwargs or dict()) + self.interaction = interaction + self.storage = storage + self.logger = logging.getLogger('bcfg2-report-collector') + + def run(self): + """Call the database storage procedure (aka import)""" + try: + start = time.time() + self.storage.import_interaction(self.interaction) + self.logger.info("Imported interaction for %s in %ss" % + (self.interaction.get('hostname', '<unknown>'), + time.time() - start)) + except: + #TODO requeue? + self.logger.error("Unhandled exception in import thread %s" % + traceback.format_exc().splitlines()[-1]) + + class ReportingCollector(object): """The collecting process for reports""" @@ -77,12 +105,11 @@ class ReportingCollector(object): (self.storage.__class__.__name__, traceback.format_exc().splitlines()[-1])) - def run(self): """Startup the processing and go!""" self.terminate = threading.Event() atexit.register(self.shutdown) - self.context = daemon.DaemonContext() + self.context = daemon.DaemonContext(detach_process=True) if self.setup['daemon']: self.logger.debug("Daemonizing") @@ -103,15 +130,9 @@ class ReportingCollector(object): interaction = self.transport.fetch() if not interaction: continue - try: - start = time.time() - self.storage.import_interaction(interaction) - self.logger.info("Imported interaction for %s in %ss" % - (interaction.get('hostname', '<unknown>'), - time.time() - start)) - except: - #TODO requeue? - raise + + store_thread = ReportingStoreThread(interaction, self.storage) + store_thread.start() except (SystemExit, KeyboardInterrupt): self.logger.info("Shutting down") self.shutdown() diff --git a/src/lib/Bcfg2/Reporting/models.py b/src/lib/Bcfg2/Reporting/models.py index 598e1c6ec..fc9523067 100644 --- a/src/lib/Bcfg2/Reporting/models.py +++ b/src/lib/Bcfg2/Reporting/models.py @@ -88,7 +88,7 @@ class InteractionManager(models.Manager): Returns the ids of most recent interactions for clients as of a date. Arguments: - maxdate -- datetime object. Most recent date to pull. (dafault None) + maxdate -- datetime object. Most recent date to pull. (default None) """ from django.db import connection diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index 7f1fcba3b..0b2b7dd36 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -93,7 +93,7 @@ This is needed for Django versions less than 1.5 <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.3.1</span> + <span>Bcfg2 Version 1.3.2</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Bcfg2/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 316c2f86c..ab7e56f33 100644 --- a/src/lib/Bcfg2/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -5,7 +5,6 @@ better. """ import os import sys import socket -import select import signal import logging import ssl @@ -183,7 +182,6 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): Adds support for HTTP authentication. """ - logger = logging.getLogger("Bcfg2.SSLServer.XMLRPCRequestHandler") def authenticate(self): @@ -228,22 +226,22 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): return False return True - ### need to override do_POST here def do_POST(self): try: max_chunk_size = 10 * 1024 * 1024 size_remaining = int(self.headers["content-length"]) L = [] while size_remaining: - try: - select.select([self.rfile.fileno()], [], [], 3) - except select.error: - print("got select timeout") - raise chunk_size = min(size_remaining, max_chunk_size) - L.append(self.rfile.read(chunk_size).decode('utf-8')) + chunk = self.rfile.read(chunk_size).decode('utf-8') + if not chunk: + break + L.append(chunk) size_remaining -= len(L[-1]) data = ''.join(L) + if data is None: + return # response has been sent + response = self.server._marshaled_dispatch(self.client_address, data) if sys.hexversion >= 0x03000000: @@ -251,6 +249,7 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): except: # pylint: disable=W0702 try: self.send_response(500) + self.send_header("Content-length", "0") self.end_headers() except: (etype, msg) = sys.exc_info()[:2] @@ -306,14 +305,11 @@ class XMLRPCRequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler): def finish(self): # shut down the connection - if not self.wfile.closed: - try: - self.wfile.flush() - self.wfile.close() - except socket.error: - err = sys.exc_info()[1] - self.logger.warning("Error closing connection: %s" % err) - self.rfile.close() + try: + SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.finish(self) + except socket.error: + err = sys.exc_info()[1] + self.logger.warning("Error closing connection: %s" % err) class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, @@ -431,8 +427,6 @@ class XMLRPCServer(SocketServer.ThreadingMixIn, SSLServer, self.handle_request() except socket.timeout: pass - except select.error: - pass except: self.logger.error("Got unexpected error in handle_request", exc_info=1) diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index e3648a6d0..d7285284a 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -115,7 +115,6 @@ class Compare(Bcfg2.Server.Admin.Mode): return identical def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0: self.errExit("No argument specified.\n" "Please see bcfg2-admin compare help for usage.") diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 6175d8ed0..153d7bea6 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -20,6 +20,8 @@ from Bcfg2.Compat import input # pylint: disable=W0622 CONFIG = '''[server] repository = %s plugins = %s +# Uncomment the following to listen on all interfaces +#listen_all = true [statistics] sendmailpath = %s @@ -31,7 +33,7 @@ sendmailpath = %s # 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. #name = # Or path to database file if using sqlite3. -#<repository>/bcfg2.sqlite is default path if left empty +#<repository>/etc/bcfg2.sqlite is default path if left empty #user = # Not used with sqlite3. #password = @@ -78,7 +80,7 @@ CLIENTS = '''<Clients version="3.0"> ''' # Mapping of operating system names to groups -OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), +OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/CentOS', 'redhat'), ('SUSE/SLES', 'suse'), ('Mandrake', 'mandrake'), ('Debian', 'debian'), @@ -234,8 +236,9 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_server(self): """Ask for the server name.""" - newserver = safe_input("Input the server location [%s]: " % - self.data['server_uri']) + newserver = safe_input( + "Input the server location (the server listens on a single " + "interface by default) [%s]: " % self.data['server_uri']) if newserver != '': self.data['server_uri'] = newserver diff --git a/src/lib/Bcfg2/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index c2d279391..fcb240352 100644 --- a/src/lib/Bcfg2/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -27,7 +27,6 @@ class Snapshots(Bcfg2.Server.Admin.Mode): self.cfile = self.configfile def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0 or args[0] == '-h': print(self.__usage__) raise SystemExit(0) diff --git a/src/lib/Bcfg2/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py index 1d9d25f16..2cbd7eaf6 100644 --- a/src/lib/Bcfg2/Server/Admin/Viz.py +++ b/src/lib/Bcfg2/Server/Admin/Viz.py @@ -102,6 +102,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): dotpipe.stdin.write('\tcolor="lightblue";\n') dotpipe.stdin.write('\tBundle [ shape="septagon" ];\n') dotpipe.stdin.write('\tGroup [shape="ellipse"];\n') + dotpipe.stdin.write('\tGroup Category [shape="trapezium"];\n') dotpipe.stdin.write('\tProfile [style="bold", shape="ellipse"];\n') dotpipe.stdin.write('\tHblock [label="Host1|Host2|Host3", ' 'shape="record"];\n') diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index e69a92b64..93da767c7 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -31,7 +31,8 @@ class Core(BaseCore): daemon_args = dict(uid=self.setup['daemon_uid'], gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8)) + umask=int(self.setup['umask'], 8), + detach_process=True) if self.setup['daemon']: daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], acquire_timeout=5) @@ -109,7 +110,6 @@ class Core(BaseCore): keyfile=self.setup['key'], certfile=self.setup['cert'], register=False, - timeout=1, ca=self.setup['ca'], protocol=self.setup['protocol']) except: # pylint: disable=W0702 diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index ecd68e1e4..5ec1b5bce 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -200,6 +200,10 @@ class BaseCore(object): # load plugins Bcfg2.settings.read_config(repo=self.datastore) + # mapping of group name => plugin name to record where groups + # that are created by Connector plugins came from + self._dynamic_groups = dict() + #: Whether or not it's possible to use the Django database #: backend for plugins that have that capability self._database_available = False @@ -263,6 +267,20 @@ class BaseCore(object): #: metadata self.metadata_cache = Cache() + def expire_caches_by_type(self, base_cls, key=None): + """ Expire caches for all + :class:`Bcfg2.Server.Plugin.interfaces.Caching` plugins that + are instances of ``base_cls``. + + :param base_cls: The base plugin interface class to match (see + :mod:`Bcfg2.Server.Plugin.interfaces`) + :type base_cls: type + :param key: The cache key to expire + """ + for plugin in self.plugins_by_type(base_cls): + if isinstance(plugin, Bcfg2.Server.Plugin.Caching): + plugin.expire_cache(key) + def plugins_by_type(self, base_cls): """ Return a list of loaded plugins that match the passed type. @@ -289,11 +307,12 @@ class BaseCore(object): self.logger.debug("Performance logging thread starting") while not self.terminate.isSet(): self.terminate.wait(self.setup['perflog_interval']) - for name, stats in self.get_statistics(None).items(): - self.logger.info("Performance statistics: " - "%s min=%.06f, max=%.06f, average=%.06f, " - "count=%d" % ((name, ) + stats)) - self.logger.debug("Performance logging thread terminated") + if not self.terminate.isSet(): + for name, stats in self.get_statistics(None).items(): + self.logger.info("Performance statistics: " + "%s min=%.06f, max=%.06f, average=%.06f, " + "count=%d" % ((name, ) + stats)) + self.logger.info("Performance logging thread terminated") def _file_monitor_thread(self): """ The thread that runs the @@ -310,11 +329,12 @@ class BaseCore(object): else: if not self.fam.pending(): terminate.wait(15) + if self.fam.pending(): + self._update_vcs_revision() self.fam.handle_event_set(self.lock) except: continue - self._update_vcs_revision() - self.logger.debug("File monitor thread terminated") + self.logger.info("File monitor thread terminated") @track_statistics() def _update_vcs_revision(self): @@ -430,14 +450,14 @@ class BaseCore(object): def shutdown(self): """ Perform plugin and FAM shutdown tasks. """ - self.logger.debug("Shutting down core...") + self.logger.info("Shutting down core...") if not self.terminate.isSet(): self.terminate.set() self.fam.shutdown() - self.logger.debug("FAM shut down") + self.logger.info("FAM shut down") for plugin in list(self.plugins.values()): plugin.shutdown() - self.logger.debug("All plugins shut down") + self.logger.info("All plugins shut down") @property def metadata_cache_mode(self): @@ -628,10 +648,10 @@ class BaseCore(object): del entry.attrib['realname'] return ret except: - entry.set('name', oldname) self.logger.error("Failed binding entry %s:%s with altsrc %s" % - (entry.tag, entry.get('name'), - entry.get('altsrc'))) + (entry.tag, entry.get('realname'), + entry.get('name'))) + entry.set('name', oldname) self.logger.error("Falling back to %s:%s" % (entry.tag, entry.get('name'))) @@ -726,7 +746,27 @@ class BaseCore(object): if event.code2str() == 'deleted': return self.setup.reparse() - self.metadata_cache.expire() + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + + def block_for_fam_events(self, handle_events=False): + """ Block until all fam events have been handleed, optionally + handling events as well. (Setting ``handle_events=True`` is + useful for local server cores that don't spawn an event + handling thread.)""" + slept = 0 + log_interval = 3 + if handle_events: + self.fam.handle_events_in_interval(1) + slept += 1 + if self.setup['fam_blocking']: + time.sleep(1) + slept += 1 + while self.fam.pending() != 0: + time.sleep(1) + slept += 1 + if slept % log_interval == 0: + self.logger.debug("Sleeping to handle FAM events...") + self.logger.debug("Slept %s seconds while handling FAM events" % slept) def run(self): """ Run the server core. This calls :func:`_daemonize`, @@ -777,13 +817,9 @@ class BaseCore(object): self.shutdown() raise - if self.setup['fam_blocking']: - time.sleep(1) - while self.fam.pending() != 0: - time.sleep(1) - if self.debug_flag: self.set_debug(None, self.debug_flag) + self.block_for_fam_events() self._block() def _daemonize(self): @@ -842,17 +878,52 @@ class BaseCore(object): imd = self.metadata_cache.get(client_name, None) if not imd: self.logger.debug("Building metadata for %s" % client_name) - imd = self.metadata.get_initial_metadata(client_name) + try: + imd = self.metadata.get_initial_metadata(client_name) + except MetadataConsistencyError: + self.critical_error( + "Client metadata resolution error for %s: %s" % + (client_name, sys.exc_info()[1])) connectors = self.plugins_by_type(Connector) for conn in connectors: - grps = conn.get_additional_groups(imd) - self.metadata.merge_additional_groups(imd, grps) + groups = conn.get_additional_groups(imd) + groupnames = [] + for group in groups: + if hasattr(group, "name"): + groupname = group.name + if groupname in self._dynamic_groups: + if self._dynamic_groups[groupname] == conn.name: + self.metadata.groups[groupname] = group + else: + self.logger.warning( + "Refusing to clobber dynamic group %s " + "defined by %s" % + (self._dynamic_groups[groupname], + groupname)) + elif groupname in self.metadata.groups: + # not recorded as a dynamic group, but + # present in metadata.groups -- i.e., a + # static group + self.logger.warning( + "Refusing to clobber predefined group %s" % + groupname) + else: + self.metadata.groups[groupname] = group + self._dynamic_groups[groupname] = conn.name + groupnames.append(groupname) + else: + groupnames.append(group) + + self.metadata.merge_additional_groups(imd, groupnames) for conn in connectors: data = conn.get_additional_data(imd) self.metadata.merge_additional_data(imd, conn.name, data) imd.query.by_name = self.build_metadata if self.metadata_cache_mode in ['cautious', 'aggressive']: self.metadata_cache[client_name] = imd + else: + self.logger.debug("Using cached metadata object for %s" % + client_name) return imd def process_statistics(self, client_name, statistics): @@ -880,6 +951,7 @@ class BaseCore(object): state.get('state'))) self.client_run_hook("end_statistics", meta) + @track_statistics() def resolve_client(self, address, cleanup_cache=False, metadata=True): """ Given a client address, get the client hostname and optionally metadata. @@ -932,15 +1004,19 @@ class BaseCore(object): raise xmlrpclib.Fault(xmlrpclib.APPLICATION_ERROR, "Critical failure: %s" % message) + def _get_rmi_objects(self): + """ Get a dict (name: object) of all objects that may have RMI + calls. Currently, that includes all plugins and the FAM. """ + rv = {self.fam.__class__.__name__: self.fam} + rv.update(self.plugins) + return rv + def _get_rmi(self): """ Get a list of RMI calls exposed by plugins """ rmi = dict() - for pname, pinst in list(self.plugins.items()): + for pname, pinst in self._get_rmi_objects().items(): for mname in pinst.__rmi__: rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) - famname = self.fam.__class__.__name__ - for mname in self.fam.__rmi__: - rmi["%s.%s" % (famname, mname)] = getattr(self.fam, mname) return rmi def _resolve_exposed_method(self, method_name): @@ -1031,6 +1107,7 @@ class BaseCore(object): for plugin in self.plugins_by_type(Probing): for probe in plugin.GetProbes(metadata): resp.append(probe) + self.logger.debug("Sending probe list to %s" % client) return lxml.etree.tostring(resp, xml_declaration=False).decode('UTF-8') except: @@ -1056,7 +1133,7 @@ class BaseCore(object): # that's created for RecvProbeData doesn't get cached. # I.e., the next metadata object that's built, after probe # data is processed, is cached. - self.metadata_cache.expire(client) + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) try: xpdata = lxml.etree.XML(probedata.encode('utf-8'), parser=Bcfg2.Server.XMLParser) @@ -1253,9 +1330,14 @@ class BaseCore(object): self.logger.info("Core: debug = %s" % debug) levels = self._loglevels[self.debug_flag] for handler in logging.root.handlers: - level = levels.get(handler.name, levels['default']) - self.logger.debug("Setting %s log handler to %s" % - (handler.name, logging.getLevelName(level))) + try: + level = levels.get(handler.name, levels['default']) + self.logger.debug("Setting %s log handler to %s" % + (handler.name, logging.getLevelName(level))) + except AttributeError: + level = levels['default'] + self.logger.debug("Setting unknown log handler %s to %s" % + (handler, logging.getLevelName(level))) handler.setLevel(level) return self.debug_flag diff --git a/src/lib/Bcfg2/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 09f3f3d25..c537877a0 100644 --- a/src/lib/Bcfg2/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -47,6 +47,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): "Decisions/*.xml": "decisions.xsd", "Packages/sources.xml": "packages.xsd", "GroupPatterns/config.xml": "grouppatterns.xsd", + "AWSTags/config.xml": "awstags.xsd", "NagiosGen/config.xml": "nagiosgen.xsd", "FileProbes/config.xml": "fileprobes.xsd", "SSLCA/**/cert.xml": "sslca-cert.xsd", diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 81fba7092..3cc308b1c 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -2,14 +2,133 @@ :mod:`Bcfg2.Server.BuiltinCore` that uses the Python :mod:`multiprocessing` library to offload work to multiple child processes. As such, it requires Python 2.6+. + +The parent communicates with the children over +:class:`multiprocessing.Queue` objects via a +:class:`Bcfg2.Server.MultiprocessingCore.RPCQueue` object. + +A method being called via the RPCQueue must be exposed by the child by +decorating it with :func:`Bcfg2.Server.Core.exposed`. """ +import time import threading import lxml.etree import multiprocessing -from Bcfg2.Compat import Queue +import Bcfg2.Server.Plugin +from itertools import cycle +from Bcfg2.Cache import Cache +from Bcfg2.Compat import Queue, Empty, wraps from Bcfg2.Server.Core import BaseCore, exposed from Bcfg2.Server.BuiltinCore import Core as BuiltinCore +from multiprocessing.connection import Listener, Client + + +class DispatchingCache(Cache, Bcfg2.Server.Plugin.Debuggable): + """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates + cache expiration events to child nodes. """ + + #: The method to send over the pipe to expire the cache + method = "expire_metadata_cache" + + def __init__(self, *args, **kwargs): + self.rpc_q = kwargs.pop("queue") + Bcfg2.Server.Plugin.Debuggable.__init__(self) + Cache.__init__(self, *args, **kwargs) + + def expire(self, key=None): + self.rpc_q.publish(self.method, args=[key]) + Cache.expire(self, key=key) + + +class RPCQueue(Bcfg2.Server.Plugin.Debuggable): + """ An implementation of a :class:`multiprocessing.Queue` designed + for several additional use patterns: + + * Random-access reads, based on a key that identifies the data; + * Publish-subscribe, where a datum is sent to all hosts. + + The subscribers can deal with this as a normal Queue with no + special handling. + """ + poll_wait = 3.0 + + def __init__(self): + Bcfg2.Server.Plugin.Debuggable.__init__(self) + self._terminate = threading.Event() + self._queues = dict() + self._available_listeners = Queue() + self._blocking_listeners = [] + + def add_subscriber(self, name): + """ Add a subscriber to the queue. This returns the + :class:`multiprocessing.Queue` object that the subscriber + should read from. """ + self._queues[name] = multiprocessing.Queue() + return self._queues[name] + + def publish(self, method, args=None, kwargs=None): + """ Publish an RPC call to the queue for consumption by all + subscribers. """ + for queue in self._queues.values(): + queue.put((None, (method, args or [], kwargs or dict()))) + + def rpc(self, dest, method, args=None, kwargs=None): + """ Make an RPC call to the named subscriber, expecting a + response. This opens a + :class:`multiprocessing.connection.Listener` and passes the + Listener address to the child as part of the RPC call, so that + the child can connect to the Listener to submit its results. + + Listeners are reused when possible to minimize overhead. + """ + try: + listener = self._available_listeners.get_nowait() + self.logger.debug("Reusing existing RPC listener at %s" % + listener.address) + except Empty: + listener = Listener() + self.logger.debug("Created new RPC listener at %s" % + listener.address) + self._blocking_listeners.append(listener) + try: + self._queues[dest].put((listener.address, + (method, args or [], kwargs or dict()))) + conn = listener.accept() + self._blocking_listeners.remove(listener) + try: + while not self._terminate.is_set(): + if conn.poll(self.poll_wait): + return conn.recv() + finally: + conn.close() + finally: + self._available_listeners.put(listener) + + def close(self): + """ Close queues and connections. """ + self._terminate.set() + self.logger.debug("Closing RPC queues") + for name, queue in self._queues.items(): + self.logger.debug("Closing RPC queue to %s" % name) + queue.close() + + # close any listeners that are waiting for connections + self.logger.debug("Closing RPC connections") + for listener in self._blocking_listeners: + self.logger.debug("Closing RPC connection at %s" % + listener.address) + listener.close() + + self.logger.debug("Closing RPC listeners") + try: + while True: + listener = self._available_listeners.get_nowait() + self.logger.debug("Closing RPC listener at %s" % + listener.address) + listener.close() + except Empty: + pass class DualEvent(object): @@ -60,68 +179,153 @@ class ChildCore(BaseCore): those, though, if the pipe communication "protocol" were made more robust. """ - #: How long to wait while polling for new clients to build. This - #: doesn't affect the speed with which a client is built, but + #: How long to wait while polling for new RPC commands. This + #: doesn't affect the speed with which a command is processed, but #: setting it too high will result in longer shutdown times, since #: we only check for the termination event from the main process #: every ``poll_wait`` seconds. - poll_wait = 5.0 + poll_wait = 3.0 - def __init__(self, setup, pipe, terminate): + def __init__(self, name, setup, rpc_q, terminate): """ + :param name: The name of this child + :type name: string :param setup: A Bcfg2 options dict :type setup: Bcfg2.Options.OptionParser - :param pipe: The pipe to which client hostnames are added for - ChildCore objects to build configurations, and to - which client configurations are added after - having been built by ChildCore objects. - :type pipe: multiprocessing.Pipe + :param read_q: The queue the child will read from for RPC + communications from the parent process. + :type read_q: multiprocessing.Queue + :param write_q: The queue the child will write the results of + RPC calls to. + :type write_q: multiprocessing.Queue :param terminate: An event that flags ChildCore objects to shut themselves down. :type terminate: multiprocessing.Event """ BaseCore.__init__(self, setup) - #: The pipe to which client hostnames are added for ChildCore - #: objects to build configurations, and to which client - #: configurations are added after having been built by - #: ChildCore objects. - self.pipe = pipe + #: The name of this child + self.name = name #: The :class:`multiprocessing.Event` that will be monitored #: to determine when this child should shut down. self.terminate = terminate - def _daemonize(self): - return True + #: The queue used for RPC communication + self.rpc_q = rpc_q + + # override this setting so that the child doesn't try to write + # the pidfile + self.setup['daemon'] = False + + # ensure that the child doesn't start a perflog thread + self.perflog_thread = None + + self._rmi = dict() def _run(self): return True + def _daemonize(self): + return True + + def _dispatch(self, address, data): + """ Method dispatcher used for commands received from + the RPC queue. """ + if address is not None: + # if the key is None, then no response is expected. we + # make the return connection before dispatching the actual + # RPC call so that the parent is blocking for a connection + # as briefly as possible + self.logger.debug("Connecting to parent via %s" % address) + client = Client(address) + method, args, kwargs = data + func = None + rv = None + if "." in method: + if method in self._rmi: + func = self._rmi[method] + else: + self.logger.error("%s: Method %s does not exist" % (self.name, + method)) + elif not hasattr(self, method): + self.logger.error("%s: Method %s does not exist" % (self.name, + method)) + else: # method is not a plugin RMI, and exists + func = getattr(self, method) + if not func.exposed: + self.logger.error("%s: Method %s is not exposed" % (self.name, + method)) + func = None + if func is not None: + self.logger.debug("%s: Calling RPC method %s" % (self.name, + method)) + rv = func(*args, **kwargs) + if address is not None: + # if the key is None, then no response is expected + self.logger.debug("Returning data to parent via %s" % address) + client.send(rv) + def _block(self): - while not self.terminate.isSet(): + self._rmi = self._get_rmi() + while not self.terminate.is_set(): try: - if self.pipe.poll(self.poll_wait): - if not self.metadata.use_database: - # handle FAM events, in case (for instance) the - # client has just been added to clients.xml, or a - # profile has just been asserted. but really, you - # should be using the metadata database if you're - # using this core. - self.fam.handle_events_in_interval(0.1) - client = self.pipe.recv() - self.logger.debug("Building configuration for %s" % client) - config = \ - lxml.etree.tostring(self.BuildConfiguration(client)) - self.logger.debug("Returning configuration for %s to main " - "process" % client) - self.pipe.send(config) - self.logger.debug("Returned configuration for %s to main " - "process" % client) + address, data = self.rpc_q.get(timeout=self.poll_wait) + threadname = "-".join(str(i) for i in data) + rpc_thread = threading.Thread(name=threadname, + target=self._dispatch, + args=[address, data]) + rpc_thread.start() + except Empty: + pass except KeyboardInterrupt: break self.shutdown() + def shutdown(self): + BaseCore.shutdown(self) + self.logger.info("%s: Closing RPC command queue" % self.name) + self.rpc_q.close() + + while len(threading.enumerate()) > 1: + threads = [t for t in threading.enumerate() + if t != threading.current_thread()] + self.logger.info("%s: Waiting for %d thread(s): %s" % + (self.name, len(threads), + [t.name for t in threads])) + time.sleep(1) + self.logger.info("%s: All threads stopped" % self.name) + + def _get_rmi(self): + rmi = dict() + for pname, pinst in self._get_rmi_objects().items(): + for crmi in pinst.__child_rmi__: + if isinstance(crmi, tuple): + mname = crmi[1] + else: + mname = crmi + rmi["%s.%s" % (pname, mname)] = getattr(pinst, mname) + return rmi + + @exposed + def expire_metadata_cache(self, client=None): + """ Expire the metadata cache for a client """ + self.metadata_cache.expire(client) + + @exposed + def RecvProbeData(self, address, _): + """ Expire the probe cache for a client """ + self.expire_caches_by_type(Bcfg2.Server.Plugin.Probing, + key=self.resolve_client(address, + metadata=False)[0]) + + @exposed + def GetConfig(self, client): + """ Render the configuration for a client """ + self.logger.debug("%s: Building configuration for %s" % + (self.name, client)) + return lxml.etree.tostring(self.BuildConfiguration(client)) + class Core(BuiltinCore): """ A multiprocessing core that delegates building the actual @@ -140,65 +344,163 @@ class Core(BuiltinCore): if setup['children'] is None: setup['children'] = multiprocessing.cpu_count() - #: A dict of child name -> one end of the - #: :class:`multiprocessing.Pipe` object used to communicate - #: with that child. (The child is given the other end of the - #: Pipe.) - self.pipes = dict() - - #: A queue that keeps track of which children are available to - #: render a configuration. A child is popped from the queue - #: when it starts to render a config, then it's pushed back on - #: when it's done. This lets us use a blocking call to - #: :func:`Queue.Queue.get` when waiting for an available - #: child. - self.available_children = Queue(maxsize=self.setup['children']) - - # sigh. multiprocessing was added in py2.6, which is when the - # camelCase methods for threading objects were deprecated in - # favor of the Pythonic under_score methods. So - # multiprocessing.Event *only* has is_set(), while - # threading.Event has *both* isSet() and is_set(). In order - # to make the core work with Python 2.4+, and with both - # multiprocessing and threading Event objects, we just - # monkeypatch self.terminate to have isSet(). + #: The flag that indicates when to stop child threads and + #: processes self.terminate = DualEvent(threading_event=self.terminate) + #: A :class:`Bcfg2.Server.MultiprocessingCore.RPCQueue` object + #: used to send or publish commands to children. + self.rpc_q = RPCQueue() + + self.metadata_cache = DispatchingCache(queue=self.rpc_q) + + #: A list of children that will be cycled through + self._all_children = [] + + #: An iterator that each child will be taken from in sequence, + #: to provide a round-robin distribution of render requests + self.children = None + def _run(self): for cnum in range(self.setup['children']): name = "Child-%s" % cnum - (mainpipe, childpipe) = multiprocessing.Pipe() - self.pipes[name] = mainpipe + self.logger.debug("Starting child %s" % name) - childcore = ChildCore(self.setup, childpipe, self.terminate) + child_q = self.rpc_q.add_subscriber(name) + childcore = ChildCore(name, self.setup, child_q, self.terminate) child = multiprocessing.Process(target=childcore.run, name=name) child.start() self.logger.debug("Child %s started with PID %s" % (name, child.pid)) - self.available_children.put(name) + self._all_children.append(name) + self.logger.debug("Started %s children: %s" % (len(self._all_children), + self._all_children)) + self.children = cycle(self._all_children) return BuiltinCore._run(self) def shutdown(self): BuiltinCore.shutdown(self) - for child in multiprocessing.active_children(): - self.logger.debug("Shutting down child %s" % child.name) - child.join(self.shutdown_timeout) - if child.is_alive(): + self.logger.info("Closing RPC command queues") + self.rpc_q.close() + + def term_children(): + """ Terminate all remaining multiprocessing children. """ + for child in multiprocessing.active_children(): self.logger.error("Waited %s seconds to shut down %s, " "terminating" % (self.shutdown_timeout, child.name)) child.terminate() - else: - self.logger.debug("Child %s shut down" % child.name) - self.logger.debug("All children shut down") + + timer = threading.Timer(self.shutdown_timeout, term_children) + timer.start() + while len(multiprocessing.active_children()): + self.logger.info("Waiting for %s child(ren): %s" % + (len(multiprocessing.active_children()), + [c.name + for c in multiprocessing.active_children()])) + time.sleep(1) + timer.cancel() + self.logger.info("All children shut down") + + while len(threading.enumerate()) > 1: + threads = [t for t in threading.enumerate() + if t != threading.current_thread()] + self.logger.info("Waiting for %s thread(s): %s" % + (len(threads), [t.name for t in threads])) + time.sleep(1) + self.logger.info("Shutdown complete") + + def _get_rmi(self): + child_rmi = dict() + for pname, pinst in self._get_rmi_objects().items(): + for crmi in pinst.__child_rmi__: + if isinstance(crmi, tuple): + parentname, childname = crmi + else: + parentname = childname = crmi + child_rmi["%s.%s" % (pname, parentname)] = \ + "%s.%s" % (pname, childname) + + rmi = BuiltinCore._get_rmi(self) + for method in rmi.keys(): + if method in child_rmi: + rmi[method] = self._child_rmi_wrapper(method, + rmi[method], + child_rmi[method]) + return rmi + + def _child_rmi_wrapper(self, method, parent_rmi, child_rmi): + """ Returns a callable that dispatches a call to the given + child RMI to child processes, and calls the parent RMI locally + (i.e., in the parent process). """ + @wraps(parent_rmi) + def inner(*args, **kwargs): + """ Function that dispatches an RMI call to child + processes and to the (original) parent function. """ + self.logger.debug("Dispatching RMI call to %s to children: %s" % + (method, child_rmi)) + self.rpc_q.publish(child_rmi, args=args, kwargs=kwargs) + return parent_rmi(*args, **kwargs) + + return inner + + @exposed + def set_debug(self, address, debug): + self.rpc_q.set_debug(debug) + self.rpc_q.publish("set_debug", args=[address, debug]) + self.metadata_cache.set_debug(debug) + return BuiltinCore.set_debug(self, address, debug) + + @exposed + def RecvProbeData(self, address, probedata): + rv = BuiltinCore.RecvProbeData(self, address, probedata) + # we don't want the children to actually process probe data, + # so we don't send the data, just the fact that we got some. + self.rpc_q.publish("RecvProbeData", args=[address, None]) + return rv @exposed def GetConfig(self, address): client = self.resolve_client(address)[0] - childname = self.available_children.get() - self.logger.debug("Building configuration on child %s" % childname) - pipe = self.pipes[childname] - pipe.send(client) - config = pipe.recv() - self.available_children.put_nowait(childname) - return config + childname = self.children.next() + self.logger.debug("Building configuration for %s on %s" % (client, + childname)) + return self.rpc_q.rpc(childname, "GetConfig", args=[client]) + + @exposed + def get_statistics(self, address): + stats = dict() + + def _aggregate_statistics(newstats, prefix=None): + """ Aggregate a set of statistics from a child or parent + server core. This adds the statistics to the overall + statistics dict (optionally prepending a prefix, such as + "Child-1", to uniquely identify this set of statistics), + and aggregates it with the set of running totals that are + kept from all cores. """ + for statname, vals in newstats.items(): + if statname.startswith("ChildCore:"): + statname = statname[5:] + if prefix: + prettyname = "%s:%s" % (prefix, statname) + else: + prettyname = statname + stats[prettyname] = vals + totalname = "Total:%s" % statname + if totalname not in stats: + stats[totalname] = vals + else: + newmin = min(stats[totalname][0], vals[0]) + newmax = max(stats[totalname][1], vals[1]) + newcount = stats[totalname][3] + vals[3] + newmean = ((stats[totalname][2] * stats[totalname][3]) + + (vals[2] * vals[3])) / newcount + stats[totalname] = (newmin, newmax, newmean, newcount) + + stats = dict() + for childname in self._all_children: + _aggregate_statistics( + self.rpc_q.rpc(childname, "get_statistics", args=[address]), + prefix=childname) + _aggregate_statistics(BuiltinCore.get_statistics(self, address)) + return stats diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index c825a57b5..03feceb6f 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -12,6 +12,10 @@ class Debuggable(object): #: List of names of methods to be exposed as XML-RPC functions __rmi__ = ['toggle_debug', 'set_debug'] + #: How exposed XML-RPC functions should be dispatched to child + #: processes. + __child_rmi__ = __rmi__[:] + def __init__(self, name=None): """ :param name: The name of the logger object to get. If none is @@ -34,9 +38,6 @@ class Debuggable(object): :returns: bool - The new value of the debug flag """ self.debug_flag = debug - self.debug_log("%s: debug = %s" % (self.__class__.__name__, - self.debug_flag), - flag=True) return debug def toggle_debug(self): @@ -94,6 +95,20 @@ class Plugin(Debuggable): #: List of names of methods to be exposed as XML-RPC functions __rmi__ = Debuggable.__rmi__ + #: How exposed XML-RPC functions should be dispatched to child + #: processes, if :mod:`Bcfg2.Server.MultiprocessingCore` is in + #: use. Items ``__child_rmi__`` can either be strings (in which + #: case the same function is called on child processes as on the + #: parent) or 2-tuples, in which case the first element is the + #: name of the RPC function called on the parent process, and the + #: second element is the name of the function to call on child + #: processes. Functions that are not listed in the list will not + #: be dispatched to child processes, i.e., they will only be + #: called on the parent. A function must be listed in ``__rmi__`` + #: in order to be exposed; functions listed in ``_child_rmi__`` + #: but not ``__rmi__`` will be ignored. + __child_rmi__ = Debuggable.__child_rmi__ + def __init__(self, core, datastore): """ :param core: The Bcfg2.Server.Core initializing the plugin @@ -136,6 +151,8 @@ class Plugin(Debuggable): self.running = False def set_debug(self, debug): + self.debug_log("%s: debug = %s" % (self.name, self.debug_flag), + flag=True) for entry in self.Entries.values(): if isinstance(entry, Debuggable): entry.set_debug(debug) diff --git a/src/lib/Bcfg2/Server/Plugin/helpers.py b/src/lib/Bcfg2/Server/Plugin/helpers.py index 81dc1d736..d9e208746 100644 --- a/src/lib/Bcfg2/Server/Plugin/helpers.py +++ b/src/lib/Bcfg2/Server/Plugin/helpers.py @@ -555,16 +555,12 @@ class XMLFileBacked(FileBacked): xdata = self.xdata.getroottree() else: xdata = lxml.etree.parse(fname) - included = [el for el in xdata.findall('//' + xinclude)] - for el in included: + for el in xdata.findall('//' + xinclude): name = el.get("href") if name.startswith("/"): fpath = name else: - if fname: - rel = fname - else: - rel = self.name + rel = fname or self.name fpath = os.path.join(os.path.dirname(rel), name) # expand globs in xinclude, a bcfg2-specific extension @@ -579,12 +575,13 @@ class XMLFileBacked(FileBacked): parent = el.getparent() parent.remove(el) for extra in extras: - if extra != self.name and extra not in self.extras: - self.extras.append(extra) + if extra != self.name: lxml.etree.SubElement(parent, xinclude, href=extra) - self._follow_xincludes(fname=extra) - if extra not in self.extra_monitors: - self.add_monitor(extra) + if extra not in self.extras: + self.extras.append(extra) + self._follow_xincludes(fname=extra) + if extra not in self.extra_monitors: + self.add_monitor(extra) def Index(self): self.xdata = lxml.etree.XML(self.data, base_url=self.name, @@ -606,15 +603,16 @@ class XMLFileBacked(FileBacked): def add_monitor(self, fpath): """ Add a FAM monitor to a file that has been XIncluded. This - is only done if the constructor got both a ``fam`` object and - ``should_monitor`` set to True. + is only done if the constructor got a ``fam`` object, + regardless of whether ``should_monitor`` is set to True (i.e., + whether or not the base file is monitored). :param fpath: The full path to the file to monitor :type fpath: string :returns: None """ self.extra_monitors.append(fpath) - if self.fam and self.should_monitor: + if self.fam: self.fam.AddMonitor(fpath, self) def __iter__(self): diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index 222b94fe3..33f6d338c 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -220,10 +220,32 @@ class Connector(object): def get_additional_groups(self, metadata): # pylint: disable=W0613 """ Return a list of additional groups for the given client. + Each group can be either the name of a group (a string), or a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that defines other data besides just the name. Note that you + cannot return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + that clobbers a group defined by another plugin; the original + group will be used instead. For instance, assume the + following in ``Metadata/groups.xml``: + + .. code-block:: xml + + <Groups> + ... + <Group name="foo" public="false"/> + </Groups> + + You could not subsequently return a + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` object + with ``public=True``; a warning would be issued, and the + original (non-public) ``foo`` group would be used. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :return: list of strings + :return: list of strings or + :class:`Bcfg2.Server.Plugins.Metadata.MetadataGroup` + objects. """ return list() @@ -598,3 +620,22 @@ class ClientRunHooks(object): :returns: None """ pass + + +class Caching(object): + """ A plugin that caches more than just the data received from the + FAM. This presents a unified interface to clear the cache. """ + + def expire_cache(self, key=None): + """ Expire the cache associated with the given key. + + :param key: The key to expire the cache for. Because cache + implementations vary tremendously between plugins, + this could be any number of things, but generally + a hostname. It also may or may not be possible to + expire the cache for a single host; this interface + does not require any guarantee about that. + :type key: varies + :returns: None + """ + raise NotImplementedError diff --git a/src/lib/Bcfg2/Server/Plugins/AWSTags.py b/src/lib/Bcfg2/Server/Plugins/AWSTags.py new file mode 100644 index 000000000..147f37fbf --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/AWSTags.py @@ -0,0 +1,217 @@ +""" Query tags from AWS via boto, optionally setting group membership """ + +import os +import re +import sys +import Bcfg2.Server.Lint +import Bcfg2.Server.Plugin +from boto import connect_ec2 +from Bcfg2.Cache import Cache +from Bcfg2.Compat import ConfigParser + + +class NoInstanceFound(Exception): + """ Raised when there's no AWS instance for a given hostname """ + + +class AWSTagPattern(object): + """ Handler for a single Tag entry """ + + def __init__(self, name, value, groups): + self.name = re.compile(name) + if value is not None: + self.value = re.compile(value) + else: + self.value = value + self.groups = groups + + def get_groups(self, tags): + """ Get groups that apply to the given tag set """ + for key, value in tags.items(): + name_match = self.name.search(key) + if name_match: + if self.value is not None: + value_match = self.value.search(value) + if value_match: + return self._munge_groups(value_match) + else: + return self._munge_groups(name_match) + break + return [] + + def _munge_groups(self, match): + """ Replace backreferences (``$1``, ``$2``) in Group tags with + their values in the regex. """ + rv = [] + sub = match.groups() + for group in self.groups: + newg = group + for idx in range(len(sub)): + newg = newg.replace('$%s' % (idx + 1), sub[idx]) + rv.append(newg) + return rv + + def __str__(self): + if self.value: + return "%s: %s=%s: %s" % (self.__class__.__name__, self.name, + self.value, self.groups) + else: + return "%s: %s: %s" % (self.__class__.__name__, self.name, + self.groups) + + +class PatternFile(Bcfg2.Server.Plugin.XMLFileBacked): + """ representation of AWSTags config.xml """ + __identifier__ = None + create = 'AWSTags' + + def __init__(self, filename, core=None): + try: + fam = core.fam + except AttributeError: + fam = None + Bcfg2.Server.Plugin.XMLFileBacked.__init__(self, filename, fam=fam, + should_monitor=True) + self.core = core + self.tags = [] + + def Index(self): + Bcfg2.Server.Plugin.XMLFileBacked.Index(self) + if (self.core and + self.core.metadata_cache_mode in ['cautious', 'aggressive']): + self.core.metadata_cache.expire() + self.tags = [] + for entry in self.xdata.xpath('//Tag'): + try: + groups = [g.text for g in entry.findall('Group')] + self.tags.append(AWSTagPattern(entry.get("name"), + entry.get("value"), + groups)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to initialize pattern %s: " + "%s" % (entry.get("name"), + sys.exc_info()[1])) + + def get_groups(self, hostname, tags): + """ return a list of groups that should be added to the given + client based on patterns that match the hostname """ + ret = [] + for pattern in self.tags: + try: + ret.extend(pattern.get_groups(tags)) + except: # pylint: disable=W0702 + self.logger.error("AWSTags: Failed to process pattern %s for " + "%s" % (pattern, hostname), + exc_info=1) + return ret + + +class AWSTags(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.Connector): + """ Query tags from AWS via boto, optionally setting group membership """ + __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['expire_cache'] + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) + try: + key_id = self.core.setup.cfp.get("awstags", "access_key_id") + secret_key = self.core.setup.cfp.get("awstags", + "secret_access_key") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + err = sys.exc_info()[1] + raise Bcfg2.Server.Plugin.PluginInitError( + "AWSTags is not configured in bcfg2.conf: %s" % err) + self.debug_log("%s: Connecting to EC2" % self.name) + self._ec2 = connect_ec2(aws_access_key_id=key_id, + aws_secret_access_key=secret_key) + self._tagcache = Cache() + try: + self._keep_cache = self.core.setup.cfp.getboolean("awstags", + "cache") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self._keep_cache = True + + self.config = PatternFile(os.path.join(self.data, 'config.xml'), + core=core) + + def _load_instance(self, hostname): + """ Load an instance from EC2 whose private DNS name matches + the given hostname """ + self.debug_log("AWSTags: Loading instance with private-dns-name=%s" % + hostname) + filters = {'private-dns-name': hostname} + reservations = self._ec2.get_all_instances(filters=filters) + if reservations: + res = reservations[0] + if res.instances: + return res.instances[0] + raise NoInstanceFound( + "AWSTags: No instance found with private-dns-name=%s" % + hostname) + + def _get_tags_from_ec2(self, hostname): + """ Get tags for the given host from EC2. This does not use + the local caching layer. """ + self.debug_log("AWSTags: Getting tags for %s from AWS" % + hostname) + try: + return self._load_instance(hostname).tags + except NoInstanceFound: + self.debug_log(sys.exc_info()[1]) + return dict() + + def get_tags(self, metadata): + """ Get tags for the given host. This caches the tags locally + if 'cache' in the ``[awstags]`` section of ``bcfg2.conf`` is + true. """ + if not self._keep_cache: + return self._get_tags_from_ec2(metadata) + + if metadata.hostname not in self._tagcache: + self._tagcache[metadata.hostname] = \ + self._get_tags_from_ec2(metadata.hostname) + return self._tagcache[metadata.hostname] + + def expire_cache(self, key=None): + self._tagcache.expire(key=key) + + def start_client_run(self, metadata): + self.expire_cache(key=metadata.hostname) + + def get_additional_data(self, metadata): + return self.get_tags(metadata) + + def get_additional_groups(self, metadata): + return self.config.get_groups(metadata.hostname, + self.get_tags(metadata)) + + +class AWSTagsLint(Bcfg2.Server.Lint.ServerPlugin): + """ ``bcfg2-lint`` plugin to check all given :ref:`AWSTags + <server-plugins-connectors-awstags>` patterns for validity. """ + + def Run(self): + cfg = self.core.plugins['AWSTags'].config + for entry in cfg.xdata.xpath('//Tag'): + self.check(entry, "name") + if entry.get("value"): + self.check(entry, "value") + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize": "error"} + + def check(self, entry, attr): + """ Check a single attribute (``name`` or ``value``) of a + single entry for validity. """ + try: + re.compile(entry.get(attr)) + except re.error: + self.LintError("pattern-fails-to-initialize", + "'%s' regex could not be compiled: %s\n %s" % + (attr, sys.exc_info()[1], entry.get("name"))) diff --git a/src/lib/Bcfg2/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index eef176cca..fb327f7ef 100644 --- a/src/lib/Bcfg2/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -38,9 +38,9 @@ if HAS_GENSHI: Bcfg2.Server.Plugin.StructFile): """ Representation of a Genshi-templated bundle XML file """ - def __init__(self, name, specific, encoding): + def __init__(self, name, specific, encoding, fam=None): TemplateFile.__init__(self, name, specific, encoding) - Bcfg2.Server.Plugin.StructFile.__init__(self, name) + Bcfg2.Server.Plugin.StructFile.__init__(self, name, fam=fam) self.logger = logging.getLogger(name) def get_xml_value(self, metadata): @@ -106,13 +106,14 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, nsmap['py'] == 'http://genshi.edgewall.org/')): if HAS_GENSHI: spec = Bcfg2.Server.Plugin.Specificity() - return BundleTemplateFile(name, spec, self.encoding) + return BundleTemplateFile(name, spec, self.encoding, + fam=self.core.fam) else: raise Bcfg2.Server.Plugin.PluginExecutionError("Genshi not " "available: %s" % name) else: - return BundleFile(name, self.fam) + return BundleFile(name, fam=self.fam) def BuildStructures(self, metadata): """Build all structures for client (metadata).""" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py index 824d01023..41d5588e4 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgAuthorizedKeysGenerator.py @@ -50,27 +50,36 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): spec = self.XMLMatch(metadata) rv = [] for allow in spec.findall("Allow"): - params = '' + options = [] if allow.find("Params") is not None: - params = ",".join("=".join(p) - for p in allow.find("Params").attrib.items()) + self.logger.warning("Use of <Params> in authorized_keys.xml " + "is deprecated; use <Option> instead") + options.extend("=".join(p) + for p in allow.find("Params").attrib.items()) + + for opt in allow.findall("Option"): + if opt.get("value"): + options.append("%s=%s" % (opt.get("name"), + opt.get("value"))) + else: + options.append(opt.get("name")) pubkey_name = allow.get("from") if pubkey_name: host = allow.get("host") group = allow.get("group") + category = allow.get("category", self.category) if host: key_md = self.core.build_metadata(host) elif group: key_md = ClientMetadata("dummy", group, [group], [], set(), set(), dict(), None, None, None, None) - elif (self.category and - not metadata.group_in_category(self.category)): + elif category and not metadata.group_in_category(category): self.logger.warning("Cfg: %s ignoring Allow from %s: " "No group in category %s" % (metadata.hostname, pubkey_name, - self.category)) + category)) continue else: key_md = metadata @@ -96,6 +105,6 @@ class CfgAuthorizedKeysGenerator(CfgGenerator, StructFile): (metadata.hostname, lxml.etree.tostring(allow))) continue - rv.append(" ".join([params, pubkey]).strip()) + rv.append(" ".join([",".join(options), pubkey]).strip()) return "\n".join(rv) get_data.__doc__ = CfgGenerator.get_data.__doc__ diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py index c7b62f352..e890fdecb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPrivateKeyCreator.py @@ -159,7 +159,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): return specificity # pylint: disable=W0221 - def create_data(self, entry, metadata, return_pair=False): + def create_data(self, entry, metadata): """ Create data for the given entry on the given client :param entry: The abstract entry to create data for. This @@ -167,15 +167,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): :type entry: lxml.etree._Element :param metadata: The client metadata to create data for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :param return_pair: Return a tuple of ``(public key, private - key)`` instead of just the private key. - This is used by - :class:`Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.CfgPublicKeyCreator` - to create public keys as requested. - :type return_pair: bool :returns: string - The private key data - :returns: tuple - Tuple of ``(public key, private key)``, if - ``return_pair`` is set to True """ spec = self.XMLMatch(metadata) specificity = self.get_specificity(metadata, spec) @@ -201,11 +193,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): specificity['ext'] = '.crypt' self.write_data(privkey, **specificity) - - if return_pair: - return (pubkey, privkey) - else: - return privkey + return privkey finally: shutil.rmtree(os.path.dirname(filename)) # pylint: enable=W0221 @@ -230,7 +218,7 @@ class CfgPrivateKeyCreator(CfgCreator, StructFile): if strict: raise PluginExecutionError(msg) else: - self.logger.warning(msg) + self.logger.info(msg) Index.__doc__ = StructFile.Index.__doc__ def _decrypt(self, element): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py index 6be438462..4bd8690ed 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPublicKeyCreator.py @@ -2,7 +2,11 @@ :class:`Bcfg2.Server.Plugins.Cfg.CfgPrivateKeyCreator.CfgPrivateKeyCreator` to create SSH keys on the fly. """ +import os +import sys +import tempfile import lxml.etree +from Bcfg2.Utils import Executor from Bcfg2.Server.Plugin import StructFile, PluginExecutionError from Bcfg2.Server.Plugins.Cfg import CfgCreator, CfgCreationError, CFG @@ -27,7 +31,8 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): CfgCreator.__init__(self, fname) StructFile.__init__(self, fname) self.cfg = CFG - __init__.__doc__ = CfgCreator.__init__.__doc__ + self.core = CFG.core + self.cmd = Executor() def create_data(self, entry, metadata): if entry.get("name").endswith(".pub"): @@ -37,25 +42,51 @@ class CfgPublicKeyCreator(CfgCreator, StructFile): "%s: Filename does not end in .pub" % entry.get("name")) - if privkey not in self.cfg.entries: - raise CfgCreationError("Cfg: Could not find Cfg entry for %s " - "(private key for %s)" % (privkey, - self.name)) - eset = self.cfg.entries[privkey] + privkey_entry = lxml.etree.Element("Path", name=privkey) try: + self.core.Bind(privkey_entry, metadata) + except PluginExecutionError: + raise CfgCreationError("Cfg: Could not bind %s (private key for " + "%s): %s" % (privkey, self.name, + sys.exc_info()[1])) + + try: + eset = self.cfg.entries[privkey] creator = eset.best_matching(metadata, eset.get_handlers(metadata, CfgCreator)) + except KeyError: + raise CfgCreationError("Cfg: No private key defined for %s (%s)" % + (self.name, privkey)) except PluginExecutionError: raise CfgCreationError("Cfg: No privkey.xml defined for %s " "(private key for %s)" % (privkey, self.name)) - privkey_entry = lxml.etree.Element("Path", name=privkey) - pubkey = creator.create_data(privkey_entry, metadata, - return_pair=True)[0] - return pubkey - create_data.__doc__ = CfgCreator.create_data.__doc__ + specificity = creator.get_specificity(metadata) + fname = self.get_filename(**specificity) + + # if the private key didn't exist, then creating it may have + # created the private key, too. check for it first. + if os.path.exists(fname): + return open(fname).read() + else: + # generate public key from private key + fd, privfile = tempfile.mkstemp() + try: + os.fdopen(fd, 'w').write(privkey_entry.text) + cmd = ["ssh-keygen", "-y", "-f", privfile] + self.debug_log("Cfg: Extracting SSH public key from %s: %s" % + (privkey, " ".join(cmd))) + result = self.cmd.run(cmd) + if not result.success: + raise CfgCreationError("Cfg: Failed to extract public key " + "from %s: %s" % (privkey, + result.error)) + self.write_data(result.stdout, **specificity) + return result.stdout + finally: + os.unlink(privfile) def handle_event(self, event): CfgCreator.handle_event(self, event) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 154cd5e63..c6e2d0acb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -10,6 +10,7 @@ import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.Lint +from fnmatch import fnmatch from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ @@ -100,6 +101,8 @@ class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, experimental = False def __init__(self, name, specific, encoding): + if not self.__specific__ and not specific: + specific = Bcfg2.Server.Plugin.Specificity(all=True) Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, encoding) Bcfg2.Server.Plugin.Debuggable.__init__(self) @@ -898,6 +901,7 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): self.check_delta(basename, entry) self.check_pubkey(basename, entry) self.check_missing_files() + self.check_conflicting_handlers() @classmethod def Errors(cls): @@ -905,7 +909,8 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): "diff-file-used": "warning", "no-pubkey-xml": "warning", "unknown-cfg-files": "error", - "extra-cfg-files": "error"} + "extra-cfg-files": "error", + "multiple-global-handlers": "error"} def check_delta(self, basename, entry): """ check that no .cat or .diff files are in use """ @@ -940,22 +945,55 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): "%s has no corresponding pubkey.xml at %s" % (basename, pubkey)) + def _list_path_components(self, path): + """ Get a list of all components of a path. E.g., + ``self._list_path_components("/foo/bar/foobaz")`` would return + ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed + to be in order.""" + rv = [] + remaining, component = os.path.split(path) + while component != '': + rv.append(component) + remaining, component = os.path.split(remaining) + return rv + + def check_conflicting_handlers(self): + """ Check that a single entryset doesn't have multiple + non-specific (i.e., 'all') handlers. """ + cfg = self.core.plugins['Cfg'] + for eset in cfg.entries.values(): + alls = [e for e in eset.entries.values() + if (e.specific.all and + issubclass(e.__class__, CfgGenerator))] + if len(alls) > 1: + self.LintError("multiple-global-handlers", + "%s has multiple global handlers: %s" % + (eset.path, ", ".join(os.path.basename(e.name) + for e in alls))) + def check_missing_files(self): """ check that all files on the filesystem are known to Cfg """ cfg = self.core.plugins['Cfg'] # first, collect ignore patterns from handlers - ignore = [] + ignore = set() for hdlr in handlers(): - ignore.extend(hdlr.__ignore__) + ignore.update(hdlr.__ignore__) # next, get a list of all non-ignored files on the filesystem all_files = set() for root, _, files in os.walk(cfg.data): - all_files.update(os.path.join(root, fname) - for fname in files - if not any(fname.endswith("." + i) - for i in ignore)) + for fname in files: + fpath = os.path.join(root, fname) + # check against the handler ignore patterns and the + # global FAM ignore list + if (not any(fname.endswith("." + i) for i in ignore) and + not any(fnmatch(fpath, p) + for p in self.config['ignore']) and + not any(fnmatch(c, p) + for p in self.config['ignore'] + for c in self._list_path_components(fpath))): + all_files.add(fpath) # next, get a list of all files known to Cfg cfg_files = set() diff --git a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py index 810b273af..d74c16e8b 100644 --- a/src/lib/Bcfg2/Server/Plugins/GroupLogic.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupLogic.py @@ -3,7 +3,9 @@ template to dynamically set additional groups for clients. """ import os import lxml.etree +from threading import local import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Metadata import MetadataGroup try: from Bcfg2.Server.Plugins.Bundler import BundleTemplateFile except ImportError: @@ -35,13 +37,40 @@ class GroupLogic(Bcfg2.Server.Plugin.Plugin, """ GroupLogic is a connector plugin that lets you use an XML Genshi template to dynamically set additional groups for clients. """ + # perform grouplogic later than other Connector plugins, so it can + # use groups set by them + sort_order = 1000 def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = GroupLogicConfig(os.path.join(self.data, "groups.xml"), core.fam) + self._local = local() def get_additional_groups(self, metadata): - return [el.get("name") - for el in self.config.get_xml_value(metadata).findall("Group")] + if not hasattr(self._local, "building"): + # building is a thread-local set that tracks which + # machines GroupLogic is getting additional groups for. + # If a get_additional_groups() is called twice for a + # machine before the first call has completed, the second + # call returns an empty list. This is for infinite + # recursion protection; without this check, it'd be + # impossible to use things like metadata.query.in_group() + # in GroupLogic, since that requires building all + # metadata, which requires running + # GroupLogic.get_additional_groups() for all hosts, which + # requires building all metadata... + self._local.building = set() + if metadata.hostname in self._local.building: + return [] + self._local.building.add(metadata.hostname) + rv = [] + for el in self.config.get_xml_value(metadata).findall("Group"): + if el.get("category"): + rv.append(MetadataGroup(el.get("name"), + category=el.get("category"))) + else: + rv.append(el.get("name")) + self._local.building.discard(metadata.hostname) + return rv diff --git a/src/lib/Bcfg2/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index 4f2601f15..3c9b8a459 100644 --- a/src/lib/Bcfg2/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -37,6 +37,7 @@ class Guppy(Bcfg2.Server.Plugin.Plugin): experimental = True __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable', 'Disable'] + __child_rmi__ = __rmi__[:] def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 6934739a3..047dd4f4e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -16,7 +16,9 @@ import Bcfg2.Server.Lint import Bcfg2.Server.Plugin import Bcfg2.Server.FileMonitor from Bcfg2.Utils import locked -from Bcfg2.Compat import MutableMapping, all, wraps # pylint: disable=W0622 +# pylint: disable=W0622 +from Bcfg2.Compat import MutableMapping, all, any, wraps +# pylint: enable=W0622 from Bcfg2.version import Bcfg2VersionInfo try: @@ -219,6 +221,7 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.XMLFileBacked): sys.exc_info()[1]) self.logger.error(msg) raise Bcfg2.Server.Plugin.MetadataRuntimeError(msg) + self.load_xml() def find_xml_for_xpath(self, xpath): """Find and load xml file containing the xpath query""" @@ -487,6 +490,7 @@ class MetadataGroup(tuple): # pylint: disable=E0012,R0924 class Metadata(Bcfg2.Server.Plugin.Metadata, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.ClientRunHooks, Bcfg2.Server.Plugin.DatabaseBacked): """This class contains data for bcfg2 server metadata.""" @@ -495,6 +499,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def __init__(self, core, datastore, watch_clients=True): Bcfg2.Server.Plugin.Metadata.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) self.watch_clients = watch_clients @@ -528,21 +533,24 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.raliases = {} # mapping of groupname -> MetadataGroup object self.groups = {} - # mappings of predicate -> MetadataGroup object + # mappings of groupname -> [predicates] self.group_membership = dict() self.negated_groups = dict() + # list of group names in document order + self.ordered_groups = [] # mapping of hostname -> version string if self._use_db: self.versions = ClientVersions(core, datastore) else: self.versions = dict() + self.uuid = {} self.session_cache = {} self.default = None self.pdirty = False self.password = core.setup['password'] self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients), + self.list_clients, self.get_client_names_by_groups, self.get_client_names_by_profiles, self.get_all_group_names, @@ -672,14 +680,15 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ Generic method to modify XML data (group, client, etc.) """ node = self._search_xdata(tag, name, config.xdata, alias=alias) if node is None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = "%s \"%s\" does not exist" % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise Bcfg2.Server.Plugin.MetadataConsistencyError + msg = 'Unexpected error finding %s "%s"' % (tag, name) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) config.write_xml(xdict['filename'], xdict['xmltree']) @@ -749,7 +758,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) def remove_client(self, client_name): - """Remove a bundle.""" + """Remove a client.""" if self._use_db: try: client = MetadataClientModel.objects.get(hostname=client_name) @@ -830,51 +839,34 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if self._use_db: self.clients = self.list_clients() + def _get_condition(self, element): + """ Return a predicate that returns True if a client meets + the condition specified in the given Group or Client + element """ + negate = element.get('negate', 'false').lower() == 'true' + pname = element.get("name") + if element.tag == 'Group': + return lambda c, g, _: negate != (pname in g) + elif element.tag == 'Client': + return lambda c, g, _: negate != (pname == c) + + def _get_category_condition(self, grpname): + """ get a predicate that returns False if a client is already + a member of a group in the given group's category, True + otherwise""" + return lambda client, _, categories: \ + bool(self._check_category(client, grpname, categories)) + + def _aggregate_conditions(self, conditions): + """ aggregate all conditions on a given group declaration + into a single predicate """ + return lambda client, groups, cats: \ + all(cond(client, groups, cats) for cond in conditions) + def _handle_groups_xml_event(self, _): # pylint: disable=R0912 """ re-read groups.xml on any event on it """ self.groups = {} - # these three functions must be separate functions in order to - # ensure that the scope is right for the closures they return - def get_condition(element): - """ Return a predicate that returns True if a client meets - the condition specified in the given Group or Client - element """ - negate = element.get('negate', 'false').lower() == 'true' - pname = element.get("name") - if element.tag == 'Group': - return lambda c, g, _: negate != (pname in g) - elif element.tag == 'Client': - return lambda c, g, _: negate != (pname == c) - - def get_category_condition(category, gname): - """ get a predicate that returns False if a client is - already a member of a group in the given category, True - otherwise """ - def in_cat(client, groups, categories): # pylint: disable=W0613 - """ return True if the client is already a member of a - group in the category given in the enclosing function, - False otherwise """ - if category in categories: - if (gname not in self.groups or - client not in self.groups[gname].warned): - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, gname, category, - client, categories[category])) - if gname in self.groups: - self.groups[gname].warned.append(client) - return False - return True - return in_cat - - def aggregate_conditions(conditions): - """ aggregate all conditions on a given group declaration - into a single predicate """ - return lambda client, groups, cats: \ - all(cond(client, groups, cats) for cond in conditions) - # first, we get a list of all of the groups declared in the # file. we do this in two stages because the old way of # parsing groups.xml didn't support nested groups; in the old @@ -900,6 +892,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.group_membership = dict() self.negated_groups = dict() + self.ordered_groups = [] # confusing loop condition; the XPath query asks for all # elements under a Group tag under a Groups tag; that is @@ -910,37 +903,47 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, # XPath. We do the same thing for Client tags. for el in self.groups_xml.xdata.xpath("//Groups/Group//*") + \ self.groups_xml.xdata.xpath("//Groups/Client//*"): - if ((el.tag != 'Group' and el.tag != 'Client') or - el.getchildren()): + if (el.tag != 'Group' and el.tag != 'Client') or el.getchildren(): continue conditions = [] for parent in el.iterancestors(): - cond = get_condition(parent) + cond = self._get_condition(parent) if cond: conditions.append(cond) gname = el.get("name") if el.get("negate", "false").lower() == "true": - self.negated_groups[aggregate_conditions(conditions)] = \ - self.groups[gname] + self.negated_groups.setdefault(gname, []) + self.negated_groups[gname].append( + self._aggregate_conditions(conditions)) else: if self.groups[gname].category: - conditions.append( - get_category_condition(self.groups[gname].category, - gname)) + conditions.append(self._get_category_condition(gname)) - self.group_membership[aggregate_conditions(conditions)] = \ - self.groups[gname] + if gname not in self.ordered_groups: + self.ordered_groups.append(gname) + self.group_membership.setdefault(gname, []) + self.group_membership[gname].append( + self._aggregate_conditions(conditions)) self.states['groups.xml'] = True + def expire_cache(self, key=None): + self.core.metadata_cache.expire(key) + def HandleEvent(self, event): """Handle update events for data files.""" for handles, event_handler in self.handlers.items(): if handles(event): # clear the entire cache when we get an event for any # metadata file - self.core.metadata_cache.expire() + self.expire_cache() + + # clear out the list of category suppressions that + # have been warned about, since this may change when + # clients.xml or groups.xml changes. + for group in self.groups.values(): + group.warned = [] event_handler(event) if False not in list(self.states.values()) and self.debug_flag: @@ -978,17 +981,21 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - profiles = [g for g in self.clientgroups[client] - if g in self.groups and self.groups[g].is_profile] - self.logger.info("Changing %s profile from %s to %s" % - (client, profiles, profile)) - self.update_client(client, dict(profile=profile)) - if client in self.clientgroups: - for prof in profiles: - self.clientgroups[client].remove(prof) - self.clientgroups[client].append(profile) + metadata = self.core.build_metadata(client) + if metadata.profile != profile: + self.logger.info("Changing %s profile from %s to %s" % + (client, metadata.profile, profile)) + self.update_client(client, dict(profile=profile)) + if client in self.clientgroups: + if metadata.profile in self.clientgroups[client]: + self.clientgroups[client].remove(metadata.profile) + self.clientgroups[client].append(profile) + else: + self.clientgroups[client] = [profile] else: - self.clientgroups[client] = [profile] + self.logger.debug( + "Ignoring %s request to change profile from %s to %s" + % (client, metadata.profile, profile)) else: self.logger.info("Creating new client: %s, profile %s" % (client, profile)) @@ -1004,8 +1011,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.add_client(client, dict(profile=profile)) self.clients.append(client) self.clientgroups[client] = [profile] - if not self._use_db: - self.clients_xml.write() + if not self._use_db: + self.clients_xml.write() def set_version(self, client, version): """Set version for provided client.""" @@ -1060,7 +1067,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, if cname in self.aliases: return self.aliases[cname] return cname - except socket.herror: + except (socket.gaierror, socket.herror): err = "Address resolution error for %s: %s" % (address, sys.exc_info()[1]) self.logger.error(err) @@ -1075,22 +1082,77 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, categories = dict() while numgroups != len(groups): numgroups = len(groups) - for predicate, group in self.group_membership.items(): - if group.name in groups: + newgroups = set() + removegroups = set() + for grpname in self.ordered_groups: + if grpname in groups: continue - if predicate(client, groups, categories): - groups.add(group.name) - if group.category: - categories[group.category] = group.name - for predicate, group in self.negated_groups.items(): - if group.name not in groups: + if any(p(client, groups, categories) + for p in self.group_membership[grpname]): + newgroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + categories[self.groups[grpname].category] = grpname + groups.update(newgroups) + for grpname, predicates in self.negated_groups.items(): + if grpname not in groups: continue - if predicate(client, groups, categories): - groups.remove(group.name) - if group.category: - del categories[group.category] + if any(p(client, groups, categories) for p in predicates): + removegroups.add(grpname) + if (grpname in self.groups and + self.groups[grpname].category): + del categories[self.groups[grpname].category] + groups.difference_update(removegroups) return (groups, categories) + def _check_category(self, client, grpname, categories): + """ Determine if the given client is already a member of a + group in the same category as the named group. + + The return value is one of three possibilities: + + * If the client is already a member of a group in the same + category, then False is returned (i.e., the category check + failed); + * If the group is not in any categories, then True is returned; + * If the group is not a member of a group in the category, + then the name of the category is returned. This makes it + easy to add the category to the ClientMetadata object (or + other category list). + + If a pure boolean value is required, you can do + ``bool(self._check_category(...))``. + """ + if grpname not in self.groups: + return True + category = self.groups[grpname].category + if not category: + return True + if category in categories: + if client not in self.groups[grpname].warned: + self.logger.warning("%s: Group %s suppressed by category %s; " + "%s already a member of %s" % + (self.name, grpname, category, + client, categories[category])) + self.groups[grpname].warned.append(client) + return False + return category + + def _check_and_add_category(self, client, grpname, categories): + """ If the client is not a member of a group in the same + category as the named group, then the category is added to + ``categories``. + :func:`Bcfg2.Server.Plugins.Metadata._check_category` is used + to determine if the category can be added. + + If the category check failed, returns False; otherwise, + returns True. """ + rv = self._check_category(client, grpname, categories) + if rv and rv is not True: + categories[rv] = grpname + return True + return rv + def get_initial_metadata(self, client): # pylint: disable=R0914,R0912 """Return the metadata for a given client.""" if False in list(self.states.values()): @@ -1112,39 +1174,37 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, Handles setting categories and category suppression. Returns the new profile for the client (which might be unchanged). """ - groups.add(grpname) if grpname in self.groups: - group = self.groups[grpname] - category = group.category - if category: - if category in categories: - self.logger.warning("%s: Group %s suppressed by " - "category %s; %s already a member " - "of %s" % - (self.name, grpname, category, - client, categories[category])) - return - categories[category] = grpname - if not profile and group.is_profile: + if not self._check_and_add_category(client, grpname, + categories): + return profile + groups.add(grpname) + if not profile and self.groups[grpname].is_profile: return grpname else: return profile + else: + groups.add(grpname) + return profile if client not in self.clients: pgroup = None if client in self.clientgroups: pgroup = self.clientgroups[client][0] + self.debug_log("%s: Adding new client with profile %s" % + (self.name, pgroup)) elif self.default: pgroup = self.default + self.debug_log("%s: Adding new client with default profile %s" + % (self.name, pgroup)) if pgroup: self.set_profile(client, pgroup, (None, None), require_public=False) profile = _add_group(pgroup) else: - msg = "Cannot add new client %s; no default group set" % client - self.logger.error(msg) - raise Bcfg2.Server.Plugin.MetadataConsistencyError(msg) + raise Bcfg2.Server.Plugin.MetadataConsistencyError( + "Cannot add new client %s; no default group set" % client) for cgroup in self.clientgroups.get(client, []): if cgroup in groups: @@ -1153,6 +1213,9 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, self.groups[cgroup] = MetadataGroup(cgroup) profile = _add_group(cgroup) + # we do this before setting the default because there may be + # groups set in <Client> tags in groups.xml that we want to + # set groups, categories = self._merge_groups(client, groups, categories=categories) @@ -1201,8 +1264,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, """ return a list of all group names """ all_groups = set() all_groups.update(self.groups.keys()) - all_groups.update([g.name for g in self.group_membership.values()]) - all_groups.update([g.name for g in self.negated_groups.values()]) + all_groups.update(self.group_membership.keys()) + all_groups.update(self.negated_groups.keys()) for grp in self.clientgroups.values(): all_groups.update(grp) return all_groups @@ -1215,7 +1278,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_profiles(self, profiles): """ return a list of names of clients in the given profile groups """ rv = [] - for client in list(self.clients): + for client in self.list_clients(): mdata = self.core.build_metadata(client) if mdata.profile in profiles: rv.append(client) @@ -1223,34 +1286,33 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, def get_client_names_by_groups(self, groups): """ return a list of names of clients in the given groups """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.groups.issuperset(groups): + rv.append(client) + return rv def get_client_names_by_bundles(self, bundles): """ given a list of bundles, return a list of names of clients that use those bundles """ - mdata = [self.core.build_metadata(client) for client in self.clients] - return [md.hostname for md in mdata if md.bundles.issuperset(bundles)] + rv = [] + for client in self.list_clients(): + mdata = self.core.build_metadata(client) + if mdata.bundles.issuperset(bundles): + rv.append(client) + return rv def merge_additional_groups(self, imd, groups): for group in groups: if group in imd.groups: continue - if group in self.groups and self.groups[group].category: - category = self.groups[group].category - if self.groups[group].category in imd.categories: - self.logger.warning("%s: Group %s suppressed by category " - "%s; %s already a member of %s" % - (self.name, group, category, - imd.hostname, - imd.categories[category])) - continue - imd.categories[category] = group + if not self._check_and_add_category(imd.hostname, group, + imd.categories): + continue imd.groups.add(group) - self._merge_groups(imd.hostname, imd.groups, - categories=imd.categories) - + self._merge_groups(imd.hostname, imd.groups, categories=imd.categories) for group in imd.groups: if group in self.groups: imd.bundles.update(self.groups[group].bundles) @@ -1398,7 +1460,7 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, viz_str.extend(self._viz_groups(egroups, bundles, clientmeta)) if key: for category in categories: - viz_str.append('"%s" [label="%s", shape="record", ' + viz_str.append('"%s" [label="%s", shape="trapezium", ' 'style="filled", fillcolor="%s"];' % (category, category, categories[category])) return "\n".join("\t" + s for s in viz_str) @@ -1412,8 +1474,8 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, instances = {} rv = [] - for client in list(self.clients): - if include_client(client): + for client in list(self.list_clients()): + if not include_client(client): continue if client in self.clientgroups: grps = self.clientgroups[client] @@ -1441,9 +1503,10 @@ class Metadata(Bcfg2.Server.Plugin.Metadata, the graph""" return not clientmeta or bundle in clientmeta.bundles - bundles = list(set(bund.get('name')) - for bund in self.groups_xml.xdata.findall('.//Bundle') - if include_bundle(bund.get('name'))) + bundles = \ + list(set(bund.get('name') + for bund in self.groups_xml.xdata.findall('.//Bundle') + if include_bundle(bund.get('name')))) bundles.sort() return ['"bundle-%s" [ label="%s", shape="septagon"];' % (bundle, bundle) @@ -1589,15 +1652,35 @@ class MetadataLint(Bcfg2.Server.Lint.ServerPlugin): "client") def duplicate_groups(self): - """ Check for groups that are defined more than once. We - count a group tag as a definition if it a) has profile or - public set; or b) has any children.""" - allgroups = [ - g - for g in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + - self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group") - if g.get("profile") or g.get("public") or g.getchildren()] - self.duplicate_entries(allgroups, "group") + """ Check for groups that are defined more than once. There + are two ways this can happen: + + 1. The group is listed twice with contradictory options. + 2. The group is listed with no options *first*, and then with + options later. + + In this context, 'first' refers to the order in which groups + are parsed; see the loop condition below and + _handle_groups_xml_event above for details. """ + groups = dict() + duplicates = dict() + for grp in self.metadata.groups_xml.xdata.xpath("//Groups/Group") + \ + self.metadata.groups_xml.xdata.xpath("//Groups/Group//Group"): + grpname = grp.get("name") + if grpname in duplicates: + duplicates[grpname].append(grp) + elif len(grp.attrib) > 1: # group has options + if grpname in groups: + duplicates[grpname] = [grp, groups[grpname]] + else: + groups[grpname] = grp + else: # group has no options + groups[grpname] = grp + for grpname, grps in duplicates.items(): + self.LintError("duplicate-group", + "Group %s is defined multiple times:\n%s" % + (grpname, + "\n".join(self.RenderXML(g) for g in grps))) def duplicate_entries(self, allentries, etype): """ Generic duplicate entry finder. diff --git a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index 466665382..8f1d03586 100644 --- a/src/lib/Bcfg2/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -24,9 +24,9 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, 'config.xml'), core.fam, should_monitor=True, create=self.name) - self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} + self.Entries = { + 'Path': {'/etc/nagiosgen.status': self.createhostconfig, + '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} self.client_attrib = {'encoding': 'ascii', 'owner': 'root', diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index b25cb0fc4..39c51f351 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -614,6 +614,10 @@ class Collection(list, Bcfg2.Server.Plugin.Debuggable): self.filter_unknown(unknown) return packages, unknown + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, + list.__repr__(self)) + def get_collection_class(source_type): """ Given a source type, determine the class of Collection object diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 332f0bbab..c47e18201 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -88,13 +88,12 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, :type event: Bcfg2.Server.FileMonitor.Event :returns: None """ - Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if event and event.filename != self.name: for fpath in self.extras: if fpath == os.path.abspath(event.filename): self.parsed.add(fpath) break - + Bcfg2.Server.Plugin.StructFile.HandleEvent(self, event=event) if self.loaded: self.logger.info("Reloading Packages plugin") self.pkg_obj.Reload() @@ -111,10 +110,11 @@ class PackagesSources(Bcfg2.Server.Plugin.StructFile, def Index(self): Bcfg2.Server.Plugin.StructFile.Index(self) self.entries = [] - for xsource in self.xdata.findall('.//Source'): - source = self.source_from_xml(xsource) - if source is not None: - self.entries.append(source) + if self.loaded: + for xsource in self.xdata.findall('.//Source'): + source = self.source_from_xml(xsource) + if source is not None: + self.entries.append(source) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ + """ ``Index`` is responsible for calling :func:`source_from_xml` diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 4608bcca5..66f8e9dbe 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,13 +53,15 @@ The Yum Backend import os import re import sys +import time import copy import errno import socket import logging import lxml.etree -from subprocess import Popen, PIPE import Bcfg2.Server.Plugin +from lockfile import FileLock +from Bcfg2.Utils import Executor # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -102,9 +104,6 @@ FL = '{http://linux.duke.edu/metadata/filelists}' PULPSERVER = None PULPCONFIG = None -#: The path to bcfg2-yum-helper -HELPER = None - def _setup_pulp(setup): """ Connect to a Pulp server and pass authentication credentials. @@ -263,6 +262,8 @@ class YumCollection(Collection): .. private-include: _add_gpg_instances, _get_pulp_consumer """ + _helper = None + #: Options that are included in the [packages:yum] section of the #: config but that should not be included in the temporary #: yum.conf we write out @@ -277,18 +278,25 @@ class YumCollection(Collection): debug=debug) self.keypath = os.path.join(self.cachepath, "keys") + #: A :class:`Bcfg2.Utils.Executor` object to use to run + #: external commands + self.cmd = Executor() + if self.use_yum: #: Define a unique cache file for this collection to use #: for cached yum metadata self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) #: The path to the server-side config file used when #: resolving packages with the Python yum libraries self.cfgfile = os.path.join(self.cachefile, "yum.conf") - self.write_config() + + if not os.path.exists(self.cachefile): + self.debug_log("Creating common cache %s" % self.cachefile) + os.mkdir(self.cachefile) + if not self.disableMetaData: + self.setup_data() else: self.cachefile = None @@ -309,7 +317,28 @@ class YumCollection(Collection): self.logger.error("Could not create Pulp consumer " "cert directory at %s: %s" % (certdir, err)) - self.pulp_cert_set = PulpCertificateSet(certdir, self.fam) + self.__class__.pulp_cert_set = PulpCertificateSet(certdir, + self.fam) + + @property + def disableMetaData(self): + """ Report whether or not metadata processing is enabled. + This duplicates code in Packages/__init__.py, and can probably + be removed in Bcfg2 1.4 when we have a module-level setup + object. """ + if self.setup is None: + return True + try: + return not self.setup.cfp.getboolean("packages", "resolver") + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return False + except ValueError: + # for historical reasons we also accept "enabled" and + # "disabled" + return self.setup.cfp.get( + "packages", + "metadata", + default="enabled").lower() == "disabled" @property def __package_groups__(self): @@ -323,20 +352,21 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ - global HELPER - if not HELPER: + if not self._helper: + # pylint: disable=W0212 try: - HELPER = self.setup.cfp.get("packages:yum", "helper") + self.__class__._helper = self.setup.cfp.get("packages:yum", + "helper") except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") - Popen(['bcfg2-yum-helper'], - stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() - HELPER = 'bcfg2-yum-helper' + self.cmd.run(['bcfg2-yum-helper']) + self.__class__._helper = 'bcfg2-yum-helper' except OSError: - HELPER = "/usr/sbin/bcfg2-yum-helper" - return HELPER + self.__class__._helper = "/usr/sbin/bcfg2-yum-helper" + # pylint: enable=W0212 + return self._helper @property def use_yum(self): @@ -374,6 +404,7 @@ class YumCollection(Collection): # the rpmdb is so hopelessly intertwined with yum that we # have to totally reinvent the dependency resolver. mainopts = dict(cachedir='/', + persistdir='/', installroot=self.cachefile, keepcache="0", debuglevel="0", @@ -840,6 +871,17 @@ class YumCollection(Collection): if not self.use_yum: return Collection.complete(self, packagelist) + lock = FileLock(os.path.join(self.cachefile, "lock")) + slept = 0 + while lock.is_locked(): + if slept > 30: + self.logger.warning("Packages: Timeout waiting for yum cache " + "to release its lock") + return set(), set() + self.logger.debug("Packages: Yum cache is locked, waiting...") + time.sleep(3) + slept += 3 + if packagelist: try: result = self.call_helper( @@ -888,40 +930,30 @@ class YumCollection(Collection): cmd.append("-v") cmd.append(command) self.debug_log("Packages: running %s" % " ".join(cmd)) - try: - helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to execute %s: %s" % - (" ".join(cmd), err)) - return None if inputdata: - idata = json.dumps(inputdata) - (stdout, stderr) = helper.communicate(idata) + result = self.cmd.run(cmd, timeout=self.setup['client_timeout'], + inputdata=json.dumps(inputdata)) else: - (stdout, stderr) = helper.communicate() - rv = helper.wait() - errlines = stderr.splitlines() - if rv: - if not errlines: - errlines.append("No error output") - self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, errlines[0])) - for line in errlines[1:]: - self.logger.error("Packages: %s" % line) - elif errlines: + result = self.cmd.run(cmd, timeout=self.setup['client_timeout']) + if not result.success: + self.logger.error("Packages: error running bcfg2-yum-helper: %s" % + result.error) + elif result.stderr: self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - errlines[0]) - for line in errlines[1:]: - self.debug_log("Packages: %s" % line) + result.stderr) try: - return json.loads(stdout) + return json.loads(result.stdout) except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: error reading bcfg2-yum-helper " - "output: %s" % err) + if result.stdout: + err = sys.exc_info()[1] + self.logger.error("Packages: Error reading bcfg2-yum-helper " + "output: %s" % err) + self.logger.error("Packages: bcfg2-yum-helper output: %s" % + result.stdout) + else: + self.logger.error("Packages: No bcfg2-yum-helper output") raise def setup_data(self, force_update=False): @@ -934,8 +966,7 @@ class YumCollection(Collection): If using the yum Python libraries, this cleans up cached yum metadata, regenerates the server-side yum config (in order to catch any new sources that have been added to this server), - and then cleans up cached yum metadata again, in case the new - config has any preexisting cache. + then regenerates the yum cache. :param force_update: Ignore all local cache and setup data from its original upstream sources (i.e., @@ -946,23 +977,22 @@ class YumCollection(Collection): return Collection.setup_data(self, force_update) if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config + # clean up data from the old config try: self.call_helper("clean") except ValueError: # error reported by call_helper pass - os.unlink(self.cfgfile) + if os.path.exists(self.cfgfile): + os.unlink(self.cfgfile) self.write_config() - if force_update: - try: - self.call_helper("clean") - except ValueError: - # error reported by call_helper - pass + try: + self.call_helper("makecache") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index f93bd0932..479138ef1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -9,7 +9,8 @@ import shutil import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin -from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError +from Bcfg2.Compat import ConfigParser, urlopen, HTTPError, URLError, \ + MutableMapping from Bcfg2.Server.Plugins.Packages.Collection import Collection, \ get_collection_class from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources @@ -22,7 +23,54 @@ APT_CONFIG_DEFAULT = \ "/etc/apt/sources.list.d/bcfg2-packages-generated-sources.list" +class OnDemandDict(MutableMapping): + """ This maps a set of keys to a set of value-getting functions; + the values are populated on-the-fly by the functions as the values + are needed (and not before). This is used by + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_additional_data`; + see the docstring for that function for details on why. + + Unlike a dict, you should not specify values for for the righthand + side of this mapping, but functions that get values. E.g.: + + .. code-block:: python + + d = OnDemandDict(foo=load_foo, + bar=lambda: "bar"); + """ + + def __init__(self, **getters): + self._values = dict() + self._getters = dict(**getters) + + def __getitem__(self, key): + if key not in self._values: + self._values[key] = self._getters[key]() + return self._values[key] + + def __setitem__(self, key, getter): + self._getters[key] = getter + + def __delitem__(self, key): + del self._values[key] + del self._getters[key] + + def __len__(self): + return len(self._getters) + + def __iter__(self): + return iter(self._getters.keys()) + + def __repr__(self): + rv = dict(self._values) + for key in self._getters.keys(): + if key not in rv: + rv[key] = 'unknown' + return str(rv) + + class Packages(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.Connector, @@ -45,8 +93,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: and :func:`Reload` __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] + __child_rmi__ = Bcfg2.Server.Plugin.Plugin.__child_rmi__ + \ + [('Refresh', 'expire_cache'), ('Reload', 'expire_cache')] + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) @@ -110,8 +162,21 @@ class Packages(Bcfg2.Server.Plugin.Plugin, #: object when one is requested, so each entry is very #: short-lived -- it's purged at the end of each client run. self.clients = dict() - # pylint: enable=C0301 + #: groupcache caches group lookups. It maps Collections (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of package groups, and thence to the packages + #: indicated by those groups. + self.groupcache = dict() + + #: pkgcache caches complete package sets. It maps Collections + #: (via + #: :attr:`Bcfg2.Server.Plugins.Packages.Collection.Collection.cachekey`) + #: to sets of initial packages, and thence to the final + #: (complete) package selections resolved from the initial + #: packages + self.pkgcache = dict() + # pylint: enable=C0301 __init__.__doc__ = Bcfg2.Server.Plugin.Plugin.__init__.__doc__ def set_debug(self, debug): @@ -355,14 +420,24 @@ class Packages(Bcfg2.Server.Plugin.Plugin, for el in to_remove: el.getparent().remove(el) - gpkgs = collection.get_groups(groups) - for pkgs in gpkgs.values(): + groups.sort() + # check for this set of groups in the group cache + gkey = hash(tuple(groups)) + if gkey not in self.groupcache[collection.cachekey]: + self.groupcache[collection.cachekey][gkey] = \ + collection.get_groups(groups) + for pkgs in self.groupcache[collection.cachekey][gkey].values(): base.update(pkgs) # essential pkgs are those marked as such by the distribution base.update(collection.get_essential()) - packages, unknown = collection.complete(base) + # check for this set of packages in the package cache + pkey = hash(tuple(base)) + if pkey not in self.pkgcache[collection.cachekey]: + self.pkgcache[collection.cachekey][pkey] = \ + collection.complete(base) + packages, unknown = self.pkgcache[collection.cachekey][pkey] if unknown: self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) @@ -388,6 +463,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self._load_config() return True + def expire_cache(self, _=None): + self.Reload() + def _load_config(self, force_update=False): """ Load the configuration data and setup sources @@ -415,9 +493,11 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.disableMetaData: collection.setup_data(force_update) - # clear Collection caches + # clear Collection and package caches self.clients = dict() self.collections = dict() + self.groupcache = dict() + self.pkgcache = dict() for source in self.sources.entries: cachefiles.add(source.cachefile) @@ -493,8 +573,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if not self.sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object - return Collection(metadata, [], self.cachepath, self.data, - self.core.fam) + collection = Collection(metadata, [], self.cachepath, self.data, + self.core.fam) + ckey = collection.cachekey + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) + return collection if metadata.hostname in self.clients: return self.collections[self.clients[metadata.hostname]] @@ -531,24 +615,47 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if cclass != Collection: self.clients[metadata.hostname] = ckey self.collections[ckey] = collection + self.groupcache.setdefault(ckey, dict()) + self.pkgcache.setdefault(ckey, dict()) return collection def get_additional_data(self, metadata): """ Return additional data for the given client. This will be - a dict containing a single key, ``sources``, whose value is a - list of data returned from - :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, - namely, a list of - :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` - data. + an :class:`Bcfg2.Server.Plugins.Packages.OnDemandDict` + containing two keys: + + * ``sources``, whose value is a list of data returned from + :func:`Bcfg2.Server.Plugins.Packages.Collection.Collection.get_additional_data`, + namely, a list of + :attr:`Bcfg2.Server.Plugins.Packages.Source.Source.url_map` + data; and + * ``get_config``, whose value is the + :func:`Bcfg2.Server.Plugins.Packages.Packages.get_config` + function, which can be used to get the Packages config for + other systems. + + This uses an OnDemandDict instead of just a normal dict + because loading a source collection can be a fairly + time-consuming process, particularly for the first time. As a + result, when all metadata objects are built at once (such as + after the server is restarted, or far more frequently if + Metadata caching is disabled), this function would be a major + bottleneck if we tried to build all collections at the same + time. Instead, they're merely built on-demand. :param metadata: The client metadata :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata :return: dict of lists of ``url_map`` data """ - collection = self.get_collection(metadata) - return dict(sources=collection.get_additional_data(), - get_config=self.get_config) + def get_sources(): + """ getter for the 'sources' key of the OnDemandDict + returned by this function. This delays calling + get_collection() until it's absolutely necessary. """ + return self.get_collection(metadata).get_additional_data() + + return OnDemandDict( + sources=get_sources, + get_config=lambda: self.get_config) def end_client_run(self, metadata): """ Hook to clear the cache for this client in diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 0974184b4..fd6fd3bd1 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -9,6 +9,7 @@ import operator import lxml.etree import Bcfg2.Server import Bcfg2.Server.Plugin +from Bcfg2.Compat import unicode # pylint: disable=W0622 try: from django.db import models @@ -64,7 +65,10 @@ class ProbeData(str): # pylint: disable=E0012,R0924 .json, and .yaml properties to provide convenient ways to use ProbeData objects as XML, JSON, or YAML data """ def __new__(cls, data): - return str.__new__(cls, data) + if isinstance(data, unicode): + return str.__new__(cls, data.encode('utf-8')) + else: + return str.__new__(cls, data) def __init__(self, data): # pylint: disable=W0613 str.__init__(self) @@ -181,14 +185,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): class Probes(Bcfg2.Server.Plugin.Probing, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Connector, Bcfg2.Server.Plugin.DatabaseBacked): """ A plugin to gather information from a client machine """ __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.Caching.__init__(self) + Bcfg2.Server.Plugin.Connector.__init__(self) Bcfg2.Server.Plugin.DatabaseBacked.__init__(self, core, datastore) try: @@ -223,9 +229,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, lxml.etree.SubElement(top, 'Client', name=client, timestamp=str(int(probedata.timestamp))) for probe in sorted(probedata): - lxml.etree.SubElement( - ctag, 'Probe', name=probe, - value=self.probedata[client][probe]) + try: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str( + self.probedata[client][probe]).decode('utf-8')) + except AttributeError: + lxml.etree.SubElement( + ctag, 'Probe', name=probe, + value=str(self.probedata[client][probe])) for group in sorted(self.cgroups[client]): lxml.etree.SubElement(ctag, "Group", name=group) try: @@ -249,7 +261,7 @@ class Probes(Bcfg2.Server.Plugin.Probing, ProbesDataModel.objects.filter( hostname=client.hostname).exclude( - probe__in=self.probedata[client.hostname]).delete() + probe__in=self.probedata[client.hostname]).delete() for group in self.cgroups[client.hostname]: try: @@ -264,14 +276,19 @@ class Probes(Bcfg2.Server.Plugin.Probing, group=group) ProbesGroupsModel.objects.filter( hostname=client.hostname).exclude( - group__in=self.cgroups[client.hostname]).delete() + group__in=self.cgroups[client.hostname]).delete() + + def expire_cache(self, key=None): + self.load_data(client=key) - def load_data(self): + def load_data(self, client=None): """ Load probe data from the appropriate backend (probed.xml or the database) """ if self._use_db: - return self._load_data_db() + return self._load_data_db(client=client) else: + # the XML backend doesn't support loading data for single + # clients, so it reloads all data return self._load_data_xml() def _load_data_xml(self): @@ -296,20 +313,36 @@ class Probes(Bcfg2.Server.Plugin.Probing, elif pdata.tag == 'Group': self.cgroups[client.get('name')].append(pdata.get('name')) - def _load_data_db(self): + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + + def _load_data_db(self, client=None): """ Load probe data from the database """ - self.probedata = {} - self.cgroups = {} - for pdata in ProbesDataModel.objects.all(): + if client is None: + self.probedata = {} + self.cgroups = {} + probedata = ProbesDataModel.objects.all() + groupdata = ProbesGroupsModel.objects.all() + else: + self.probedata.pop(client, None) + self.cgroups.pop(client, None) + probedata = ProbesDataModel.objects.filter(hostname=client) + groupdata = ProbesGroupsModel.objects.filter(hostname=client) + + for pdata in probedata: if pdata.hostname not in self.probedata: self.probedata[pdata.hostname] = ClientProbeDataSet( timestamp=time.mktime(pdata.timestamp.timetuple())) self.probedata[pdata.hostname][pdata.probe] = ProbeData(pdata.data) - for pgroup in ProbesGroupsModel.objects.all(): + for pgroup in groupdata: if pgroup.hostname not in self.cgroups: self.cgroups[pgroup.hostname] = [] self.cgroups[pgroup.hostname].append(pgroup.group) + if self.core.metadata_cache_mode in ['cautious', 'aggressive']: + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) + @Bcfg2.Server.Plugin.track_statistics() def GetProbes(self, meta): return self.probes.get_probe_data(meta) diff --git a/src/lib/Bcfg2/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index e97f66675..89f2d21ff 100644 --- a/src/lib/Bcfg2/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -223,7 +223,7 @@ class XMLPropertyFile(Bcfg2.Server.Plugin.StructFile, PropertyFile): if strict: raise PluginExecutionError(msg) else: - LOGGER.warning(msg) + LOGGER.info(msg) Index.__doc__ = Bcfg2.Server.Plugin.StructFile.Index.__doc__ def _decrypt(self, element): diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py index 801e7006d..072f3f7e7 100644 --- a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -127,7 +127,7 @@ class PuppetENC(Bcfg2.Server.Plugin.Plugin, self.logger.warning("PuppetENC is incompatible with aggressive " "client metadata caching, try 'cautious' or " "'initial' instead") - self.core.cache.expire() + self.core.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) def end_statistics(self, metadata): self.end_client_run(self, metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index d8b3104b7..2deea5f07 100644 --- a/src/lib/Bcfg2/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -92,6 +92,7 @@ class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): class SSHbase(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Caching, Bcfg2.Server.Plugin.Generator, Bcfg2.Server.Plugin.PullTarget): """ @@ -125,6 +126,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Caching.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.PullTarget.__init__(self) self.ipcache = {} @@ -149,6 +151,9 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, HostKeyEntrySet(keypattern, self.data) self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk + def expire_cache(self, key=None): + self.__skn = False + def get_skn(self): """Build memory cache of the ssh known hosts file.""" if not self.__skn: diff --git a/src/lib/Bcfg2/Server/models.py b/src/lib/Bcfg2/Server/models.py index 1f64111e7..7e2f5b09d 100644 --- a/src/lib/Bcfg2/Server/models.py +++ b/src/lib/Bcfg2/Server/models.py @@ -57,7 +57,7 @@ def load_models(plugins=None, cfile='/etc/bcfg2.conf', quiet=True): # the second attempt. LOGGER.error("Failed to load plugin %s: %s" % (plugin, err)) - continue + continue for sym in dir(mod): obj = getattr(mod, sym) if hasattr(obj, "__bases__") and models.Model in obj.__bases__: diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 4293f3f69..ab1276178 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -244,9 +244,9 @@ class Executor(object): # py3k fixes if not isinstance(stdout, str): - stdout = stdout.decode('utf-8') + stdout = stdout.decode('utf-8') # pylint: disable=E1103 if not isinstance(stderr, str): - stderr = stderr.decode('utf-8') + stderr = stderr.decode('utf-8') # pylint: disable=E1103 for line in stdout.splitlines(): # pylint: disable=E1103 self.logger.debug('< %s' % line) diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 9adfd66bf..82a3bdb2a 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -26,6 +26,8 @@ DATABASE_USER = None DATABASE_PASSWORD = None DATABASE_HOST = None DATABASE_PORT = None +DATABASE_OPTIONS = None +DATABASE_SCHEMA = None TIME_ZONE = None @@ -58,8 +60,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ - DATABASE_HOST, DATABASE_PORT, DEBUG, TEMPLATE_DEBUG, TIME_ZONE, \ - MEDIA_URL + DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DATABASE_SCHEMA, \ + DEBUG, TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): @@ -86,7 +88,9 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): USER=setup['db_user'], PASSWORD=setup['db_password'], HOST=setup['db_host'], - PORT=setup['db_port']) + PORT=setup['db_port'], + OPTIONS=setup['db_options'], + SCHEMA=setup['db_schema']) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: DATABASE_ENGINE = setup['db_engine'] @@ -95,6 +99,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): DATABASE_PASSWORD = DATABASES['default']['PASSWORD'] DATABASE_HOST = DATABASES['default']['HOST'] DATABASE_PORT = DATABASES['default']['PORT'] + DATABASE_OPTIONS = DATABASES['default']['OPTIONS'] + DATABASE_SCHEMA = DATABASES['default']['SCHEMA'] # dropping the version check. This was added in 1.1.2 TIME_ZONE = setup['time_zone'] diff --git a/src/lib/Bcfg2/version.py b/src/lib/Bcfg2/version.py index 12fc584fe..140fb6937 100644 --- a/src/lib/Bcfg2/version.py +++ b/src/lib/Bcfg2/version.py @@ -2,7 +2,7 @@ import re -__version__ = "1.3.1" +__version__ = "1.3.2" class Bcfg2VersionInfo(tuple): # pylint: disable=E0012,R0924 diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt index aad89882f..0ba84fa0a 100755 --- a/src/sbin/bcfg2-crypt +++ b/src/sbin/bcfg2-crypt @@ -18,291 +18,168 @@ except ImportError: raise SystemExit(1) -class EncryptionChunkingError(Exception): - """ error raised when Encryptor cannot break a file up into chunks - to be encrypted, or cannot reassemble the chunks """ - pass +class PassphraseError(Exception): + """ Exception raised when there's a problem determining the + passphrase to encrypt or decrypt with """ -class Encryptor(object): - """ Generic encryptor for all files """ - - def __init__(self, setup): +class CryptoTool(object): + """ Generic decryption/encryption interface base object """ + def __init__(self, filename, setup): self.setup = setup - self.passphrase = None - self.pname = None self.logger = logging.getLogger(self.__class__.__name__) + self.passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - def get_encrypted_filename(self, plaintext_filename): - """ get the name of the file encrypted data should be written to """ - return plaintext_filename - - def get_plaintext_filename(self, encrypted_filename): - """ get the name of the file decrypted data should be written to """ - return encrypted_filename - - def chunk(self, data): - """ generator to break the file up into smaller chunks that - will each be individually encrypted or decrypted """ - yield data - - def unchunk(self, data, original): # pylint: disable=W0613 - """ given chunks of a file, reassemble then into the whole file """ + self.filename = filename try: - return data[0] - except IndexError: - raise EncryptionChunkingError("No data to unchunk") + self.data = open(self.filename).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % (filename, + err)) + return False - def set_passphrase(self): - """ set the passphrase for the current file """ + self.pname, self.passphrase = self._get_passphrase() + + def _get_passphrase(self): + """ get the passphrase for the current file """ if (not self.setup.cfp.has_section(Bcfg2.Encryption.CFG_SECTION) or len(Bcfg2.Encryption.get_passphrases(self.setup)) == 0): - self.logger.error("No passphrases available in %s" % - self.setup['configfile']) - return False - - if self.passphrase: - self.logger.debug("Using previously determined passphrase %s" % - self.pname) - return True + raise PassphraseError("No passphrases available in %s" % + self.setup['configfile']) + pname = None if self.setup['passphrase']: - self.pname = self.setup['passphrase'] + pname = self.setup['passphrase'] - if self.pname: + if pname: if self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - self.pname): - self.passphrase = \ - self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - self.pname) + pname): + passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, + pname) self.logger.debug("Using passphrase %s specified on command " - "line" % self.pname) - return True + "line" % pname) + return (pname, passphrase) else: - self.logger.error("Could not find passphrase %s in %s" % - (self.pname, self.setup['configfile'])) - return False + raise PassphraseError("Could not find passphrase %s in %s" % + (pname, self.setup['configfile'])) else: pnames = Bcfg2.Encryption.get_passphrases(self.setup) if len(pnames) == 1: - self.pname = pnames.keys()[0] - self.passphrase = pnames[self.pname] - self.logger.info("Using passphrase %s" % self.pname) - return True + pname = pnames.keys()[0] + passphrase = pnames[pname] + self.logger.info("Using passphrase %s" % pname) + return (pname, passphrase) elif len(pnames) > 1: - self.logger.warning("Multiple passphrases found in %s, " - "specify one on the command line with -p" % - self.setup['configfile']) - self.logger.info("No passphrase could be determined") - return False - - def encrypt(self, fname): - """ encrypt the given file, returning the encrypted data """ + return (None, None) + raise PassphraseError("No passphrase could be determined") + + def get_destination_filename(self, original_filename): + """ Get the filename where data should be written """ + return original_filename + + def write(self, data): + """ write data to disk """ + new_fname = self.get_destination_filename(self.filename) try: - plaintext = open(fname).read() + self._write(new_fname, data) + self.logger.info("Wrote data to %s" % new_fname) + return True except IOError: err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (fname, err)) + self.logger.error("Error writing data from %s to %s: %s" % + (self.filename, new_fname, err)) return False - if not self.set_passphrase(): - return False + def _write(self, filename, data): + """ Perform the actual write of data. This is separate from + :func:`CryptoTool.write` so it can be easily + overridden. """ + open(filename, "wb").write(data) - crypted = [] - try: - for chunk in self.chunk(plaintext): - try: - passphrase, pname = self.get_passphrase(chunk) - except TypeError: - return False - crypted.append(self._encrypt(chunk, passphrase, name=pname)) - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error getting data to encrypt from %s: %s" % - (fname, err)) - return False - return self.unchunk(crypted, plaintext) +class Decryptor(CryptoTool): + """ Decryptor interface """ + def decrypt(self): + """ decrypt the file, returning the encrypted data """ + raise NotImplementedError - # pylint: disable=W0613 - def _encrypt(self, plaintext, passphrase, name=None): - """ encrypt a single chunk of a file """ - return Bcfg2.Encryption.ssl_encrypt( - plaintext, passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)) - # pylint: enable=W0613 - def decrypt(self, fname): - """ decrypt the given file, returning the plaintext data """ - try: - crypted = open(fname).read() - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error reading %s, skipping: %s" % (fname, err)) - return False +class Encryptor(CryptoTool): + """ encryptor interface """ + def encrypt(self): + """ encrypt the file, returning the encrypted data """ + raise NotImplementedError - self.set_passphrase() - plaintext = [] - try: - for chunk in self.chunk(crypted): - try: - passphrase, pname = self.get_passphrase(chunk) - try: - plaintext.append(self._decrypt(chunk, passphrase)) - except Bcfg2.Encryption.EVPError: - self.logger.info("Could not decrypt %s with the " - "specified passphrase" % fname) - continue - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (fname, err)) - continue - except TypeError: - pchunk = None - passphrases = Bcfg2.Encryption.get_passphrases(self.setup) - for pname, passphrase in passphrases.items(): - self.logger.debug("Trying passphrase %s" % pname) - try: - pchunk = self._decrypt(chunk, passphrase) - break - except Bcfg2.Encryption.EVPError: - pass - except: - err = sys.exc_info()[1] - self.logger.error("Error decrypting %s: %s" % - (fname, err)) - if pchunk is not None: - plaintext.append(pchunk) - else: - self.logger.error("Could not decrypt %s with any " - "passphrase in %s" % - (fname, self.setup['configfile'])) - continue - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error getting encrypted data from %s: %s" % - (fname, err)) - return False +class CfgEncryptor(Encryptor): + """ encryptor class for Cfg files """ - try: - return self.unchunk(plaintext, crypted) - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error assembling plaintext data from %s: %s" % - (fname, err)) - return False + def __init__(self, filename, setup): + Encryptor.__init__(self, filename, setup) + if self.passphrase is None: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + self.setup['configfile']) - def _decrypt(self, crypted, passphrase): - """ decrypt a single chunk """ - return Bcfg2.Encryption.ssl_decrypt( - crypted, passphrase, + def encrypt(self): + return Bcfg2.Encryption.ssl_encrypt( + self.data, self.passphrase, Bcfg2.Encryption.get_algorithm(self.setup)) - def write_encrypted(self, fname, data=None): - """ write encrypted data to disk """ - if data is None: - data = self.decrypt(fname) - new_fname = self.get_encrypted_filename(fname) - try: - open(new_fname, "wb").write(data) - self.logger.info("Wrote encrypted data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing encrypted data from %s to %s: %s" - % (fname, new_fname, err)) - return False - except EncryptionChunkingError: - err = sys.exc_info()[1] - self.logger.error("Error assembling encrypted data from %s: %s" % - (fname, err)) - return False + def get_destination_filename(self, original_filename): + return original_filename + ".crypt" - def write_decrypted(self, fname, data=None): - """ write decrypted data to disk """ - if data is None: - data = self.decrypt(fname) - new_fname = self.get_plaintext_filename(fname) - try: - open(new_fname, "wb").write(data) - self.logger.info("Wrote decrypted data to %s" % new_fname) - return True - except IOError: - err = sys.exc_info()[1] - self.logger.error("Error writing encrypted data from %s to %s: %s" - % (fname, new_fname, err)) - return False - def get_passphrase(self, chunk): - """ get the passphrase for a chunk of a file """ - pname = self._get_passphrase(chunk) - if not self.pname: - if not pname: - self.logger.info("No passphrase given on command line or " - "found in file") +class CfgDecryptor(Decryptor): + """ Decrypt Cfg files """ + + def decrypt(self): + """ decrypt the given file, returning the plaintext data """ + if self.passphrase: + try: + return Bcfg2.Encryption.ssl_decrypt( + self.data, self.passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)) + except Bcfg2.Encryption.EVPError: + self.logger.info("Could not decrypt %s with the " + "specified passphrase" % self.filename) return False - elif self.setup.cfp.has_option(Bcfg2.Encryption.CFG_SECTION, - pname): - passphrase = self.setup.cfp.get(Bcfg2.Encryption.CFG_SECTION, - pname) - else: - self.logger.error("Could not find passphrase %s in %s" % - (pname, self.setup['configfile'])) + except: + err = sys.exc_info()[1] + self.logger.error("Error decrypting %s: %s" % + (self.filename, err)) + return False + else: # no passphrase given, brute force + try: + return Bcfg2.Encryption.bruteforce_decrypt( + self.data, passphrases=self.passphrases.values(), + algorithm=Bcfg2.Encryption.get_algorithm(self.setup)) + except Bcfg2.Encryption.EVPError: + self.logger.info("Could not decrypt %s with any passphrase" % + self.filename) return False - else: - pname = self.pname - passphrase = self.passphrase - if self.pname != pname: - self.logger.warning("Passphrase given on command line (%s) " - "differs from passphrase embedded in " - "file (%s), using command-line option" % - (self.pname, pname)) - return (passphrase, pname) - - def _get_passphrase(self, chunk): # pylint: disable=W0613 - """ get the passphrase for a chunk of a file """ - return None - - -class CfgEncryptor(Encryptor): - """ encryptor class for Cfg files """ - - def get_encrypted_filename(self, plaintext_filename): - return plaintext_filename + ".crypt" - def get_plaintext_filename(self, encrypted_filename): - if encrypted_filename.endswith(".crypt"): - return encrypted_filename[:-6] + def get_destination_filename(self, original_filename): + if original_filename.endswith(".crypt"): + return original_filename[:-6] else: - return Encryptor.get_plaintext_filename(self, encrypted_filename) + return Decryptor.get_plaintext_filename(self, original_filename) -class PropertiesEncryptor(Encryptor): - """ encryptor class for Properties files """ - - def _encrypt(self, plaintext, passphrase, name=None): - # plaintext is an lxml.etree._Element - if name is None: - name = "true" - if plaintext.text and plaintext.text.strip(): - plaintext.text = Bcfg2.Encryption.ssl_encrypt( - plaintext.text, - passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - plaintext.set("encrypted", name) - return plaintext +class PropertiesCryptoMixin(object): + """ Mixin to provide some common methods for Properties crypto """ + default_xpath = '//*' - def chunk(self, data): - xdata = lxml.etree.XML(data, parser=XMLParser) + def _get_elements(self, xdata): + """ Get the list of elements to encrypt or decrypt """ if self.setup['xpath']: elements = xdata.xpath(self.setup['xpath']) if not elements: - raise EncryptionChunkingError("XPath expression %s matched no " - "elements" % self.setup['xpath']) + self.logger.warning("XPath expression %s matched no " + "elements" % self.setup['xpath']) else: - elements = xdata.xpath('//*[@encrypted]') + elements = xdata.xpath(self.default_xpath) if not elements: elements = list(xdata.getiterator(tag=lxml.etree.Element)) @@ -329,50 +206,85 @@ class PropertiesEncryptor(Encryptor): ans = input("Encrypt this element? [y/N] ") if not ans.lower().startswith("y"): elements.remove(element) + return elements + + def _get_element_passphrase(self, element): + """ Get the passphrase to use to encrypt or decrypt a given + element """ + pname = element.get("encrypted") + if pname in self.passphrases: + passphrase = self.passphrases[pname] + elif self.passphrase: + if pname: + self.logger.warning("Passphrase %s not found in %s, " + "using passphrase given on command line" + % (pname, self.setup['configfile'])) + passphrase = self.passphrase + pname = self.pname + else: + raise PassphraseError("Multiple passphrases found in %s, " + "specify one on the command line with -p" % + self.setup['configfile']) + return (pname, passphrase) - # this is not a good use of a generator, but we need to - # generate the full list of elements in order to ensure that - # some exist before we know what to return - for elt in elements: - yield elt - - def unchunk(self, data, original): - # Properties elements are modified in-place, so we don't - # actually need to unchunk anything - xdata = data[0] - # find root element - while xdata.getparent() is not None: - xdata = xdata.getparent() - return lxml.etree.tostring(xdata, - xml_declaration=False, - pretty_print=True).decode('UTF-8') - - def _get_passphrase(self, chunk): - pname = chunk.get("encrypted") - if pname and pname.lower() != "true": - return pname - return None - - def _decrypt(self, crypted, passphrase): - # crypted is in lxml.etree._Element - if not crypted.text or not crypted.text.strip(): - self.logger.warning("Skipping empty element %s" % crypted.tag) - return crypted - decrypted = Bcfg2.Encryption.ssl_decrypt( - crypted.text, - passphrase, - Bcfg2.Encryption.get_algorithm(self.setup)).strip() - try: - crypted.text = decrypted.encode('ascii', 'xmlcharrefreplace') - except UnicodeDecodeError: - # we managed to decrypt the value, but it contains content - # that can't even be encoded into xml entities. what - # probably happened here is that we coincidentally could - # decrypt a value encrypted with a different key, and - # wound up with gibberish. - self.logger.warning("Decrypted %s to gibberish, skipping" % - crypted.tag) - return crypted + def _write(self, filename, data): + """ Write the data """ + data.getroottree().write(filename, + xml_declaration=False, + pretty_print=True) + + +class PropertiesEncryptor(Encryptor, PropertiesCryptoMixin): + """ encryptor class for Properties files """ + + def encrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + elt.text = Bcfg2.Encryption.ssl_encrypt( + elt.text, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)).strip() + elt.set("encrypted", pname) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) + + +class PropertiesDecryptor(Decryptor, PropertiesCryptoMixin): + """ decryptor class for Properties files """ + default_xpath = '//*[@encrypted]' + + def decrypt(self): + xdata = lxml.etree.XML(self.data, parser=XMLParser) + for elt in self._get_elements(xdata): + try: + pname, passphrase = self._get_element_passphrase(elt) + except PassphraseError: + self.logger.error(str(sys.exc_info()[1])) + return False + decrypted = Bcfg2.Encryption.ssl_decrypt( + elt.text, passphrase, + Bcfg2.Encryption.get_algorithm(self.setup)).strip() + try: + elt.text = decrypted.encode('ascii', 'xmlcharrefreplace') + elt.set("encrypted", pname) + except UnicodeDecodeError: + # we managed to decrypt the value, but it contains + # content that can't even be encoded into xml + # entities. what probably happened here is that we + # coincidentally could decrypt a value encrypted with + # a different key, and wound up with gibberish. + self.logger.warning("Decrypted %s to gibberish, skipping" % + elt.tag) + return xdata + + def _write(self, filename, data): + PropertiesCryptoMixin._write(self, filename, data) def main(): # pylint: disable=R0912,R0915 @@ -422,9 +334,6 @@ def main(): # pylint: disable=R0912,R0915 logger.error("--remove cannot be used with --properties, ignoring") setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default - props_crypt = PropertiesEncryptor(setup) - cfg_crypt = CfgEncryptor(setup) - for fname in setup['args']: if not os.path.exists(fname): logger.error("%s does not exist, skipping" % fname) @@ -454,10 +363,10 @@ def main(): # pylint: disable=R0912,R0915 props = False if props: - encryptor = props_crypt if setup['remove']: logger.info("Cannot use --remove with Properties file %s, " "ignoring for this file" % fname) + tools = (PropertiesEncryptor, PropertiesDecryptor) else: if setup['xpath']: logger.info("Cannot use xpath with Cfg file %s, ignoring " @@ -465,31 +374,52 @@ def main(): # pylint: disable=R0912,R0915 if setup['interactive']: logger.info("Cannot use interactive mode with Cfg file %s, " "ignoring -I for this file" % fname) - encryptor = cfg_crypt + tools = (CfgEncryptor, CfgDecryptor) data = None + mode = None if setup['encrypt']: - xform = encryptor.encrypt - write = encryptor.write_encrypted + try: + tool = tools[0](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "encrypt" elif setup['decrypt']: - xform = encryptor.decrypt - write = encryptor.write_decrypted + try: + tool = tools[1](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "decrypt" else: logger.info("Neither --encrypt nor --decrypt specified, " "determining mode") - data = encryptor.decrypt(fname) - if data: - write = encryptor.write_decrypted - else: - logger.info("Failed to decrypt %s, trying encryption" % fname) + try: + tool = tools[1](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + + try: + data = tool.decrypt() + mode = "decrypt" + except: # pylint: disable=W0702 + pass + if data is False: data = None - xform = encryptor.encrypt - write = encryptor.write_encrypted + logger.info("Failed to decrypt %s, trying encryption" % fname) + try: + tool = tools[0](fname, setup) + except PassphraseError: + logger.error(str(sys.exc_info()[1])) + return 2 + mode = "encrypt" if data is None: - data = xform(fname) + data = getattr(tool, mode)() if not data: - logger.error("Failed to %s %s, skipping" % (xform.__name__, fname)) + logger.error("Failed to %s %s, skipping" % (mode, fname)) continue if setup['crypt_stdout']: if len(setup['args']) > 1: @@ -498,10 +428,10 @@ def main(): # pylint: disable=R0912,R0915 if len(setup['args']) > 1: print("") else: - write(fname, data=data) + tool.write(data) if (setup['remove'] and - encryptor.get_encrypted_filename(fname) != fname): + tool.get_destination_filename(fname) != fname): try: os.unlink(fname) except IOError: diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 133e1ccb3..6008f8896 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -437,7 +437,7 @@ Bcfg2 client itself.""") pname, client = alist automatch = self.setup.cfp.getboolean("properties", "automatch", default=False) - pfile = self.plugins['Properties'].store.entries[pname] + pfile = self.plugins['Properties'].entries[pname] if (not force and not automatch and pfile.xdata.get("automatch", "false").lower() != "true"): @@ -482,6 +482,17 @@ Bcfg2 client itself.""") ('Logging', self.setup['logging'])] print_tabular(output) + def do_expirecache(self, args): + """ expirecache [<hostname> [<hostname> ...]]- Expire the + metadata cache """ + alist = args.split() + if len(alist): + for client in self._get_client_list(alist): + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata, + key=client) + else: + self.expire_caches_by_type(Bcfg2.Server.Plugin.Metadata) + def do_probes(self, args): """ probes [-p] <hostname> - Get probe list for the given host, in XML (the default) or human-readable pretty (with -p) @@ -717,7 +728,7 @@ Bcfg2 client itself.""") def run(self, args): # pylint: disable=W0221 try: self.load_plugins() - self.fam.handle_events_in_interval(1) + self.block_for_fam_events(handle_events=True) if args: self.onecmd(" ".join(args)) else: @@ -758,7 +769,8 @@ USAGE = build_usage() def main(): optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, interactive=Bcfg2.Options.INTERACTIVE, - interpreter=Bcfg2.Options.INTERPRETER) + interpreter=Bcfg2.Options.INTERPRETER, + command_timeout=Bcfg2.Options.CLIENT_COMMAND_TIMEOUT) optinfo.update(Bcfg2.Options.INFO_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index ab3b6450f..9ceb1dd04 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -73,7 +73,7 @@ def load_server(setup): """ load server """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index d7a1894f0..7c38a65d8 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -157,7 +157,7 @@ def get_core(setup): """ Get a server core, with events handled """ core = Bcfg2.Server.Core.BaseCore(setup) core.load_plugins() - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) return core @@ -298,8 +298,8 @@ def main(): for client in clients: yield ClientTest(core, client, ignore) - TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], - suite=LazySuite(generate_tests), exit=False) + result = TestProgram(argv=sys.argv[:1] + core.setup['noseopts'], + suite=LazySuite(generate_tests), exit=False) # block until all children have completed -- should be # immediate since we've already gotten all the results we @@ -308,7 +308,10 @@ def main(): child.join() core.shutdown() - os._exit(0) # pylint: disable=W0212 + if result.success: + os._exit(0) # pylint: disable=W0212 + else: + os._exit(1) # pylint: disable=W0212 if __name__ == "__main__": diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 4ef531d39..49baeb9c3 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -10,6 +10,8 @@ import sys import yum import logging import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout from optparse import OptionParser try: import json @@ -42,8 +44,8 @@ def pkgtup_to_string(package): return ''.join(str(e) for e in rv) -class DepSolver(object): - """ Yum dependency solver """ +class YumHelper(object): + """ Yum helper base object """ def __init__(self, cfgfile, verbose=1): self.cfgfile = cfgfile @@ -57,6 +59,16 @@ class DepSolver(object): self.yumbase._getConfig(cfgfile, debuglevel=verbose) # pylint: enable=E1121,W0212 self.logger = logging.getLogger(self.__class__.__name__) + + +class DepSolver(YumHelper): + """ Yum dependency solver. This is used for operations that only + read from the yum cache, and thus operates in cacheonly mode. """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + # internally, yum uses an integer, not a boolean, for conf.cache + self.yumbase.conf.cache = 1 self._groups = None def get_groups(self): @@ -181,6 +193,45 @@ class DepSolver(object): packages.add(txmbr.pkgtup) return list(packages), list(unknown) + +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + +class CacheManager(YumHelper): + """ Yum cache manager. Unlike :class:`DepSolver`, this can write + to the yum cache, and so is used for operations that muck with the + cache. (Technically, :func:`CacheManager.clean_cache` could be in + either DepSolver or CacheManager, but for consistency I've put it + here.) """ + + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock def clean_cache(self): """ clean the yum cache """ for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", @@ -193,6 +244,27 @@ class DepSolver(object): if not msg.startswith("0 "): self.logger.info(msg) + @acquire_lock + def populate_cache(self): + """ populate the yum cache """ + for repo in self.yumbase.repos.findRepos('*'): + repo.metadata_expire = 0 + repo.mdpolicy = "group:all" + self.yumbase.doRepoSetup() + self.yumbase.repos.doSetup() + for repo in self.yumbase.repos.listEnabled(): + # this populates the cache as a side effect + repo.repoXML # pylint: disable=W0104 + try: + repo.getGroups() + except yum.Errors.RepoMDError: + pass # this repo has no groups + self.yumbase.repos.populateSack(mdtype='metadata', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='filelists', cacheonly=1) + self.yumbase.repos.populateSack(mdtype='otherdata', cacheonly=1) + # this does something with the groups cache as a side effect + self.yumbase.comps # pylint: disable=W0104 + def main(): parser = OptionParser() @@ -223,17 +295,28 @@ def main(): # pylint: disable=W0702 rv = 0 - depsolver = DepSolver(options.config, options.verbose) if cmd == "clean": + cachemgr = CacheManager(options.config, options.verbose) try: - depsolver.clean_cache() + cachemgr.clean_cache() print(json.dumps(True)) except: logger.error("Unexpected error cleaning cache: %s" % sys.exc_info()[1], exc_info=1) print(json.dumps(False)) rv = 2 + elif cmd == "makecache": + cachemgr = CacheManager(options.config, options.verbose) + try: + # this code copied from yumcommands.py + cachemgr.populate_cache() + print(json.dumps(True)) + except yum.Errors.YumBaseError: + logger.error("Unexpected error creating cache: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(False)) elif cmd == "complete": + depsolver = DepSolver(options.config, options.verbose) try: data = json.loads(sys.stdin.read()) except: @@ -252,6 +335,7 @@ def main(): print(json.dumps(dict(packages=[], unknown=data['packages']))) rv = 2 elif cmd == "get_groups": + depsolver = DepSolver(options.config, options.verbose) try: data = json.loads(sys.stdin.read()) rv = dict() diff --git a/testsuite/Testschema/test_schema.py b/testsuite/Testschema/test_schema.py index ddfe4775f..cd9b74cdf 100644 --- a/testsuite/Testschema/test_schema.py +++ b/testsuite/Testschema/test_schema.py @@ -41,7 +41,7 @@ class TestSchemas(Bcfg2TestCase): xmllint = Popen(['xmllint', '--xinclude', '--noout', '--schema', self.schema_url] + schemas, stdout=PIPE, stderr=STDOUT) - print(xmllint.communicate()[0]) + print(xmllint.communicate()[0].decode()) self.assertEqual(xmllint.wait(), 0) def test_duplicates(self): diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index e0406fd92..d2f383f42 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -711,12 +711,13 @@ class TestPOSIXTool(TestTool): gather_data_rv[idx] = val ptool._gather_data.return_value = tuple(gather_data_rv) + stat_mode = 17407 mtime = 1344430414 + stat_rv = (stat_mode, Mock(), Mock(), Mock(), Mock(), Mock(), Mock(), + Mock(), mtime, Mock()) + gather_data_rv[0] = stat_rv entry = reset() entry.set("mtime", str(mtime)) - stat_rv = MagicMock() - stat_rv.__getitem__.return_value = mtime - gather_data_rv[0] = stat_rv ptool._gather_data.return_value = tuple(gather_data_rv) self.assertTrue(ptool._verify_metadata(entry)) ptool._gather_data.assert_called_with(entry.get("name")) @@ -788,7 +789,7 @@ class TestPOSIXTool(TestTool): ptool._gather_data.assert_called_with(entry.get("name")) ptool._verify_acls.assert_called_with(entry, path=entry.get("name")) - mock_matchpathcon.assert_called_with(entry.get("name"), 0) + mock_matchpathcon.assert_called_with(entry.get("name"), stat_mode) self.assertEqual(entry.get("current_exists", 'true'), 'true') for attr, idx, val in expected: self.assertEqual(entry.get(attr), val) @@ -803,7 +804,7 @@ class TestPOSIXTool(TestTool): ptool._gather_data.assert_called_with(entry.get("name")) ptool._verify_acls.assert_called_with(entry, path=entry.get("name")) - mock_matchpathcon.assert_called_with(entry.get("name"), 0) + mock_matchpathcon.assert_called_with(entry.get("name"), stat_mode) self.assertEqual(entry.get("current_exists", 'true'), 'true') for attr, idx, val in expected: self.assertEqual(entry.get(attr), val) @@ -1009,7 +1010,7 @@ class TestPOSIXTool(TestTool): else: return True ptool._set_perms.side_effect = set_perms_rv - self.assertFalse(ptool._makedirs(entry)) + self.assertTrue(ptool._makedirs(entry)) self.assertItemsEqual(mock_exists.call_args_list, [call("/test"), call("/test/foo"), call("/test/foo/bar")]) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py index 9478f7071..c207900f1 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIXUsers.py @@ -27,7 +27,14 @@ class TestPOSIXUsers(TestTool): def get_obj(self, logger=None, setup=None, config=None): if setup is None: setup = MagicMock() - setup.__getitem__.return_value = [] + def getitem(key): + if key == 'encoding': + return 'UTF-8' + else: + return [] + + setup.__getitem__.side_effect = getitem + return TestTool.get_obj(self, logger, setup, config) @patch("pwd.getpwall") @@ -381,15 +388,15 @@ class TestPOSIXUsers(TestTool): (lxml.etree.Element("POSIXUser", name="test", group="test", home="/home/test", shell="/bin/zsh", gecos="Test McTest"), - ["-m", "-g", "test", "-d", "/home/test", "-s", "/bin/zsh", + ["-g", "test", "-d", "/home/test", "-s", "/bin/zsh", "-c", "Test McTest"]), (lxml.etree.Element("POSIXUser", name="test", group="test", home="/home/test", shell="/bin/zsh", gecos="Test McTest", uid="1001"), - ["-m", "-u", "1001", "-g", "test", "-d", "/home/test", + ["-u", "1001", "-g", "test", "-d", "/home/test", "-s", "/bin/zsh", "-c", "Test McTest"]), (entry, - ["-m", "-g", "test", "-G", "wheel,users", "-d", "/home/test", + ["-g", "test", "-G", "wheel,users", "-d", "/home/test", "-s", "/bin/zsh", "-c", "Test McTest"])] for entry, expected in cases: for action in ["add", "mod", "del"]: diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py index e26c26d41..870983f60 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py @@ -29,16 +29,11 @@ class TestDebuggable(Bcfg2TestCase): def test_set_debug(self): d = self.get_obj() - d.debug_log = Mock() self.assertEqual(True, d.set_debug(True)) self.assertEqual(d.debug_flag, True) - self.assertTrue(d.debug_log.called) - - d.debug_log.reset_mock() self.assertEqual(False, d.set_debug(False)) self.assertEqual(d.debug_flag, False) - self.assertTrue(d.debug_log.called) def test_toggle_debug(self): d = self.get_obj() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py index 94866cf39..5ae0dfcba 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testhelpers.py @@ -623,17 +623,9 @@ class TestXMLFileBacked(TestFileBacked): self.assertIn("/test/test2.xml", xfb.extra_monitors) fam = Mock() - if self.should_monitor is not True: - fam.reset_mock() - xfb = self.get_obj(fam=fam) - fam.reset_mock() - xfb.add_monitor("/test/test3.xml") - self.assertFalse(fam.AddMonitor.called) - self.assertIn("/test/test3.xml", xfb.extra_monitors) - - if self.should_monitor is not False: - fam.reset_mock() - xfb = self.get_obj(fam=fam, should_monitor=True) + fam.reset_mock() + xfb = self.get_obj(fam=fam) + if xfb.fam: xfb.add_monitor("/test/test4.xml") fam.AddMonitor.assert_called_with("/test/test4.xml", xfb) self.assertIn("/test/test4.xml", xfb.extra_monitors) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py new file mode 100644 index 000000000..05e0bb9a1 --- /dev/null +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestAWSTags.py @@ -0,0 +1,140 @@ +import os +import sys +import lxml.etree +import Bcfg2.Server.Plugin +from mock import Mock, MagicMock, patch +try: + from Bcfg2.Server.Plugins.AWSTags import * + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + +# add all parent testsuite directories to sys.path to allow (most) +# relative imports in python 2.4 +path = os.path.dirname(__file__) +while path != "/": + if os.path.basename(path).lower().startswith("test"): + sys.path.append(path) + if os.path.basename(path) == "testsuite": + break + path = os.path.dirname(path) +from common import * +from TestPlugin import TestPlugin, TestConnector, TestClientRunHooks + +config = ''' +<AWSTags> + <Tag name="name-only"> + <Group>group1</Group> + <Group>group2</Group> + </Tag> + <Tag name="name-and-value" value="value"> + <Group>group3</Group> + </Tag> + <Tag name="regex-(.*)"> + <Group>group-$1</Group> + </Tag> + <Tag name="regex-value" value="(.*)"> + <Group>group-$1</Group> + </Tag> +</AWSTags> +''' + +tags = { + "empty.example.com": {}, + "no-matches.example.com": {"nameonly": "foo", + "Name": "no-matches", + "foo": "bar"}, + "foo.example.com": {"name-only": "name-only", + "name-and-value": "wrong", + "regex-name": "foo"}, + "bar.example.com": {"name-and-value": "value", + "regex-value": "bar"}} + +groups = { + "empty.example.com": [], + "no-matches.example.com": [], + "foo.example.com": ["group1", "group2", "group-name"], + "bar.example.com": ["group3", "group-value", "group-bar"]} + + +def make_instance(name): + rv = Mock() + rv.private_dns_name = name + rv.tags = tags[name] + return rv + + +instances = [make_instance(n) for n in tags.keys()] + + +def get_all_instances(filters=None): + insts = [i for i in instances + if i.private_dns_name == filters['private-dns-name']] + res = Mock() + res.instances = insts + return [res] + + +if HAS_BOTO: + class TestAWSTags(TestPlugin, TestClientRunHooks, TestConnector): + test_obj = AWSTags + + def get_obj(self, core=None): + @patchIf(not isinstance(Bcfg2.Server.Plugins.AWSTags.connect_ec2, + Mock), + "Bcfg2.Server.Plugins.AWSTags.connect_ec2", Mock()) + @patch("lxml.etree.Element", Mock()) + def inner(): + obj = TestPlugin.get_obj(self, core=core) + obj.config.data = config + obj.config.Index() + return obj + return inner() + + @patch("Bcfg2.Server.Plugins.AWSTags.connect_ec2") + def test_connect(self, mock_connect_ec2): + """ Test connection to EC2 """ + key_id = "a09sdbipasdf" + access_key = "oiilb234ipwe9" + + def cfp_get(section, option): + if option == "access_key_id": + return key_id + elif option == "secret_access_key": + return access_key + else: + return Mock() + + core = Mock() + core.setup.cfp.get = Mock(side_effect=cfp_get) + awstags = self.get_obj(core=core) + mock_connect_ec2.assert_called_with( + aws_access_key_id=key_id, + aws_secret_access_key=access_key) + + def test_get_additional_data(self): + """ Test AWSTags.get_additional_data() """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in tags.items(): + metadata = Mock() + metadata.hostname = hostname + self.assertItemsEqual(awstags.get_additional_data(metadata), + expected) + + def test_get_additional_groups_caching(self): + """ Test AWSTags.get_additional_groups() with caching enabled """ + awstags = self.get_obj() + awstags._ec2.get_all_instances = \ + Mock(side_effect=get_all_instances) + + for hostname, expected in groups.items(): + metadata = Mock() + metadata.hostname = hostname + actual = awstags.get_additional_groups(metadata) + msg = """%s has incorrect groups: +actual: %s +expected: %s""" % (hostname, actual, expected) + self.assertItemsEqual(actual, expected, msg) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py index dc4b11241..e139a592b 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPrivateKeyCreator.py @@ -31,6 +31,7 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): should_monitor = False def get_obj(self, name=None, fam=None): + Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.CFG = Mock() return TestCfgCreator.get_obj(self, name=name) @patch("Bcfg2.Server.Plugins.Cfg.CfgCreator.handle_event") @@ -259,24 +260,6 @@ class TestCfgPrivateKeyCreator(TestCfgCreator, TestStructFile): pkc.write_data.assert_called_with("privatekey", group="foo") mock_rmtree.assert_called_with(datastore) - reset() - self.assertEqual(pkc.create_data(entry, metadata, return_pair=True), - ("ssh-rsa publickey pubkey.filename\n", - "privatekey")) - pkc.XMLMatch.assert_called_with(metadata) - pkc.get_specificity.assert_called_with(metadata, - pkc.XMLMatch.return_value) - pkc._gen_keypair.assert_called_with(metadata, - pkc.XMLMatch.return_value) - self.assertItemsEqual(mock_open.call_args_list, - [call(privkey + ".pub"), call(privkey)]) - pkc.pubkey_creator.get_filename.assert_called_with(group="foo") - pkc.pubkey_creator.write_data.assert_called_with( - "ssh-rsa publickey pubkey.filename\n", - group="foo") - pkc.write_data.assert_called_with("privatekey", group="foo") - mock_rmtree.assert_called_with(datastore) - inner() if HAS_CRYPTO: diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py index 04772cf9a..ef4610fae 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/TestCfgPublicKeyCreator.py @@ -26,6 +26,7 @@ class TestCfgPublicKeyCreator(TestCfgCreator, TestStructFile): should_monitor = False def get_obj(self, name=None, fam=None): + Bcfg2.Server.Plugins.Cfg.CfgPublicKeyCreator.CFG = Mock() return TestCfgCreator.get_obj(self, name=name) @patch("Bcfg2.Server.Plugins.Cfg.CfgCreator.handle_event") @@ -37,41 +38,117 @@ class TestCfgPublicKeyCreator(TestCfgCreator, TestStructFile): mock_HandleEvent.assert_called_with(pkc, evt) mock_handle_event.assert_called_with(pkc, evt) - def test_create_data(self): + @patch("os.unlink") + @patch("os.path.exists") + @patch("tempfile.mkstemp") + @patch("os.fdopen", Mock()) + @patch("%s.open" % builtins) + def test_create_data(self, mock_open, mock_mkstemp, mock_exists, + mock_unlink): metadata = Mock() pkc = self.get_obj() pkc.cfg = Mock() + pkc.core = Mock() + pkc.cmd = Mock() + pkc.write_data = Mock() + pubkey = "public key data" privkey_entryset = Mock() privkey_creator = Mock() - pubkey = Mock() - privkey = Mock() - privkey_creator.create_data.return_value = (pubkey, privkey) - privkey_entryset.best_matching.return_value = privkey_creator + privkey_creator.get_specificity = Mock() + privkey_creator.get_specificity.return_value = dict() + fileloc = pkc.get_filename() pkc.cfg.entries = {"/home/foo/.ssh/id_rsa": privkey_entryset} + def reset(): + privkey_creator.reset_mock() + pkc.cmd.reset_mock() + pkc.core.reset_mock() + pkc.write_data.reset_mock() + mock_exists.reset_mock() + mock_unlink.reset_mock() + mock_mkstemp.reset_mock() + mock_open.reset_mock() + # public key doesn't end in .pub entry = lxml.etree.Element("Path", name="/home/bar/.ssh/bogus") self.assertRaises(CfgCreationError, pkc.create_data, entry, metadata) + self.assertFalse(pkc.write_data.called) + + # cannot bind private key + reset() + pkc.core.Bind.side_effect = PluginExecutionError + entry = lxml.etree.Element("Path", name="/home/foo/.ssh/id_rsa.pub") + self.assertRaises(CfgCreationError, + pkc.create_data, entry, metadata) + self.assertFalse(pkc.write_data.called) # private key not in cfg.entries + reset() + pkc.core.Bind.side_effect = None + pkc.core.Bind.return_value = "private key data" entry = lxml.etree.Element("Path", name="/home/bar/.ssh/id_rsa.pub") self.assertRaises(CfgCreationError, pkc.create_data, entry, metadata) + self.assertFalse(pkc.write_data.called) - # successful operation + # no privkey.xml defined + reset() + privkey_entryset.best_matching.side_effect = PluginExecutionError + entry = lxml.etree.Element("Path", name="/home/foo/.ssh/id_rsa.pub") + self.assertRaises(CfgCreationError, + pkc.create_data, entry, metadata) + self.assertFalse(pkc.write_data.called) + + # successful operation, create new key + reset() + pkc.cmd.run.return_value = Mock() + pkc.cmd.run.return_value.success = True + pkc.cmd.run.return_value.stdout = pubkey + mock_mkstemp.return_value = (Mock(), str(Mock())) + mock_exists.return_value = False + privkey_entryset.best_matching.side_effect = None + privkey_entryset.best_matching.return_value = privkey_creator entry = lxml.etree.Element("Path", name="/home/foo/.ssh/id_rsa.pub") self.assertEqual(pkc.create_data(entry, metadata), pubkey) + self.assertTrue(pkc.core.Bind.called) + (privkey_entry, md) = pkc.core.Bind.call_args[0] + self.assertXMLEqual(privkey_entry, + lxml.etree.Element("Path", + name="/home/foo/.ssh/id_rsa")) + self.assertEqual(md, metadata) + privkey_entryset.get_handlers.assert_called_with(metadata, CfgCreator) - privkey_entryset.best_matching.assert_called_with(metadata, - privkey_entryset.get_handlers.return_value) - self.assertXMLEqual(privkey_creator.create_data.call_args[0][0], + privkey_entryset.best_matching.assert_called_with( + metadata, + privkey_entryset.get_handlers.return_value) + mock_exists.assert_called_with(fileloc) + pkc.cmd.run.assert_called_with(["ssh-keygen", "-y", "-f", + mock_mkstemp.return_value[1]]) + self.assertEqual(pkc.write_data.call_args[0][0], pubkey) + mock_unlink.assert_called_with(mock_mkstemp.return_value[1]) + self.assertFalse(mock_open.called) + + # successful operation, no need to create new key + reset() + mock_exists.return_value = True + mock_open.return_value = Mock() + mock_open.return_value.read.return_value = pubkey + pkc.cmd.run.return_value.stdout = None + self.assertEqual(pkc.create_data(entry, metadata), pubkey) + self.assertTrue(pkc.core.Bind.called) + (privkey_entry, md) = pkc.core.Bind.call_args[0] + self.assertXMLEqual(privkey_entry, lxml.etree.Element("Path", name="/home/foo/.ssh/id_rsa")) - self.assertEqual(privkey_creator.create_data.call_args[0][1], metadata) + self.assertEqual(md, metadata) - # no privkey.xml - privkey_entryset.best_matching.side_effect = PluginExecutionError - self.assertRaises(CfgCreationError, - pkc.create_data, entry, metadata) + privkey_entryset.get_handlers.assert_called_with(metadata, CfgCreator) + privkey_entryset.best_matching.assert_called_with( + metadata, + privkey_entryset.get_handlers.return_value) + mock_exists.assert_called_with(fileloc) + mock_open.assert_called_with(fileloc) + self.assertFalse(mock_mkstemp.called) + self.assertFalse(pkc.write_data.called) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py index ea3549c1b..fdfb3a9f7 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py @@ -560,7 +560,7 @@ class TestCfgEntrySet(TestEntrySet): def reset(): for e in eset.entries.values(): - if e.specific is not None: + if hasattr(e.specific, "reset_mock"): e.specific.reset_mock() metadata = Mock() @@ -577,7 +577,7 @@ class TestCfgEntrySet(TestEntrySet): [eset.entries['test1.txt'], eset.entries['test3.txt']]) for entry in eset.entries.values(): - if entry.specific is not None: + if hasattr(entry.specific.matches, "called"): self.assertFalse(entry.specific.matches.called) reset() @@ -585,20 +585,22 @@ class TestCfgEntrySet(TestEntrySet): [eset.entries['test6.txt']]) eset.entries['test6.txt'].specific.matches.assert_called_with(metadata) for ename, entry in eset.entries.items(): - if ename != 'test6.txt' and entry.specific is not None: + if (ename != 'test6.txt' and + hasattr(entry.specific.matches, "called")): self.assertFalse(entry.specific.matches.called) reset() self.assertItemsEqual(eset.get_handlers(metadata, CfgFilter), []) eset.entries['test7.txt'].specific.matches.assert_called_with(metadata) for ename, entry in eset.entries.items(): - if ename != 'test7.txt' and entry.specific is not None: + if (ename != 'test7.txt' and + hasattr(entry.specific.matches, "called")): self.assertFalse(entry.specific.matches.called) reset() self.assertItemsEqual(eset.get_handlers(metadata, Mock), []) for ename, entry in eset.entries.items(): - if entry.specific is not None: + if hasattr(entry.specific.matches, "called"): self.assertFalse(entry.specific.matches.called) def test_bind_info_to_entry(self): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py index e2da6ec5d..a07fffba1 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestMetadata.py @@ -339,6 +339,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): @patch('Bcfg2.Utils.locked', Mock(return_value=False)) @patch('fcntl.lockf', Mock()) + @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml") @patch('os.open') @patch('os.fdopen') @patch('os.unlink') @@ -346,7 +347,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): @patch('os.path.islink') @patch('os.readlink') def test_write_xml(self, mock_readlink, mock_islink, mock_rename, - mock_unlink, mock_fdopen, mock_open): + mock_unlink, mock_fdopen, mock_open, mock_load_xml): fname = "clients.xml" config = self.get_obj(fname) fpath = os.path.join(self.metadata.data, fname) @@ -360,6 +361,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): mock_unlink.reset_mock() mock_fdopen.reset_mock() mock_open.reset_mock() + mock_load_xml.reset_mock() mock_islink.return_value = False @@ -371,6 +373,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertTrue(mock_fdopen.return_value.write.called) mock_islink.assert_called_with(fpath) mock_rename.assert_called_with(tmpfile, fpath) + mock_load_xml.assert_called_with() # test: clients.xml.new is locked the first time we write it def rv(fname, mode): @@ -389,6 +392,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): self.assertTrue(mock_fdopen.return_value.write.called) mock_islink.assert_called_with(fpath) mock_rename.assert_called_with(tmpfile, fpath) + mock_load_xml.assert_called_with() # test writing a symlinked clients.xml reset() @@ -397,6 +401,7 @@ class TestXMLMetadataConfig(TestXMLFileBacked): mock_readlink.return_value = linkdest config.write_xml(fpath, get_clients_test_tree()) mock_rename.assert_called_with(tmpfile, linkdest) + mock_load_xml.assert_called_with() # test failure of os.rename() reset() @@ -830,21 +835,18 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): self.assertEqual(metadata.groups['group4'].category, 'category1') self.assertEqual(metadata.default, "group1") - all_groups = [] - negated_groups = [] + all_groups = set() + negated_groups = set() for group in get_groups_test_tree().xpath("//Groups/Client//*") + \ get_groups_test_tree().xpath("//Groups/Group//*"): if group.tag == 'Group' and not group.getchildren(): if group.get("negate", "false").lower() == 'true': - negated_groups.append(group.get("name")) + negated_groups.add(group.get("name")) else: - all_groups.append(group.get("name")) - self.assertItemsEqual([g.name - for g in metadata.group_membership.values()], - all_groups) - self.assertItemsEqual([g.name - for g in metadata.negated_groups.values()], - negated_groups) + all_groups.add(group.get("name")) + self.assertItemsEqual(metadata.ordered_groups, all_groups) + self.assertItemsEqual(metadata.group_membership.keys(), all_groups) + self.assertItemsEqual(metadata.negated_groups.keys(), negated_groups) @patch("Bcfg2.Server.Plugins.Metadata.XMLMetadataConfig.load_xml", Mock()) def test_set_profile(self): @@ -885,10 +887,13 @@ class TestMetadata(_TestMetadata, TestClientRunHooks, TestDatabaseBacked): metadata = self.load_clients_data(metadata=self.load_groups_data()) if not metadata._use_db: metadata.clients_xml.write = Mock() + metadata.core.build_metadata = Mock() + metadata.core.build_metadata.side_effect = \ + lambda c: metadata.get_initial_metadata(c) + metadata.set_profile("client1", "group2", None) mock_update_client.assert_called_with("client1", dict(profile="group2")) - metadata.clients_xml.write.assert_any_call() self.assertEqual(metadata.clientgroups["client1"], ["group2"]) metadata.clients_xml.write.reset_mock() diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py index 0794db62e..a87628eaf 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestProbes.py @@ -466,7 +466,7 @@ text def test_load_data_db(self): probes = self.get_probes_object(use_db=True) probes.load_data() - probes._load_data_db.assert_any_call() + probes._load_data_db.assert_any_call(client=None) self.assertFalse(probes._load_data_xml.called) @patch("lxml.etree.parse") diff --git a/testsuite/Testsrc/test_code_checks.py b/testsuite/Testsrc/test_code_checks.py index a38710fd4..e1214a942 100644 --- a/testsuite/Testsrc/test_code_checks.py +++ b/testsuite/Testsrc/test_code_checks.py @@ -79,7 +79,9 @@ no_checks = { "TCheetah.py", "TGenshi.py"], } - +if sys.version_info < (2, 6): + # multiprocessing core requires py2.6 + no_checks['lib/Bcfg2/Server'].append('MultiprocessingCore.py') try: any @@ -186,7 +188,7 @@ class CodeTestCase(Bcfg2TestCase): cmd = self.command + self.full_args + extra_args + \ [os.path.join(srcpath, f) for f in files] proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, env=self.get_env()) - print(proc.communicate()[0]) + print(proc.communicate()[0].decode()) self.assertEqual(proc.wait(), 0) def _test_errors(self, files, extra_args=None): @@ -198,7 +200,7 @@ class CodeTestCase(Bcfg2TestCase): cmd = self.command + self.error_args + extra_args + \ [os.path.join(srcpath, f) for f in files] proc = Popen(cmd, stdout=PIPE, stderr=STDOUT, env=self.get_env()) - print(proc.communicate()[0]) + print(proc.communicate()[0].decode()) self.assertEqual(proc.wait(), 0) @skipIf(not os.path.exists(srcpath), "%s does not exist" % srcpath) @@ -321,7 +323,7 @@ class TestPylint(CodeTestCase): args = self.command + self.error_args + extra_args + \ [os.path.join(srcpath, p) for p in files] pylint = Popen(args, stdout=PIPE, stderr=STDOUT, env=self.get_env()) - output = pylint.communicate()[0] + output = pylint.communicate()[0].decode() rv = pylint.wait() for line in output.splitlines(): diff --git a/testsuite/install.sh b/testsuite/install.sh index 1ca89f40f..3f066b2bc 100755 --- a/testsuite/install.sh +++ b/testsuite/install.sh @@ -7,15 +7,17 @@ pip install -r testsuite/requirements.txt --use-mirrors PYVER=$(python -c 'import sys;print(".".join(str(v) for v in sys.version_info[0:2]))') if [[ "$WITH_OPTIONAL_DEPS" == "yes" ]]; then - pip install --use-mirrors genshi PyYAML pyinotify + pip install --use-mirrors genshi PyYAML pyinotify boto if [[ $PYVER == "2.5" ]]; then # markdown 2.2+ doesn't work on py2.5 - pip install --use-mirrors simplejson 'markdown<2.2' + pip install --use-mirrors simplejson 'markdown<2.2' 'django<1.4.9' + else + pip install 'django<1.5' fi if [[ ${PYVER:0:1} == "2" ]]; then # django supports py3k, but South doesn't, and the django bits # in bcfg2 require South - pip install cheetah 'django<1.5' 'South<0.8' M2Crypto + pip install cheetah 'South<0.8' M2Crypto fi else # python < 2.6 requires M2Crypto for SSL communication, not just diff --git a/testsuite/requirements.txt b/testsuite/requirements.txt index 2d6dbc557..898249389 100644 --- a/testsuite/requirements.txt +++ b/testsuite/requirements.txt @@ -2,6 +2,6 @@ lxml nose mock sphinx -pylint +pylint<1.0 pep8 python-daemon diff --git a/tools/README b/tools/README index 9e7f667e3..8fe2c1a28 100644 --- a/tools/README +++ b/tools/README @@ -61,6 +61,9 @@ export.sh generate-manpages.bash - Generate man pages from the Sphinx source +git_commit.py + - Trigger script to commit local changes back to a git repository + hostbasepush.py - Call the Hostbase.rebuildState XML-RPC method @@ -100,4 +103,3 @@ upgrade yum-listpkgs-xml.py - Produces a list of all packages installed and available in a format suitable for use by Packages or Pkgmgr - diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index f4069e454..2b0ca6d63 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -67,7 +67,7 @@ def main(): logger.info("Bcfg2 server core loaded") core.load_plugins() logger.debug("Plugins loaded") - core.fam.handle_events_in_interval(0.1) + core.block_for_fam_events(handle_events=True) logger.debug("Repository events processed") if setup['args']: diff --git a/tools/bcfg2_local.py b/tools/bcfg2_local.py index 8c164e52e..78a46ba5c 100755 --- a/tools/bcfg2_local.py +++ b/tools/bcfg2_local.py @@ -20,7 +20,7 @@ class LocalCore(BaseCore): Bcfg2.Server.Core.BaseCore.__init__(self, setup=setup) setup['syslog'], setup['logging'] = saved self.load_plugins() - self.fam.handle_events_in_interval(0.1) + self.block_for_fam_events(handle_events=True) def _daemonize(self): return True @@ -47,7 +47,10 @@ class LocalProxy(object): func = getattr(self.core, attr) if func.exposed: def inner(*args, **kwargs): - args = ((self.ipaddr, self.hostname), ) + args + # the port portion of the addresspair tuple isn't + # actually used, so it's safe to hardcode 6789 + # here. + args = ((self.ipaddr, 6789), ) + args return func(*args, **kwargs) return inner raise AttributeError(attr) diff --git a/tools/git_commit.py b/tools/git_commit.py new file mode 100755 index 000000000..cc4061f25 --- /dev/null +++ b/tools/git_commit.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +""" Trigger script to commit selected changes to a local repository +back to git. To use this script, enable the Trigger plugin, put this +script in /var/lib/bcfg2/Trigger/, and create /etc/bcfg2-commit.conf. + +The config file, /etc/bcfg2-commit.conf, may contain four options in +the [global] section: + +* "config" is the path to the Bcfg2 server config file. (Default: + /etc/bcfg2.conf) +* "commit" is a comma-separated list of globs giving the paths that + should be committed back to the repository. Default is 'SSLCA/*, + SSHbase/*, Cfg/*', which will commit data back for SSLCA, SSHbase, + Cfg, FileProbes, etc., but not, for instance, Probes/probed.xml. + You may wish to add Metadata/clients.xml to the commit list. +* "debug" and "verbose" let you set the log level for git_commit.py + itself. +""" + + +import os +import sys +import git +import logging +import Bcfg2.Logger +import Bcfg2.Options +from Bcfg2.Compat import ConfigParser +from fnmatch import fnmatch + +# config file path +CONFIG = "/etc/bcfg2-commit.conf" + +# config defaults. all config options are in the [global] section +DEFAULTS = dict(config='/etc/bcfg2.conf', + commit="SSLCA/*, SSHbase/*, Cfg/*") + + +def list_changed_files(repo): + return [d for d in repo.index.diff(None) + if (d.a_blob is not None and not d.deleted_file and + not d.renamed and not d.new_file)] + + +def add_to_commit(patterns, path, repo, relpath): + progname = os.path.basename(sys.argv[0]) + logger = logging.getLogger(progname) + for pattern in patterns: + if fnmatch(path, os.path.join(relpath, pattern)): + logger.debug("%s: Adding %s to commit" % (progname, path)) + repo.index.add([path]) + return True + return False + + +def parse_options(): + config = ConfigParser.SafeConfigParser(DEFAULTS) + config.read(CONFIG) + + optinfo = dict( + profile=Bcfg2.Options.CLIENT_PROFILE, + dryrun=Bcfg2.Options.CLIENT_DRYRUN, + groups=Bcfg2.Options.Option("Groups", + default=[], + cmd="-g", + odesc='<group>:<group>', + cook=Bcfg2.Options.colon_split)) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + argv = [Bcfg2.Options.CFILE.cmd, config.get("global", "config")] + argv.extend(sys.argv[1:]) + setup = Bcfg2.Options.OptionParser(optinfo, argv=argv) + setup.parse(argv) + + setup['commit'] = Bcfg2.Options.list_split(config.get("global", + "commit")) + for opt in ['debug', 'verbose']: + try: + setup[opt] = config.getboolean("global", opt) + except ConfigParser.NoOptionError: + pass + + try: + hostname = setup['args'][0] + except IndexError: + print(setup.hm) + raise SystemExit(1) + return (setup, hostname) + + +def setup_logging(setup): + progname = os.path.basename(sys.argv[0]) + log_args = dict(to_syslog=setup['syslog'], to_console=sys.stdout.isatty(), + to_file=setup['logging'], level=logging.WARNING) + if setup['debug']: + log_args['level'] = logging.DEBUG + elif setup['verbose']: + log_args['level'] = logging.INFO + Bcfg2.Logger.setup_logging(progname, **log_args) + return logging.getLogger(progname) + + +def main(): + progname = os.path.basename(sys.argv[0]) + setup, hostname = parse_options() + logger = setup_logging(setup) + if setup['dryrun']: + logger.info("%s: In dry-run mode, changes will not be committed" % + progname) + + if setup['vcs_root']: + gitroot = os.path.realpath(setup['vcs_root']) + else: + gitroot = os.path.realpath(setup['repo']) + logger.info("%s: Using Git repo at %s" % (progname, gitroot)) + try: + repo = git.Repo(gitroot) + except: # pylint: disable=W0702 + logger.error("%s: Error setting up Git repo at %s: %s" % + (progname, gitroot, sys.exc_info()[1])) + return 1 + + # canonicalize the repo path so that git will recognize it as + # being inside the git repo + bcfg2root = os.path.realpath(setup['repo']) + + if not bcfg2root.startswith(gitroot): + logger.error("%s: Bcfg2 repo %s is not inside Git repo %s" % + (progname, bcfg2root, gitroot)) + return 1 + + # relative path to Bcfg2 root from VCS root + if gitroot == bcfg2root: + relpath = '' + else: + relpath = bcfg2root[len(gitroot) + 1:] + + new = 0 + changed = 0 + logger.debug("%s: Untracked files: %s" % (progname, repo.untracked_files)) + for path in repo.untracked_files: + if add_to_commit(setup['commit'], path, repo, relpath): + new += 1 + else: + logger.debug("%s: Not adding %s to commit" % (progname, path)) + logger.debug("%s: Untracked files after building commit: %s" % + (progname, repo.untracked_files)) + + changes = list_changed_files(repo) + logger.info("%s: Changed files: %s" % (progname, + [d.a_blob.path for d in changes])) + for diff in changes: + if add_to_commit(setup['commit'], diff.a_blob.path, repo, relpath): + changed += 1 + else: + logger.debug("%s: Not adding %s to commit" % (progname, + diff.a_blob.path)) + logger.info("%s: Changed files after building commit: %s" % + (progname, [d.a_blob.path for d in list_changed_files(repo)])) + + if new + changed > 0: + logger.debug("%s: Committing %s new files and %s changed files" % + (progname, new, changed)) + if setup['dryrun']: + logger.warning("%s: In dry-run mode, skipping commit and push" % + progname) + else: + output = repo.index.commit("Auto-commit with %s from %s run" % + (progname, hostname)) + if output: + logger.debug("%s: %s" % (progname, output)) + remote = repo.remote() + logger.debug("%s: Pushing to remote %s at %s" % (progname, remote, + remote.url)) + output = remote.push() + if output: + logger.debug("%s: %s" % (progname, output)) + else: + logger.info("%s: No changes to commit" % progname) + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/posixusers_baseline.py b/tools/posixusers_baseline.py index c45e54f1a..4c78a757e 100755 --- a/tools/posixusers_baseline.py +++ b/tools/posixusers_baseline.py @@ -46,8 +46,8 @@ def main(): config = lxml.etree.parse(setup['file']).getroot() else: config = lxml.etree.Element("Configuration") - users = POSIXUsers(logging.getLogger('posixusers_baseline.py'), - setup, config) + logger = logging.getLogger('posixusers_baseline.py') + users = POSIXUsers(logger, setup, config) baseline = lxml.etree.Element("Bundle", name="posixusers_baseline") for entry in users.FindExtra(): @@ -59,7 +59,12 @@ def main(): continue entry.set(attr, str(data[idx])) if entry.tag == 'POSIXUser': - entry.set("group", grp.getgrgid(data[3])[0]) + try: + entry.set("group", grp.getgrgid(data[3])[0]) + except KeyError: + logger.warning("User %s is a member of nonexistent group %s" % + (entry.get("name"), data[3])) + entry.set("group", str(data[3])) for group in users.user_supplementary_groups(entry): memberof = lxml.etree.SubElement(entry, "MemberOf", group=group[0]) |