From cf1a79203cd5e52a17f1e3228c137c8bd20132fc Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 13:23:33 -0400 Subject: bcfg2-info: added expirecache command to expire metadata cache --- src/sbin/bcfg2-info | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 539ae3115..badf637e8 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -482,6 +482,16 @@ Bcfg2 client itself.""") ('Logging', self.setup['logging'])] print_tabular(output) + def do_expirecache(self, args): + """ expirecache [ [ ...]]- Expire the + metadata cache """ + alist = args.split() + if len(alist): + for client in self._get_client_list(alist): + self.metadata_cache.expire(client) + else: + self.metadata_cache.expire() + def do_probes(self, args): """ probes [-p] - Get probe list for the given host, in XML (the default) or human-readable pretty (with -p) -- cgit v1.2.3-1-g7c22 From 20794ebf60a9cc63d762b047ba24a2ad5c6a115f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 14:19:00 -0400 Subject: bcfg2-info: fixed expirecache --- src/sbin/bcfg2-info | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index badf637e8..451d8e49c 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -489,8 +489,8 @@ Bcfg2 client itself.""") if len(alist): for client in self._get_client_list(alist): self.metadata_cache.expire(client) - else: - self.metadata_cache.expire() + else: + self.metadata_cache.expire() def do_probes(self, args): """ probes [-p] - Get probe list for the given -- cgit v1.2.3-1-g7c22 From 827d0a83b8c9148598c23cb550862c0cf50b5a23 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 14:22:11 -0400 Subject: Packages: added lock to yum cache update --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 13 +++++++++++ src/sbin/bcfg2-yum-helper | 32 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 7c950a435..7a90f4f2e 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -53,11 +53,13 @@ The Yum Backend import os import re import sys +import time import copy import errno import socket import logging import lxml.etree +from lockfile import FileLock from subprocess import Popen, PIPE import Bcfg2.Server.Plugin # pylint: disable=W0622 @@ -864,6 +866,17 @@ class YumCollection(Collection): if not self.use_yum: return Collection.complete(self, packagelist) + lock = FileLock(os.path.join(self.cachefile, "lock")) + slept = 0 + while lock.is_locked(): + if slept > 30: + self.logger.warning("Packages: Timeout waiting for yum cache " + "to release its lock") + return set(), set() + self.logger.debug("Packages: Yum cache is locked, waiting...") + time.sleep(3) + slept += 3 + if packagelist: try: result = self.call_helper( diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 414606abb..1f22c0ff9 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -10,6 +10,8 @@ import sys import yum import logging import Bcfg2.Logger +from Bcfg2.Compat import wraps +from lockfile import FileLock, LockTimeout from optparse import OptionParser try: import json @@ -192,6 +194,27 @@ class DepSolver(YumHelper): return list(packages), list(unknown) +def acquire_lock(func): + """ decorator for CacheManager methods that gets and release a + lock while the method runs """ + @wraps(func) + def inner(self, *args, **kwargs): + self.logger.debug("Acquiring lock at %s" % self.lockfile) + while not self.lock.i_am_locking(): + try: + self.lock.acquire(timeout=60) # wait up to 60 seconds + except LockTimeout: + self.lock.break_lock() + self.lock.acquire() + try: + func(self, *args, **kwargs) + finally: + self.lock.release() + self.logger.debug("Released lock at %s" % self.lockfile) + + return inner + + class CacheManager(YumHelper): """ Yum cache manager. Unlike :class:`DepSolver`, this can write to the yum cache, and so is used for operations that muck with the @@ -199,6 +222,14 @@ class CacheManager(YumHelper): either DepSolver or CacheManager, but for consistency I've put it here.) """ + def __init__(self, cfgfile, verbose=1): + YumHelper.__init__(self, cfgfile, verbose=verbose) + self.lockfile = \ + os.path.join(os.path.dirname(self.yumbase.conf.config_file_path), + "lock") + self.lock = FileLock(self.lockfile) + + @acquire_lock def clean_cache(self): """ clean the yum cache """ for mdtype in ["Headers", "Packages", "Sqlite", "Metadata", @@ -211,6 +242,7 @@ class CacheManager(YumHelper): if not msg.startswith("0 "): self.logger.info(msg) + @acquire_lock def populate_cache(self): """ populate the yum cache """ for repo in self.yumbase.repos.findRepos('*'): -- cgit v1.2.3-1-g7c22 From 6cc24e1f7fdddda46e8665c6bb2fbca4b6f6a57f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Thu, 18 Jul 2013 14:50:52 -0400 Subject: bcfg2-yum-helper: added missing docstring --- src/sbin/bcfg2-yum-helper | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 1f22c0ff9..161aa3e50 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -199,6 +199,8 @@ def acquire_lock(func): lock while the method runs """ @wraps(func) def inner(self, *args, **kwargs): + """ Get and release a lock while running the function this + wraps. """ self.logger.debug("Acquiring lock at %s" % self.lockfile) while not self.lock.i_am_locking(): try: -- cgit v1.2.3-1-g7c22 From 99c64abc91eb7c8e65b5d53cdfdebd31b25b5d21 Mon Sep 17 00:00:00 2001 From: Alexander Sulfrian Date: Fri, 19 Jul 2013 15:43:35 +0200 Subject: man/bcfg2.conf: add documentation for fam_blocking option --- doc/man/bcfg2.conf.txt | 6 ++++++ man/bcfg2.conf.5 | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/man/bcfg2.conf.txt b/doc/man/bcfg2.conf.txt index 3a0217aef..58e68ebc8 100644 --- a/doc/man/bcfg2.conf.txt +++ b/doc/man/bcfg2.conf.txt @@ -46,6 +46,12 @@ filemonitor fam pseudo +fam_blocking + Whether the server should block at startup until the file monitor + backend has processed all events. This can cause a slower startup, + but ensure that all files are recognized before the first client + is handled. + ignore_files A comma-separated list of globs that should be ignored by the file monitor. Default values are:: diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index b0db91a5b..48f49685d 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -1,4 +1,4 @@ -.TH "BCFG2.CONF" "5" "March 18, 2013" "1.3" "Bcfg2" +.TH "BCFG2.CONF" "5" "July 19, 2013" "1.3" "Bcfg2" .SH NAME bcfg2.conf \- Configuration parameters for Bcfg2 . @@ -76,6 +76,13 @@ pseudo .UNINDENT .UNINDENT .TP +.B fam_blocking +. +Whether the server should block at startup until the file monitor +backend has processed all events. This can cause a slower startup, +but ensure that all files are recognized before the first client +is handled. +.TP .B ignore_files A comma\-separated list of globs that should be ignored by the file monitor. Default values are: -- cgit v1.2.3-1-g7c22 From 1450bd76eec59d409bdf93e5e9982ca3ec852f2b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 19 Jul 2013 17:23:41 -0500 Subject: Admin/Init: Add listen_all option Some of these changes were suggested in github issue #29. Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Admin/Init.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 6175d8ed0..c76e54569 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -20,6 +20,8 @@ from Bcfg2.Compat import input # pylint: disable=W0622 CONFIG = '''[server] repository = %s plugins = %s +# Uncomment the following to listen on all interfaces +#listen_all = true [statistics] sendmailpath = %s @@ -78,7 +80,7 @@ CLIENTS = ''' ''' # Mapping of operating system names to groups -OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), +OS_LIST = [('Red Hat/Fedora/RHEL/RHAS/CentOS', 'redhat'), ('SUSE/SLES', 'suse'), ('Mandrake', 'mandrake'), ('Debian', 'debian'), @@ -234,8 +236,9 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_server(self): """Ask for the server name.""" - newserver = safe_input("Input the server location [%s]: " % - self.data['server_uri']) + newserver = safe_input( + "Input the server location (the server listens on a single " + "interface by default) [%s]: " % self.data['server_uri']) if newserver != '': self.data['server_uri'] = newserver -- cgit v1.2.3-1-g7c22 From c256efb6b4780615aadfc5634cd35df8b2c0a2c6 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 19 Jul 2013 17:27:00 -0500 Subject: doc: Update ubuntu quickstart guide This commit fixes #29 along with 1450bd7. Signed-off-by: Sol Jerome --- doc/appendix/guides/ubuntu.txt | 741 ++++++++++++++++++++++---------- doc/server/plugins/generators/rules.txt | 4 + 2 files changed, 522 insertions(+), 223 deletions(-) diff --git a/doc/appendix/guides/ubuntu.txt b/doc/appendix/guides/ubuntu.txt index 5a67d0a37..30ecbc3d7 100644 --- a/doc/appendix/guides/ubuntu.txt +++ b/doc/appendix/guides/ubuntu.txt @@ -1,4 +1,5 @@ .. -*- mode: rst -*- +.. vim: ft=rst .. _appendix-guides-ubuntu: @@ -8,7 +9,7 @@ Ubuntu .. note:: - This particular how to was done on lucid, but should apply to any + This particular how to was done on saucy, but should apply to any other `stable`__ version of Ubuntu. __ ubuntu-releases_ @@ -23,11 +24,6 @@ version available in the ubuntu archives, but it is not as up to date). .. _PPA: https://launchpad.net/~bcfg2/+archive/ppa -Add the Ubuntu PPA listing to your APT sources ----------------------------------------------- - -See http://trac.mcs.anl.gov/projects/bcfg2/wiki/PrecompiledPackages#UbuntuLucid - Install bcfg2-server -------------------- :: @@ -36,7 +32,7 @@ Install bcfg2-server Remove the default configuration preseeded by the ubuntu package:: - root@lucid:~# rm -rf /etc/bcfg2* /var/lib/bcfg2 + root@saucy:~# rm -rf /etc/bcfg2* /etc/ssl/bcfg2* /var/lib/bcfg2 Initialize your repository ========================== @@ -45,63 +41,95 @@ Now that you're done with the install, you need to intialize your repository and setup your bcfg2.conf. bcfg2-admin init is a tool which allows you to automate this process.:: - root@lucid:~# bcfg2-admin init - Store bcfg2 configuration in [/etc/bcfg2.conf]: - Location of bcfg2 repository [/var/lib/bcfg2]: + root@saucy:~# bcfg2-admin init + Store Bcfg2 configuration in [/etc/bcfg2.conf]: + Location of Bcfg2 repository [/var/lib/bcfg2]: Input password used for communication verification (without echoing; leave blank for a random): - What is the server's hostname: [lucid] - Input the server location [https://lucid:6789]: + What is the server's hostname: [saucy] + Input the server location (the server listens on a single interface by default) [https://saucy:6789]: Input base Operating System for clients: - 1: Redhat/Fedora/RHEL/RHAS/Centos + 1: Redhat/Fedora/RHEL/RHAS/CentOS 2: SUSE/SLES 3: Mandrake 4: Debian 5: Ubuntu 6: Gentoo 7: FreeBSD + 8: Arch : 5 + Path where Bcfg2 server private key will be created [/etc/ssl/bcfg2.key]: + Path where Bcfg2 server cert will be created [/etc/ssl/bcfg2.crt]: + The following questions affect SSL certificate generation. + If no data is provided, the default values are used. + Country name (2 letter code) for certificate: US + State or Province Name (full name) for certificate: Illinois + Locality Name (eg, city) for certificate: Argonne + Repository created successfuly in /var/lib/bcfg2 Generating a 2048 bit RSA private key - ......................................................................................+++ - ...+++ - writing new private key to '/etc/bcfg2.key' + ....................................................................................................................+++ + ..............................+++ + writing new private key to '/etc/ssl/bcfg2.key' ----- Signature ok - subject=/C=US/ST=Illinois/L=Argonne/CN=lucid + subject=/C=US/ST=Illinois/L=Argonne/CN=saucy Getting Private key - Repository created successfuly in /var/lib/bcfg2 - Of course, change responses as necessary. Start the server ================ +Before you start the server, you need to fix your network resolution for +this host. The short and easy way is to remove the 127.0.1.1 line in +``/etc/hosts`` and move your hostname to the 127.0.0.1 line. + +:: + + 127.0.0.1 saucy localhost + + # The following lines are desirable for IPv6 capable hosts + ... + +.. _Debian Manual: http://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution + +.. note:: + + This configuration is not recommended except as a quick hack to get + you through this guide. Ideally you'd add a line containing the + host's actual IP address. More information on why this is broken + can be found in the `Debian Manual`_. + You are now ready to start your bcfg2 server for the first time.:: - root@lucid:~# /etc/init.d/bcfg2-server start - root@lucid:~# tail /var/log/syslog - Dec 17 22:07:02 lucid bcfg2-server[17523]: serving bcfg2-server at https://lucid:6789 - Dec 17 22:07:02 lucid bcfg2-server[17523]: serve_forever() [start] - Dec 17 22:07:02 lucid bcfg2-server[17523]: Processed 16 fam events in 0.502 seconds. 0 coalesced + root@saucy:~# /etc/init.d/bcfg2-server start + Starting Configuration Management Server: * bcfg2-server + root@saucy:~# tail /var/log/syslog + Jul 18 17:50:48 saucy bcfg2-server[5872]: Reconnected to syslog + Jul 18 17:50:48 saucy bcfg2-server[5872]: bcfg2-server daemonized + Jul 18 17:50:48 saucy bcfg2-server[5872]: service available at https://saucy:6789 + Jul 18 17:50:48 saucy bcfg2-server[5872]: serving bcfg2-server at https://saucy:6789 + Jul 18 17:50:48 saucy bcfg2-server[5872]: serve_forever() [start] + Jul 18 17:50:48 saucy bcfg2-server[5872]: Handled 13 events in 0.006s Run bcfg2 to be sure you are able to communicate with the server:: - root@lucid:~# bcfg2 -vqn + root@saucy:~# bcfg2 -vqn + Starting Bcfg2 client run at 1374188552.53 Loaded tool drivers: - APT Action DebInit POSIX - + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers Phase: initial Correct entries: 0 Incorrect entries: 0 Total managed entries: 0 - Unmanaged entries: 382 - - + Unmanaged entries: 590 Phase: final Correct entries: 0 Incorrect entries: 0 Total managed entries: 0 - Unmanaged entries: 382 + Unmanaged entries: 590 + Finished Bcfg2 client run at 1374188563.26 Bring your first machine under Bcfg2 control ============================================ @@ -114,92 +142,101 @@ Setup the :ref:`server-plugins-generators-packages` plugin Replace Pkgmgr with Packages in the plugins line of ``bcfg2.conf``:: - root@lucid:~# cat /etc/bcfg2.conf + root@saucy:~# cat /etc/bcfg2.conf [server] repository = /var/lib/bcfg2 - plugins = SSHbase,Cfg,Packages,Rules,Metadata,Base,Bundler + plugins = Bundler,Cfg,Metadata,Packages,Rules,SSHbase + # Uncomment the following to listen on all interfaces + #listen_all = true [statistics] sendmailpath = /usr/lib/sendmail + #web_debug = False + #time_zone = [database] - engine = sqlite3 + #engine = sqlite3 # 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. - name = + #name = # Or path to database file if using sqlite3. - #/etc/brpt.sqlite is default path if left empty - user = + #/bcfg2.sqlite is default path if left empty + #user = # Not used with sqlite3. - password = + #password = # Not used with sqlite3. - host = + #host = # Not used with sqlite3. - port = + #port = + + [reporting] + transport = LocalFilesystem [communication] protocol = xmlrpc/ssl password = secret - certificate = /etc/bcfg2.crt - key = /etc/bcfg2.key - ca = /etc/bcfg2.crt + certificate = /etc/ssl/bcfg2.crt + key = /etc/ssl/bcfg2.key + ca = /etc/ssl/bcfg2.crt [components] - bcfg2 = https://lucid:6789 + bcfg2 = https://saucy:6789 Create Packages layout (as per :ref:`packages-exampleusage`) in ``/var/lib/bcfg2`` .. code-block:: xml - root@lucid:~# mkdir /var/lib/bcfg2/Packages - root@lucid:~# cat /var/lib/bcfg2/Packages/packages.conf + root@saucy:~# mkdir /var/lib/bcfg2/Packages + root@saucy:~# cat /var/lib/bcfg2/Packages/packages.conf [global] - root@lucid:~# cat /var/lib/bcfg2/Packages/sources.xml + root@saucy:~# cat /var/lib/bcfg2/Packages/sources.xml - - + + main multiverse restricted universe amd64 + bcfg2 + bcfg2-server - + main multiverse restricted universe amd64 + bcfg2 + bcfg2-server - + main multiverse restricted universe amd64 + bcfg2 + bcfg2-server + + + main + amd64 -Due to the :ref:`server-plugins-generators-packages-magic-groups`, -we need to modify our Metadata. Let's add an **ubuntu-lucid** -group which inherits the **ubuntu** group already present in -``/var/lib/bcfg2/Metadata/groups.xml``. The resulting file should look -something like this - -.. note:: - - The reason we are creating a release-specific group in this case is - that the APTSource above is specific to the lucid release of ubuntu. - That is, it should not apply to other releases (hardy, maverick, etc). +Above, we have grouped our package sources under **ubuntu-saucy**. We +need to add this group to our ``/var/lib/bcfg2/Metadata/groups.xml`` so +that our client is able to obtain these sources. .. code-block:: xml - + - + @@ -214,7 +251,7 @@ something like this .. note:: When editing your xml files by hand, it is useful to occasionally run - `bcfg2-lint` to ensure that your xml validates properly. + ``bcfg2-lint -v`` to ensure that your xml validates properly. The last thing we need is for the client to have the proper arch group membership. For this, we will make use of the @@ -223,13 +260,13 @@ Probes to your plugins line in ``bcfg2.conf`` and create the Probe. .. code-block:: sh - root@lucid:~# grep plugins /etc/bcfg2.conf - plugins = Base,Bundler,Cfg,...,Probes - root@lucid:~# mkdir /var/lib/bcfg2/Probes - root@lucid:~# cat /var/lib/bcfg2/Probes/groups + root@saucy:~# grep plugins /etc/bcfg2.conf + plugins = Bundler,Cfg,Metadata,...,Probes + root@saucy:~# mkdir /var/lib/bcfg2/Probes + root@saucy:~# cat /var/lib/bcfg2/Probes/groups #!/bin/sh - ARCH=`uname -m` + ARCH=$(uname -m) case "$ARCH" in "x86_64") echo "group:amd64" @@ -241,33 +278,36 @@ Probes to your plugins line in ``bcfg2.conf`` and create the Probe. Now we restart the bcfg2-server:: - root@lucid:~# /etc/init.d/bcfg2-server restart + root@saucy:~# /etc/init.d/bcfg2-server restart Stopping Configuration Management Server: * bcfg2-server Starting Configuration Management Server: * bcfg2-server - root@lucid:~# tail /var/log/syslog - Dec 17 22:36:47 lucid bcfg2-server[17937]: Packages: File read failed; falling back to file download - Dec 17 22:36:47 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/main/binary-amd64/Packages.gz - Dec 17 22:36:54 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/multiverse/binary-amd64/Packages.gz - Dec 17 22:36:55 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/restricted/binary-amd64/Packages.gz - Dec 17 22:36:56 lucid bcfg2-server[17937]: Packages: Updating http://us.archive.ubuntu.com/ubuntu//dists/lucid/universe/binary-amd64/Packages.gz - Dec 17 22:37:27 lucid bcfg2-server[17937]: Failed to read file probed.xml - Dec 17 22:37:27 lucid bcfg2-server[17937]: Loading experimental plugin(s): Packages - Dec 17 22:37:27 lucid bcfg2-server[17937]: NOTE: Interfaces subject to change - Dec 17 22:37:27 lucid bcfg2-server[17937]: service available at https://lucid:6789 - Dec 17 22:37:27 lucid bcfg2-server[17937]: serving bcfg2-server at https://lucid:6789 - Dec 17 22:37:27 lucid bcfg2-server[17937]: serve_forever() [start] - Dec 17 22:37:28 lucid bcfg2-server[17937]: Processed 17 fam events in 0.502 seconds. 0 coalesced + root@saucy:~# tail /var/log/syslog + Jul 18 18:43:22 saucy bcfg2-server[6215]: Reconnected to syslog + Jul 18 18:43:22 saucy bcfg2-server[6215]: bcfg2-server daemonized + Jul 18 18:43:22 saucy bcfg2-server[6215]: service available at https://saucy:6789 + Jul 18 18:43:22 saucy bcfg2-server[6215]: Failed to read file probed.xml: Error reading file '/var/lib/bcfg2/Probes/probed.xml': failed to load external entity "/var/lib/bcfg2/Probes/probed.xml" + Jul 18 18:43:22 saucy bcfg2-server[6215]: serving bcfg2-server at https://saucy:6789 + Jul 18 18:43:22 saucy bcfg2-server[6215]: serve_forever() [start] + Jul 18 18:43:22 saucy bcfg2-server[6215]: Reloading Packages plugin + Jul 18 18:43:22 saucy bcfg2-server[6215]: Handled 15 events in 0.205s + +.. note:: + + The error regarding *probed.xml* is non-fatal and just telling you + that the file doesn't yet exist. It will be populated once you have + run a client with the Probes plugin enabled. Start managing packages ----------------------- -Add a base-packages bundle. Let's see what happens when we just populate -it with the ubuntu-standard package. +Add a base-saucy (or whatever release you happen to be using) +bundle. Let's see what happens when we just populate it with the +ubuntu-standard package. .. code-block:: xml - root@lucid:~# cat /var/lib/bcfg2/Bundler/base-packages.xml - + root@saucy:~# cat /var/lib/bcfg2/Bundler/base-saucy.xml + @@ -277,218 +317,473 @@ profile group might look something like this .. code-block:: xml - - + + Now if we run the client in debug mode (-d), we can see what this has done for us.:: - root@lucid:~# bcfg2 -vqdn + root@saucy:/var/lib/bcfg2# bcfg2 -vqdn + Configured logging: DEBUG to console; DEBUG to syslog + {'help': False, 'extra': False, 'ppath': '/var/cache/bcfg2', 'ca': '/etc/ssl/bcfg2.crt', 'rpm_version_fail_action': 'upgrade', 'yum_version_fail_action': 'upgrade', 'retry_delay': '1', 'posix_uid_whitelist': [], 'rpm_erase_flags': ['allmatches'], 'verbose': True, 'certificate': '/etc/ssl/bcfg2.crt', 'paranoid': False, 'rpm_installonly': ['kernel', 'kernel-bigmem', 'kernel-enterprise', 'kernel-smp', 'kernel-modules', 'kernel-debug', 'kernel-unsupported', 'kernel-devel', 'kernel-source', 'kernel-default', 'kernel-largesmp-devel', 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], 'cache': None, 'yum24_autodep': True, 'yum_pkg_verify': True, 'probe_timeout': None, 'yum_installed_action': 'install', 'rpm_verify_fail_action': 'reinstall', 'dryrun': True, 'retries': '3', 'apt_install_path': '/usr', 'quick': True, 'password': 'secret', 'yum24_installed_action': 'install', 'kevlar': False, 'max_copies': 1, 'syslog': True, 'decision_list': False, 'configfile': '/etc/bcfg2.conf', 'remove': None, 'server': 'https://saucy:6789', 'encoding': 'UTF-8', 'timeout': 90, 'debug': True, 'yum24_installonly': ['kernel', 'kernel-bigmem', 'kernel-enterprise', 'kernel-smp', 'kernel-modules', 'kernel-debug', 'kernel-unsupported', 'kernel-devel', 'kernel-source', 'kernel-default', 'kernel-largesmp-devel', 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], 'yum24_erase_flags': ['allmatches'], 'yum24_pkg_checks': True, 'interactive': False, 'apt_etc_path': '/etc', 'rpm_installed_action': 'install', 'yum24_verify_fail_action': 'reinstall', 'omit_lock_check': False, 'yum24_pkg_verify': True, 'serverCN': None, 'file': None, 'apt_var_path': '/var', 'posix_gid_whitelist': [], 'posix_gid_blacklist': [], 'indep': False, 'decision': 'none', 'servicemode': 'default', 'version': False, 'rpm_pkg_checks': True, 'profile': None, 'yum_pkg_checks': True, 'args': [], 'bundle': [], 'posix_uid_blacklist': [], 'user': 'root', 'key': '/etc/ssl/bcfg2.key', 'command_timeout': None, 'probe_exit': True, 'lockfile': '/var/lock/bcfg2.run', 'yum_verify_fail_action': 'reinstall', 'yum24_version_fail_action': 'upgrade', 'yum_verify_flags': [], 'logging': None, 'rpm_pkg_verify': True, 'bundle_quick': False, 'rpm_verify_flags': [], 'yum24_verify_flags': [], 'skipindep': False, 'skipbundle': [], 'portage_binpkgonly': False, 'drivers': ['APK', 'APT', 'Action', 'Blast', 'Chkconfig', 'DebInit', 'Encap', 'FreeBSDInit', 'FreeBSDPackage', 'IPS', 'MacPorts', 'OpenCSW', 'POSIX', 'POSIXUsers', 'Pacman', 'Portage', 'RPM', 'RPMng', 'RcUpdate', 'SELinux', 'SMF', 'SYSV', 'Systemd', 'Upstart', 'VCS', 'YUM', 'YUM24', 'YUMng', 'launchd']} + Starting Bcfg2 client run at 1374191628.88 Running probe groups + Running: /tmp/tmpEtgdwo + < group:amd64 Probe groups has result: - amd64 + group:amd64 + + POSIX: Handlers loaded: nonexistent, directory, hardlink, symlink, file, device, permissions Loaded tool drivers: - APT Action DebInit POSIX + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers The following packages are specified in bcfg2: ubuntu-standard The following packages are prereqs added by Packages: - adduser debconf hdparm libdevmapper1.02.1 libk5crypto3 libparted1.8-12 libxml2 passwd upstart - apt debianutils info libdns53 libkeyutils1 libpci3 logrotate pciutils usbutils - aptitude dmidecode install-info libelf1 libkrb5-3 libpopt0 lsb-base perl-base wget - at dnsutils iptables libept0 libkrb5support0 libreadline5 lshw popularity-contest zlib1g - base-files dosfstools libacl1 libgcc1 liblwres50 libreadline6 lsof psmisc - base-passwd dpkg libattr1 libgdbm3 libmagic1 libselinux1 ltrace readline-common - bsdmainutils ed libbind9-50 libgeoip1 libmpfr1ldbl libsigc++-2.0-0c2a man-db rsync - bsdutils file libc-bin libgmp3c2 libncurses5 libssl0.9.8 memtest86+ sed - cpio findutils libc6 libgssapi-krb5-2 libncursesw5 libstdc++6 mime-support sensible-utils - cpp ftp libcap2 libisc50 libpam-modules libusb-0.1-4 ncurses-bin strace - cpp-4.4 gcc-4.4-base libcomerr2 libisccc50 libpam-runtime libuuid1 netbase time - cron groff-base libcwidget3 libisccfg50 libpam0g libxapian15 parted tzdata - + accountsservice libdrm2 libusb-1.0-0 + adduser libedit2 libustr-1.0-1 + apparmor libelf1 libuuid1 + apt libexpat1 libwind0-heimdal + apt-transport-https libffi6 libx11-6 + apt-utils libfribidi0 libx11-data + base-files libfuse2 libxau6 + base-passwd libgcc1 libxcb1 + bash libgck-1-0 libxdmcp6 + bash-completion libgcr-3-common libxext6 + bsdmainutils libgcr-base-3-1 libxml2 + bsdutils libgcrypt11 libxmuu1 + busybox-initramfs libgdbm3 libxtables10 + busybox-static libgeoip1 locales + ca-certificates libglib2.0-0 login + command-not-found libglib2.0-data logrotate + command-not-found-data libgnutls26 lsb-base + coreutils libgpg-error0 lsb-release + cpio libgpm2 lshw + cron libgssapi-krb5-2 lsof + dash libgssapi3-heimdal ltrace + dbus libhcrypto4-heimdal makedev + debconf libheimbase1-heimdal man-db + debconf-i18n libheimntlm0-heimdal manpages + debianutils libhx509-5-heimdal memtest86+ + diffutils libidn11 mime-support + dmidecode libisc92 mlocate + dmsetup libisccc90 module-init-tools + dnsutils libisccfg90 mount + dosfstools libjson-c2 mountall + dpkg libjson0 mtr-tiny + e2fslibs libk5crypto3 multiarch-support + e2fsprogs libkeyutils1 nano + ed libklibc ncurses-base + file libkmod2 ncurses-bin + findutils libkrb5-26-heimdal netbase + friendly-recovery libkrb5-3 ntfs-3g + ftp libkrb5support0 openssh-client + fuse libldap-2.4-2 openssl + gcc-4.8-base liblocale-gettext-perl parted + geoip-database liblwres90 passwd + gettext-base liblzma5 pciutils + gnupg libmagic1 perl-base + gpgv libmount1 plymouth + grep libncurses5 plymouth-theme-ubuntu-text + groff-base libncursesw5 popularity-contest + gzip libnewt0.52 powermgmt-base + hdparm libnfnetlink0 ppp + hostname libnih-dbus1 pppconfig + ifupdown libnih1 pppoeconf + info libnuma1 procps + initramfs-tools libp11-kit0 psmisc + initramfs-tools-bin libpam-modules python-apt-common + initscripts libpam-modules-bin python3 + insserv libpam-runtime python3-apt + install-info libpam-systemd python3-commandnotfound + iproute libpam0g python3-dbus + iproute2 libparted0debian1 python3-distupgrade + iptables libpcap0.8 python3-gdbm + iputils-tracepath libpci3 python3-minimal + irqbalance libpcre3 python3-update-manager + iso-codes libpipeline1 python3.3 + klibc-utils libplymouth2 python3.3-minimal + kmod libpng12-0 readline-common + krb5-locales libpolkit-gobject-1-0 rsync + language-selector-common libpopt0 sed + libaccountsservice0 libprocps0 sensible-utils + libacl1 libpython3-stdlib sgml-base + libapparmor-perl libpython3.3-minimal shared-mime-info + libapparmor1 libpython3.3-stdlib strace + libapt-inst1.5 libreadline6 systemd-services + libapt-pkg4.12 libroken18-heimdal sysv-rc + libasn1-8-heimdal librtmp0 sysvinit-utils + libasprintf0c2 libsasl2-2 tar + libatm1 libsasl2-modules tcpdump + libattr1 libselinux1 telnet + libaudit-common libsemanage-common time + libaudit1 libsemanage1 tzdata + libbind9-90 libsepol1 ubuntu-keyring + libblkid1 libslang2 ubuntu-release-upgrader-core + libbsd0 libsqlite3-0 ucf + libbz2-1.0 libss2 udev + libc-bin libssl1.0.0 ufw + libc6 libstdc++6 update-manager-core + libcap-ng0 libsystemd-daemon0 upstart + libcap2 libsystemd-login0 usbutils + libcomerr2 libtasn1-3 util-linux + libcurl3-gnutls libtext-charwidth-perl uuid-runtime + libdb5.1 libtext-iconv-perl wget + libdbus-1-3 libtext-wrapi18n-perl whiptail + libdbus-glib-1-2 libtinfo5 xauth + libdevmapper1.02.1 libudev1 xml-core + libdns95 libusb-0.1-4 zlib1g Phase: initial - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 - - + Total managed entries: 280 + Unmanaged entries: 313 + Installing entries in the following bundle(s): + base-saucy + Bundle base-saucy was not modified Phase: final - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 + Total managed entries: 280 + Unmanaged entries: 313 + Finished Bcfg2 client run at 1374191642.69 As you can see, the Packages plugin has generated the dependencies required for the ubuntu-standard package for us automatically. The ultimate goal should be to move all the packages from the **Unmanaged** entries section to the **Managed** entries section. So, what exactly *are* -those Unmanaged entries?:: +those Unmanaged entries? + +:: - root@lucid:~# bcfg2 -vqen + Starting Bcfg2 client run at 1374192077.76 Running probe groups Probe groups has result: - amd64 - Loaded tool drivers: - APT Action DebInit POSIX + group:amd64 + Loaded tool drivers: + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers Phase: initial - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 - - + Total managed entries: 280 + Unmanaged entries: 313 Phase: final - Correct entries: 101 + Correct entries: 280 Incorrect entries: 0 - Total managed entries: 101 - Unmanaged entries: 281 - Package:apparmor - Package:apparmor-utils - Package:apport - ... - -Now you can go through these and continue adding the packages you want to -your Bundle. Note that ``aptitude why`` is useful when trying to figure -out the reason for a package being installed. Also, deborphan is helpful -for removing leftover dependencies which are no longer needed. After a -while, I ended up with a minimal bundle that looks like this + Total managed entries: 280 + Unmanaged entries: 313 + POSIXGroup:adm + POSIXGroup:audio + POSIXGroup:backup + ... + Package:deb:apt-xapian-index + Package:deb:aptitude + Package:deb:aptitude-common + ... + +Now you can go through these and continue adding the packages you want +to your Bundle. Note that ``aptitude why`` is useful when trying to +figure out the reason for a package being installed. Also, ``deborphan`` +is helpful for removing leftover dependencies which are no longer +needed. After a while, I ended up with a minimal bundle that looks +like this: .. code-block:: xml - - + + - + + + + + - - - - - - - - - + - - - - - - - + + + + + + - - - - - - - - - - - - - - - -As you can see below, I no longer have any unmanaged packages. :: +Once your ``bcfg2 -vqen`` output no longer shows Package entries, you +can move on to the next section. - root@lucid:~# bcfg2 -vqen - Running probe groups - Probe groups has result: - amd64 - Loaded tool drivers: - APT Action DebInit POSIX +Manage users +------------ - Phase: initial - Correct entries: 247 - Incorrect entries: 0 - Total managed entries: 247 - Unmanaged entries: 10 +The default setting in ``login.defs`` is for system accounts to be UIDs +< 1000. We will ignore those accounts for now (you can manage them if +you like at a later time). +To ignore system UID/GIDs, add the following lines to ``bcfg2.conf`` +(we will also ignore the nobody uid and nogroup gid--65534). - Phase: final - Correct entries: 247 - Incorrect entries: 0 - Total managed entries: 247 - Unmanaged entries: 10 - Service:bcfg2 Service:fam Service:killprocs Service:rc.local Service:single - Service:bcfg2-server Service:grub-common Service:ondemand Service:rsync Service:ssh +:: + + [POSIXUsers] + uid_blacklist = 0-999,65534 + gid_blacklist = 0-999,65534 + +If you run the client again with ``bcfg2 -vqen``, you should now see a +:ref:`POSIXUser ` entry +and :ref:`POSIXUser ` +entry for your user account (assuming this is a fresh install with a +regular user). + +You can manage this user by adding the following to your bundle. + +.. code-block:: xml + + + adm + cdrom + dip + lpadmin + plugdev + sambashare + sudo + Manage services --------------- -Now let's clear up the unmanaged service entries by adding the following -entries to our bundle... +To clear up the unmanaged service entries, you will need to add the +entries to your bundle. Here's an example of what that might look like. .. code-block:: xml - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - -...and bind them in Rules + + + + + + + + + + + + + + + + + + + + + + +Add the literal entries in Rules to bind the Service entries from above. .. code-block:: xml - root@lucid:~# cat /var/lib/bcfg2/Rules/services.xml + root@saucy:~# cat /var/lib/bcfg2/Rules/services.xml - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -Now we run the client and see there are no more unmanaged entries! :: +Now we run the client and see there are no more unmanaged entries! - root@lucid:~# bcfg2 -vqn +:: + + root@saucy:~# bcfg2 -vqn + Starting Bcfg2 client run at 1374271524.83 Running probe groups Probe groups has result: - amd64 - Loaded tool drivers: - APT Action DebInit POSIX + group:amd64 + Loaded tool drivers: + APT Action DebInit POSIX POSIXUsers Upstart VCS + Loaded experimental tool drivers: + POSIXUsers Phase: initial - Correct entries: 257 + Correct entries: 519 Incorrect entries: 0 - Total managed entries: 257 + Total managed entries: 519 Unmanaged entries: 0 - - All entries correct. - Phase: final - Correct entries: 257 + Correct entries: 519 Incorrect entries: 0 - Total managed entries: 257 + Total managed entries: 519 Unmanaged entries: 0 - All entries correct. + Finished Bcfg2 client run at 1374271541.56 .. warning:: diff --git a/doc/server/plugins/generators/rules.txt b/doc/server/plugins/generators/rules.txt index 2493be53f..2c5204ba9 100644 --- a/doc/server/plugins/generators/rules.txt +++ b/doc/server/plugins/generators/rules.txt @@ -376,6 +376,8 @@ SEModule Tag See also :ref:`server-plugins-generators-semodules`. +.. _server-plugins-generators-rules-posixuser-tag: + POSIXUser Tag ------------- @@ -411,6 +413,8 @@ Defaults plugin `. See :ref:`client-tools-posixusers` for more information on managing users and groups. +.. _server-plugins-generators-rules-posixgroup-tag: + POSIXGroup Tag -------------- -- cgit v1.2.3-1-g7c22 From 76b878ca116f3fadc3f109a6f2d53ba927f03824 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 19 Jul 2013 17:34:05 -0500 Subject: doc: Fix typo Signed-off-by: Sol Jerome --- doc/appendix/guides/ubuntu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/appendix/guides/ubuntu.txt b/doc/appendix/guides/ubuntu.txt index 30ecbc3d7..f68c8b9ad 100644 --- a/doc/appendix/guides/ubuntu.txt +++ b/doc/appendix/guides/ubuntu.txt @@ -540,7 +540,7 @@ To ignore system UID/GIDs, add the following lines to ``bcfg2.conf`` If you run the client again with ``bcfg2 -vqen``, you should now see a :ref:`POSIXUser ` entry -and :ref:`POSIXUser ` +and :ref:`POSIXGroup ` entry for your user account (assuming this is a fresh install with a regular user). -- cgit v1.2.3-1-g7c22 From 240b4b423257c5e8d5dd5b3e32398f840bb8a66b Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 19 Jul 2013 17:43:43 -0500 Subject: debian: Add south dependency Signed-off-by: Sol Jerome --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index edc4003dd..37b72a9f4 100644 --- a/debian/control +++ b/debian/control @@ -43,7 +43,7 @@ Description: Configuration management server Package: bcfg2-web Architecture: all -Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django, +Depends: ${python:Depends}, ${misc:Depends}, bcfg2-server (= ${binary:Version}), python-django, python-django-south (>= 0.7.5) Suggests: python-mysqldb, python-psycopg2, python-sqlite, libapache2-mod-wsgi Description: Configuration management web interface Bcfg2 is a configuration management system that generates configuration sets -- cgit v1.2.3-1-g7c22 From 2f45bf09380595da88ebb1757042b7e799d41f57 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Fri, 19 Jul 2013 17:50:07 -0500 Subject: Admin/Init: Fix default db location Signed-off-by: Sol Jerome --- src/lib/Bcfg2/Server/Admin/Init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index c76e54569..153d7bea6 100644 --- a/src/lib/Bcfg2/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -33,7 +33,7 @@ sendmailpath = %s # 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. #name = # Or path to database file if using sqlite3. -#/bcfg2.sqlite is default path if left empty +#/etc/bcfg2.sqlite is default path if left empty #user = # Not used with sqlite3. #password = -- cgit v1.2.3-1-g7c22 From 360ba2e77865d2a292568ede99d8896ef7742056 Mon Sep 17 00:00:00 2001 From: Arto Jantunen Date: Sun, 21 Jul 2013 10:35:50 +0300 Subject: Make the server process always detach, even if started by init Otherwise startup with systemd in type=forking fails since the daemon will not fork. --- src/lib/Bcfg2/Server/BuiltinCore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/BuiltinCore.py b/src/lib/Bcfg2/Server/BuiltinCore.py index e69a92b64..2dd83289d 100644 --- a/src/lib/Bcfg2/Server/BuiltinCore.py +++ b/src/lib/Bcfg2/Server/BuiltinCore.py @@ -31,7 +31,8 @@ class Core(BaseCore): daemon_args = dict(uid=self.setup['daemon_uid'], gid=self.setup['daemon_gid'], - umask=int(self.setup['umask'], 8)) + umask=int(self.setup['umask'], 8), + detach_process=True) if self.setup['daemon']: daemon_args['pidfile'] = TimeoutPIDLockFile(self.setup['daemon'], acquire_timeout=5) -- cgit v1.2.3-1-g7c22 From 7d544a289852c761fc209772064f794fb0472198 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 22 Jul 2013 09:08:32 -0400 Subject: Packages: Added timeout to bcfg2-yum-helper calls This involved making the Yum backend use Bcfg2.Utils.Executor to call bcfg2-yum-helper instead of subprocess.Popen directly. This was cherry-picked (kinda) from 3d06f311274d6b942ee89d8cdb13b2ecc99af1b0, so will likely break the maint -> master merge in spectacular ways. --- src/lib/Bcfg2/Options.py | 3 +- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 54 +++++++++++----------------- 2 files changed, 22 insertions(+), 35 deletions(-) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index dba3e96ef..3d105bf30 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1221,7 +1221,8 @@ SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, authentication=SERVER_AUTHENTICATION, perflog=LOG_PERFORMANCE, perflog_interval=PERFLOG_INTERVAL, - children=SERVER_CHILDREN) + children=SERVER_CHILDREN, + client_timeout=CLIENT_TIMEOUT) CRYPT_OPTIONS = dict(encrypt=ENCRYPT, decrypt=DECRYPT, diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 7a90f4f2e..e0002ef34 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -59,9 +59,9 @@ import errno import socket import logging import lxml.etree -from lockfile import FileLock -from subprocess import Popen, PIPE import Bcfg2.Server.Plugin +from lockfile import FileLock +from Bcfg2.Utils import Executor # pylint: disable=W0622 from Bcfg2.Compat import StringIO, cPickle, HTTPError, URLError, \ ConfigParser, any @@ -279,6 +279,7 @@ class YumCollection(Collection): debug=debug) self.keypath = os.path.join(self.cachepath, "keys") + self._helper = None if self.use_yum: #: Define a unique cache file for this collection to use #: for cached yum metadata @@ -294,8 +295,10 @@ class YumCollection(Collection): os.mkdir(self.cachefile) if not self.disableMetaData: self.setup_data() + self.cmd = Executor() else: self.cachefile = None + self.cmd = None if HAS_PULP and self.has_pulp_sources: _setup_pulp(self.setup) @@ -348,20 +351,18 @@ class YumCollection(Collection): a call to it; I wish there was a way to do this without forking, but apparently not); finally we check in /usr/sbin, the default location. """ - global HELPER - if not HELPER: + if not self._helper: try: - HELPER = self.setup.cfp.get("packages:yum", "helper") + self._helper = self.setup.cfp.get("packages:yum", "helper") except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): # first see if bcfg2-yum-helper is in PATH try: self.debug_log("Checking for bcfg2-yum-helper in $PATH") - Popen(['bcfg2-yum-helper'], - stdin=PIPE, stdout=PIPE, stderr=PIPE).wait() - HELPER = 'bcfg2-yum-helper' + self.cmd.run(['bcfg2-yum-helper']) + self._helper = 'bcfg2-yum-helper' except OSError: - HELPER = "/usr/sbin/bcfg2-yum-helper" - return HELPER + self._helper = "/usr/sbin/bcfg2-yum-helper" + return self._helper @property def use_yum(self): @@ -925,36 +926,21 @@ class YumCollection(Collection): cmd.append("-v") cmd.append(command) self.debug_log("Packages: running %s" % " ".join(cmd)) - try: - helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to execute %s: %s" % - (" ".join(cmd), err)) - return None if inputdata: - idata = json.dumps(inputdata) - (stdout, stderr) = helper.communicate(idata) + result = self.cmd.run(cmd, timeout=self.setup['client_timeout'], + inputdata=json.dumps(inputdata)) else: - (stdout, stderr) = helper.communicate() - rv = helper.wait() - errlines = stderr.splitlines() - if rv: - if not errlines: - errlines.append("No error output") - self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, errlines[0])) - for line in errlines[1:]: - self.logger.error("Packages: %s" % line) - elif errlines: + result = self.cmd.run(cmd, timeout=self.setup['client_timeout']) + if not result.success: + self.logger.error("Packages: error running bcfg2-yum-helper: %s" % + result.error) + elif result.stderr: self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - errlines[0]) - for line in errlines[1:]: - self.debug_log("Packages: %s" % line) + result.stderr) try: - return json.loads(stdout) + return json.loads(result.stdout) except ValueError: err = sys.exc_info()[1] self.logger.error("Packages: error reading bcfg2-yum-helper " -- cgit v1.2.3-1-g7c22 From 752da22a2247892f647c0a9c46e7b0faf9351ea6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 22 Jul 2013 11:44:23 -0400 Subject: Packages: instantiate Executor before determining path to helper --- src/lib/Bcfg2/Server/Plugins/Packages/Yum.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index e0002ef34..48c5b1f65 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -104,9 +104,6 @@ FL = '{http://linux.duke.edu/metadata/filelists}' PULPSERVER = None PULPCONFIG = None -#: The path to bcfg2-yum-helper -HELPER = None - def _setup_pulp(setup): """ Connect to a Pulp server and pass authentication credentials. @@ -279,6 +276,10 @@ class YumCollection(Collection): debug=debug) self.keypath = os.path.join(self.cachepath, "keys") + #: A :class:`Bcfg2.Utils.Executor` object to use to run + #: external commands + self.cmd = Executor() + self._helper = None if self.use_yum: #: Define a unique cache file for this collection to use @@ -295,10 +296,8 @@ class YumCollection(Collection): os.mkdir(self.cachefile) if not self.disableMetaData: self.setup_data() - self.cmd = Executor() else: self.cachefile = None - self.cmd = None if HAS_PULP and self.has_pulp_sources: _setup_pulp(self.setup) -- cgit v1.2.3-1-g7c22 From 7d17d1c283f2718ae86a4b2db03726f4ae802889 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 22 Jul 2013 14:26:49 -0400 Subject: MultiprocessingCore: Dispatch metadata cache expiration to children When the broker in a multiprocessing configuration expires its metadata cache (e.g., when probe data is received), it must dispatch that expiration call to its children. This also makes the protocol for communication between the broker and its children into a real RPC protocol, so we can do even more stuff in the future. --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 102 +++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 10 deletions(-) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 81fba7092..066519774 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -2,16 +2,71 @@ :mod:`Bcfg2.Server.BuiltinCore` that uses the Python :mod:`multiprocessing` library to offload work to multiple child processes. As such, it requires Python 2.6+. + +The parent communicates with the children over a +:class:`multiprocessing.Pipe` that implements a very simple RPC +protocol. Each command passed to a child over the Pipe must be a +tuple with the format:: + + (, , ) + +The method must be exposed by the child by decorating it with +:func:`Bcfg2.Server.Core.exposed`. + +The RPC call always returns a value via the pipe, so the caller *must* +read the return value in order to keep the pipe consistent. """ +import logging import threading import lxml.etree import multiprocessing +from Bcfg2.Cache import Cache from Bcfg2.Compat import Queue from Bcfg2.Server.Core import BaseCore, exposed +from Bcfg2.Server.Plugin import Debuggable from Bcfg2.Server.BuiltinCore import Core as BuiltinCore +class DispatchingCache(Cache, Debuggable): + """ Implementation of :class:`Bcfg2.Cache.Cache` that propagates + cache expiration events to child nodes. """ + + #: The method to send over the pipe to expire the cache + method = "expire_cache" + + def __init__(self, *args, **kwargs): + #: A dict of : :class:`multiprocessing.Pipe` + #: objects that should be given a cache expiration command any + #: time an item is expired. + self.pipes = kwargs.pop("pipes", dict()) + + #: A :class:`logging.Logger` object this cache object can use + self.logger = logging.getLogger(self.__class__.__name__) + Cache.__init__(self, *args, **kwargs) + + def expire(self, key=None): + if (key and key in self) or (not key and len(self)): + # dispatching cache expiration to children can be + # expensive, so only do it if there's something to expire + for child, pipe in self.pipes.items(): + if key: + self.logger.debug("Expiring metadata cache for %s on %s" % + (key, child)) + else: + self.logger.debug("Expiring metadata cache on %s" % child) + pipe.send((self.method, [key], dict())) + pipe.recv() + Cache.expire(self, key=key) + + +class NoSuchMethod(Exception): + """ Exception raised by a child process if it's asked to execute a + method that doesn't exist or that isn't exposed via the + :class:`multiprocessing.Pipe` RPC interface. """ + pass + + class DualEvent(object): """ DualEvent is a clone of :class:`threading.Event` that internally implements both :class:`threading.Event` and @@ -98,6 +153,33 @@ class ChildCore(BaseCore): def _run(self): return True + def rpc_dispatch(self): + """ Dispatch a method received via the + :class:`multiprocessing.Pipe` RPC interface. + + :param data: The tuple of ``(, , )`` + :type data: tuple + """ + method, args, kwargs = self.pipe.recv() + if hasattr(self, method): + func = getattr(self, method) + if func.exposed: + self.pipe.send(func(*args, **kwargs)) + else: + raise NoSuchMethod(method) + else: + raise NoSuchMethod(method) + + @exposed + def GetConfig(self, client): + self.logger.debug("Building configuration for %s" % client) + return lxml.etree.tostring(self.BuildConfiguration(client)) + + @exposed + def expire_cache(self, client=None): + """ Expire the metadata cache for a client """ + self.metadata_cache.expire(client) + def _block(self): while not self.terminate.isSet(): try: @@ -109,15 +191,7 @@ class ChildCore(BaseCore): # should be using the metadata database if you're # using this core. self.fam.handle_events_in_interval(0.1) - client = self.pipe.recv() - self.logger.debug("Building configuration for %s" % client) - config = \ - lxml.etree.tostring(self.BuildConfiguration(client)) - self.logger.debug("Returning configuration for %s to main " - "process" % client) - self.pipe.send(config) - self.logger.debug("Returned configuration for %s to main " - "process" % client) + self.rpc_dispatch() except KeyboardInterrupt: break self.shutdown() @@ -164,11 +238,14 @@ class Core(BuiltinCore): # monkeypatch self.terminate to have isSet(). self.terminate = DualEvent(threading_event=self.terminate) + self.metadata_cache = DispatchingCache() + def _run(self): for cnum in range(self.setup['children']): name = "Child-%s" % cnum (mainpipe, childpipe) = multiprocessing.Pipe() self.pipes[name] = mainpipe + self.metadata_cache.pipes[name] = mainpipe self.logger.debug("Starting child %s" % name) childcore = ChildCore(self.setup, childpipe, self.terminate) child = multiprocessing.Process(target=childcore.run, name=name) @@ -192,13 +269,18 @@ class Core(BuiltinCore): self.logger.debug("Child %s shut down" % child.name) self.logger.debug("All children shut down") + @exposed + def set_debug(self, address, debug): + BuiltinCore.set_debug(self, address, debug) + self.metadata_cache.set_debug(debug) + @exposed def GetConfig(self, address): client = self.resolve_client(address)[0] childname = self.available_children.get() self.logger.debug("Building configuration on child %s" % childname) pipe = self.pipes[childname] - pipe.send(client) + pipe.send(("GetConfig", [client], dict())) config = pipe.recv() self.available_children.put_nowait(childname) return config -- cgit v1.2.3-1-g7c22 From 01693c1905aab3cc11978a193003537f6ca35444 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 22 Jul 2013 14:49:43 -0400 Subject: MultiprocessingCore: Call Debuggable.__init__ on DispatchingCache --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 066519774..76a80f5fb 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -41,8 +41,7 @@ class DispatchingCache(Cache, Debuggable): #: time an item is expired. self.pipes = kwargs.pop("pipes", dict()) - #: A :class:`logging.Logger` object this cache object can use - self.logger = logging.getLogger(self.__class__.__name__) + Debuggable.__init__(self) Cache.__init__(self, *args, **kwargs) def expire(self, key=None): -- cgit v1.2.3-1-g7c22 From 2ba89f93a9146eabad02828eb1a41b0a97dd2038 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 22 Jul 2013 15:20:42 -0400 Subject: MultiprocessingCore: removed unused import --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index 76a80f5fb..af5eb0f3a 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -17,7 +17,6 @@ The RPC call always returns a value via the pipe, so the caller *must* read the return value in order to keep the pipe consistent. """ -import logging import threading import lxml.etree import multiprocessing -- cgit v1.2.3-1-g7c22 From a9ea92aa595c5df63eff25ff545927078f7651e6 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Mon, 22 Jul 2013 15:43:34 -0400 Subject: MultiprocessingCore: Fixed return value from set_debug --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index af5eb0f3a..c9d7fc8c0 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -269,8 +269,8 @@ class Core(BuiltinCore): @exposed def set_debug(self, address, debug): - BuiltinCore.set_debug(self, address, debug) self.metadata_cache.set_debug(debug) + return BuiltinCore.set_debug(self, address, debug) @exposed def GetConfig(self, address): -- cgit v1.2.3-1-g7c22 From 9084b0e889407956227ae8d65bceff5148f7ee1f Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 23 Jul 2013 08:23:59 -0400 Subject: MultiprocessingCore: rewrote parent-child RPC to be thread-safe (and less powerful) --- src/lib/Bcfg2/Server/MultiprocessingCore.py | 157 ++++++++++++++++------------ 1 file changed, 90 insertions(+), 67 deletions(-) diff --git a/src/lib/Bcfg2/Server/MultiprocessingCore.py b/src/lib/Bcfg2/Server/MultiprocessingCore.py index c9d7fc8c0..02710ab99 100644 --- a/src/lib/Bcfg2/Server/MultiprocessingCore.py +++ b/src/lib/Bcfg2/Server/MultiprocessingCore.py @@ -3,18 +3,24 @@ :mod:`multiprocessing` library to offload work to multiple child processes. As such, it requires Python 2.6+. -The parent communicates with the children over a -:class:`multiprocessing.Pipe` that implements a very simple RPC -protocol. Each command passed to a child over the Pipe must be a -tuple with the format:: +The parent communicates with the children over two constructs: + +* A :class:`multiprocessing.Pipe` is used to process render requests. + The pipe is locked when in use (i.e., between the time that a client + is submitted to be rendered and the time that its configuration is + returned) to keep things thread-safe. (This is accomplished through + the use of + :attr:`Bcfg2.Server.MultiprocessingCore.available_children.) +* A :class:`multiprocessing.Queue` is used to submit other commands in + a thread-safe, non-blocking fashion. (Note that, since it is a + queue, no results can be returned.) It implements a very simple RPC + protocol. Each command passed to a child over the Pipe must be a + tuple with the format:: (, , ) -The method must be exposed by the child by decorating it with -:func:`Bcfg2.Server.Core.exposed`. - -The RPC call always returns a value via the pipe, so the caller *must* -read the return value in order to keep the pipe consistent. + The method must be exposed by the child by decorating it with + :func:`Bcfg2.Server.Core.exposed`. """ import threading @@ -35,10 +41,10 @@ class DispatchingCache(Cache, Debuggable): method = "expire_cache" def __init__(self, *args, **kwargs): - #: A dict of : :class:`multiprocessing.Pipe` + #: A dict of : :class:`multiprocessing.Queue` #: objects that should be given a cache expiration command any #: time an item is expired. - self.pipes = kwargs.pop("pipes", dict()) + self.command_queues = kwargs.pop("pipes", dict()) Debuggable.__init__(self) Cache.__init__(self, *args, **kwargs) @@ -47,24 +53,16 @@ class DispatchingCache(Cache, Debuggable): if (key and key in self) or (not key and len(self)): # dispatching cache expiration to children can be # expensive, so only do it if there's something to expire - for child, pipe in self.pipes.items(): + for child, cmd_q in self.command_queues.items(): if key: self.logger.debug("Expiring metadata cache for %s on %s" % (key, child)) else: self.logger.debug("Expiring metadata cache on %s" % child) - pipe.send((self.method, [key], dict())) - pipe.recv() + cmd_q.put((self.method, [key], dict())) Cache.expire(self, key=key) -class NoSuchMethod(Exception): - """ Exception raised by a child process if it's asked to execute a - method that doesn't exist or that isn't exposed via the - :class:`multiprocessing.Pipe` RPC interface. """ - pass - - class DualEvent(object): """ DualEvent is a clone of :class:`threading.Event` that internally implements both :class:`threading.Event` and @@ -120,7 +118,7 @@ class ChildCore(BaseCore): #: every ``poll_wait`` seconds. poll_wait = 5.0 - def __init__(self, setup, pipe, terminate): + def __init__(self, setup, render_pipe, command_queue, terminate): """ :param setup: A Bcfg2 options dict :type setup: Bcfg2.Options.OptionParser @@ -139,61 +137,75 @@ class ChildCore(BaseCore): #: objects to build configurations, and to which client #: configurations are added after having been built by #: ChildCore objects. - self.pipe = pipe + self.render_pipe = render_pipe + + #: The queue from which other commands are received + self.command_queue = command_queue #: The :class:`multiprocessing.Event` that will be monitored #: to determine when this child should shut down. self.terminate = terminate + #: The :class:`threading.Thread` used to process commands + #: received via the :class:`multiprocessing.Queue` RPC + #: interface + self.command_thread = \ + threading.Thread(name="CommandThread", + target=self._command_queue_thread) + def _daemonize(self): return True def _run(self): + try: + self.command_thread.start() + except: + self.shutdown() + raise return True - def rpc_dispatch(self): - """ Dispatch a method received via the - :class:`multiprocessing.Pipe` RPC interface. - - :param data: The tuple of ``(, , )`` - :type data: tuple - """ - method, args, kwargs = self.pipe.recv() - if hasattr(self, method): - func = getattr(self, method) - if func.exposed: - self.pipe.send(func(*args, **kwargs)) - else: - raise NoSuchMethod(method) - else: - raise NoSuchMethod(method) - - @exposed - def GetConfig(self, client): - self.logger.debug("Building configuration for %s" % client) - return lxml.etree.tostring(self.BuildConfiguration(client)) - - @exposed - def expire_cache(self, client=None): - """ Expire the metadata cache for a client """ - self.metadata_cache.expire(client) + def render(self): + """ Process client configuration render requests """ + if self.render_pipe.poll(self.poll_wait): + if not self.metadata.use_database: + # handle FAM events, in case (for instance) the + # client has just been added to clients.xml, or a + # profile has just been asserted. but really, you + # should be using the metadata database if you're + # using this core. + self.fam.handle_events_in_interval(0.1) + client = self.render_pipe.recv() + self.logger.debug("Building configuration for %s" % client) + self.render_pipe.send( + lxml.etree.tostring(self.BuildConfiguration(client))) def _block(self): while not self.terminate.isSet(): try: - if self.pipe.poll(self.poll_wait): - if not self.metadata.use_database: - # handle FAM events, in case (for instance) the - # client has just been added to clients.xml, or a - # profile has just been asserted. but really, you - # should be using the metadata database if you're - # using this core. - self.fam.handle_events_in_interval(0.1) - self.rpc_dispatch() + self.render() except KeyboardInterrupt: break self.shutdown() + def _command_queue_thread(self): + """ Process commands received on the command queue thread """ + while not self.terminate.isSet(): + method, args, kwargs = self.command_queue.get() + if hasattr(self, method): + func = getattr(self, method) + if func.exposed: + self.logger.debug("Child calling RPC method %s" % method) + func(*args, **kwargs) + else: + self.logger.error("Method %s is not exposed" % method) + else: + self.logger.error("Method %s does not exist" % method) + + @exposed + def expire_cache(self, client=None): + """ Expire the metadata cache for a client """ + self.metadata_cache.expire(client) + class Core(BuiltinCore): """ A multiprocessing core that delegates building the actual @@ -213,10 +225,14 @@ class Core(BuiltinCore): setup['children'] = multiprocessing.cpu_count() #: A dict of child name -> one end of the - #: :class:`multiprocessing.Pipe` object used to communicate - #: with that child. (The child is given the other end of the - #: Pipe.) - self.pipes = dict() + #: :class:`multiprocessing.Pipe` object used to submit render + #: requests to that child. (The child is given the other end + #: of the Pipe.) + self.render_pipes = dict() + + #: A dict of child name -> :class:`multiprocessing.Queue` + #: object used to pass commands to that child. + self.command_queues = dict() #: A queue that keeps track of which children are available to #: render a configuration. A child is popped from the queue @@ -241,11 +257,18 @@ class Core(BuiltinCore): def _run(self): for cnum in range(self.setup['children']): name = "Child-%s" % cnum + + # create Pipe for render requests and results (mainpipe, childpipe) = multiprocessing.Pipe() - self.pipes[name] = mainpipe - self.metadata_cache.pipes[name] = mainpipe + self.render_pipes[name] = mainpipe + + # create Queue for other commands + cmd_q = multiprocessing.Queue() + self.command_queues[name] = cmd_q + self.metadata_cache.command_queues[name] = cmd_q + self.logger.debug("Starting child %s" % name) - childcore = ChildCore(self.setup, childpipe, self.terminate) + childcore = ChildCore(self.setup, childpipe, cmd_q, self.terminate) child = multiprocessing.Process(target=childcore.run, name=name) child.start() self.logger.debug("Child %s started with PID %s" % (name, @@ -277,8 +300,8 @@ class Core(BuiltinCore): client = self.resolve_client(address)[0] childname = self.available_children.get() self.logger.debug("Building configuration on child %s" % childname) - pipe = self.pipes[childname] - pipe.send(("GetConfig", [client], dict())) + pipe = self.render_pipes[childname] + pipe.send(client) config = pipe.recv() self.available_children.put_nowait(childname) return config -- cgit v1.2.3-1-g7c22 From ca9be467a0a4460a2dacadebb2ba9e45433297a4 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 23 Jul 2013 14:26:27 -0400 Subject: Cfg: unknown-cfg-files lint check honors FAM ignore list --- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 154cd5e63..44455ff13 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -10,6 +10,7 @@ import lxml.etree import Bcfg2.Options import Bcfg2.Server.Plugin import Bcfg2.Server.Lint +from fnmatch import fnmatch from Bcfg2.Server.Plugin import PluginExecutionError # pylint: disable=W0622 from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ @@ -940,22 +941,37 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): "%s has no corresponding pubkey.xml at %s" % (basename, pubkey)) + def _list_path_components(self, path): + rv = [] + remaining, component = os.path.split(path) + while component != '': + rv.append(component) + remaining, component = os.path.split(remaining) + return rv + def check_missing_files(self): """ check that all files on the filesystem are known to Cfg """ cfg = self.core.plugins['Cfg'] # first, collect ignore patterns from handlers - ignore = [] + ignore = set() for hdlr in handlers(): - ignore.extend(hdlr.__ignore__) + ignore.update(hdlr.__ignore__) # next, get a list of all non-ignored files on the filesystem all_files = set() for root, _, files in os.walk(cfg.data): - all_files.update(os.path.join(root, fname) - for fname in files - if not any(fname.endswith("." + i) - for i in ignore)) + for fname in files: + fpath = os.path.join(root, fname) + # check against the handler ignore patterns and the + # global FAM ignore list + if (not any(fname.endswith("." + i) for i in ignore) and + not any(fnmatch(fpath, p) + for p in self.config['ignore']) and + not any(fnmatch(c, p) + for p in self.config['ignore'] + for c in self._list_path_components(fpath))): + all_files.add(fpath) # next, get a list of all files known to Cfg cfg_files = set() -- cgit v1.2.3-1-g7c22 From 084c9293050eb78a6da9f6dac41f71507b8098a2 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Tue, 23 Jul 2013 14:52:37 -0400 Subject: Cfg: added missing docstring --- src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 44455ff13..7f271fc7f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -942,6 +942,10 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): (basename, pubkey)) def _list_path_components(self, path): + """ Get a list of all components of a path. E.g., + ``self._list_path_components("/foo/bar/foobaz")`` would return + ``["foo", "bar", "foo", "baz"]``. The list is not guaranteed + to be in order.""" rv = [] remaining, component = os.path.split(path) while component != '': -- cgit v1.2.3-1-g7c22 From df87f88840841ef4ec6b14eaef92cd11b6ad8710 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 24 Jul 2013 15:52:46 -0400 Subject: settings: allow setting database schema --- src/lib/Bcfg2/Options.py | 5 ++++- src/lib/Bcfg2/settings.py | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 3d105bf30..2b9a8706f 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -673,12 +673,15 @@ DB_PORT = \ default='', cf=('database', 'port'), deprecated_cf=('statistics', 'database_port')) - DB_OPTIONS = \ Option('Database options', default=dict(), cf=('database', 'options'), cook=dict_split) +DB_PORT = \ + Option('Database schema', + default='', + cf=('database', 'schema')) # Django options WEB_CFILE = \ diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index 6e718a079..82a3bdb2a 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -27,6 +27,7 @@ DATABASE_PASSWORD = None DATABASE_HOST = None DATABASE_PORT = None DATABASE_OPTIONS = None +DATABASE_SCHEMA = None TIME_ZONE = None @@ -59,8 +60,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): """ read the config file and set django settings based on it """ # pylint: disable=W0602,W0603 global DATABASE_ENGINE, DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, \ - DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DEBUG, \ - TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL + DATABASE_HOST, DATABASE_PORT, DATABASE_OPTIONS, DATABASE_SCHEMA, \ + DEBUG, TEMPLATE_DEBUG, TIME_ZONE, MEDIA_URL # pylint: enable=W0602,W0603 if not os.path.exists(cfile) and os.path.exists(DEFAULT_CONFIG): @@ -88,7 +89,8 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): PASSWORD=setup['db_password'], HOST=setup['db_host'], PORT=setup['db_port'], - OPTIONS=setup['db_options']) + OPTIONS=setup['db_options'], + SCHEMA=setup['db_schema']) if HAS_DJANGO and django.VERSION[0] == 1 and django.VERSION[1] < 2: DATABASE_ENGINE = setup['db_engine'] @@ -98,6 +100,7 @@ def read_config(cfile=DEFAULT_CONFIG, repo=None, quiet=False): DATABASE_HOST = DATABASES['default']['HOST'] DATABASE_PORT = DATABASES['default']['PORT'] DATABASE_OPTIONS = DATABASES['default']['OPTIONS'] + DATABASE_SCHEMA = DATABASES['default']['SCHEMA'] # dropping the version check. This was added in 1.1.2 TIME_ZONE = setup['time_zone'] -- cgit v1.2.3-1-g7c22 From 1c86a3668d017bcaeda602cbc5c5bee84d701647 Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 24 Jul 2013 16:06:25 -0400 Subject: settings: fixed db schema option --- src/lib/Bcfg2/Options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 2b9a8706f..51af84cb1 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -678,7 +678,7 @@ DB_OPTIONS = \ default=dict(), cf=('database', 'options'), cook=dict_split) -DB_PORT = \ +DB_SCHEMA = \ Option('Database schema', default='', cf=('database', 'schema')) @@ -1318,6 +1318,7 @@ DATABASE_COMMON_OPTIONS = dict(web_configfile=WEB_CFILE, db_host=DB_HOST, db_port=DB_PORT, db_options=DB_OPTIONS, + db_schema=DB_SCHEMA, time_zone=DJANGO_TIME_ZONE, django_debug=DJANGO_DEBUG, web_prefix=DJANGO_WEB_PREFIX) -- cgit v1.2.3-1-g7c22 From 330f3ba3fa22b39ee1dcc6bbbbbe29a0a18778ab Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 27 Jul 2013 15:56:23 -0500 Subject: debian: Apply patches from Arto Jantunen Get rid of some stuff that's no longer necessary and doesn't exist in the downstream Debian packages. Signed-off-by: Sol Jerome --- debian/bcfg2-server.postinst | 17 ----------------- debian/bcfg2.default | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/debian/bcfg2-server.postinst b/debian/bcfg2-server.postinst index 2f65fe847..77dea5f22 100644 --- a/debian/bcfg2-server.postinst +++ b/debian/bcfg2-server.postinst @@ -40,21 +40,4 @@ esac #DEBHELPER# -# We do a restart manually here because with autogenerated code -# we get this traceback (eg something isn't done yet): -# This happens due to debhelper bug #546293, fixed in version 7.4.2. -## Setting up bcfg2-server (1.0.0~rc3+r5542-0.1+dctest8) ... -## Starting Configuration Management Server: Traceback (most recent call last): -## File "/usr/sbin/bcfg2-server", line 12, in -## import Bcfg2.Server.Plugins.Metadata -## ImportError: No module named Server.Plugins.Metadata -## * bcfg2-server -if [ -x "/etc/init.d/bcfg2-server" ]; then - if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then - invoke-rc.d bcfg2-server start || exit $? - else - /etc/init.d/bcfg2-server start || exit $? - fi -fi - exit 0 diff --git a/debian/bcfg2.default b/debian/bcfg2.default index 0164e5531..8ed0da74a 100644 --- a/debian/bcfg2.default +++ b/debian/bcfg2.default @@ -20,7 +20,7 @@ #BCFG2_INIT=1 # BCFG2_AGENT: -# Bcfg2 no longer supports agent mode please use the Agent+SSH method +# Bcfg2 no longer supports agent mode, please see NEWS.Debian # BCFG2_CRON: # Set the frequency of cron runs. -- cgit v1.2.3-1-g7c22 From 3435963a7c715bd3e6e912c6224fc8b893b1abe4 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 27 Jul 2013 16:51:38 -0500 Subject: man: Add bcfg2-report-collector manpage Signed-off-by: Sol Jerome --- doc/conf.py | 2 + doc/man/bcfg2-report-collector.txt | 40 +++++++++++++++++++ doc/man/bcfg2-server.txt | 3 +- man/bcfg2-report-collector.8 | 79 ++++++++++++++++++++++++++++++++++++++ man/bcfg2-server.8 | 5 +-- 5 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 doc/man/bcfg2-report-collector.txt create mode 100644 man/bcfg2-report-collector.8 diff --git a/doc/conf.py b/doc/conf.py index 9862603d7..d1bb029d2 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -254,6 +254,8 @@ man_pages = [ [], 8), ('man/bcfg2-lint.conf', 'bcfg2-lint.conf', 'Configuration parameters for bcfg2-lint', [], 5), + ('man/bcfg2-report-collector', 'bcfg2-report-collector', + 'Reports collection daemon', [], 8), ('man/bcfg2-reports', 'bcfg2-reports', 'Query reporting system for client status', [], 8), ('man/bcfg2-server', 'bcfg2-server', diff --git a/doc/man/bcfg2-report-collector.txt b/doc/man/bcfg2-report-collector.txt new file mode 100644 index 000000000..07c618537 --- /dev/null +++ b/doc/man/bcfg2-report-collector.txt @@ -0,0 +1,40 @@ +.. -*- mode: rst -*- +.. vim: ft=rst + + +bcfg2-report-collector +====================== + +.. program:: bcfg2-report-collector + +Synopsis +-------- + +**bcfg2-report-collector** [*options*] + +Description +----------- + +:program:`bcfg2-report-collector` runs a daemon to collect logs from the +LocalFilesystem :ref:`Bcfg2 Reports ` transport object +and add them to the Reporting storage backend. + +Options +------- + +-C configfile Specify alternate bcfg2.conf location. +-D pidfile Daemonize, placing the program pid in *pidfile*. +-E encoding Specify the encoding of config files. +-Q path Specify the path to the server repository. +-W configfile Specify the path to the web interface + configuration file. +-d Enable debugging output. +-h Print usage information. +-o path Set path of file log +-v Run in verbose mode. +--version Print the version and exit + +See Also +-------- + +:manpage:`bcfg2-server(8)`, :manpage:`bcfg2-reports(8)` diff --git a/doc/man/bcfg2-server.txt b/doc/man/bcfg2-server.txt index d5945cad6..3f8f3ea21 100644 --- a/doc/man/bcfg2-server.txt +++ b/doc/man/bcfg2-server.txt @@ -23,8 +23,7 @@ Options ------- -C configfile Specify alternate bcfg2.conf location. --D pidfile Daemonize, placing the program pid in the specified - pidfile. +-D pidfile Daemonize, placing the program pid in *pidfile*. -E encoding Specify the encoding of config files. -Q path Specify the path to the server repository. -S server Manually specify the server location (as opposed to diff --git a/man/bcfg2-report-collector.8 b/man/bcfg2-report-collector.8 new file mode 100644 index 000000000..195b15ec8 --- /dev/null +++ b/man/bcfg2-report-collector.8 @@ -0,0 +1,79 @@ +.TH "BCFG2-REPORT-COLLECTOR" "8" "July 27, 2013" "1.3" "Bcfg2" +.SH NAME +bcfg2-report-collector \- Reports collection daemon +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.\" Man page generated from reStructuredText. +. +.SH SYNOPSIS +.sp +\fBbcfg2\-report\-collector\fP [\fIoptions\fP] +.SH DESCRIPTION +.sp +\fBbcfg2\-report\-collector\fP runs a daemon to collect logs from the +LocalFilesystem \fIBcfg2 Reports\fP transport object +and add them to the Reporting storage backend. +.SH OPTIONS +.INDENT 0.0 +.TP +.BI \-C \ configfile +Specify alternate bcfg2.conf location. +.TP +.BI \-D \ pidfile +Daemonize, placing the program pid in \fIpidfile\fP. +.TP +.BI \-E \ encoding +Specify the encoding of config files. +.TP +.BI \-Q \ path +Specify the path to the server repository. +.TP +.BI \-W \ configfile +Specify the path to the web interface +configuration file. +.TP +.B \-d +Enable debugging output. +.TP +.B \-h +Print usage information. +.TP +.BI \-o \ path +Set path of file log +.TP +.B \-v +Run in verbose mode. +.TP +.B \-\-version +Print the version and exit +.UNINDENT +.SH SEE ALSO +.sp +\fIbcfg2\-server(8)\fP, \fIbcfg2\-reports(8)\fP +.\" Generated by docutils manpage writer. +. diff --git a/man/bcfg2-server.8 b/man/bcfg2-server.8 index 27f6a7b01..dcec03252 100644 --- a/man/bcfg2-server.8 +++ b/man/bcfg2-server.8 @@ -1,4 +1,4 @@ -.TH "BCFG2-SERVER" "8" "March 18, 2013" "1.3" "Bcfg2" +.TH "BCFG2-SERVER" "8" "July 27, 2013" "1.3" "Bcfg2" .SH NAME bcfg2-server \- Server for client configuration specifications . @@ -46,8 +46,7 @@ configurations to clients based on the data in its repository. Specify alternate bcfg2.conf location. .TP .BI \-D \ pidfile -Daemonize, placing the program pid in the specified -pidfile. +Daemonize, placing the program pid in \fIpidfile\fP. .TP .BI \-E \ encoding Specify the encoding of config files. -- cgit v1.2.3-1-g7c22