diff options
73 files changed, 815 insertions, 379 deletions
@@ -159,3 +159,5 @@ add themselves to this file. See LICENSE for the full license. - Michael Fenn <fennm@deshawresearch.com> fixed various small bugs related to bcfg2 on CentOS 5 + +- Alexander Sulfrian <alexander@sulfrian.net> fixed various bugs. diff --git a/debian/bcfg2-report-collector.init b/debian/bcfg2-server.bcfg2-report-collector.init index df7b751cb..df7b751cb 100755 --- a/debian/bcfg2-report-collector.init +++ b/debian/bcfg2-server.bcfg2-report-collector.init diff --git a/debian/control b/debian/control index 20cef93c8..7b27b27ed 100644 --- a/debian/control +++ b/debian/control @@ -11,9 +11,11 @@ Build-Depends: debhelper (>= 7.0.50~), python-daemon, python-cherrypy, python-gamin, + python-genshi, python-pyinotify, python-m2crypto, python-doc, + python-mock, python-mock-doc Build-Depends-Indep: python-support (>= 0.5.3) Standards-Version: 3.8.0.0 diff --git a/debian/rules b/debian/rules index 5694e4e37..eaf80a4d7 100755 --- a/debian/rules +++ b/debian/rules @@ -1,13 +1,20 @@ #!/usr/bin/make -f +# Lucid does not have dh_python2, but we would like to be able to use +# this rules file to build on lucid as well. +WITH_PYTHON2 = $(shell test -f /usr/bin/dh_python2 && echo "--with python2") +WITH_SPHINXDOC = $(shell test -f /usr/bin/dh_sphinxdoc && echo "--with sphinxdoc") + %: - dh $@ --with python-support,sphinxdoc + dh $@ ${WITH_PYTHON2} ${WITH_SPHINXDOC} override_dh_installinit: # Install bcfg2 initscript without starting it on postinst dh_installinit --package=bcfg2 --no-start # Install bcfg2-server initscript without starting it on postinst dh_installinit --package=bcfg2-server --no-start + # Install bcfg2-report-collector initscript without starting it on postinst + dh_installinit --package=bcfg2-server --name=bcfg2-report-collector --no-start override_dh_auto_build: dh_auto_build diff --git a/doc/reports/dynamic.txt b/doc/reports/dynamic.txt index 9de3f868f..6b8a1f467 100644 --- a/doc/reports/dynamic.txt +++ b/doc/reports/dynamic.txt @@ -39,7 +39,7 @@ Prerequisites * sqlite3 * pysqlite2 (if using python 2.4) -* `Django <http://www.djangoproject.com>`_ >= 1.2 +* `Django <http://www.djangoproject.com>`_ >= 1.3 * mod-wsgi .. warning:: diff --git a/doc/server/database.txt b/doc/server/database.txt index 87d3e3afe..b0ec7b571 100644 --- a/doc/server/database.txt +++ b/doc/server/database.txt @@ -34,9 +34,10 @@ of ``/etc/bcfg2.conf``. +-------------+------------------------------------------------------------+-------------------------------+ | Option name | Description | Default | +=============+============================================================+===============================+ -| engine | The full name of the Django database backend to use. See | "django.db.backends.sqlite3" | +| engine | The name of the Django database backend to use. See | "sqlite3" | | | https://docs.djangoproject.com/en/dev/ref/settings/#engine | | -| | for available options | | +| | for available options (note that django.db.backends is not | | +| | included in the engine name) | | +-------------+------------------------------------------------------------+-------------------------------+ | name | The name of the database | "/var/lib/bcfg2/bcfg2.sqlite" | +-------------+------------------------------------------------------------+-------------------------------+ diff --git a/misc/bcfg2.spec b/misc/bcfg2.spec index 7655752dd..aef61f816 100644 --- a/misc/bcfg2.spec +++ b/misc/bcfg2.spec @@ -99,8 +99,6 @@ Requires: bcfg2 = %{version} Requires: python-ssl %endif Requires: python-lxml >= 1.2.1 -%if "%{_vendor}" == "redhat" -%endif %if 0%{?suse_version} Requires: python-pyinotify Requires: python-python-daemon @@ -151,7 +149,7 @@ Group: System Tools Requires: bcfg2 = %{version} Requires: bcfg2-server = %{version} -# cherrypy 3.2.3 actually doesn't exist yet, but 3.2.2 has bugs that +# cherrypy 3.3 actually doesn't exist yet, but 3.2 has bugs that # prevent it from working: # https://bitbucket.org/cherrypy/cherrypy/issue/1154/assertionerror-in-recv-when-ssl-is-enabled Requires: python-cherrypy > 3.3 @@ -297,6 +295,8 @@ This package includes the Bcfg2 reports web frontend. %{?pythonpath: export PYTHONPATH="%{pythonpath}"} %{__python}%{pythonversion} setup.py build_sphinx +sed -i "s/apache2/httpd/g" misc/apache/bcfg2.conf + %install rm -rf %{buildroot} %{__python}%{pythonversion} setup.py install --root=%{buildroot} --record=INSTALLED_FILES --prefix=/usr @@ -316,7 +316,7 @@ mkdir -p %{buildroot}%{_defaultdocdir}/bcfg2-server-%{version} %{__mv} %{buildroot}%{_bindir}/bcfg2* %{buildroot}%{_sbindir} %{__install} -m 755 debian/bcfg2.init %{buildroot}%{_initrddir}/bcfg2 %{__install} -m 755 debian/bcfg2-server.init %{buildroot}%{_initrddir}/bcfg2-server -%{__install} -m 755 debian/bcfg2-report-collector.init %{buildroot}%{_initrddir}/bcfg2-report-collector +%{__install} -m 755 debian/bcfg2-server.bcfg2-report-collector.init %{buildroot}%{_initrddir}/bcfg2-report-collector %{__install} -m 755 debian/bcfg2.default %{buildroot}%{_sysconfdir}/default/bcfg2 %{__install} -m 755 debian/bcfg2-server.default %{buildroot}%{_sysconfdir}/default/bcfg2-server %{__install} -m 755 debian/bcfg2.cron.daily %{buildroot}%{_sysconfdir}/cron.daily/bcfg2 @@ -333,7 +333,6 @@ cp -r tools/* %{buildroot}%{_defaultdocdir}/bcfg2-server-%{version} cp -r build/sphinx/html/* %{buildroot}%{_defaultdocdir}/bcfg2-doc-%{version} %{__install} -d %{buildroot}%{apache_conf}/conf.d -sed -i "s/apache2/httpd/g" misc/apache/bcfg2.conf %{__install} -m 644 misc/apache/bcfg2.conf %{buildroot}%{apache_conf}/conf.d/wsgi_bcfg2.conf %{__mkdir_p} %{buildroot}%{_localstatedir}/cache/%{name} diff --git a/schemas/types.xsd b/schemas/types.xsd index 81b1e3927..dbb0c0390 100644 --- a/schemas/types.xsd +++ b/schemas/types.xsd @@ -119,14 +119,14 @@ </xsd:documentation> </xsd:annotation> - <xsd:attribute type='ActionTimingEnum' name='timing' use='required'> + <xsd:attribute type='ActionTimingEnum' name='timing'> <xsd:annotation> <xsd:documentation> When the action is run. </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type='ActionWhenEnum' name='when' use='required'> + <xsd:attribute type='ActionWhenEnum' name='when'> <xsd:annotation> <xsd:documentation> If the action is always run, or is only run when a bundle @@ -136,7 +136,7 @@ </xsd:documentation> </xsd:annotation> </xsd:attribute> - <xsd:attribute type='ActionStatusEnum' name='status' use='required'> + <xsd:attribute type='ActionStatusEnum' name='status'> <xsd:annotation> <xsd:documentation> Whether or not to check the return code of the action. If @@ -162,8 +162,16 @@ <xsd:attribute type='xsd:string' name='command' use='required'> <xsd:annotation> <xsd:documentation> - The command to run. The command is executed within a shell, - so flow control and other shell-specific things can be used. + The command to run. + </xsd:documentation> + </xsd:annotation> + </xsd:attribute> + <xsd:attribute type='xsd:boolean' name='shell'> + <xsd:annotation> + <xsd:documentation> + Whether the command string should be executeed within a shell. + If enabled flow control and other shell-specific things can + be used. </xsd:documentation> </xsd:annotation> </xsd:attribute> diff --git a/solaris-ips/MANIFEST.bcfg2-server.header b/solaris-ips/MANIFEST.bcfg2-server.header new file mode 100644 index 000000000..efa11181f --- /dev/null +++ b/solaris-ips/MANIFEST.bcfg2-server.header @@ -0,0 +1,5 @@ +license ../../LICENSE license=simplified_bsd +set name=description value="Configuration management server" +set name=pkg.summary value="Configuration management server" +set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@1.3.1" + diff --git a/solaris-ips/MANIFEST.bcfg2.header b/solaris-ips/MANIFEST.bcfg2.header new file mode 100644 index 000000000..8358aafca --- /dev/null +++ b/solaris-ips/MANIFEST.bcfg2.header @@ -0,0 +1,6 @@ +license ../../LICENSE license=simplified_bsd +set name=description value="Configuration management client" +set name=pkg.summary value="Configuration management client" +set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.3.1" + +file usr/bin/bcfg2 group=bin mode=0755 owner=root path=usr/bin/bcfg2 diff --git a/solaris-ips/Makefile b/solaris-ips/Makefile new file mode 100644 index 000000000..343150dc5 --- /dev/null +++ b/solaris-ips/Makefile @@ -0,0 +1,20 @@ +#!/usr/bin/gmake + +VERS=1.2.4-1 +PYVERSION := $(shell python -c "import sys; print sys.version[0:3]") + +default: clean package + +package: + -mkdir tmp tmp/bcfg2-server tmp/bcfg2 + -mkdir -p build/lib/$(PYVERSION)/site-packages + -cd ../ && PYTHONPATH=$(PYTHONPATH):$(PWD)/build/lib/python2.6/site-packages/ python setup.py install --single-version-externally-managed --record=/dev/null --prefix=$(PWD)/build/usr + #setuptools appears to use a restictive umask + -chmod -R o+r build/ + -chmod +x build/usr/bin/bcfg2 + -sh ./gen-manifests.sh + +clean: + -rm -rf tmp build + -rm -rf MANIFEST.bcfg2 + -rm -rf MANIFEST.bcfg2-server diff --git a/solaris-ips/README b/solaris-ips/README new file mode 100644 index 000000000..24021b992 --- /dev/null +++ b/solaris-ips/README @@ -0,0 +1,22 @@ +BUILDING +-------- + +Dependancies: + gmake + +Usage: + gmake + + +PUBLISHING +---------- + +Modify MANIFEST.bcfg2 and MANIFEST.bcfg2-server to set your publisher name in the fmri, e.g. Change + set name=pkg.fmri value="pkg://bcfg2/bcfg2@1.2.4" +to + set name=pkg.fmri value="pkg://example.com/bcfg2@1.2.4" + + +Then run the pkgsend publish, i.e. + pkgsend publish -s http://example.com/path/to/repo -d build MANIFEST.bcfg2 + pkgsend publish -s http://example.com/path/to/repo -d build MANIFEST.bcfg2-server diff --git a/solaris-ips/gen-manifests.sh b/solaris-ips/gen-manifests.sh new file mode 100644 index 000000000..3b4cd30df --- /dev/null +++ b/solaris-ips/gen-manifests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/sh + +#bcfg2 +cat MANIFEST.bcfg2.header > MANIFEST.bcfg2 +pkgsend generate build | grep man[15] >> MANIFEST.bcfg2 +pkgsend generate build | grep Bcfg2/[^/]*.py$ >> MANIFEST.bcfg2 +pkgsend generate build | grep Bcfg2/Client/.*.py$ >> MANIFEST.bcfg2 + +#bcfg2-server +cat MANIFEST.bcfg2-server.header > MANIFEST.bcfg2-server +pkgsend generate build | grep man[8] >> MANIFEST.bcfg2-server +pkgsend generate build | grep share/bcfg2 >> MANIFEST.bcfg2-server +pkgsend generate build | grep bin/bcfg2- >> MANIFEST.bcfg2-server +pkgsend generate build | grep Bcfg2/Server/.*.py$ >> MANIFEST.bcfg2-server + diff --git a/solaris-ips/pkginfo.bcfg2 b/solaris-ips/pkginfo.bcfg2 new file mode 100644 index 000000000..90c628c53 --- /dev/null +++ b/solaris-ips/pkginfo.bcfg2 @@ -0,0 +1,10 @@ +PKG="SCbcfg2" +NAME="bcfg2" +ARCH="sparc" +VERSION="1.2.4" +CATEGORY="application" +VENDOR="Argonne National Labratory" +EMAIL="bcfg-dev@mcs.anl.gov" +PSTAMP="Bcfg2 Developers" +BASEDIR="/opt/csw" +CLASSES="none" diff --git a/solaris-ips/pkginfo.bcfg2-server b/solaris-ips/pkginfo.bcfg2-server new file mode 100644 index 000000000..0e865522c --- /dev/null +++ b/solaris-ips/pkginfo.bcfg2-server @@ -0,0 +1,10 @@ +PKG="SCbcfg2-server" +NAME="bcfg2-server" +ARCH="sparc" +VERSION="1.2.4" +CATEGORY="application" +VENDOR="Argonne National Labratory" +EMAIL="bcfg-dev@mcs.anl.gov" +PSTAMP="Bcfg2 Developers" +BASEDIR="/usr" +CLASSES="none" diff --git a/src/lib/Bcfg2/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index b24b46dbc..a668a0870 100644 --- a/src/lib/Bcfg2/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -111,8 +111,8 @@ class Frame(object): self.logger.warning(deprecated) experimental = [tool.name for tool in self.tools if tool.experimental] if experimental: - self.logger.warning("Loaded experimental tool drivers:") - self.logger.warning(experimental) + self.logger.info("Loaded experimental tool drivers:") + self.logger.info(experimental) # find entries not handled by any tools self.unhandled = [entry for struct in config diff --git a/src/lib/Bcfg2/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index fd2c467d7..05e35befc 100644 --- a/src/lib/Bcfg2/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -31,10 +31,17 @@ class Action(Bcfg2.Client.Tools.Tool): def RunAction(self, entry): """This method handles command execution and status return.""" + shell = False + shell_string = '' + if entry.get('shell', 'false') == 'true': + shell = True + shell_string = '(in shell) ' + if not self.setup['dryrun']: if self.setup['interactive']: - prompt = ('Run Action %s, %s: (y/N): ' % - (entry.get('name'), entry.get('command'))) + prompt = ('Run Action %s%s, %s: (y/N): ' % + (shell_string, entry.get('name'), + entry.get('command'))) # flush input buffer while len(select.select([sys.stdin.fileno()], [], [], 0.0)[0]) > 0: @@ -47,8 +54,9 @@ class Action(Bcfg2.Client.Tools.Tool): self.logger.debug("Action: Deferring execution of %s due " "to build mode" % entry.get('command')) return False - self.logger.debug("Running Action %s" % (entry.get('name'))) - rv = self.cmd.run(entry.get('command')) + self.logger.debug("Running Action %s %s" % + (shell_string, entry.get('name'))) + rv = self.cmd.run(entry.get('command'), shell=shell) self.logger.debug("Action: %s got return code %s" % (entry.get('command'), rv.retval)) entry.set('rc', str(rv.retval)) diff --git a/src/lib/Bcfg2/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index 0f5f32302..156f76159 100644 --- a/src/lib/Bcfg2/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -57,27 +57,29 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): return True current_bootstatus = self.verify_bootstatus(entry, bootstatus) - svcstatus = self.check_service(entry) - if entry.get('status') == 'on': - if svcstatus: - current_srvstatus = True - else: - current_srvstatus = False - elif entry.get('status') == 'off': - if svcstatus: - current_srvstatus = False - else: - current_srvstatus = True - else: + if entry.get('status') == 'ignore': # 'ignore' should verify - current_srvstatus = True + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True if svcstatus: entry.set('current_status', 'on') else: entry.set('current_status', 'off') - return current_bootstatus and current_srvstatus + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" diff --git a/src/lib/Bcfg2/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 116d4f8b0..761c51db7 100644 --- a/src/lib/Bcfg2/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -18,6 +18,9 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): svcre = \ re.compile(r'/etc/.*/(?P<action>[SK])(?P<sequence>\d+)(?P<name>\S+)') + def get_svc_command(self, service, action): + return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) + def verify_bootstatus(self, entry, bootstatus): """Verify bootstatus for entry.""" rawfiles = glob.glob("/etc/rc*.d/[SK]*%s" % (entry.get('name'))) @@ -78,27 +81,29 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): return True current_bootstatus = self.verify_bootstatus(entry, bootstatus) - svcstatus = self.check_service(entry) - if entry.get('status') == 'on': - if svcstatus: - current_srvstatus = True - else: - current_srvstatus = False - elif entry.get('status') == 'off': - if svcstatus: - current_srvstatus = False - else: - current_srvstatus = True - else: + if entry.get('status') == 'ignore': # 'ignore' should verify - current_srvstatus = True + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True if svcstatus: entry.set('current_status', 'on') else: entry.set('current_status', 'off') - return current_bootstatus and current_srvstatus + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" @@ -165,6 +170,3 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): # Extra service removal is nonsensical # Extra services need to be reflected in the config return - - def get_svc_command(self, service, action): - return '/usr/sbin/invoke-rc.d %s %s' % (service.get('name'), action) diff --git a/src/lib/Bcfg2/Client/Tools/POSIX/File.py b/src/lib/Bcfg2/Client/Tools/POSIX/File.py index 168c35c98..9f47fb53a 100644 --- a/src/lib/Bcfg2/Client/Tools/POSIX/File.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX/File.py @@ -146,8 +146,8 @@ class POSIXFile(POSIXTool): return POSIXTool.install(self, entry) and rv - def _get_diffs(self, entry, interactive=False, sensitive=False, - is_binary=False, content=None): + def _get_diffs(self, entry, interactive=False, # pylint: disable=R0912 + sensitive=False, is_binary=False, content=None): """ generate the necessary diffs for entry """ if not interactive and sensitive: return @@ -163,6 +163,8 @@ class POSIXFile(POSIXTool): # prompts for -I and the reports try: content = open(entry.get('name')).read() + except UnicodeDecodeError: + content = open(entry.get('name'), encoding='utf-8').read() except IOError: self.logger.error("POSIX: Failed to read %s: %s" % (entry.get("name"), sys.exc_info()[1])) diff --git a/src/lib/Bcfg2/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index d6329256e..8e9626521 100644 --- a/src/lib/Bcfg2/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -62,27 +62,29 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): entry.get('name')) return False - svcstatus = self.check_service(entry) - if entry.get('status') == 'on': - if svcstatus: - current_srvstatus = True - else: - current_srvstatus = False - elif entry.get('status') == 'off': - if svcstatus: - current_srvstatus = False - else: - current_srvstatus = True - else: + if entry.get('status') == 'ignore': # 'ignore' should verify - current_srvstatus = True + current_svcstatus = True + svcstatus = True + else: + svcstatus = self.check_service(entry) + if entry.get('status') == 'on': + if svcstatus: + current_svcstatus = True + else: + current_svcstatus = False + elif entry.get('status') == 'off': + if svcstatus: + current_svcstatus = False + else: + current_svcstatus = True if svcstatus: entry.set('current_status', 'on') else: entry.set('current_status', 'off') - return current_bootstatus and current_srvstatus + return current_bootstatus and current_svcstatus def InstallService(self, entry): """Install Service entry.""" diff --git a/src/lib/Bcfg2/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index 1ab867215..aca5dbbc7 100644 --- a/src/lib/Bcfg2/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py @@ -1,14 +1,15 @@ """VCS support.""" # TODO: -# * git_write_index # * add svn support # * integrate properly with reports missing = [] +import errno import os import shutil import sys +import stat # python-dulwich git imports try: @@ -26,6 +27,38 @@ except ImportError: import Bcfg2.Client.Tools +def cleanup_mode(mode): + """Cleanup a mode value. + + This will return a mode that can be stored in a tree object. + + :param mode: Mode to clean up. + """ + if stat.S_ISLNK(mode): + return stat.S_IFLNK + elif stat.S_ISDIR(mode): + return stat.S_IFDIR + elif dulwich.index.S_ISGITLINK(mode): + return dulwich.index.S_IFGITLINK + ret = stat.S_IFREG | int('644', 8) + ret |= (mode & int('111', 8)) + return ret + + +def index_entry_from_stat(stat_val, hex_sha, flags, mode=None): + """Create a new index entry from a stat value. + + :param stat_val: POSIX stat_result instance + :param hex_sha: Hex sha of the object + :param flags: Index flags + """ + if mode is None: + mode = cleanup_mode(stat_val.st_mode) + return (stat_val.st_ctime, stat_val.st_mtime, stat_val.st_dev, + stat_val.st_ino, mode, stat_val.st_uid, + stat_val.st_gid, stat_val.st_size, hex_sha, flags) + + class VCS(Bcfg2.Client.Tools.Tool): """VCS support.""" __handles__ = [('Path', 'vcs')] @@ -47,11 +80,24 @@ class VCS(Bcfg2.Client.Tools.Tool): self.logger.info("Repository %s does not exist" % entry.get('name')) return False - cur_rev = repo.head() - if cur_rev != entry.get('revision'): + try: + expected_rev = entry.get('revision') + cur_rev = repo.head() + except: + return False + + try: + client, path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) + remote_refs = client.fetch_pack(path, (lambda x: None), None, None, None) + if expected_rev in remote_refs: + expected_rev = remote_refs[expected_rev] + except: + pass + + if cur_rev != expected_rev: self.logger.info("At revision %s need to go to revision %s" % - (cur_rev, entry.get('revision'))) + (cur_rev.strip(), expected_rev.strip())) return False return True @@ -71,45 +117,64 @@ class VCS(Bcfg2.Client.Tools.Tool): destname) return False - destr = dulwich.repo.Repo.init(destname, mkdir=True) + dulwich.file.ensure_dir_exists(destname) + destr = dulwich.repo.Repo.init(destname) cl, host_path = dulwich.client.get_transport_and_path(entry.get('sourceurl')) remote_refs = cl.fetch(host_path, destr, determine_wants=destr.object_store.determine_wants_all, progress=sys.stdout.write) - destr.refs['refs/heads/master'] = entry.get('revision') - dtree = destr[entry.get('revision')].tree - obj_store = destr.object_store - for fname, mode, sha in obj_store.iter_tree_contents(dtree): - fullpath = os.path.join(destname, fname) - try: - f = open(os.path.join(destname, fname), 'wb') - except IOError: - dir = os.path.split(fullpath)[0] - os.makedirs(dir) - f = open(os.path.join(destname, fname), 'wb') - f.write(destr[sha].data) - f.close() - os.chmod(os.path.join(destname, fname), mode) + + if entry.get('revision') in remote_refs: + destr.refs['HEAD'] = remote_refs[entry.get('revision')] + else: + destr.refs['HEAD'] = entry.get('revision') + + dtree = destr['HEAD'].tree + index = dulwich.index.Index(destr.index_path()) + for fname, mode, sha in destr.object_store.iter_tree_contents(dtree): + full_path = os.path.join(destname, fname) + dulwich.file.ensure_dir_exists(os.path.dirname(full_path)) + + if stat.S_ISLNK(mode): + src_path = destr[sha].as_raw_string() + try: + os.symlink(src_path, full_path) + except OSError: + e = sys.exc_info()[1] + if e.errno == errno.EEXIST: + os.unlink(full_path) + os.symlink(src_path, full_path) + else: + raise + else: + file = open(full_path, 'wb') + file.write(destr[sha].as_raw_string()) + file.close() + os.chmod(full_path, mode) + + st = os.lstat(full_path) + index[fname] = index_entry_from_stat(st, sha, 0) + + index.write() return True - # FIXME: figure out how to write the git index properly - #iname = "%s/.git/index" % entry.get('name') - #f = open(iname, 'w+') - #entries = obj_store[sha].iteritems() - #try: - # dulwich.index.write_index(f, entries) - #finally: - # f.close() def Verifysvn(self, entry, _): """Verify svn repositories""" + headrev = pysvn.Revision( pysvn.opt_revision_kind.head ) client = pysvn.Client() try: cur_rev = str(client.info(entry.get('name')).revision.number) + server = client.info2(entry.get('sourceurl'), headrev, recurse=False) + if server: + server_rev = str(server[0][1].rev.number) except: self.logger.info("Repository %s does not exist" % entry.get('name')) return False + if entry.get('revision') == 'latest' and cur_rev == server_rev: + return True + if cur_rev != entry.get('revision'): self.logger.info("At revision %s need to go to revision %s" % (cur_rev, entry.get('revision'))) diff --git a/src/lib/Bcfg2/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index 3bc261f2f..25603186e 100644 --- a/src/lib/Bcfg2/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -22,8 +22,5 @@ def prompt(msg): ans = input(msg) return ans in ['y', 'Y'] except EOFError: - # python 2.4.3 on CentOS doesn't like ^C for some reason - return False - except: - print("Error while reading input: %s" % sys.exc_info()[1]) - return False + # handle ^C on rhel-based platforms + raise SystemExit(1) diff --git a/src/lib/Bcfg2/Compat.py b/src/lib/Bcfg2/Compat.py index d034c0777..049236e03 100644 --- a/src/lib/Bcfg2/Compat.py +++ b/src/lib/Bcfg2/Compat.py @@ -79,10 +79,7 @@ except NameError: def u_str(string, encoding=None): """ print to file compatibility """ if sys.hexversion >= 0x03000000: - if encoding is not None: - return string.encode(encoding) - else: - return string + return string else: if encoding is not None: return unicode(string, encoding) diff --git a/src/lib/Bcfg2/Reporting/Collector.py b/src/lib/Bcfg2/Reporting/Collector.py index df82248d0..3d224432e 100644 --- a/src/lib/Bcfg2/Reporting/Collector.py +++ b/src/lib/Bcfg2/Reporting/Collector.py @@ -125,7 +125,9 @@ class ReportingCollector(object): # this wil be missing if called from bcfg2-admin self.terminate.set() if self.transport: - self.transport.shutdown() + try: + self.transport.shutdown() + except OSError: + pass if self.storage: self.storage.shutdown() - diff --git a/src/lib/Bcfg2/Reporting/templates/base.html b/src/lib/Bcfg2/Reporting/templates/base.html index c73339911..7f1fcba3b 100644 --- a/src/lib/Bcfg2/Reporting/templates/base.html +++ b/src/lib/Bcfg2/Reporting/templates/base.html @@ -1,4 +1,8 @@ {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} <?xml version="1.0"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> @@ -25,8 +29,9 @@ <div id="header"> <a href="http://bcfg2.org"><img src='{% to_media_url bcfg2_logo.png %}' - height='115' width='300' alt='Bcfg2' style='float:left; height: 115px' /></a> - </div> + height='115' width='300' alt='Bcfg2' + style='float:left; height: 115px' /></a> + </div> <div id="document"> <div id="content"><div id="contentwrapper"> @@ -46,26 +51,26 @@ <li>Overview</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_summary %}">Summary</a></li> - <li><a href="{% url reports_history %}">Recent Interactions</a></li> - <li><a href="{% url reports_timing %}">Timing</a></li> + <li><a href="{% url "reports_summary" %}">Summary</a></li> + <li><a href="{% url "reports_history" %}">Recent Interactions</a></li> + <li><a href="{% url "reports_timing" %}">Timing</a></li> </ul> <ul class='menu-level1'> <li>Clients</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_grid_view %}">Grid View</a></li> - <li><a href="{% url reports_detailed_list %}">Detailed List</a></li> - <li><a href="{% url reports_client_manage %}">Manage</a></li> + <li><a href="{% url "reports_grid_view" %}">Grid View</a></li> + <li><a href="{% url "reports_detailed_list" %}">Detailed List</a></li> + <li><a href="{% url "reports_client_manage" %}">Manage</a></li> </ul> <ul class='menu-level1'> <li>Entries Configured</li> </ul> <ul class='menu-level2'> - <li><a href="{% url reports_common_problems %}">Common problems</a></li> - <li><a href="{% url reports_item_list "bad" %}">Bad</a></li> - <li><a href="{% url reports_item_list "modified" %}">Modified</a></li> - <li><a href="{% url reports_item_list "extra" %}">Extra</a></li> + <li><a href="{% url "reports_common_problems" %}">Common problems</a></li> + <li><a href="{% url "reports_item_list" "bad" %}">Bad</a></li> + <li><a href="{% url "reports_item_list" "modified" %}">Modified</a></li> + <li><a href="{% url "reports_item_list" "extra" %}">Extra</a></li> </ul> {% comment %} TODO diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detail.html b/src/lib/Bcfg2/Reporting/templates/clients/detail.html index 4608ce6f1..e890589a7 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detail.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detail.html @@ -1,24 +1,28 @@ {% extends "base.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Client {{client.name}}{% endblock %} {% block extra_header_info %} <style type="text/css"> .node_data { - border: 1px solid #98DBCC; - margin: 10px; - padding-left: 18px; + border: 1px solid #98DBCC; + margin: 10px; + padding-left: 18px; } .node_data td { - padding: 1px 20px 1px 2px; + padding: 1px 20px 1px 2px; } span.history_links { - font-size: 90%; - margin-left: 50px; + font-size: 90%; + margin-left: 50px; } span.history_links a { - font-size: 90%; + font-size: 90%; } </style> {% endblock %} @@ -30,12 +34,12 @@ span.history_links a { {% block content %} <div class='detail_header'> <h2>{{client.name}}</h2> - <a href='{% url reports_client_manage %}#{{ client.name }}'>[manage]</a> - <span class='history_links'><a href="{% url reports_client_history client.name %}">View History</a> | Jump to + <a href='{% url "reports_client_manage" %}#{{ client.name }}'>[manage]</a> + <span class='history_links'><a href="{% url "reports_client_history" client.name %}">View History</a> | Jump to <select id="quick" name="quick" onchange="javascript:pageJump('quick');"> <option value="" selected="selected">--- Time ---</option> {% for i in client.interactions.all|slice:":25" %} - <option value="{% url reports_client_detail_pk hostname=client.name, pk=i.id %}">{{i.timestamp|date:"c"}}</option> + <option value="{% url "reports_client_detail_pk" hostname=client.name pk=i.id %}">{{i.timestamp|date:"c"}}</option> {% endfor %} </select></span> </div> @@ -110,7 +114,7 @@ span.history_links a { {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{entry.entry_type}}</td> - <td><a href="{% url reports_item entry.class_name entry.pk interaction.pk %}"> + <td><a href="{% url "reports_item" entry.class_name entry.pk interaction.pk %}"> {{entry.name}}</a></td> </tr> {% endfor %} @@ -129,7 +133,7 @@ span.history_links a { {% for failure in interaction.failures.all %} <tr class='{% cycle listview,listview_alt %}'> <td class='entry_list_type'>{{failure.entry_type}}</td> - <td><a href="{% url reports_item failure.class_name failure.pk interaction.pk %}"> + <td><a href="{% url "reports_item" failure.class_name failure.pk interaction.pk %}"> {{failure.name}}</a></td> </tr> {% endfor %} @@ -140,11 +144,11 @@ span.history_links a { {% if entry_list %} <div class="entry_list recent_history_wrapper"> <div class="entry_list_head" style="border-bottom: 2px solid #98DBCC;"> - <h4 style="display: inline"><a href="{% url reports_client_history client.name %}">Recent Interactions</a></h4> + <h4 style="display: inline"><a href="{% url "reports_client_history" client.name %}">Recent Interactions</a></h4> </div> <div class='recent_history_box'> {% include "widgets/interaction_list.inc" %} - <div style='padding-left: 5px'><a href="{% url reports_client_history client.name %}">more...</a></div> + <div style='padding-left: 5px'><a href="{% url "reports_client_history" client.name %}">more...</a></div> </div> </div> {% endif %} diff --git a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html index fd9a545ce..33c78a5f0 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/detailed-list.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Detailed Client Listing{% endblock %} {% block pagebanner %}Clients - Detailed View{% endblock %} @@ -21,7 +25,7 @@ </tr> {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> - <td class='left_column'><a href='{% url Bcfg2.Reporting.views.client_detail hostname=entry.client.name, pk=entry.id %}'>{{ entry.client.name }}</a></td> + <td class='left_column'><a href='{% url "Bcfg2.Reporting.views.client_detail" hostname=entry.client.name pk=entry.id %}'>{{ entry.client.name }}</a></td> <td class='right_column' style='width:75px'><a href='{% add_url_filter state=entry.state %}' class='{{entry|determine_client_state}}'>{{ entry.state }}</a></td> <td class='right_column_narrow'>{{ entry.good_count }}</td> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/index.html b/src/lib/Bcfg2/Reporting/templates/clients/index.html index d9c415c20..eba83670b 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/index.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/index.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block extra_header_info %} {% endblock%} @@ -17,9 +21,9 @@ <td class='{{ inter|determine_client_state }}'> <a href="{% spaceless %} {% if not timestamp %} - {% url reports_client_detail inter.client.name %} + {% url "reports_client_detail" inter.client.name %} {% else %} - {% url reports_client_detail_pk inter.client.name,inter.id %} + {% url "reports_client_detail_pk" inter.client.name inter.id %} {% endif %} {% endspaceless %}">{{ inter.client.name }}</a> </td> diff --git a/src/lib/Bcfg2/Reporting/templates/clients/manage.html b/src/lib/Bcfg2/Reporting/templates/clients/manage.html index 443ec8ccb..03918aad7 100644 --- a/src/lib/Bcfg2/Reporting/templates/clients/manage.html +++ b/src/lib/Bcfg2/Reporting/templates/clients/manage.html @@ -1,4 +1,8 @@ {% extends "base.html" %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block extra_header_info %} {% endblock%} @@ -24,10 +28,10 @@ <td><span id="{{ client.name }}"> </span> <span id="ttag-{{ client.name }}"> </span> <span id="s-ttag-{{ client.name }}"> </span> - <a href="{% url reports_client_detail client.name %}">{{ client.name }}</a></td> + <a href='{% url "reports_client_detail" client.name %}'>{{ client.name }}</a></td> <td>{% firstof client.expiration 'Active' %}</td> <td> - <form method="post" action="{% url reports_client_manage %}"> + <form method="post" action='{% url "reports_client_manage" %}'> <div> {# here for no reason other then to validate #} <input type="hidden" name="client_name" value="{{ client.name }}" /> <input type="hidden" name="client_action" value="{% if client.expiration %}unexpire{% else %}expire{% endif %}" /> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/common.html b/src/lib/Bcfg2/Reporting/templates/config_items/common.html index 57191ec39..91f37d7dc 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/common.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/common.html @@ -1,5 +1,6 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% load url from future %} {% block title %}Bcfg2 - Common Problems{% endblock %} @@ -29,9 +30,9 @@ {% for item in type_list %} <tr class='{% cycle listview,listview_alt %}'> <td>{{ item.ENTRY_TYPE }}</td> - <td><a href="{% url reports_entry item.class_name, item.pk %}">{{ item.name }}</a></td> + <td><a href='{% url "reports_entry" item.class_name item.pk %}'>{{ item.name }}</a></td> <td>{{ item.num_entries }}</td> - <td><a href="{% url reports_item item.ENTRY_TYPE, item.pk %}">{{ item.short_list|join:"," }}</a></td> + <td><a href='{% url "reports_item" item.ENTRY_TYPE item.pk %}'>{{ item.short_list|join:"," }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html index e940889ab..e3befb0eb 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/entry_status.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Entry Status{% endblock %} @@ -17,10 +21,10 @@ {% for item, inters in items %} {% for inter in inters %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href='{% url reports_client_detail hostname=inter.client.name %}'>{{inter.client.name}}</a></td> - <td><a href='{% url reports_client_detail_pk hostname=inter.client.name, pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> + <td><a href='{% url "reports_client_detail" hostname=inter.client.name %}'>{{inter.client.name}}</a></td> + <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.pk %}'>{{inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe}}</a></td> <td>{{ item.get_state_display }}</td> - <td style='white-space: nowrap'><a href="{% url reports_item entry_type=item.class_name pk=item.pk %}">({{item.pk}}) {{item.short_list|join:","}}</a></td> + <td style='white-space: nowrap'><a href='{% url "reports_item" entry_type=item.class_name pk=item.pk %}'>({{item.pk}}) {{item.short_list|join:","}}</a></td> </tr> {% endfor %} {% endfor %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/item.html b/src/lib/Bcfg2/Reporting/templates/config_items/item.html index 259414399..b03d48045 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/item.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/item.html @@ -1,6 +1,10 @@ {% extends "base.html" %} {% load split %} {% load syntax_coloring %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Element Details{% endblock %} @@ -9,20 +13,20 @@ {% block extra_header_info %} <style type="text/css"> #table_list_header { - font-size: 100%; + font-size: 100%; } table.entry_list { - width: auto; + width: auto; } div.information_wrapper { - margin: 15px; + margin: 15px; } div.diff_wrapper { - overflow: auto; + overflow: auto; } div.entry_list h3 { - font-size: 90%; - padding: 5px; + font-size: 90%; + padding: 5px; } </style> {% endblock%} @@ -131,9 +135,9 @@ div.entry_list h3 { {% if associated_list %} <table class="entry_list" cellpadding="3"> {% for inter in associated_list %} - <tr><td><a href="{% url reports_client_detail inter.client.name %}" + <tr><td><a href='{% url "reports_client_detail" inter.client.name %}' >{{inter.client.name}}</a></td> - <td><a href="{% url reports_client_detail_pk hostname=inter.client.name,pk=inter.id %}" + <td><a href='{% url "reports_client_detail_pk" hostname=inter.client.name pk=inter.id %}' >{{inter.timestamp}}</a></td> </tr> {% endfor %} diff --git a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html index 864392754..0e4812e85 100644 --- a/src/lib/Bcfg2/Reporting/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Reporting/templates/config_items/listing.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Element Listing{% endblock %} @@ -21,9 +25,9 @@ <tr style='text-align: left' ><th>Name</th><th>Count</th><th>Reason</th></tr> {% for entry in type_data %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_entry entry.class_name entry.pk %}">{{entry.name}}</a></td> + <td><a href='{% url "reports_entry" entry.class_name entry.pk %}'>{{entry.name}}</a></td> <td>{{entry.num_entries}}</td> - <td><a href="{% url reports_item entry.class_name entry.pk %}">{{entry.short_list|join:","}}</a></td> + <td><a href='{% url "reports_item" entry.class_name entry.pk %}'>{{entry.short_list|join:","}}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/displays/summary.html b/src/lib/Bcfg2/Reporting/templates/displays/summary.html index b9847cf96..ffafd52e0 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/summary.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/summary.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Client Summary{% endblock %} {% block pagebanner %}Clients - Summary{% endblock %} @@ -30,7 +34,7 @@ hide_tables[{{ forloop.counter0 }}] = "table_{{ summary.name }}"; <table id='table_{{ summary.name }}' class='entry_list'> {% for node in summary.nodes|sort_interactions_by_name %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_client_detail_pk hostname=node.client.name,pk=node.id %}">{{ node.client.name }}</a></td> + <td><a href='{% url "reports_client_detail_pk" hostname=node.client.name pk=node.id %}'>{{ node.client.name }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Bcfg2/Reporting/templates/displays/timing.html b/src/lib/Bcfg2/Reporting/templates/displays/timing.html index ff775ded5..8ac5e49bb 100644 --- a/src/lib/Bcfg2/Reporting/templates/displays/timing.html +++ b/src/lib/Bcfg2/Reporting/templates/displays/timing.html @@ -1,5 +1,9 @@ {% extends "base-timeview.html" %} {% load bcfg2_tags %} +{% comment %} +This is needed for Django versions less than 1.5 +{% endcomment %} +{% load url from future %} {% block title %}Bcfg2 - Performance Metrics{% endblock %} {% block pagebanner %}Performance Metrics{% endblock %} @@ -12,7 +16,7 @@ <div class='client_list_box'> {% if metrics %} <table cellpadding="3"> - <tr id='table_list_header' class='listview'> + <tr id='table_list_header' class='listview'> <td>Name</td> <td>Parse</td> <td>Probe</td> @@ -21,15 +25,15 @@ <td>Config</td> <td>Total</td> </tr> - {% for metric in metrics|dictsort:"name" %} + {% for metric in metrics|dictsort:"name" %} <tr class='{% cycle listview,listview_alt %}'> <td><a style='font-size: 100%' - href="{% url reports_client_detail hostname=metric.name %}">{{ metric.name }}</a></td> + href='{% url "reports_client_detail" hostname=metric.name %}'>{{ metric.name }}</a></td> {% for mitem in metric|build_metric_list %} <td>{{ mitem }}</td> {% endfor %} - </tr> - {% endfor %} + </tr> + {% endfor %} </table> {% else %} <p>No metric data available</p> diff --git a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py index f5f2e7528..489682f30 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Reporting/templatetags/bcfg2_tags.py @@ -5,9 +5,8 @@ from django import template from django.conf import settings from django.core.urlresolvers import resolve, reverse, \ Resolver404, NoReverseMatch -from django.template.loader import get_template, \ - get_template_from_string,TemplateDoesNotExist -from django.utils.encoding import smart_unicode, smart_str +from django.template.loader import get_template_from_string +from django.utils.encoding import smart_str from django.utils.safestring import mark_safe from datetime import datetime, timedelta from Bcfg2.Reporting.utils import filter_list @@ -133,19 +132,22 @@ def filter_navigator(context): del myargs[filter] filters.append((filter, reverse(view, args=args, kwargs=myargs) + qs)) - filters.sort(lambda x, y: cmp(x[0], y[0])) + filters.sort(key=lambda x: x[0]) myargs = kwargs.copy() - selected=True + selected = True if 'group' in myargs: del myargs['group'] - selected=False - groups = [('---', reverse(view, args=args, kwargs=myargs) + qs, selected)] + selected = False + groups = [('---', + reverse(view, args=args, kwargs=myargs) + qs, + selected)] for group in Group.objects.values('name'): myargs['group'] = group['name'] - groups.append((group['name'], reverse(view, args=args, kwargs=myargs) + qs, - group['name'] == kwargs.get('group', ''))) - + groups.append((group['name'], + reverse(view, args=args, kwargs=myargs) + qs, + group['name'] == kwargs.get('group', ''))) + return {'filters': filters, 'groups': groups} except (Resolver404, NoReverseMatch, ValueError, KeyError): pass @@ -205,7 +207,7 @@ def sort_interactions_by_name(value): Sort an interaction list by client name """ inters = list(value) - inters.sort(lambda a, b: cmp(a.client.name, b.client.name)) + inters.sort(key=lambda a: a.client.name) return inters @@ -223,7 +225,7 @@ class AddUrlFilter(template.Node): filter_value = self.filter_value.resolve(context, True) if filter_value: filter_name = smart_str(self.filter_name) - filter_value = smart_unicode(filter_value) + filter_value = smart_str(filter_value) kwargs[filter_name] = filter_value # These two don't make sense if filter_name == 'server' and 'hostname' in kwargs: @@ -306,6 +308,7 @@ def to_media_url(parser, token): return MediaTag(filter_value) + @register.filter def determine_client_state(entry): """ @@ -338,10 +341,11 @@ def do_qs(parser, token): try: tag, name, value = token.split_contents() except ValueError: - raise template.TemplateSyntaxError, "%r tag requires exactly two arguments" \ - % token.contents.split()[0] + raise template.TemplateSyntaxError("%r tag requires exactly two arguments" + % token.contents.split()[0]) return QsNode(name, value) + class QsNode(template.Node): def __init__(self, name, value): self.name = template.Variable(name) @@ -359,7 +363,7 @@ class QsNode(template.Node): return '' except KeyError: if settings.TEMPLATE_DEBUG: - raise Exception, "'qs' tag requires context['request']" + raise Exception("'qs' tag requires context['request']") return '' except: return '' @@ -380,6 +384,7 @@ def sort_link(parser, token): return SortLinkNode(sort_key, text) + class SortLinkNode(template.Node): __TMPL__ = "{% load bcfg2_tags %}<a href='{% qs 'sort' key %}'>{{ text }}</a>" @@ -420,4 +425,3 @@ class SortLinkNode(template.Node): raise raise return '' - diff --git a/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py index 2712d6395..22700689f 100644 --- a/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py +++ b/src/lib/Bcfg2/Reporting/templatetags/syntax_coloring.py @@ -1,11 +1,8 @@ -import sys from django import template -from django.utils.encoding import smart_unicode +from django.utils.encoding import smart_str from django.utils.html import conditional_escape from django.utils.safestring import mark_safe -from Bcfg2.Compat import u_str - register = template.Library() # pylint: disable=E0611 @@ -33,9 +30,9 @@ def syntaxhilight(value, arg="diff", autoescape=None): if colorize: try: - output = u_str('<style type="text/css">') \ - + smart_unicode(HtmlFormatter().get_style_defs('.highlight')) \ - + u_str('</style>') + output = smart_str('<style type="text/css">') \ + + smart_str(HtmlFormatter().get_style_defs('.highlight')) \ + + smart_str('</style>') lexer = get_lexer_by_name(arg) output += highlight(value, lexer, HtmlFormatter()) @@ -43,6 +40,7 @@ def syntaxhilight(value, arg="diff", autoescape=None): except: return value else: - return mark_safe(u_str('<div class="note-box">Tip: Install pygments ' - 'for highlighting</div><pre>%s</pre>') % value) + return mark_safe(smart_str( + '<div class="note-box">Tip: Install pygments ' + 'for highlighting</div><pre>%s</pre>') % value) syntaxhilight.needs_autoescape = True diff --git a/src/lib/Bcfg2/Reporting/views.py b/src/lib/Bcfg2/Reporting/views.py index 6cba7bf8c..c7c2a503f 100644 --- a/src/lib/Bcfg2/Reporting/views.py +++ b/src/lib/Bcfg2/Reporting/views.py @@ -338,6 +338,8 @@ def client_detail(request, hostname=None, pk=None): for label in etypes.values(): edict[label] = [] for ekind in inter.entry_types: + if ekind == 'failures': + continue for ent in getattr(inter, ekind).all(): edict[etypes[ent.state]].append(ent) context['entry_types'] = edict diff --git a/src/lib/Bcfg2/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py index 570e993ed..187ccfd71 100644 --- a/src/lib/Bcfg2/Server/Admin/Client.py +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -18,19 +18,15 @@ class Client(Bcfg2.Server.Admin.MetadataCore): try: self.metadata.add_client(args[1]) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in adding client: %s" % err) - raise SystemExit(1) + self.errExit("Error in adding client: %s" % sys.exc_info()[1]) elif args[0] in ['delete', 'remove', 'del', 'rm']: try: self.metadata.remove_client(args[1]) except MetadataConsistencyError: - err = sys.exc_info()[1] - print("Error in deleting client: %s" % err) - raise SystemExit(1) + self.errExit("Error in deleting client: %s" % + sys.exc_info()[1]) elif args[0] in ['list', 'ls']: for client in self.metadata.list_clients(): print(client) else: - print("No command specified") - raise SystemExit(1) + self.errExit("No command specified") diff --git a/src/lib/Bcfg2/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index d0831362c..6bb15cafd 100644 --- a/src/lib/Bcfg2/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -144,5 +144,4 @@ class Compare(Bcfg2.Server.Admin.Mode): (old, new) = args return self.compareSpecifications(new, old) except IndexError: - print(self.__call__.__doc__) - raise SystemExit(1) + self.errExit(self.__call__.__doc__) diff --git a/src/lib/Bcfg2/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index 93e42305c..37ca74894 100644 --- a/src/lib/Bcfg2/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -20,9 +20,8 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): "Please see bcfg2-admin minestruct help for usage.") try: (opts, args) = getopt.getopt(args, 'f:g:h') - except: - self.log.error(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) client = args[0] output = sys.stdout @@ -33,8 +32,7 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): try: output = open(optarg, 'w') except IOError: - self.log.error("Failed to open file: %s" % (optarg)) - raise SystemExit(1) + self.errExit("Failed to open file: %s" % (optarg)) elif opt == '-g': groups = optarg.split(':') @@ -43,10 +41,9 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): for source in self.bcore.plugins_by_type(PullSource): for item in source.GetExtra(client): extra.add(item) - except: - self.log.error("Failed to find extra entry info for client %s" % - client) - raise SystemExit(1) + except: # pylint: disable=W0702 + self.errExit("Failed to find extra entry info for client %s" % + client) root = lxml.etree.Element("Base") self.log.info("Found %d extra entries" % (len(extra))) add_point = root diff --git a/src/lib/Bcfg2/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index e883c432f..8f84cd87d 100644 --- a/src/lib/Bcfg2/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -32,9 +32,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): use_stdin = False try: opts, gargs = getopt.getopt(args, 'vfIs') - except: - print(self.__doc__) - raise SystemExit(1) + except getopt.GetoptError: + self.errExit(self.__doc__) for opt in opts: if opt[0] == '-v': self.log = True diff --git a/src/lib/Bcfg2/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index bb5ee352b..d21d66a22 100644 --- a/src/lib/Bcfg2/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -79,8 +79,7 @@ class Reports(Bcfg2.Server.Admin.Mode): def __call__(self, args): if len(args) == 0 or args[0] == '-h': - print(self.__usage__) - raise SystemExit(0) + self.errExit(self.__usage__) # FIXME - dry run @@ -101,9 +100,7 @@ class Reports(Bcfg2.Server.Admin.Mode): management.call_command("syncdb", verbosity=vrb) management.call_command("migrate", verbosity=vrb) except: - print("Update failed: %s" % - traceback.format_exc().splitlines()[-1]) - raise SystemExit(1) + self.errExit("Update failed: %s" % sys.exc_info()[1]) elif args[0] == 'purge': expired = False client = None @@ -124,22 +121,20 @@ class Reports(Bcfg2.Server.Admin.Mode): maxdate = datetime.datetime.now() - \ datetime.timedelta(days=int(args[i + 1])) except: - self.log.error("Invalid number of days: %s" % - args[i + 1]) - raise SystemExit(-1) + self.errExit("Invalid number of days: %s" % + args[i + 1]) i = i + 1 elif args[i] == '--expired': expired = True i = i + 1 if expired: if state: - self.log.error("--state is not valid with --expired") - raise SystemExit(-1) + self.errExit("--state is not valid with --expired") self.purge_expired(maxdate) else: self.purge(client, maxdate, state) else: - print("Unknown command: %s" % args[0]) + self.errExit("Unknown command: %s" % args[0]) @transaction.commit_on_success def scrub(self): @@ -155,8 +150,7 @@ class Reports(Bcfg2.Server.Admin.Mode): (start_count - cls.objects.count(), cls.__class__.__name__)) except: print("Failed to prune %s: %s" % - (cls.__class__.__name__, - traceback.format_exc().splitlines()[-1])) + (cls.__class__.__name__, sys.exc_info()[1])) def django_command_proxy(self, command): '''Call a django command''' @@ -180,8 +174,7 @@ class Reports(Bcfg2.Server.Admin.Mode): cobj = Client.objects.get(name=client) ipurge = ipurge.filter(client=cobj) except Client.DoesNotExist: - self.log.error("Client %s not in database" % client) - raise SystemExit(-1) + self.errExit("Client %s not in database" % client) self.log.debug("Filtering by client: %s" % client) if maxdate: diff --git a/src/lib/Bcfg2/Server/Admin/Syncdb.py b/src/lib/Bcfg2/Server/Admin/Syncdb.py index 84ad93ae0..2722364f7 100644 --- a/src/lib/Bcfg2/Server/Admin/Syncdb.py +++ b/src/lib/Bcfg2/Server/Admin/Syncdb.py @@ -3,6 +3,7 @@ import Bcfg2.settings import Bcfg2.Options import Bcfg2.Server.Admin import Bcfg2.Server.models +from django.core.exceptions import ImproperlyConfigured from django.core.management import setup_environ, call_command @@ -24,10 +25,7 @@ class Syncdb(Bcfg2.Server.Admin.Mode): call_command("syncdb", interactive=False, verbosity=0) self._database_available = True except ImproperlyConfigured: - err = sys.exc_info()[1] - self.log.error("Django configuration problem: %s" % err) - raise SystemExit(1) + self.errExit("Django configuration problem: %s" % + sys.exc_info()[1]) except: - err = sys.exc_info()[1] - self.log.error("Database update failed: %s" % err) - raise SystemExit(1) + self.errExit("Database update failed: %s" % sys.exc_info()[1]) diff --git a/src/lib/Bcfg2/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index ba4777c93..2613f74ac 100644 --- a/src/lib/Bcfg2/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -1,10 +1,10 @@ """ XML-RPC Command Interface for bcfg2-admin""" import sys +import xmlrpclib import Bcfg2.Options import Bcfg2.Client.Proxy import Bcfg2.Server.Admin -from Bcfg2.Compat import xmlrpclib class Xcmd(Bcfg2.Server.Admin.Mode): @@ -32,8 +32,7 @@ class Xcmd(Bcfg2.Server.Admin.Mode): ca=setup['ca'], timeout=setup['timeout']) if len(setup['args']) == 0: - print("Usage: xcmd <xmlrpc method> <optional arguments>") - return + self.errExit("Usage: xcmd <xmlrpc method> <optional arguments>") cmd = args[0] try: data = getattr(proxy, cmd)(*args[1:]) diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 8ef9e3e96..7aa07f2a2 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -2,14 +2,14 @@ implementations inherit from. """ import os -import sys -import time +import pwd import atexit -import select -import signal import logging -import inspect +import select +import sys import threading +import time +import inspect import lxml.etree import Bcfg2.Server import Bcfg2.Logger @@ -243,14 +243,6 @@ class BaseCore(object): #: The CA that signed the server cert self.ca = self.setup['ca'] - def hdlr(sig, frame): # pylint: disable=W0613 - """ Handle SIGINT/Ctrl-C by shutting down the core and exiting - properly. """ - self.shutdown() - os._exit(1) # pylint: disable=W0212 - - signal.signal(signal.SIGINT, hdlr) - #: The FAM :class:`threading.Thread`, #: :func:`_file_monitor_thread` self.fam_thread = \ @@ -762,6 +754,11 @@ class BaseCore(object): os.chmod(piddir, 493) # 0775 if not self._daemonize(): return False + + # rewrite $HOME. pulp stores its auth creds in ~/.pulp, so + # this is necessary to make that work when privileges are + # dropped + os.environ['HOME'] = pwd.getpwuid(self.setup['daemon_uid'])[5] else: os.umask(int(self.setup['umask'], 8)) @@ -789,7 +786,8 @@ class BaseCore(object): while self.fam.pending() != 0: time.sleep(1) - self.set_debug(None, self.debug_flag) + if self.debug_flag: + self.set_debug(None, self.debug_flag) self._block() def _daemonize(self): diff --git a/src/lib/Bcfg2/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 7c3b2d9cc..f028e225e 100644 --- a/src/lib/Bcfg2/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -143,10 +143,11 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_xml(os.path.join(self.metadata.data, "groups.xml"), self.metadata.groups_xml.data, "metadata") - if self.has_all_xincludes("clients.xml"): - self.check_xml(os.path.join(self.metadata.data, "clients.xml"), - self.metadata.clients_xml.data, - "metadata") + if hasattr(self.metadata, "clients_xml"): + if self.has_all_xincludes("clients.xml"): + self.check_xml(os.path.join(self.metadata.data, "clients.xml"), + self.metadata.clients_xml.data, + "metadata") def check_cfg(self): """ Check Cfg files and ``info.xml`` files for required diff --git a/src/lib/Bcfg2/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index a1d0b7fa1..da8da1aa4 100755 --- a/src/lib/Bcfg2/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -34,6 +34,12 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): err = sys.exc_info()[1] self.LintError("genshi-syntax-error", "Genshi syntax error: %s" % err) + except: + etype, err = sys.exc_info()[:2] + self.LintError( + "genshi-syntax-error", + "Unexpected Genshi error on %s: %s: %s" % + (entry.name, etype.__name__, err)) def check_bundler(self): """ Check templates in Bundler for syntax errors. """ diff --git a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 83b00bcb3..3bf76765b 100644 --- a/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -70,7 +70,7 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): permissions=dict(name=is_filename, owner=is_username, group=is_username, mode=is_octal_mode), vcs=dict(vcstype=lambda v: (v != 'Path' and - hasattr(Bcfg2.Client.Tools.VCS, + hasattr(Bcfg2.Client.Tools.VCS.VCS, "Install%s" % v)), revision=None, sourceurl=None)), Service={"__any__": dict(name=None), diff --git a/src/lib/Bcfg2/Server/Plugin/base.py b/src/lib/Bcfg2/Server/Plugin/base.py index ecd970b54..c825a57b5 100644 --- a/src/lib/Bcfg2/Server/Plugin/base.py +++ b/src/lib/Bcfg2/Server/Plugin/base.py @@ -87,6 +87,10 @@ class Plugin(Debuggable): #: alphabetically by their name. sort_order = 500 + #: Whether or not to automatically create a data directory for + #: this plugin + create = True + #: List of names of methods to be exposed as XML-RPC functions __rmi__ = Debuggable.__rmi__ @@ -107,7 +111,7 @@ class Plugin(Debuggable): self.Entries = {} self.core = core self.data = os.path.join(datastore, self.name) - if not os.path.exists(self.data): + if self.create and not os.path.exists(self.data): self.logger.warning("%s: %s does not exist, creating" % (self.name, self.data)) os.makedirs(self.data) diff --git a/src/lib/Bcfg2/Server/Plugin/interfaces.py b/src/lib/Bcfg2/Server/Plugin/interfaces.py index d460cc45d..7909eaa03 100644 --- a/src/lib/Bcfg2/Server/Plugin/interfaces.py +++ b/src/lib/Bcfg2/Server/Plugin/interfaces.py @@ -286,6 +286,8 @@ class Statistics(Plugin): you should avoid using Statistics and use :class:`ThreadedStatistics` instead.""" + create = False + def process_statistics(self, client, xdata): """ Process the given XML statistics data for the specified client. @@ -526,6 +528,8 @@ class GoalValidator(object): class Version(Plugin): """ Version plugins interact with various version control systems. """ + create = False + #: The path to the VCS metadata file or directory, relative to the #: base of the Bcfg2 repository. E.g., for Subversion this would #: be ".svn" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py index 7af69ec81..8a787751c 100644 --- a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -24,6 +24,24 @@ from Bcfg2.Compat import u_str, unicode, b64encode, walk_packages, \ #: facility for passing it otherwise. CFG = None +_HANDLERS = [] + + +def handlers(): + """ A list of Cfg handler classes. Loading the handlers must + be done at run-time, not at compile-time, or it causes a + circular import and Bad Things Happen.""" + if not _HANDLERS: + for submodule in walk_packages(path=__path__, prefix=__name__ + "."): + mname = submodule[1].rsplit('.', 1)[-1] + module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, + mname) + hdlr = getattr(module, mname) + if issubclass(hdlr, CfgBaseFileMatcher): + _HANDLERS.append(hdlr) + _HANDLERS.sort(key=operator.attrgetter("__priority__")) + return _HANDLERS + class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData, Bcfg2.Server.Plugin.Debuggable): @@ -432,24 +450,6 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, entry.set_debug(debug) return rv - @property - def handlers(self): - """ A list of Cfg handler classes. Loading the handlers must - be done at run-time, not at compile-time, or it causes a - circular import and Bad Things Happen.""" - if self._handlers is None: - self._handlers = [] - for submodule in walk_packages(path=__path__, - prefix=__name__ + "."): - mname = submodule[1].rsplit('.', 1)[-1] - module = getattr(__import__(submodule[1]).Server.Plugins.Cfg, - mname) - hdlr = getattr(module, mname) - if CfgBaseFileMatcher in hdlr.__mro__: - self._handlers.append(hdlr) - self._handlers.sort(key=operator.attrgetter("__priority__")) - return self._handlers - def handle_event(self, event): """ Dispatch a FAM event to :func:`entry_init` or the appropriate child handler object. @@ -466,7 +466,7 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, # process a bogus changed event like a created return - for hdlr in self.handlers: + for hdlr in handlers(): if hdlr.handles(event, basename=self.path): if action == 'changed': # warn about a bogus 'changed' event, but @@ -546,10 +546,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, def bind_entry(self, entry, metadata): self.bind_info_to_entry(entry, metadata) - data = self._generate_data(entry, metadata) - - for fltr in self.get_handlers(metadata, CfgFilter): - data = fltr.modify_data(entry, metadata, data) + data, generator = self._generate_data(entry, metadata) + + if generator is not None: + # apply no filters if the data was created by a CfgCreator + for fltr in self.get_handlers(metadata, CfgFilter): + if fltr.specific <= generator.specific: + # only apply filters that are as specific or more + # specific than the generator used for this entry. + # Note that specificity comparison is backwards in + # this sense, since it's designed to sort from + # most specific to least specific. + data = fltr.modify_data(entry, metadata, data) if self.setup['validate']: try: @@ -658,7 +666,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, :type entry: lxml.etree._Element :param metadata: The client metadata to generate data for :type metadata: Bcfg2.Server.Plugins.Metadata.ClientMetadata - :returns: string - the data for the entry + :returns: tuple of (string, generator) - the data for the + entry and the generator used to generate it (or + None, if data was created) """ try: generator = self.best_matching(metadata, @@ -667,10 +677,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet, except PluginExecutionError: # if no creators or generators exist, _create_data() # raises an appropriate exception - return self._create_data(entry, metadata) + return (self._create_data(entry, metadata), None) try: - return generator.get_data(entry, metadata) + return (generator.get_data(entry, metadata), generator) except: msg = "Cfg: Error rendering %s: %s" % (entry.get("name"), sys.exc_info()[1]) @@ -837,10 +847,13 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): def Run(self): for basename, entry in list(self.core.plugins['Cfg'].entries.items()): self.check_pubkey(basename, entry) + self.check_missing_files() @classmethod def Errors(cls): - return {"no-pubkey-xml": "warning"} + return {"no-pubkey-xml": "warning", + "unknown-cfg-files": "error", + "extra-cfg-files": "error"} def check_pubkey(self, basename, entry): """ check that privkey.xml files have corresponding pubkey.xml @@ -862,3 +875,41 @@ class CfgLint(Bcfg2.Server.Lint.ServerPlugin): self.LintError("no-pubkey-xml", "%s has no corresponding pubkey.xml at %s" % (basename, pubkey)) + + def check_missing_files(self): + """ check that all files on the filesystem are known to Cfg """ + cfg = self.core.plugins['Cfg'] + + # first, collect ignore patterns from handlers + ignore = [] + for hdlr in handlers(): + ignore.extend(hdlr.__ignore__) + + # next, get a list of all non-ignored files on the filesystem + all_files = set() + for root, _, files in os.walk(cfg.data): + all_files.update(os.path.join(root, fname) + for fname in files + if not any(fname.endswith("." + i) + for i in ignore)) + + # next, get a list of all files known to Cfg + cfg_files = set() + for root, eset in cfg.entries.items(): + cfg_files.update(os.path.join(cfg.data, root.lstrip("/"), fname) + for fname in eset.entries.keys()) + + # finally, compare the two + unknown_files = all_files - cfg_files + extra_files = cfg_files - all_files + if unknown_files: + self.LintError( + "unknown-cfg-files", + "Files on the filesystem could not be understood by Cfg: %s" % + "; ".join(unknown_files)) + if extra_files: + self.LintError( + "extra-cfg-files", + "Cfg has entries for files that do not exist on the " + "filesystem: %s\nThis is probably a bug." % + "; ".join(extra_files)) diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 507973fa6..a9b622637 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -40,6 +40,8 @@ if HAS_DJANGO: """ dict-like object to make it easier to access client bcfg2 versions from the database """ + create = False + def __getitem__(self, key): try: return MetadataClientModel.objects.get(hostname=key).version diff --git a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py index 1736becc7..71128d64c 100644 --- a/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/POSIXCompat.py @@ -9,6 +9,8 @@ class POSIXCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """POSIXCompat is a goal validator plugin for POSIX entries.""" + create = False + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.GoalValidator.__init__(self) diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index a4b17f05a..98add2ccf 100644 --- a/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -674,7 +674,10 @@ class YumCollection(Collection): gdicts.append(dict(group=group, type=ptype)) if self.use_yum: - return self.call_helper("get_groups", inputdata=gdicts) + try: + return self.call_helper("get_groups", inputdata=gdicts) + except ValueError: + return dict() else: pkgs = dict() for gdict in gdicts: @@ -837,12 +840,13 @@ class YumCollection(Collection): return Collection.complete(self, packagelist) if packagelist: - result = \ - self.call_helper("complete", - dict(packages=list(packagelist), - groups=list(self.get_relevant_groups()))) - if not result: - # some sort of error, reported by call_helper() + try: + result = self.call_helper( + "complete", + dict(packages=list(packagelist), + groups=list(self.get_relevant_groups()))) + except ValueError: + # error reported by call_helper() return set(), packagelist # json doesn't understand sets or tuples, so we get back a # lists of lists (packages) and a list of unicode strings @@ -873,28 +877,39 @@ class YumCollection(Collection): ``bcfg2-yum-helper`` command. """ cmd = [self.helper, "-c", self.cfgfile] - verbose = self.debug_flag or self.setup['verbose'] - if verbose: + if self.setup['verbose']: + cmd.append("-v") + if self.debug_flag: + if not self.setup['verbose']: + # ensure that running in debug gets -vv, even if + # verbose is not enabled + cmd.append("-v") cmd.append("-v") cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd), flag=verbose) + self.debug_log("Packages: running %s" % " ".join(cmd)) if inputdata: result = self.cmd.run(cmd, inputdata=json.dumps(inputdata)) else: result = self.cmd.run(cmd) if not result.success: + errlines = result.error.splitlines() self.logger.error("Packages: error running bcfg2-yum-helper: %s" % - result.error) + errlines[0]) + for line in errlines[1:]: + self.logger.error("Packages: %s" % line) elif result.stderr: + errlines = result.stderr.splitlines() self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - result.stderr, flag=verbose) + errlines[0]) + for line in errlines[1:]: + self.debug_log("Packages: %s" % line) try: return json.loads(result.stdout) except ValueError: err = sys.exc_info()[1] self.logger.error("Packages: error reading bcfg2-yum-helper " "output: %s" % err) - return None + raise def setup_data(self, force_update=False): """ Do any collection-level data setup tasks. This is called @@ -920,13 +935,21 @@ class YumCollection(Collection): if force_update: # we call this twice: one to clean up data from the old # config, and once to clean up data from the new config - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass os.unlink(self.cfgfile) self.write_config() if force_update: - self.call_helper("clean") + try: + self.call_helper("clean") + except ValueError: + # error reported by call_helper + pass class YumSource(Source): diff --git a/src/lib/Bcfg2/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index e97607093..6827c3d1f 100644 --- a/src/lib/Bcfg2/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -14,6 +14,7 @@ from Bcfg2.Server.Statistics import track_statistics try: from django.db import models + from django.core.exceptions import MultipleObjectsReturned HAS_DJANGO = True class ProbesDataModel(models.Model, @@ -254,12 +255,15 @@ class Probes(Bcfg2.Server.Plugin.Probing, for group in self.cgroups[client.hostname]: try: - ProbesGroupsModel.objects.get(hostname=client.hostname, - group=group) - except ProbesGroupsModel.DoesNotExist: - grp = ProbesGroupsModel(hostname=client.hostname, - group=group) - grp.save() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) + except MultipleObjectsReturned: + ProbesGroupsModel.objects.filter(hostname=client.hostname, + group=group).delete() + ProbesGroupsModel.objects.get_or_create( + hostname=client.hostname, + group=group) ProbesGroupsModel.objects.filter( hostname=client.hostname).exclude( group__in=self.cgroups[client.hostname]).delete() diff --git a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py index c3a2221f6..41e6bf8b5 100644 --- a/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py +++ b/src/lib/Bcfg2/Server/Plugins/ServiceCompat.py @@ -6,7 +6,9 @@ import Bcfg2.Server.Plugin class ServiceCompat(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.GoalValidator): """ Use old-style service modes for older clients """ - name = 'ServiceCompat' + + create = False + __author__ = 'bcfg-dev@mcs.anl.gov' mode_map = {('true', 'true'): 'default', ('interactive', 'true'): 'interactive_only', diff --git a/src/lib/Bcfg2/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 34a6e89e0..dfe864d48 100644 --- a/src/lib/Bcfg2/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -60,9 +60,48 @@ class Svn(Bcfg2.Server.Plugin.Version): self.client.callback_conflict_resolver = \ self.get_conflict_resolver(choice) + try: + if self.core.setup.cfp.get( + "svn", + "always_trust").lower() == "true": + self.client.callback_ssl_server_trust_prompt = \ + self.ssl_server_trust_prompt + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.debug("Svn: Using subversion cache for SSL " + "certificate trust") + + try: + if (self.core.setup.cfp.get("svn", "user") and + self.core.setup.cfp.get("svn", "password")): + self.client.callback_get_login = \ + self.get_login + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + self.logger.info("Svn: Using subversion cache for " + "password-based authetication") + self.logger.debug("Svn: Initialized svn plugin with SVN directory %s" % self.vcs_path) + # pylint: disable=W0613 + def get_login(self, realm, username, may_save): + """ PySvn callback to get credentials for HTTP basic authentication """ + self.logger.debug("Svn: Logging in with username: %s" % + self.core.setup.cfp.get("svn", "user")) + return True, \ + self.core.setup.cfp.get("svn", "user"), \ + self.core.setup.cfp.get("svn", "password"), \ + False + # pylint: enable=W0613 + + def ssl_server_trust_prompt(self, trust_dict): + """ PySvn callback to always trust SSL certificates from SVN server """ + self.logger.debug("Svn: Trusting SSL certificate from %s, " + "issued by %s for realm %s" % + (trust_dict['hostname'], + trust_dict['issuer_dname'], + trust_dict['realm'])) + return True, trust_dict['failures'], False + def get_conflict_resolver(self, choice): """ Get a PySvn conflict resolution callback """ def callback(conflict_description): diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py index 050ba3b3e..77bdd6576 100644 --- a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -114,7 +114,7 @@ class TemplateHelperLint(Bcfg2.Server.Lint.ServerPlugin): def Run(self): for helper in self.core.plugins['TemplateHelper'].entries.values(): - if self.HandlesFile(helper): + if self.HandlesFile(helper.name): self.check_helper(helper.name) def check_helper(self, helper): diff --git a/src/lib/Bcfg2/Utils.py b/src/lib/Bcfg2/Utils.py index 69c3264f9..d087f4f87 100644 --- a/src/lib/Bcfg2/Utils.py +++ b/src/lib/Bcfg2/Utils.py @@ -81,9 +81,6 @@ class PackedDigitRange(object): # pylint: disable=E0012,R0924 def __str__(self): return "[%s]" % self.str - def __len__(self): - return sum(r[1] - r[0] + 1 for r in self.ranges) + len(self.ints) - def locked(fd): """ Acquire a lock on a file. diff --git a/src/lib/Bcfg2/settings.py b/src/lib/Bcfg2/settings.py index f3697d66f..c06074845 100644 --- a/src/lib/Bcfg2/settings.py +++ b/src/lib/Bcfg2/settings.py @@ -32,6 +32,8 @@ TIME_ZONE = None DEBUG = False TEMPLATE_DEBUG = DEBUG +ALLOWED_HOSTS = ['*'] + MEDIA_URL = '/site_media/' diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 3bce7fdab..0e1e34c60 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -83,7 +83,7 @@ def main(): raise SystemExit(1) mode = mode_cls() try: - mode(setup['args'][1:]) + return mode(setup['args'][1:]) finally: mode.shutdown() else: @@ -93,6 +93,6 @@ def main(): if __name__ == '__main__': try: - main() + sys.exit(main()) except KeyboardInterrupt: raise SystemExit(1) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 853c98845..1fd9bc067 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -472,7 +472,6 @@ Bcfg2 client itself.""") ('Path Bcfg2 repository', self.setup['repo']), ('Plugins', self.setup['plugins']), ('Password', self.setup['password']), - ('Server Metadata Connector', self.setup['mconnect']), ('Filemonitor', self.setup['filemonitor']), ('Server address', self.setup['location']), ('Path to key', self.setup['key']), diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 2ae5e02d5..4987b8034 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -3,6 +3,7 @@ """This tool examines your Bcfg2 specifications for errors.""" import sys +import time import inspect import logging import Bcfg2.Logger @@ -51,7 +52,11 @@ def run_plugin(plugin, plugin_name, errorhandler=None, args=None, files=None): args.append(setup) # python 2.5 doesn't support mixing *magic and keyword arguments - return plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() + start = time.time() + rv = plugin(*args, **dict(files=files, errorhandler=errorhandler)).Run() + LOGGER.debug(" Ran %s in %0.2f seconds" % (plugin_name, + time.time() - start)) + return rv def get_errorhandler(): diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 2065825ec..735a6c49c 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -5,6 +5,7 @@ without failures""" import os import sys +import signal import fnmatch import logging import Bcfg2.Logger @@ -191,9 +192,23 @@ def run_child(setup, clients, queue): core.shutdown() +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr + + def parse_args(): """ Parse command line arguments. """ optinfo = dict(Bcfg2.Options.TEST_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.load_option_parser(optinfo) @@ -247,6 +262,7 @@ def main(): setup = parse_args() logger = logging.getLogger(sys.argv[0]) core = get_core(setup) + signal.signal(signal.SIGINT, get_sigint_handler(core)) if setup['args']: clients = setup['args'] diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index 7dbdad16b..4ef531d39 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -221,29 +221,58 @@ def main(): logger.error("Config file %s not found" % options.config) return 1 + # pylint: disable=W0702 + rv = 0 depsolver = DepSolver(options.config, options.verbose) if cmd == "clean": - depsolver.clean_cache() - print(json.dumps(True)) + try: + depsolver.clean_cache() + print(json.dumps(True)) + except: + logger.error("Unexpected error cleaning cache: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(False)) + rv = 2 elif cmd == "complete": - data = json.loads(sys.stdin.read()) - depsolver.groups = data['groups'] - (packages, unknown) = depsolver.complete([pkg_to_tuple(p) - for p in data['packages']]) - print(json.dumps(dict(packages=list(packages), - unknown=list(unknown)))) + try: + data = json.loads(sys.stdin.read()) + except: + logger.error("Unexpected error decoding JSON input: %s" % + sys.exc_info()[1]) + rv = 2 + try: + depsolver.groups = data['groups'] + (packages, unknown) = depsolver.complete( + [pkg_to_tuple(p) for p in data['packages']]) + print(json.dumps(dict(packages=list(packages), + unknown=list(unknown)))) + except: + logger.error("Unexpected error completing package set: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(dict(packages=[], unknown=data['packages']))) + rv = 2 elif cmd == "get_groups": - data = json.loads(sys.stdin.read()) - rv = dict() - for gdata in data: - if "type" in gdata: - packages = depsolver.get_group(gdata['group'], - ptype=gdata['type']) - else: - packages = depsolver.get_group(gdata['group']) - rv[gdata['group']] = list(packages) - print(json.dumps(rv)) - + try: + data = json.loads(sys.stdin.read()) + rv = dict() + for gdata in data: + if "type" in gdata: + packages = depsolver.get_group(gdata['group'], + ptype=gdata['type']) + else: + packages = depsolver.get_group(gdata['group']) + rv[gdata['group']] = list(packages) + print(json.dumps(rv)) + except: + logger.error("Unexpected error getting groups: %s" % + sys.exc_info()[1], exc_info=1) + print(json.dumps(dict())) + rv = 2 + else: + logger.error("Unknown command %s" % cmd) + print(json.dumps(None)) + rv = 2 + return rv if __name__ == '__main__': sys.exit(main()) diff --git a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py index 49e9be2ba..e0406fd92 100644 --- a/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestClient/TestTools/TestPOSIX/Testbase.py @@ -897,7 +897,7 @@ class TestPOSIXTool(TestTool): filedef_rv.__iter__.return_value = iter(file_acls) defacls = acls - for akey, perms in acls.items(): + for akey, perms in list(acls.items()): defacls[('default', akey[1], akey[2])] = perms self.assertItemsEqual(ptool._list_file_acls(path), defacls) mock_isdir.assert_called_with(path) diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py index 318f5ceaa..e26c26d41 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugin/Testbase.py @@ -80,24 +80,25 @@ class TestPlugin(TestDebuggable): @patch("os.makedirs") @patch("os.path.exists") def test__init(self, mock_exists, mock_makedirs): - core = Mock() - core.setup = MagicMock() - - mock_exists.return_value = True - p = self.get_obj(core=core) - self.assertEqual(p.data, os.path.join(datastore, p.name)) - self.assertEqual(p.core, core) - mock_exists.assert_any_call(p.data) - self.assertFalse(mock_makedirs.called) - - mock_exists.reset_mock() - mock_makedirs.reset_mock() - mock_exists.return_value = False - p = self.get_obj(core=core) - self.assertEqual(p.data, os.path.join(datastore, p.name)) - self.assertEqual(p.core, core) - mock_exists.assert_any_call(p.data) - mock_makedirs.assert_any_call(p.data) + if self.test_obj.create: + core = Mock() + core.setup = MagicMock() + + mock_exists.return_value = True + p = self.get_obj(core=core) + self.assertEqual(p.data, os.path.join(datastore, p.name)) + self.assertEqual(p.core, core) + mock_exists.assert_any_call(p.data) + self.assertFalse(mock_makedirs.called) + + mock_exists.reset_mock() + mock_makedirs.reset_mock() + mock_exists.return_value = False + p = self.get_obj(core=core) + self.assertEqual(p.data, os.path.join(datastore, p.name)) + self.assertEqual(p.core, core) + mock_exists.assert_any_call(p.data) + mock_makedirs.assert_any_call(p.data) @patch("os.makedirs") def test_init_repo(self, mock_makedirs): diff --git a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py index ab383e4f3..07a1b120e 100644 --- a/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py +++ b/testsuite/Testsrc/Testlib/TestServer/TestPlugins/TestCfg/Test_init.py @@ -6,7 +6,7 @@ import Bcfg2.Options from Bcfg2.Compat import walk_packages from mock import Mock, MagicMock, patch from Bcfg2.Server.Plugins.Cfg import * -from Bcfg2.Server.Plugin import PluginExecutionError +from Bcfg2.Server.Plugin import PluginExecutionError, Specificity # add all parent testsuite directories to sys.path to allow (most) # relative imports in python 2.4 @@ -280,21 +280,20 @@ class TestCfgEntrySet(TestEntrySet): for submodule in walk_packages(path=Bcfg2.Server.Plugins.Cfg.__path__, prefix="Bcfg2.Server.Plugins.Cfg."): expected.append(submodule[1].rsplit('.', 1)[-1]) - eset = self.get_obj() - self.assertItemsEqual(expected, - [h.__name__ for h in eset.handlers]) + self.assertItemsEqual(expected, [h.__name__ for h in handlers()]) - def test_handle_event(self): + @patch("Bcfg2.Server.Plugins.Cfg.handlers") + def test_handle_event(self, mock_handlers): eset = self.get_obj() eset.entry_init = Mock() - eset._handlers = [Mock(), Mock(), Mock()] - for hdlr in eset.handlers: + mock_handlers.return_value = [Mock(), Mock(), Mock()] + for hdlr in mock_handlers.return_value: hdlr.__name__ = "handler" eset.entries = dict() def reset(): eset.entry_init.reset_mock() - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: hdlr.reset_mock() # test that a bogus deleted event is discarded @@ -304,7 +303,7 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) @@ -315,7 +314,7 @@ class TestCfgEntrySet(TestEntrySet): evt.filename = os.path.join(datastore, "test.txt") # test with no handler that handles - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: hdlr.handles.return_value = False hdlr.ignore.return_value = False @@ -323,16 +322,16 @@ class TestCfgEntrySet(TestEntrySet): eset.handle_event(evt) self.assertFalse(eset.entry_init.called) self.assertItemsEqual(eset.entries, dict()) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: hdlr.handles.assert_called_with(evt, basename=eset.path) hdlr.ignore.assert_called_with(evt, basename=eset.path) # test with a handler that handles the entry reset() - eset.handlers[-1].handles.return_value = True + mock_handlers.return_value[-1].handles.return_value = True eset.handle_event(evt) - eset.entry_init.assert_called_with(evt, eset.handlers[-1]) - for hdlr in eset.handlers: + eset.entry_init.assert_called_with(evt, mock_handlers.return_value[-1]) + for hdlr in mock_handlers.return_value: hdlr.handles.assert_called_with(evt, basename=eset.path) if not hdlr.return_value: hdlr.ignore.assert_called_with(evt, basename=eset.path) @@ -340,14 +339,14 @@ class TestCfgEntrySet(TestEntrySet): # test with a handler that ignores the entry before one # that handles it reset() - eset.handlers[0].ignore.return_value = True + mock_handlers.return_value[0].ignore.return_value = True eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - eset.handlers[0].handles.assert_called_with(evt, + mock_handlers.return_value[0].handles.assert_called_with(evt, basename=eset.path) - eset.handlers[0].ignore.assert_called_with(evt, + mock_handlers.return_value[0].ignore.assert_called_with(evt, basename=eset.path) - for hdlr in eset.handlers[1:]: + for hdlr in mock_handlers.return_value[1:]: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) @@ -359,7 +358,7 @@ class TestCfgEntrySet(TestEntrySet): eset.entries[evt.filename] = Mock() eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) eset.entries[evt.filename].handle_event.assert_called_with(evt) @@ -369,7 +368,7 @@ class TestCfgEntrySet(TestEntrySet): evt.code2str.return_value = "deleted" eset.handle_event(evt) self.assertFalse(eset.entry_init.called) - for hdlr in eset.handlers: + for hdlr in mock_handlers.return_value: self.assertFalse(hdlr.handles.called) self.assertFalse(hdlr.ignore.called) self.assertItemsEqual(eset.entries, dict()) @@ -443,7 +442,7 @@ class TestCfgEntrySet(TestEntrySet): metadata = Mock() # test basic entry, no validation, no filters, etc. - eset._generate_data.return_value = "data" + eset._generate_data.return_value = ("data", None) eset.get_handlers.return_value = [] bound = eset.bind_entry(entry, metadata) eset.bind_info_to_entry.assert_called_with(entry, metadata) @@ -456,7 +455,7 @@ class TestCfgEntrySet(TestEntrySet): # test empty entry entry = reset() - eset._generate_data.return_value = "" + eset._generate_data.return_value = ("", None) bound = eset.bind_entry(entry, metadata) eset.bind_info_to_entry.assert_called_with(entry, metadata) eset._generate_data.assert_called_with(entry, metadata) @@ -467,7 +466,9 @@ class TestCfgEntrySet(TestEntrySet): # test filters entry = reset() - eset._generate_data.return_value = "initial data" + generator = Mock() + generator.specific = Specificity(all=True) + eset._generate_data.return_value = ("initial data", generator) filters = [Mock(), Mock()] filters[0].modify_data.return_value = "modified data" filters[1].modify_data.return_value = "final data" @@ -489,7 +490,7 @@ class TestCfgEntrySet(TestEntrySet): entry.set("encoding", "base64") mock_b64encode.return_value = "base64 data" eset.get_handlers.return_value = [] - eset._generate_data.return_value = "data" + eset._generate_data.return_value = ("data", None) bound = eset.bind_entry(entry, metadata) eset.bind_info_to_entry.assert_called_with(entry, metadata) eset._generate_data.assert_called_with(entry, metadata) @@ -673,7 +674,7 @@ class TestCfgEntrySet(TestEntrySet): eset._create_data.reset_mock() # test success - self.assertEqual(eset._generate_data(entry, metadata), + self.assertEqual(eset._generate_data(entry, metadata)[0], "data") eset.get_handlers.assert_called_with(metadata, CfgGenerator) eset.best_matching.assert_called_with(metadata, @@ -690,7 +691,7 @@ class TestCfgEntrySet(TestEntrySet): reset() eset.best_matching.side_effect = PluginExecutionError self.assertEqual(eset._generate_data(entry, metadata), - eset._create_data.return_value) + (eset._create_data.return_value, None)) eset.get_handlers.assert_called_with(metadata, CfgGenerator) eset.best_matching.assert_called_with(metadata, eset.get_handlers.return_value) diff --git a/testsuite/pylintrc.conf b/testsuite/pylintrc.conf index 14ccd1d23..653c68426 100644 --- a/testsuite/pylintrc.conf +++ b/testsuite/pylintrc.conf @@ -156,7 +156,7 @@ zope=no # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. -generated-members=objects,DoesNotExist,isoformat,filter,save,count,get,add,id +generated-members=objects,DoesNotExist,isoformat,filter,save,count,get,add,id,MultipleObjectsReturned [MISCELLANEOUS] diff --git a/tools/bcfg2-profile-templates.py b/tools/bcfg2-profile-templates.py index 93314f1e3..f4069e454 100755 --- a/tools/bcfg2-profile-templates.py +++ b/tools/bcfg2-profile-templates.py @@ -5,6 +5,7 @@ import sys import time import math +import signal import logging import operator import Bcfg2.Logger @@ -17,6 +18,19 @@ def stdev(nums): return math.sqrt(sum((n - mean)**2 for n in nums) / float(len(nums))) +def get_sigint_handler(core): + """ Get a function that handles SIGINT/Ctrl-C by shutting down the + core and exiting properly.""" + + def hdlr(sig, frame): # pylint: disable=W0613 + """ Handle SIGINT/Ctrl-C by shutting down the core and exiting + properly. """ + core.shutdown() + os._exit(1) # pylint: disable=W0212 + + return hdlr + + def main(): optinfo = dict( client=Bcfg2.Options.Option("Benchmark templates for one client", @@ -49,6 +63,7 @@ def main(): logger = logging.getLogger(sys.argv[0]) core = Bcfg2.Server.Core.BaseCore(setup) + signal.signal(signal.SIGINT, get_sigint_handler(core)) logger.info("Bcfg2 server core loaded") core.load_plugins() logger.debug("Plugins loaded") diff --git a/tools/export.py b/tools/export.py index 0f4724e6b..7c3c56db2 100755 --- a/tools/export.py +++ b/tools/export.py @@ -136,8 +136,7 @@ E.G. 1.2.0pre1 is a valid version. tarname = '/tmp/%s-%s.tar.gz' % (pkgname, version) - newchangelog = \ -"""bcfg2 (%s-0.0) unstable; urgency=low + newchangelog = """bcfg2 (%s-0.0) unstable; urgency=low * New upstream release @@ -177,7 +176,9 @@ E.G. 1.2.0pre1 is a valid version. "- New upstream release\n", "\n"] # write out the new RPM changelog - specs = ["misc/bcfg2.spec", "misc/bcfg2-selinux.spec", "redhat/bcfg2.spec.in"] + specs = ["misc/bcfg2.spec", + "misc/bcfg2-selinux.spec", + "redhat/bcfg2.spec.in"] if options.dryrun: print("*** Add the following to the top of the %%changelog section in %s:\n%s\n" % (rpmchangelog, " and ".join(specs))) @@ -227,6 +228,29 @@ E.G. 1.2.0pre1 is a valid version. 'VERSION="%s"\n' % version, startswith=True, dryrun=options.dryrun) + # update solaris IPS version + find_and_replace('solaris-ips/Makefile', 'VERS=', + 'VERS=%s-1\n' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/MANIFEST.bcfg2.header', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2@', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2@%s"' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/MANIFEST.bcfg2-server.header', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@', + 'set name=pkg.fmri value="pkg://bcfg2/bcfg2-server@%s"' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/pkginfo.bcfg2', 'VERSION=', + 'VERSION="%s"\n' % version, + startswith=True, + dryrun=options.dryrun) + find_and_replace('solaris-ips/pkginfo.bcfg2-server', 'VERSION=', + 'VERSION="%s"\n' % version, + startswith=True, + dryrun=options.dryrun) # set new version in Bcfg2/version.py find_and_replace('src/lib/Bcfg2/version.py', '__version__ =', @@ -249,30 +273,30 @@ E.G. 1.2.0pre1 is a valid version. else: find_and_replace('misc/bcfg2.spec', 'Release: ', 'Release: 0.%s.%s\n' % - (version_info['build'][-1], version_info['build']), + (version_info['build'][-1], version_info['build']), dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', 'Release: ', 'Release: 0.%s.%s\n' % - (version_info['build'][-1], version_info['build']), + (version_info['build'][-1], version_info['build']), dryrun=options.dryrun) find_and_replace('misc/bcfg2.spec', '%setup', '%%setup -q -n %%{name}-%%{version}%s\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', '%setup', '%%setup -q -n %%{name}-%%{version}%s\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2.spec', 'BuildRoot', 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) find_and_replace('misc/bcfg2-selinux.spec', 'BuildRoot', 'BuildRoot: %%{_tmppath}/%%{name}-%%{version}%s-%%{release}-root-%%(%%{__id_u} -n)\n' % - version_info['build'], + version_info['build'], startswith=True, dryrun=options.dryrun) # fix pre problem noted in |