diff options
Diffstat (limited to 'src/lib')
-rw-r--r-- | src/lib/Bcfg2/Bcfg2Py3k.py (renamed from src/lib/Bcfg2Py3k.py) | 11 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Frame.py (renamed from src/lib/Client/Frame.py) | 44 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/APK.py (renamed from src/lib/Client/Tools/APK.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/APT.py (renamed from src/lib/Client/Tools/APT.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Action.py (renamed from src/lib/Client/Tools/Action.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Blast.py (renamed from src/lib/Client/Tools/Blast.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Chkconfig.py (renamed from src/lib/Client/Tools/Chkconfig.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/DebInit.py (renamed from src/lib/Client/Tools/DebInit.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Encap.py (renamed from src/lib/Client/Tools/Encap.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/FreeBSDInit.py (renamed from src/lib/Client/Tools/FreeBSDInit.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py (renamed from src/lib/Client/Tools/FreeBSDPackage.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/IPS.py (renamed from src/lib/Client/Tools/IPS.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/MacPorts.py (renamed from src/lib/Client/Tools/MacPorts.py) | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/POSIX.py (renamed from src/lib/Client/Tools/POSIX.py) | 23 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Pacman.py (renamed from src/lib/Client/Tools/Pacman.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Portage.py | 125 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/RPMng.py (renamed from src/lib/Client/Tools/RPMng.py) | 10 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/RcUpdate.py (renamed from src/lib/Client/Tools/RcUpdate.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/SMF.py (renamed from src/lib/Client/Tools/SMF.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/SYSV.py (renamed from src/lib/Client/Tools/SYSV.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Systemd.py (renamed from src/lib/Client/Tools/Systemd.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Upstart.py (renamed from src/lib/Client/Tools/Upstart.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/VCS.py (renamed from src/lib/Client/Tools/VCS.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/YUM24.py (renamed from src/lib/Client/Tools/YUM24.py) | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/YUMng.py (renamed from src/lib/Client/Tools/YUMng.py) | 86 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/__init__.py (renamed from src/lib/Client/Tools/__init__.py) | 31 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/launchd.py (renamed from src/lib/Client/Tools/launchd.py) | 54 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Client/Tools/rpmtools.py (renamed from src/lib/Client/Tools/rpmtools.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/XML.py (renamed from src/lib/Client/XML.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/__init__.py (renamed from src/lib/Client/__init__.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Component.py (renamed from src/lib/Component.py) | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Logger.py (renamed from src/lib/Logger.py) | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Options.py (renamed from src/lib/Options.py) | 27 | ||||
-rw-r--r-- | src/lib/Bcfg2/Proxy.py (renamed from src/lib/Proxy.py) | 36 | ||||
-rw-r--r-- | src/lib/Bcfg2/SSLServer.py (renamed from src/lib/SSLServer.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Backup.py (renamed from src/lib/Server/Admin/Backup.py) | 10 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Bundle.py (renamed from src/lib/Server/Admin/Bundle.py) | 16 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Client.py (renamed from src/lib/Server/Admin/Client.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Compare.py (renamed from src/lib/Server/Admin/Compare.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Group.py (renamed from src/lib/Server/Admin/Group.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Init.py (renamed from src/lib/Server/Admin/Init.py) | 9 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Minestruct.py (renamed from src/lib/Server/Admin/Minestruct.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Perf.py (renamed from src/lib/Server/Admin/Perf.py) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Pull.py (renamed from src/lib/Server/Admin/Pull.py) | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Query.py (renamed from src/lib/Server/Admin/Query.py) | 24 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Reports.py (renamed from src/lib/Server/Admin/Reports.py) | 32 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Snapshots.py (renamed from src/lib/Server/Admin/Snapshots.py) | 9 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Tidy.py (renamed from src/lib/Server/Admin/Tidy.py) | 11 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Viz.py (renamed from src/lib/Server/Admin/Viz.py) | 16 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Xcmd.py (renamed from src/lib/Server/Admin/Xcmd.py) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/__init__.py (renamed from src/lib/Server/Admin/__init__.py) | 60 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py (renamed from src/lib/Server/Core.py) | 25 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor.py (renamed from src/lib/Server/FileMonitor.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/.gitignore (renamed from src/lib/Server/Hostbase/.gitignore) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/__init__.py (renamed from src/lib/Server/Hostbase/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/backends.py (renamed from src/lib/Server/Hostbase/backends.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py (renamed from src/lib/Server/Hostbase/hostbase/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py (renamed from src/lib/Server/Hostbase/hostbase/admin.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/models.py (renamed from src/lib/Server/Hostbase/hostbase/models.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql (renamed from src/lib/Server/Hostbase/hostbase/sql/zone.sql) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py (renamed from src/lib/Server/Hostbase/hostbase/urls.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/views.py (renamed from src/lib/Server/Hostbase/hostbase/views.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/base.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/copy.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/dns.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/edit.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/errors.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/host.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/index.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/login.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/logout.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/new.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/remove.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/results.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/search.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zones.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/ldapauth.py (renamed from src/lib/Server/Hostbase/ldapauth.py) | 0 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Hostbase/manage.py (renamed from src/lib/Server/Hostbase/manage.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/base.css (renamed from src/lib/Server/Hostbase/media/base.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css (renamed from src/lib/Server/Hostbase/media/boxypastel.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/global.css (renamed from src/lib/Server/Hostbase/media/global.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/layout.css (renamed from src/lib/Server/Hostbase/media/layout.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/nisauth.py (renamed from src/lib/Server/Hostbase/nisauth.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/regex.py (renamed from src/lib/Server/Hostbase/regex.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/settings.py (renamed from src/lib/Server/Hostbase/settings.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl (renamed from src/lib/Server/Hostbase/templates/batchadd.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head (renamed from src/lib/Server/Hostbase/templates/dhcpd.conf.head) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl (renamed from src/lib/Server/Hostbase/templates/dhcpd.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl (renamed from src/lib/Server/Hostbase/templates/hosts.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl (renamed from src/lib/Server/Hostbase/templates/hostsappend.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl (renamed from src/lib/Server/Hostbase/templates/named.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl (renamed from src/lib/Server/Hostbase/templates/namedviews.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl (renamed from src/lib/Server/Hostbase/templates/reverseappend.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl (renamed from src/lib/Server/Hostbase/templates/reversesoa.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl (renamed from src/lib/Server/Hostbase/templates/zone.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/harness.py (renamed from src/lib/Server/Hostbase/test/harness.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/test_environ_settings.py (renamed from src/lib/Server/Hostbase/test/test_environ_settings.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/test_ldapauth.py (renamed from src/lib/Server/Hostbase/test/test_ldapauth.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/test_settings.py (renamed from src/lib/Server/Hostbase/test/test_settings.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/urls.py (renamed from src/lib/Server/Hostbase/urls.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Bundles.py (renamed from src/lib/Server/Lint/Bundles.py) | 20 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Comments.py (renamed from src/lib/Server/Lint/Comments.py) | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Deltas.py | 23 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Duplicates.py (renamed from src/lib/Server/Lint/Duplicates.py) | 9 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Lint/Genshi.py (renamed from src/lib/Server/Lint/Genshi.py) | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/GroupPatterns.py | 34 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/InfoXML.py (renamed from src/lib/Server/Lint/InfoXML.py) | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/MergeFiles.py (renamed from src/lib/Server/Lint/MergeFiles.py) | 16 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Pkgmgr.py (renamed from src/lib/Server/Lint/Pkgmgr.py) | 24 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/RequiredAttrs.py (renamed from src/lib/Server/Lint/RequiredAttrs.py) | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/TemplateHelper.py | 63 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py (renamed from src/lib/Server/Lint/Validate.py) | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/__init__.py (renamed from src/lib/Server/Lint/__init__.py) | 66 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin.py (renamed from src/lib/Server/Plugin.py) | 242 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Account.py (renamed from src/lib/Server/Plugins/Account.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/BB.py (renamed from src/lib/Server/Plugins/BB.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Base.py (renamed from src/lib/Server/Plugins/Base.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Bundler.py (renamed from src/lib/Server/Plugins/Bundler.py) | 42 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Bzr.py (renamed from src/lib/Server/Plugins/Bzr.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg.py (renamed from src/lib/Server/Plugins/Cfg.py) | 87 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cvs.py (renamed from src/lib/Server/Plugins/Cvs.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/DBStats.py (renamed from src/lib/Server/Plugins/DBStats.py) | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Darcs.py (renamed from src/lib/Server/Plugins/Darcs.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Decisions.py (renamed from src/lib/Server/Plugins/Decisions.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Defaults.py (renamed from src/lib/Server/Plugins/Defaults.py) | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Deps.py (renamed from src/lib/Server/Plugins/Deps.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Editor.py (renamed from src/lib/Server/Plugins/Editor.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/FileProbes.py (renamed from src/lib/Server/Plugins/FileProbes.py) | 156 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Fossil.py (renamed from src/lib/Server/Plugins/Fossil.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Git.py (renamed from src/lib/Server/Plugins/Git.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/GroupPatterns.py (renamed from src/lib/Server/Plugins/GroupPatterns.py) | 25 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Guppy.py (renamed from src/lib/Server/Plugins/Guppy.py) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Hg.py (renamed from src/lib/Server/Plugins/Hg.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Hostbase.py (renamed from src/lib/Server/Plugins/Hostbase.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Ldap.py (renamed from src/lib/Server/Plugins/Ldap.py) | 53 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py (renamed from src/lib/Server/Plugins/Metadata.py) | 639 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/NagiosGen.py (renamed from src/lib/Server/Plugins/NagiosGen.py) | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Ohai.py (renamed from src/lib/Server/Plugins/Ohai.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Apt.py (renamed from src/lib/Server/Plugins/Packages/Apt.py) | 13 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py (renamed from src/lib/Server/Plugins/Packages/Collection.py) | 56 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Pac.py (renamed from src/lib/Server/Plugins/Packages/Pac.py) | 19 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py (renamed from src/lib/Server/Plugins/Packages/PackagesSources.py) | 46 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py (renamed from src/lib/Server/Plugins/Packages/Source.py) | 74 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py (renamed from src/lib/Server/Plugins/Packages/Yum.py) | 185 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py (renamed from src/lib/Server/Plugins/Packages/__init__.py) | 88 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Pkgmgr.py (renamed from src/lib/Server/Plugins/Pkgmgr.py) | 10 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Probes.py (renamed from src/lib/Server/Plugins/Probes.py) | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Properties.py (renamed from src/lib/Server/Plugins/Properties.py) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Rules.py (renamed from src/lib/Server/Plugins/Rules.py) | 28 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SGenshi.py (renamed from src/lib/Server/Plugins/SGenshi.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SSHbase.py (renamed from src/lib/Server/Plugins/SSHbase.py) | 18 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SSLCA.py (renamed from src/lib/Server/Plugins/SSLCA.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Snapshots.py (renamed from src/lib/Server/Plugins/Snapshots.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Statistics.py (renamed from src/lib/Server/Plugins/Statistics.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Svcmgr.py (renamed from src/lib/Server/Plugins/Svcmgr.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Svn.py (renamed from src/lib/Server/Plugins/Svn.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Svn2.py (renamed from src/lib/Server/Plugins/Svn2.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TCheetah.py (renamed from src/lib/Server/Plugins/TCheetah.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TGenshi.py (renamed from src/lib/Server/Plugins/TGenshi.py) | 12 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 83 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Trigger.py (renamed from src/lib/Server/Plugins/Trigger.py) | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/__init__.py (renamed from src/lib/Server/Plugins/__init__.py) | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/__init__.py (renamed from src/lib/Server/Reports/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/backends.py (renamed from src/lib/Server/Reports/backends.py) | 0 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Reports/importscript.py (renamed from src/lib/Server/Reports/importscript.py) | 11 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Reports/manage.py (renamed from src/lib/Server/Reports/manage.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/nisauth.py (renamed from src/lib/Server/Reports/nisauth.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/__init__.py (renamed from src/lib/Server/Reports/reports/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/fixtures/initial_version.xml (renamed from src/lib/Server/Reports/reports/fixtures/initial_version.xml) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/models.py (renamed from src/lib/Server/Reports/reports/models.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/sql/client.sql (renamed from src/lib/Server/Reports/reports/sql/client.sql) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/404.html (renamed from src/lib/Server/Reports/reports/templates/404.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html (renamed from src/lib/Server/Reports/reports/templates/base-timeview.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/base.html (renamed from src/lib/Server/Reports/reports/templates/base.html) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html (renamed from src/lib/Server/Reports/reports/templates/clients/detail.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html (renamed from src/lib/Server/Reports/reports/templates/clients/detailed-list.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html (renamed from src/lib/Server/Reports/reports/templates/clients/history.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html (renamed from src/lib/Server/Reports/reports/templates/clients/index.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html (renamed from src/lib/Server/Reports/reports/templates/clients/manage.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html (renamed from src/lib/Server/Reports/reports/templates/config_items/item.html) | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html (renamed from src/lib/Server/Reports/reports/templates/config_items/listing.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html (renamed from src/lib/Server/Reports/reports/templates/displays/summary.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html (renamed from src/lib/Server/Reports/reports/templates/displays/timing.html) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html (renamed from src/lib/Server/Reports/reports/templates/widgets/filter_bar.html) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc (renamed from src/lib/Server/Reports/reports/templates/widgets/interaction_list.inc) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html (renamed from src/lib/Server/Reports/reports/templates/widgets/page_bar.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py (renamed from src/lib/Server/Reports/reports/templatetags/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py (renamed from src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py (renamed from src/lib/Server/Reports/reports/templatetags/syntax_coloring.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/urls.py (renamed from src/lib/Server/Reports/reports/urls.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/views.py (renamed from src/lib/Server/Reports/reports/views.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/settings.py (renamed from src/lib/Server/Reports/settings.py) | 35 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/updatefix.py (renamed from src/lib/Server/Reports/updatefix.py) | 97 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/urls.py (renamed from src/lib/Server/Reports/urls.py) | 0 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Reports/utils.py (renamed from src/lib/Server/Reports/utils.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Snapshots/__init__.py (renamed from src/lib/Server/Snapshots/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Snapshots/model.py (renamed from src/lib/Server/Snapshots/model.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/__init__.py (renamed from src/lib/Server/__init__.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Statistics.py (renamed from src/lib/Statistics.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/__init__.py (renamed from src/lib/__init__.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2Py3Incompat.py | 2 | ||||
-rw-r--r-- | src/lib/Client/Tools/Portage.py | 72 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Packages/PackagesConfig.py | 33 |
215 files changed, 1939 insertions, 1413 deletions
diff --git a/src/lib/Bcfg2Py3k.py b/src/lib/Bcfg2/Bcfg2Py3k.py index ee05b7e41..6af8b3e5c 100644 --- a/src/lib/Bcfg2Py3k.py +++ b/src/lib/Bcfg2/Bcfg2Py3k.py @@ -75,17 +75,6 @@ def u_str(string, encoding=None): else: return unicode(string) -""" -In order to use the new syntax for printing to a file, we need to do -a conditional import because there is a syntax incompatibility between -the two versions of python. -""" -if sys.hexversion >= 0x03000000: - from Bcfg2.Bcfg2Py3Incompat import fprint -else: - def fprint(s, f): - print >> f, s - if sys.hexversion >= 0x03000000: from io import FileIO as file else: diff --git a/src/lib/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index eca8960c1..9ad669ad6 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -2,7 +2,6 @@ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ -__revision__ = '$Revision$' import logging import sys @@ -240,25 +239,25 @@ class Frame: if self.setup['remove'] == 'all': self.removal = self.extra elif self.setup['remove'] in ['services', 'Services']: - self.removal = [entry for entry in self.extra \ + self.removal = [entry for entry in self.extra if entry.tag == 'Service'] elif self.setup['remove'] in ['packages', 'Packages']: - self.removal = [entry for entry in self.extra \ + self.removal = [entry for entry in self.extra if entry.tag == 'Package'] - candidates = [entry for entry in self.states \ + candidates = [entry for entry in self.states if not self.states[entry]] if self.dryrun: if self.whitelist: self.logger.info("In dryrun mode: suppressing entry installation for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) for entry \ - in self.whitelist]) + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.whitelist]) self.whitelist = [] if self.removal: self.logger.info("In dryrun mode: suppressing entry removal for:") - self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) for entry \ - in self.removal]) + self.logger.info(["%s:%s" % (entry.tag, entry.get('name')) + for entry in self.removal]) self.removal = [] return # Here is where most of the work goes @@ -270,13 +269,13 @@ class Frame: for bundle in self.setup['bundle']: if bundle not in all_bundle_names: self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = [b for b in self.config.findall('./Bundle') \ + bundles = [b for b in self.config.findall('./Bundle') if b.get('name') in self.setup['bundle']] - self.whitelist = [e for e in self.whitelist if \ - True in [e in b for b in bundles]] + self.whitelist = [e for e in self.whitelist + if True in [e in b for b in bundles]] elif self.setup['indep']: - bundles = [nb for nb in self.config.getchildren() if nb.tag != \ - 'Bundle'] + bundles = [nb for nb in self.config.getchildren() + if nb.tag != 'Bundle'] else: bundles = self.config.getchildren() @@ -284,22 +283,24 @@ class Frame: for bundle in bundles[:]: if bundle.tag != 'Bundle': continue - actions = [a for a in bundle.findall('./Action') \ - if a.get('timing') != 'post'] - # now we process all "always actions" bmodified = len([item for item in bundle if item in self.whitelist]) - for action in actions: - if bmodified or action.get('when') == 'always': - self.DispatchInstallCalls([action]) + actions = [a for a in bundle.findall('./Action') + if (a.get('timing') != 'post' and + (bmodified or a.get('when') == 'always'))] + # now we process all "always actions" + if self.setup['interactive']: + promptFilter(prompt, actions) + self.DispatchInstallCalls(actions) + # need to test to fail entries in whitelist if False in [self.states[a] for a in actions]: # then display bundles forced off with entries - self.logger.info("Bundle %s failed prerequisite action" % \ + self.logger.info("Bundle %s failed prerequisite action" % (bundle.get('name'))) bundles.remove(bundle) b_to_remv = [ent for ent in self.whitelist if ent in bundle] if b_to_remv: - self.logger.info("Not installing entries from Bundle %s" % \ + self.logger.info("Not installing entries from Bundle %s" % (bundle.get('name'))) self.logger.info(["%s:%s" % (e.tag, e.get('name')) for e in b_to_remv]) @@ -425,7 +426,6 @@ class Frame: stats = Bcfg2.Client.XML.SubElement(feedback, 'Statistics', total=str(len(self.states)), - client_version=__revision__, version='2.0', revision=self.config.get('revision', '-1')) good = len([key for key, val in list(self.states.items()) if val]) diff --git a/src/lib/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index 6a6fd51b3..aaaf2472f 100644 --- a/src/lib/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -1,5 +1,4 @@ """This provides Bcfg2 support for Alpine Linux APK packages.""" -__revision__ = '$Revision$' import Bcfg2.Client.Tools diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index db8cd56e7..6b839ffbc 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -1,5 +1,4 @@ """This is the Bcfg2 support for apt-get.""" -__revision__ = '$Revision$' # suppress apt API warnings import warnings diff --git a/src/lib/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index c089cde1d..dc49347e9 100644 --- a/src/lib/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -1,5 +1,4 @@ """Action driver""" -__revision__ = '$Revision$' import Bcfg2.Client.Tools from Bcfg2.Client.Frame import matches_white_list, passes_black_list diff --git a/src/lib/Client/Tools/Blast.py b/src/lib/Bcfg2/Client/Tools/Blast.py index 29cdfa116..5d5e74ab2 100644 --- a/src/lib/Client/Tools/Blast.py +++ b/src/lib/Bcfg2/Client/Tools/Blast.py @@ -1,6 +1,4 @@ -# This is the bcfg2 support for blastwave packages (pkg-get) """This provides Bcfg2 support for Blastwave.""" -__revision__ = '$Revision$' import tempfile import Bcfg2.Client.Tools.SYSV diff --git a/src/lib/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index 17e8bf09b..12ea5f132 100644 --- a/src/lib/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -1,8 +1,6 @@ # This is the bcfg2 support for chkconfig -# $Id$ """This is chkconfig support.""" -__revision__ = '$Revision$' import os diff --git a/src/lib/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 022332602..ca6fc439e 100644 --- a/src/lib/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -1,5 +1,4 @@ """Debian Init Support for Bcfg2""" -__revision__ = '$Revision$' import glob import os diff --git a/src/lib/Client/Tools/Encap.py b/src/lib/Bcfg2/Client/Tools/Encap.py index 92062a750..fa09c3ec7 100644 --- a/src/lib/Client/Tools/Encap.py +++ b/src/lib/Bcfg2/Client/Tools/Encap.py @@ -1,7 +1,5 @@ """Bcfg2 Support for Encap Packages""" -__revision__ = '$Revision$' - import glob import re import Bcfg2.Client.Tools diff --git a/src/lib/Client/Tools/FreeBSDInit.py b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py index 10f0f2e93..10f0f2e93 100644 --- a/src/lib/Client/Tools/FreeBSDInit.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py diff --git a/src/lib/Client/Tools/FreeBSDPackage.py b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py index 04c05adaa..3e6f2b6bb 100644 --- a/src/lib/Client/Tools/FreeBSDPackage.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py @@ -1,5 +1,4 @@ """This is the Bcfg2 tool for the FreeBSD package system.""" -__revision__ = '$Rev$' # TODO # - actual package installation diff --git a/src/lib/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py index 9afd23143..e30bbd2a4 100644 --- a/src/lib/Client/Tools/IPS.py +++ b/src/lib/Bcfg2/Client/Tools/IPS.py @@ -1,5 +1,4 @@ """This is the Bcfg2 support for OpenSolaris packages.""" -__revision__ = '$Revision$' import pkg.client.image as image import pkg.client.progress as progress diff --git a/src/lib/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index 23b536451..9724fab57 100644 --- a/src/lib/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -1,5 +1,4 @@ """This provides Bcfg2 support for macports packages.""" -__revision__ = '$Revision$' import Bcfg2.Client.Tools @@ -11,7 +10,7 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): __handles__ = [('Package', 'macport')] __req__ = {'Package': ['name', 'version']} pkgtype = 'macport' - pkgtool = ("/opt/local/bin/port install %s") + pkgtool = ('/opt/local/bin/port install %s', ('%s', ['name'])) def __init__(self, logger, setup, config): Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) @@ -27,7 +26,7 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): continue pkgname = pkg.split('@')[0].strip() version = pkg.split('@')[1].split(' ')[0] - self.logger.info(" pkgname: %s\n version: %s" % (pkgname, version)) + self.logger.info(" pkgname: %s version: %s" % (pkgname, version)) self.installed[pkgname] = version def VerifyPackage(self, entry, modlist): @@ -38,13 +37,20 @@ class MacPorts(Bcfg2.Client.Tools.PkgTool): return False if entry.attrib['name'] in self.installed: - if self.installed[entry.attrib['name']] == entry.attrib['version']: + if (self.installed[entry.attrib['name']] == entry.attrib['version'] or + entry.attrib['version'] == 'any'): #if not self.setup['quick'] and \ # entry.get('verify', 'true') == 'true': #FIXME: We should be able to check this once # http://trac.macports.org/ticket/15709 is implemented return True else: + self.logger.info(" %s: Wrong version installed. " + "Want %s, but have %s" % (entry.get("name"), + entry.get("version"), + self.installed[entry.get("name")], + )) + entry.set('current_version', self.installed[entry.get('name')]) return False entry.set('current_exists', 'false') diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Bcfg2/Client/Tools/POSIX.py index 3591c33ad..0d67dbbab 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX.py @@ -1,5 +1,4 @@ """All POSIX Type client support for Bcfg2.""" -__revision__ = '$Revision$' import binascii from datetime import datetime @@ -21,7 +20,7 @@ import Bcfg2.Client.Tools import Bcfg2.Options from Bcfg2.Client import XML -log = logging.getLogger('posix') +log = logging.getLogger('POSIX') # map between dev_type attribute and stat constants device_map = {'block': stat.S_IFBLK, @@ -259,8 +258,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): if entry.get('perms') == None or \ entry.get('owner') == None or \ entry.get('group') == None: - self.logger.error('Entry %s not completely specified. ' - 'Try running bcfg2-lint.' % (entry.get('name'))) + self.logger.error("POSIX: Entry %s not completely specified. " + "Try running bcfg2-lint." % (entry.get('name'))) return False while len(entry.get('perms', '')) < 4: entry.set('perms', '0' + entry.get('perms', '')) @@ -268,15 +267,15 @@ class POSIX(Bcfg2.Client.Tools.Tool): ondisk = os.stat(entry.get('name')) except OSError: entry.set('current_exists', 'false') - self.logger.debug("%s %s does not exist" % + self.logger.info("POSIX: %s %s does not exist" % (entry.tag, entry.get('name'))) return False try: owner = str(ondisk[stat.ST_UID]) group = str(ondisk[stat.ST_GID]) except (OSError, KeyError): - self.logger.error('User/Group resolution failed for path %s' % \ - entry.get('name')) + self.logger.info("POSIX: User/Group resolution failed " + "for path %s" % entry.get('name')) owner = 'root' group = '0' finfo = os.stat(entry.get('name')) @@ -301,11 +300,11 @@ class POSIX(Bcfg2.Client.Tools.Tool): ex_ents = [e for e in entries if e not in modlist] if ex_ents: pruneTrue = False - self.logger.debug("Directory %s contains extra entries:" % \ - entry.get('name')) - self.logger.debug(ex_ents) + self.logger.info("POSIX: Directory %s contains " + "extra entries:" % entry.get('name')) + self.logger.info(ex_ents) nqtext = entry.get('qtext', '') + '\n' - nqtext += "Directory %s contains extra entries:" % \ + nqtext += "Directory %s contains extra entries: " % \ entry.get('name') nqtext += ":".join(ex_ents) entry.set('qtest', nqtext) @@ -496,7 +495,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): (err.filename, err)) return False different = content != tempdata - + if different: if self.setup['interactive']: prompt = [entry.get('qtext', '')] diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index c8c05061c..c8c05061c 100644 --- a/src/lib/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py new file mode 100644 index 000000000..4516f419d --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -0,0 +1,125 @@ +"""This is the Bcfg2 tool for the Gentoo Portage system.""" + +import re +import Bcfg2.Client.Tools +from Bcfg2.Bcfg2Py3k import ConfigParser + + +class Portage(Bcfg2.Client.Tools.PkgTool): + """The Gentoo toolset implements package and service operations and + inherits the rest from Toolset.Toolset.""" + name = 'Portage' + __execs__ = ['/usr/bin/emerge', '/usr/bin/equery'] + __handles__ = [('Package', 'ebuild')] + __req__ = {'Package': ['name', 'version']} + pkgtype = 'ebuild' + # requires a working PORTAGE_BINHOST in make.conf + _binpkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', \ + ['name', 'version'])) + pkgtool = ('emerge %s', ('=%s-%s', ['name', 'version'])) + + def __init__(self, logger, cfg, setup): + self._initialised = False + Bcfg2.Client.Tools.PkgTool.__init__(self, logger, cfg, setup) + self._initialised = True + self.__important__ = self.__important__ + ['/etc/make.conf'] + self._pkg_pattern = re.compile('(.*)-(\d.*)') + self._ebuild_pattern = re.compile('(ebuild|binary)') + self.cfg = cfg + self.installed = {} + self._binpkgonly = True + + # Used to get options from configuration file + parser = ConfigParser.ConfigParser() + parser.read(self.setup.get('setup')) + for opt in ['binpkgonly']: + if parser.has_option(self.name, opt): + setattr(self, ('_%s' % opt), + self._StrToBoolIfBool(parser.get(self.name, opt))) + + if self._binpkgonly: + self.pkgtool = self._binpkgtool + self.RefreshPackages() + + def _StrToBoolIfBool(self, s): + """Returns a boolean if the string specifies a boolean value. + Returns a string otherwise""" + if s.lower() in ('true', 'yes', 't', 'y', '1'): + return True + elif s.lower() in ('false', 'no', 'f', 'n', '0'): + return False + else: + return s + + def RefreshPackages(self): + """Refresh memory hashes of packages.""" + if not self._initialised: + return + self.logger.info('Getting list of installed packages') + cache = self.cmd.run("equery -q list '*'")[1] + self.installed = {} + for pkg in cache: + if self._pkg_pattern.match(pkg): + name = self._pkg_pattern.match(pkg).group(1) + version = self._pkg_pattern.match(pkg).group(2) + self.installed[name] = version + else: + self.logger.info("Failed to parse pkg name %s" % pkg) + + def VerifyPackage(self, entry, modlist): + """Verify package for entry.""" + if not 'version' in entry.attrib: + self.logger.info("Cannot verify unversioned package %s" % + (entry.get('name'))) + return False + + if not (entry.get('name') in self.installed): + # Can't verify package that isn't installed + entry.set('current_exists', 'false') + return False + + # get the installed version + version = self.installed[entry.get('name')] + entry.set('current_version', version) + + if not self.setup['quick']: + if ('verify' not in entry.attrib) or \ + self._StrToBoolIfBool(entry.get('verify')): + + # Check the package if: + # - Not running in quick mode + # - No verify option is specified in the literal configuration + # OR + # - Verify option is specified and is true + + self.logger.debug('Running equery check on %s' % + entry.get('name')) + output = self.cmd.run("/usr/bin/equery -N check '=%s-%s' " + "2>&1 | grep '!!!' | awk '{print $2}'" + % ((entry.get('name'), version)))[1] + if [filename for filename in output \ + if filename not in modlist]: + return False + + # By now the package must be in one of the following states: + # - Not require checking + # - Have no files modified at all + # - Have modified files in the modlist only + if self.installed[entry.get('name')] == version: + # Specified package version is installed + # Specified package version may be any in literal configuration + return True + + # Something got skipped. Indicates a bug + return False + + def RemovePackages(self, packages): + """Deal with extra configuration detected.""" + pkgnames = " ".join([pkg.get('name') for pkg in packages]) + if len(packages) > 0: + self.logger.info('Removing packages:') + self.logger.info(pkgnames) + self.cmd.run("emerge --unmerge --quiet %s" % + " ".join(pkgnames.split(' '))) + self.RefreshPackages() + self.extra = self.FindExtraPackages() diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Bcfg2/Client/Tools/RPMng.py index 5376118c2..00dd00d71 100644 --- a/src/lib/Client/Tools/RPMng.py +++ b/src/lib/Bcfg2/Client/Tools/RPMng.py @@ -1,7 +1,5 @@ """Bcfg2 Support for RPMS""" -__revision__ = '$Revision$' - import os.path import rpm import rpmtools @@ -9,12 +7,6 @@ import Bcfg2.Client.Tools # Compatibility import from Bcfg2.Bcfg2Py3k import ConfigParser -# Fix for python2.3 -try: - set -except NameError: - from sets import Set as set - class RPMng(Bcfg2.Client.Tools.PkgTool): """Support for RPM packages.""" name = 'RPMng' @@ -461,7 +453,7 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): packages is a list of Package Entries with Instances generated by FindExtraPackages(). - + """ self.logger.debug('Running RPMng.RemovePackages()') diff --git a/src/lib/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index d832d98a8..1b9a29478 100644 --- a/src/lib/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -1,5 +1,4 @@ """This is rc-update support.""" -__revision__ = '$Revision$' import os import Bcfg2.Client.Tools diff --git a/src/lib/Client/Tools/SMF.py b/src/lib/Bcfg2/Client/Tools/SMF.py index 944408326..f824410ad 100644 --- a/src/lib/Client/Tools/SMF.py +++ b/src/lib/Bcfg2/Client/Tools/SMF.py @@ -1,5 +1,4 @@ """SMF support for Bcfg2""" -__revision__ = '$Revision$' import glob import os diff --git a/src/lib/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index b5e1f1c59..eb4a13dfb 100644 --- a/src/lib/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -1,6 +1,4 @@ -# This is the bcfg2 support for solaris sysv packages """This provides bcfg2 support for Solaris SYSV packages.""" -__revision__ = '$Revision$' import tempfile diff --git a/src/lib/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index e3f6a4169..e3f6a4169 100644 --- a/src/lib/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py diff --git a/src/lib/Client/Tools/Upstart.py b/src/lib/Bcfg2/Client/Tools/Upstart.py index 41a585c23..7afc8edd7 100644 --- a/src/lib/Client/Tools/Upstart.py +++ b/src/lib/Bcfg2/Client/Tools/Upstart.py @@ -1,5 +1,4 @@ """Upstart support for Bcfg2.""" -__revision__ = '$Revision$' import glob import re diff --git a/src/lib/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index e6081dc1c..e6081dc1c 100644 --- a/src/lib/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Bcfg2/Client/Tools/YUM24.py index 66768fb34..4e488b9da 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Bcfg2/Client/Tools/YUM24.py @@ -1,5 +1,4 @@ """This provides bcfg2 support for yum.""" -__revision__ = '$Revision: $' import copy import os.path @@ -10,12 +9,6 @@ import Bcfg2.Client.Tools.RPMng # Compatibility import from Bcfg2.Bcfg2Py3k import ConfigParser -# Fix for python2.3 -try: - set -except NameError: - from sets import Set as set - YAD = True CP = ConfigParser.ConfigParser() try: diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Bcfg2/Client/Tools/YUMng.py index a018e68fb..244b66cf4 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Bcfg2/Client/Tools/YUMng.py @@ -1,5 +1,4 @@ """This provides bcfg2 support for yum.""" -__revision__ = '$Revision$' import copy import os.path @@ -16,12 +15,6 @@ import Bcfg2.Client.Tools # Compatibility import from Bcfg2.Bcfg2Py3k import ConfigParser -# Fix for python2.3 -try: - set -except NameError: - from sets import Set as set - def build_yname(pkgname, inst): """Build yum appropriate package name.""" @@ -146,21 +139,7 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): conflicts = ['YUM24', 'RPMng'] def __init__(self, logger, setup, config): - self.yb = yum.YumBase() - - if setup['debug']: - debuglevel = 3 - elif setup['verbose']: - debuglevel = 2 - else: - debuglevel = 1 - - try: - self.yb.preconf.debuglevel = debuglevel - except AttributeError: - self.yb._getConfig(self.yb.config_file_path, - debuglevel=debuglevel) - + self._loadYumBase(setup=setup, logger=logger) Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) self.ignores = [entry.get('name') for struct in config \ for entry in struct \ @@ -179,18 +158,6 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): or entry.get('name') == '/etc/yum.conf'] self.yum_avail = dict() self.yum_installed = dict() - try: - self.yb.doConfigSetup() - self.yb.doTsSetup() - self.yb.doRpmDBSetup() - except yum.Errors.RepoError: - e = sys.exc_info()[1] - self.logger.error("YUMng Repository error: %s" % e) - raise Bcfg2.Client.Tools.toolInstantiationError - except Exception: - e = sys.exc_info()[1] - self.logger.error("YUMng error: %s" % e) - raise Bcfg2.Client.Tools.toolInstantiationError yup = self.yb.doPackageLists(pkgnarrow='updates') if hasattr(self.yb.rpmdb, 'pkglist'): @@ -211,6 +178,50 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): else: dest[pname] = dict(data) + def _loadYumBase(self, setup=None, logger=None): + ''' this may be called before PkgTool.__init__() is called on + this object (when the YUMng object is first instantiated; + PkgTool.__init__() calls RefreshPackages(), which requires a + YumBase object already exist), or after __init__() has + completed, when we reload the yum config before installing + packages. Consequently, we support both methods by allowing + setup and logger, the only object properties we use in this + function, to be passed as keyword arguments or to be omitted + and drawn from the object itself.''' + self.yb = yum.YumBase() + + if setup is None: + setup = self.setup + if logger is None: + logger = self.logger + + if setup['debug']: + debuglevel = 3 + elif setup['verbose']: + debuglevel = 2 + else: + debuglevel = 0 + + try: + self.yb.preconf.debuglevel = debuglevel + self.yb._getConfig() + except AttributeError: + self.yb._getConfig(self.yb.conf.config_file_path, + debuglevel=debuglevel) + + try: + self.yb.doConfigSetup() + self.yb.doTsSetup() + self.yb.doRpmDBSetup() + except yum.Errors.RepoError: + err = sys.exc_info()[1] + logger.error("YUMng Repository error: %s" % err) + raise Bcfg2.Client.Tools.toolInstantiationError + except Exception: + err = sys.exc_info()[1] + logger.error("YUMng error: %s" % err) + raise Bcfg2.Client.Tools.toolInstantiationError + def _loadConfig(self): # Process the YUMng section from the config file. CP = Parser() @@ -742,7 +753,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): else: self.logger.error("Second pass yum install failed.") self.logger.debug(" %s" % restring) - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) self.yb.conf.skip_broken = skipBroken @@ -841,6 +853,10 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): pkg = self.instance_status[gpg_keys[0]].get('pkg') states[pkg] = self.VerifyPackage(pkg, []) + # We want to reload all Yum configuration in case we've + # deployed new .repo files we should consider + self._loadYumBase() + # Install packages. if len(install_pkgs) > 0: self.logger.info("Attempting to install packages") diff --git a/src/lib/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index 9d0c69892..c6cb6e239 100644 --- a/src/lib/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -1,8 +1,4 @@ """This contains all Bcfg2 Tool modules""" -# suppress popen2 warnings for python 2.3 -import warnings -warnings.filterwarnings("ignore", "The popen2 module is deprecated.*", - DeprecationWarning) import os import stat import sys @@ -10,7 +6,6 @@ from subprocess import Popen, PIPE import time import Bcfg2.Client.XML -__revision__ = '$Revision$' __all__ = [tool.split('.')[0] \ for tool in os.listdir(os.path.dirname(__file__)) \ @@ -288,6 +283,10 @@ class SvcTool(Tool): """This class defines basic Service behavior""" name = 'SvcTool' + def __init__(self, logger, setup, config): + Tool.__init__(self, logger, setup, config) + self.restarted = [] + def get_svc_command(self, service, action): """Return the basename of the command used to start/stop services.""" return '/etc/init.d/%s %s' % (service.get('name'), action) @@ -309,6 +308,13 @@ class SvcTool(Tool): # not supported for this driver return 0 + def Remove(self, services): + """ Dummy implementation of service removal method """ + if self.setup['servicemode'] != 'disabled': + for entry in services: + entry.set("status", "off") + self.InstallService(entry) + def BundleUpdated(self, bundle, states): """The Bundle has been updated.""" if self.setup['servicemode'] == 'disabled': @@ -316,17 +322,20 @@ class SvcTool(Tool): for entry in [ent for ent in bundle if self.handlesEntry(ent)]: mode = entry.get('mode', 'default') - if mode == 'manual' or \ - (mode == 'interactive_only' and not self.setup['interactive']): + if (mode == 'manual' or + (mode == 'interactive_only' and + not self.setup['interactive'])): continue # need to handle servicemode = (build|default) # need to handle mode = (default|supervised) + rc = None if entry.get('status') == 'on': if self.setup['servicemode'] == 'build': rc = self.stop_service(entry) - else: + elif entry.get('name') not in self.restarted: if self.setup['interactive']: - prompt = 'Restart service %s?: (y/N): ' % entry.get('name') + prompt = ('Restart service %s?: (y/N): ' % + entry.get('name')) # py3k compatibility try: ans = raw_input(prompt) @@ -335,8 +344,10 @@ class SvcTool(Tool): if ans not in ['y', 'Y']: continue rc = self.restart_service(entry) + if not rc: + self.restarted.append(entry.get('name')) else: rc = self.stop_service(entry) if rc: - self.logger.error("Failed to manipulate service %s" % \ + self.logger.error("Failed to manipulate service %s" % (entry.get('name'))) diff --git a/src/lib/Client/Tools/launchd.py b/src/lib/Bcfg2/Client/Tools/launchd.py index 03dd97e71..c022d32ae 100644 --- a/src/lib/Client/Tools/launchd.py +++ b/src/lib/Bcfg2/Client/Tools/launchd.py @@ -1,8 +1,6 @@ """launchd support for Bcfg2.""" -__revision__ = '$Revision$' import os -import popen2 import Bcfg2.Client.Tools @@ -27,7 +25,8 @@ class launchd(Bcfg2.Client.Tools.Tool): /Library/LaunchDaemons System wide daemons provided by the administrator. /System/Library/LaunchAgents Mac OS X Per-user agents. /System/Library/LaunchDaemons Mac OS X System wide daemons.''' - plistLocations = ["/Library/LaunchDaemons", "/System/Library/LaunchDaemons"] + plistLocations = ["/Library/LaunchDaemons", + "/System/Library/LaunchDaemons"] self.plistMapping = {} for directory in plistLocations: for daemon in os.listdir(directory): @@ -36,11 +35,12 @@ class launchd(Bcfg2.Client.Tools.Tool): d = daemon[:-6] else: d = daemon - (stdout, _) = popen2.popen2('defaults read %s/%s Label' % (directory, d)) - label = stdout.read().strip() + label = self.cmd.run('defaults read %s/%s Label' % + (directory, d))[1][0] self.plistMapping[label] = "%s/%s" % (directory, daemon) - except KeyError: #perhaps this could be more robust - pass + except KeyError: + self.logger.warning("Could not get label from %s/%s" % + (directory, daemon)) def FindPlist(self, entry): return self.plistMapping.get(entry.get('name'), None) @@ -61,20 +61,26 @@ class launchd(Bcfg2.Client.Tools.Tool): """Verify launchd service entry.""" try: services = self.cmd.run("/bin/launchctl list")[1] - except IndexError:#happens when no services are running (should be never) + except IndexError: + # happens when no services are running (should be never) services = [] # launchctl output changed in 10.5 - # It is now three columns, with the last column being the name of the # service + # It is now three columns, with the last + # column being the name of the # service version = self.os_version() if version.startswith('10.5') or version.startswith('10.6'): services = [s.split()[-1] for s in services] - if entry.get('name') in services:#doesn't check if non-spawning services are Started + if entry.get('name') in services: + # doesn't check if non-spawning services are Started return entry.get('status') == 'on' else: - self.logger.debug("Didn't find service Loaded (launchd running under same user as bcfg)") + self.logger.debug("Launchd: Didn't find service Loaded " + "(launchd running under same user as bcfg)") return entry.get('status') == 'off' - try: #Perhaps add the "-w" flag to load and unload to modify the file itself! + try: + # Perhaps add the "-w" flag to load and + # unload to modify the file itself! self.cmd.run("/bin/launchctl load -w %s" % self.FindPlist(entry)) except IndexError: return 'on' @@ -90,12 +96,14 @@ class launchd(Bcfg2.Client.Tools.Tool): name = entry.get('name') if entry.get('status') == 'on': self.logger.error("Installing service %s" % name) - cmdrc = self.cmd.run("/bin/launchctl load -w %s" % self.FindPlist(entry)) + cmdrc = self.cmd.run("/bin/launchctl load -w %s" % + self.FindPlist(entry)) cmdrc = self.cmd.run("/bin/launchctl start %s" % name) else: self.logger.error("Uninstalling service %s" % name) cmdrc = self.cmd.run("/bin/launchctl stop %s" % name) - cmdrc = self.cmd.run("/bin/launchctl unload -w %s" % self.FindPlist(entry)) + cmdrc = self.cmd.run("/bin/launchctl unload -w %s" % + self.FindPlist(entry)) return cmdrc[0] == 0 def Remove(self, svcs): @@ -120,17 +128,23 @@ class launchd(Bcfg2.Client.Tools.Tool): """Reload launchd plist.""" for entry in [entry for entry in bundle if self.handlesEntry(entry)]: if not self.canInstall(entry): - self.logger.error("Insufficient information to restart service %s" % (entry.get('name'))) + self.logger.error("Insufficient information to restart service %s" % + (entry.get('name'))) else: name = entry.get('name') if entry.get('status') == 'on' and self.FindPlist(entry): self.logger.info("Reloading launchd service %s" % name) - #stop? + # stop? self.cmd.run("/bin/launchctl stop %s" % name) - self.cmd.run("/bin/launchctl unload -w %s" % (self.FindPlist(entry)))#what if it disappeared? how do we stop services that are currently running but the plist disappeared?! - self.cmd.run("/bin/launchctl load -w %s" % (self.FindPlist(entry))) + # what if it disappeared? how do we stop services + # that are currently running but the plist disappeared?! + self.cmd.run("/bin/launchctl unload -w %s" % + (self.FindPlist(entry))) + self.cmd.run("/bin/launchctl load -w %s" % + (self.FindPlist(entry))) self.cmd.run("/bin/launchctl start %s" % name) else: - #only if necessary.... + # only if necessary.... self.cmd.run("/bin/launchctl stop %s" % name) - self.cmd.run("/bin/launchctl unload -w %s" % (self.FindPlist(entry))) + self.cmd.run("/bin/launchctl unload -w %s" % + (self.FindPlist(entry))) diff --git a/src/lib/Client/Tools/rpmtools.py b/src/lib/Bcfg2/Client/Tools/rpmtools.py index 3cd2b7014..7441b2c06 100755 --- a/src/lib/Client/Tools/rpmtools.py +++ b/src/lib/Bcfg2/Client/Tools/rpmtools.py @@ -18,7 +18,6 @@ Run 'rpmtools' -h for the options. """ -__revision__ = '$Revision$' import grp import optparse diff --git a/src/lib/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 42b1017ac..858479611 100644 --- a/src/lib/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -1,5 +1,4 @@ '''XML lib compatibility layer for the Bcfg2 client''' -__revision__ = '$Revision$' # library will use lxml, then builtin xml.etree, then ElementTree diff --git a/src/lib/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index ea60a4259..6ed37b257 100644 --- a/src/lib/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -1,4 +1,3 @@ """This contains all Bcfg2 Client modules""" -__revision__ = '$Revision$' __all__ = ["Frame", "Tools", "XML"] diff --git a/src/lib/Component.py b/src/lib/Bcfg2/Component.py index caea3eda9..eb9ea166a 100644 --- a/src/lib/Component.py +++ b/src/lib/Bcfg2/Component.py @@ -1,7 +1,5 @@ """Cobalt component base.""" -__revision__ = '$Revision$' - __all__ = ["Component", "exposed", "automatic", "run_component"] import inspect @@ -16,7 +14,7 @@ import Bcfg2.Logger from Bcfg2.Statistics import Statistics from Bcfg2.SSLServer import XMLRPCServer # Compatibility import -from Bcfg2.Bcfg2Py3k import xmlrpclib, urlparse, fprint +from Bcfg2.Bcfg2Py3k import xmlrpclib, urlparse logger = logging.getLogger() @@ -57,7 +55,7 @@ def run_component(component_cls, listen_all, location, daemon, pidfile_name, os.chdir(os.sep) pidfile = open(pidfile_name or "/dev/null", "w") - fprint(os.getpid(), pidfile) + pidfile.write("%s\n" % os.getpid()) pidfile.close() component = component_cls(cfile=cfile, **cls_kwargs) diff --git a/src/lib/Logger.py b/src/lib/Bcfg2/Logger.py index 0acafb24c..06aae615e 100644 --- a/src/lib/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -1,5 +1,4 @@ """Bcfg2 logging support""" -__revision__ = '$Revision$' import copy import fcntl @@ -88,7 +87,7 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): record.exc_info = None msgdata = record.msg while msgdata: - newrec = copy.deepcopy(record) + newrec = copy.copy(record) newrec.msg = msgdata[:250] msgs.append(newrec) msgdata = msgdata[250:] diff --git a/src/lib/Options.py b/src/lib/Bcfg2/Options.py index fcd9107a9..f6273924a 100644 --- a/src/lib/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -1,9 +1,9 @@ """Option parsing library for utilities.""" -__revision__ = '$Revision$' import getopt import os import sys +import shlex import Bcfg2.Client.Tools # Compatibility imports from Bcfg2.Bcfg2Py3k import ConfigParser @@ -111,6 +111,7 @@ class Option(object): self.value = self.get_cooked_value(os.environ[self.env]) return if self.cf: + # FIXME: This is potentially masking a lot of errors try: self.value = self.get_cooked_value(self.cfp.get(*self.cf)) return @@ -133,7 +134,12 @@ class OptionSet(dict): def buildHelpMessage(self): if hasattr(self, 'hm'): return self.hm - return ' '.join([opt.buildHelpMessage() for opt in list(self.values())]) + hlist = [] # list of _non-empty_ help messages + for opt in list(self.values()): + hm = opt.buildHelpMessage() + if hm != '': + hlist.append(hm) + return ' '.join(hlist) def helpExit(self, msg='', code=1): if msg: @@ -347,7 +353,16 @@ CLIENT_SERVICE_MODE = Option('Set client service mode', default='default', CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90, cmd='-t', cf=('communication', 'timeout'), odesc='<timeout>') - + +# bcfg2-test options +TEST_NOSEOPTS = Option('Options to pass to nosetests', default=[], + cmd='--nose-options', cf=('bcfg2_test', 'nose_options'), + odesc='<opts>', long_arg=True, cook=shlex.split) +TEST_IGNORE = Option('Ignore these entries if they fail to build.', default=[], + cmd='--ignore', + cf=('bcfg2_test', 'ignore_entries'), long_arg=True, + odesc='<Type>:<name>,<Type>:<name>', cook=list_split) + # APT client tool options CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path', cf=('APT', 'install_path'), @@ -373,3 +388,9 @@ class OptionParser(OptionSet): Option.cfpath = self.Bootstrap['configfile'] Option.__cfp = False OptionSet.__init__(self, args) + try: + f = open(Option.cfpath, 'r') + f.close() + except IOError: + e = sys.exc_info()[1] + print("Warning! Unable to read specified configuration file: %s" % e) diff --git a/src/lib/Proxy.py b/src/lib/Bcfg2/Proxy.py index e1406bd99..2e653f47e 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -8,9 +8,6 @@ load_config -- read configuration files """ -__revision__ = '$Revision: $' - - import logging import re import socket @@ -22,10 +19,12 @@ import socket try: import ssl SSL_LIB = 'py26_ssl' + SSL_ERROR = ssl.SSLError except ImportError: from M2Crypto import SSL import M2Crypto.SSL.Checker SSL_LIB = 'm2crypto' + SSL_ERROR = SSL.SSLError import sys @@ -49,18 +48,20 @@ class ProxyError(Exception): the various xmlrpclib errors that might arise (mainly ProtocolError and Fault) """ def __init__(self, err): + msg = None if isinstance(err, xmlrpclib.ProtocolError): # cut out the password in the URL url = re.sub(r'([^:]+):(.*?)@([^@]+:\d+/)', r'\1:******@\3', err.url) - self.message = "XML-RPC Protocol Error for %s: %s (%s)" % \ - (url, err.errmsg, err.errcode) + msg = "XML-RPC Protocol Error for %s: %s (%s)" % (url, + err.errmsg, + err.errcode) elif isinstance(err, xmlrpclib.Fault): - self.message = "XML-RPC Fault: %s (%s)" % (err.faultString, - err.faultCode) + msg = "XML-RPC Fault: %s (%s)" % (err.faultString, + err.faultCode) else: - self.message = str(err) - self.args = (self.message, ) + msg = str(err) + Exception(self, msg) class CertificateError(Exception): def __init__(self, commonName): @@ -298,25 +299,20 @@ class XMLRPCTransport(xmlrpclib.Transport): def request(self, host, handler, request_body, verbose=0): """Send request to server and return response.""" h = self.make_connection(host) - self.send_request(h, handler, request_body) - self.send_host(h, host) - self.send_user_agent(h) - self.send_content(h, request_body) - - if SSL_LIB == 'py26_ssl': - catch = ssl.SSLError - elif SSL_LIB == 'm2crypto': - catch = SSL.SSLError try: + self.send_request(h, handler, request_body) + self.send_host(h, host) + self.send_user_agent(h) + self.send_content(h, request_body) errcode, errmsg, headers = h.getreply() - except catch: + except (socket.error, SSL_ERROR): err = sys.exc_info()[1] raise ProxyError(xmlrpclib.ProtocolError(host + handler, 408, str(err), self._extra_headers)) - + if errcode != 200: raise ProxyError(xmlrpclib.ProtocolError(host + handler, errcode, diff --git a/src/lib/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 32ab9933b..418e259cc 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -1,7 +1,5 @@ """Bcfg2 SSL server.""" -__revision__ = '$Revision$' - __all__ = [ "SSLServer", "XMLRPCRequestHandler", "XMLRPCServer", ] diff --git a/src/lib/Server/Admin/Backup.py b/src/lib/Bcfg2/Server/Admin/Backup.py index 9bd644ff9..3744abca3 100644 --- a/src/lib/Server/Admin/Backup.py +++ b/src/lib/Bcfg2/Server/Admin/Backup.py @@ -12,17 +12,9 @@ class Backup(Bcfg2.Server.Admin.MetadataCore): #"\n\nbcfg2-admin backup restore") __usage__ = ("bcfg2-admin backup") - def __init__(self, configfile): - Bcfg2.Server.Admin.MetadataCore.__init__(self, configfile, - self.__usage__) - def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - # Get Bcfg2 repo directory - opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY} - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - self.datastore = setup['repo'] + self.datastore = self.setup['repo'] timestamp = time.strftime('%Y%m%d%H%M%S') format = 'gz' mode = 'w:' + format diff --git a/src/lib/Server/Admin/Bundle.py b/src/lib/Bcfg2/Server/Admin/Bundle.py index 9b2a71783..89c099602 100644 --- a/src/lib/Server/Admin/Bundle.py +++ b/src/lib/Bcfg2/Server/Admin/Bundle.py @@ -15,19 +15,13 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore): "\nbcfg2-admin bundle show\n") __usage__ = ("bcfg2-admin bundle [options] [add|del] [group]") - def __init__(self, configfile): - Bcfg2.Server.Admin.MetadataCore.__init__(self, configfile, - self.__usage__) - def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - reg = '((?:[a-z][a-z\\.\\d\\-]+)\\.(?:[a-z][a-z\\-]+))(?![\\w\\.])' + rg = re.compile(r'([^.]+\.(?:[a-z][a-z\-]+))(?![\w\.])', + re.IGNORECASE | re.DOTALL) # Get all bundles out of the Bundle/ directory - opts = {'repo': Bcfg2.Options.SERVER_REPOSITORY} - setup = Bcfg2.Options.OptionParser(opts) - setup.parse(sys.argv[1:]) - repo = setup['repo'] + repo = self.setup['repo'] xml_list = glob.glob("%s/Bundler/*.xml" % repo) genshi_list = glob.glob("%s/Bundler/*.genshi" % repo) @@ -50,7 +44,6 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore): elif args[0] in ['list-xml', 'ls-xml']: bundle_name = [] for bundle_path in xml_list: - rg = re.compile(reg, re.IGNORECASE | re.DOTALL) bundle_name.append(rg.search(bundle_path).group(1)) for bundle in bundle_name: print(bundle.split('.')[0]) @@ -58,7 +51,6 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore): elif args[0] in ['list-genshi', 'ls-gen']: bundle_name = [] for bundle_path in genshi_list: - rg = re.compile(reg, re.IGNORECASE | re.DOTALL) bundle_name.append(rg.search(bundle_path).group(1)) for bundle in bundle_name: print(bundle.split('.')[0]) @@ -71,7 +63,7 @@ class Bundle(Bcfg2.Server.Admin.MetadataCore): bundle_name = [] bundle_list = xml_list + genshi_list for bundle_path in bundle_list: - rg = re.compile(reg, re.IGNORECASE | re.DOTALL) + print "matching %s" % bundle_path bundle_name.append(rg.search(bundle_path).group(1)) text = "Available bundles (Number of bundles: %s)" % \ (len(bundle_list)) diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py index c746374a2..4d580c54c 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -13,10 +13,6 @@ class Client(Bcfg2.Server.Admin.MetadataCore): "\nbcfg2-admin client del <client>\n") __usage__ = ("bcfg2-admin client [options] [add|del|update|list] [attr=val]") - def __init__(self, configfile): - Bcfg2.Server.Admin.MetadataCore.__init__(self, configfile, - self.__usage__) - def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) if len(args) == 0: diff --git a/src/lib/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index 82d0d690c..050dd69f8 100644 --- a/src/lib/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py @@ -12,8 +12,8 @@ class Compare(Bcfg2.Server.Admin.Mode): __usage__ = ("bcfg2-admin compare <old> <new>\n\n" " -r\trecursive") - def __init__(self, configfile): - Bcfg2.Server.Admin.Mode.__init__(self, configfile) + def __init__(self, setup): + Bcfg2.Server.Admin.Mode.__init__(self, setup) self.important = {'Path': ['name', 'type', 'owner', 'group', 'perms', 'important', 'paranoid', 'sensitive', 'dev_type', 'major', 'minor', 'prune', diff --git a/src/lib/Server/Admin/Group.py b/src/lib/Bcfg2/Server/Admin/Group.py index 1c5d0c12f..16a773d6f 100644 --- a/src/lib/Server/Admin/Group.py +++ b/src/lib/Bcfg2/Server/Admin/Group.py @@ -13,10 +13,6 @@ class Group(Bcfg2.Server.Admin.MetadataCore): "\nbcfg2-admin group del <group>\n") __usage__ = ("bcfg2-admin group [options] [add|del|update|list] [attr=val]") - def __init__(self, configfile): - Bcfg2.Server.Admin.MetadataCore.__init__(self, configfile, - self.__usage__) - def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) if len(args) == 0: diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index aba6bbd32..c1f9ed484 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -30,8 +30,6 @@ database_password = database_host = # Not used with sqlite3. database_port = -# Set to empty string for default. Not used with sqlite3. -web_debug = True [communication] protocol = %s @@ -57,6 +55,7 @@ groups = '''<Groups version='3.0'> <Group name='suse'/> <Group name='mandrake'/> <Group name='solaris'/> + <Group name='arch'/> </Groups> ''' @@ -73,7 +72,8 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), ('Debian', 'debian'), ('Ubuntu', 'ubuntu'), ('Gentoo', 'gentoo'), - ('FreeBSD', 'freebsd')] + ('FreeBSD', 'freebsd'), + ('Arch', 'arch')] # Complete list of plugins plugin_list = ['Account', @@ -175,9 +175,6 @@ class Init(Bcfg2.Server.Admin.Mode): repopath = "" response = "" - def __init__(self, configfile): - Bcfg2.Server.Admin.Mode.__init__(self, configfile) - def _set_defaults(self): """Set default parameters.""" self.configfile = self.opts['configfile'] diff --git a/src/lib/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index abe1d5a7a..b929a9a8c 100644 --- a/src/lib/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py @@ -18,10 +18,6 @@ class Minestruct(Bcfg2.Server.Admin.StructureMode): "-g <groups>", "only build config for groups")) - def __init__(self, configfile): - Bcfg2.Server.Admin.StructureMode.__init__(self, configfile, - self.__usage__) - def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) if len(args) == 0: diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py index d03b37d57..411442698 100644 --- a/src/lib/Server/Admin/Perf.py +++ b/src/lib/Bcfg2/Server/Admin/Perf.py @@ -10,9 +10,6 @@ class Perf(Bcfg2.Server.Admin.Mode): __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin perf\n") __usage__ = ("bcfg2-admin perf") - def __init__(self, configfile): - Bcfg2.Server.Admin.Mode.__init__(self, configfile) - def __call__(self, args): output = [('Name', 'Min', 'Max', 'Mean', 'Count')] optinfo = { @@ -25,7 +22,7 @@ class Perf(Bcfg2.Server.Admin.Mode): 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, } setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[2:]) + setup.parse(sys.argv[1:]) proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], setup['user'], setup['password'], diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index 47a8be253..daf353107 100644 --- a/src/lib/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -28,14 +28,13 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): "stdin")) allowed = ['Metadata', 'BB', "DBStats", "Statistics", "Cfg", "SSHbase"] - def __init__(self, configfile): - Bcfg2.Server.Admin.MetadataCore.__init__(self, configfile, - self.__usage__) + def __init__(self, setup): + Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) self.log = False self.mode = 'interactive' def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) use_stdin = False try: opts, gargs = getopt.getopt(args, 'vfIs') diff --git a/src/lib/Server/Admin/Query.py b/src/lib/Bcfg2/Server/Admin/Query.py index 9e1d7cc88..3dd326645 100644 --- a/src/lib/Server/Admin/Query.py +++ b/src/lib/Bcfg2/Server/Admin/Query.py @@ -1,9 +1,10 @@ +import sys import logging import Bcfg2.Logger import Bcfg2.Server.Admin -class Query(Bcfg2.Server.Admin.Mode): +class Query(Bcfg2.Server.Admin.MetadataCore): __shorthelp__ = "Query clients" __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin query [-n] [-c] " "[-f filename] g=group p=profile") @@ -18,23 +19,14 @@ class Query(Bcfg2.Server.Admin.Mode): "-f filename", "write query to file")) - def __init__(self, cfile): + def __init__(self, setup): + Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) logging.root.setLevel(100) Bcfg2.Logger.setup_logging(100, to_console=False, to_syslog=False) - Bcfg2.Server.Admin.Mode.__init__(self, cfile) - try: - self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), - ['Metadata', 'Probes'], - 'foo', False, 'UTF-8') - except Bcfg2.Server.Core.CoreInitError: - msg = sys.exc_info()[1] - self.errExit("Core load failed because %s" % msg) - self.bcore.fam.handle_events_in_interval(1) - self.meta = self.bcore.metadata def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - clients = list(self.meta.clients.keys()) + Bcfg2.Server.Admin.MetadataCore.__call__(self, args) + clients = list(self.metadata.clients.keys()) filename_arg = False filename = None for arg in args: @@ -53,9 +45,9 @@ class Query(Bcfg2.Server.Admin.Mode): print("Unknown argument %s" % arg) continue if k == 'p': - nc = self.meta.get_client_names_by_profiles(v.split(',')) + nc = self.metadata.get_client_names_by_profiles(v.split(',')) elif k == 'g': - nc = self.meta.get_client_names_by_groups(v.split(',')) + nc = self.metadata.get_client_names_by_groups(v.split(',')) # add probed groups (if present) for conn in self.bcore.connectors: if isinstance(conn, Bcfg2.Server.Plugins.Probes.Probes): diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 3f25f11af..974cdff9d 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -8,9 +8,6 @@ import pickle import platform import sys import traceback -from Bcfg2.Server.Reports.importscript import load_stats -from Bcfg2.Server.Reports.updatefix import update_database -from Bcfg2.Server.Reports.utils import * from lxml.etree import XML, XMLSyntaxError # Compatibility import @@ -22,16 +19,15 @@ if sys.version_info >= (2, 5): else: from md5 import md5 -# Load django -import django.core.management - +# Prereq issues can be signaled with ImportError, so no try needed # FIXME - settings file uses a hardcoded path for /etc/bcfg2.conf -try: - import Bcfg2.Server.Reports.settings -except Exception: - e = sys.exc_info()[1] - sys.stderr.write("Failed to load configuration settings. %s\n" % e) - raise SystemExit(1) +import Bcfg2.Server.Reports.settings + +# Load django and reports stuff _after_ we know we can load settings +import django.core.management +from Bcfg2.Server.Reports.importscript import load_stats +from Bcfg2.Server.Reports.updatefix import update_database +from Bcfg2.Server.Reports.utils import * project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) project_name = os.path.basename(project_directory) @@ -81,6 +77,7 @@ class Reports(Bcfg2.Server.Admin.Mode): '''Admin interface for dynamic reports''' __shorthelp__ = "Manage dynamic reports" __longhelp__ = (__shorthelp__) + django_commands = ['syncdb', 'sqlall', 'validate'] __usage__ = ("bcfg2-admin reports [command] [options]\n" " -v|--verbose Be verbose\n" " -q|--quiet Print only errors\n" @@ -97,14 +94,13 @@ class Reports(Bcfg2.Server.Admin.Mode): " --expired Expired clients only\n" " scrub Scrub the database for duplicate reasons and orphaned entries\n" " update Apply any updates to the reporting database\n" - "\n") + "\n" + " Django commands:\n " + "\n ".join(django_commands)) - def __init__(self, cfile): - Bcfg2.Server.Admin.Mode.__init__(self, cfile) + def __init__(self, setup): + Bcfg2.Server.Admin.Mode.__init__(self, setup) self.log.setLevel(logging.INFO) - self.django_commands = ['syncdb', 'sqlall', 'validate'] - self.__usage__ = self.__usage__ + " Django commands:\n " + \ - "\n ".join(self.django_commands) def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) diff --git a/src/lib/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index 052545b61..8bc56f1f1 100644 --- a/src/lib/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py @@ -23,11 +23,10 @@ class Snapshots(Bcfg2.Server.Admin.Mode): 'package': Package, 'snapshot': Snapshot} - def __init__(self, configfile): - Bcfg2.Server.Admin.Mode.__init__(self, configfile) - #self.session = Bcfg2.Server.Snapshots.setup_session(debug=True) - self.session = Bcfg2.Server.Snapshots.setup_session(configfile) - self.cfile = configfile + def __init__(self, setup): + Bcfg2.Server.Admin.Mode.__init__(self, setup) + self.session = Bcfg2.Server.Snapshots.setup_session(self.configfile) + self.cfile = self.configfile def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) diff --git a/src/lib/Server/Admin/Tidy.py b/src/lib/Bcfg2/Server/Admin/Tidy.py index f79991fd9..82319b93e 100644 --- a/src/lib/Server/Admin/Tidy.py +++ b/src/lib/Bcfg2/Server/Admin/Tidy.py @@ -16,9 +16,6 @@ class Tidy(Bcfg2.Server.Admin.Mode): "-I", "interactive")) - def __init__(self, cfile): - Bcfg2.Server.Admin.Mode.__init__(self, cfile) - def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) badfiles = self.buildTidyList() @@ -49,7 +46,7 @@ class Tidy(Bcfg2.Server.Admin.Mode): bad = [] # clean up unresolvable hosts in SSHbase - for name in os.listdir("%s/SSHbase" % (self.get_repo_path())): + for name in os.listdir("%s/SSHbase" % self.setup['repo']): if hostmatcher.match(name): hostname = hostmatcher.match(name).group(1) if hostname in good + bad: @@ -59,14 +56,14 @@ class Tidy(Bcfg2.Server.Admin.Mode): good.append(hostname) except: bad.append(hostname) - for name in os.listdir("%s/SSHbase" % (self.get_repo_path())): + for name in os.listdir("%s/SSHbase" % self.setup['repo']): if not hostmatcher.match(name): - to_remove.append("%s/SSHbase/%s" % (self.get_repo_path(), + to_remove.append("%s/SSHbase/%s" % (self.setup['repo'], name)) else: if hostmatcher.match(name).group(1) in bad: to_remove.append("%s/SSHbase/%s" % - (self.get_repo_path(), name)) + (self.setup['repo'], name)) # clean up file~ # clean up files without parsable names in Cfg return to_remove diff --git a/src/lib/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py index 2c618634a..2faa423c1 100644 --- a/src/lib/Server/Admin/Viz.py +++ b/src/lib/Bcfg2/Server/Admin/Viz.py @@ -32,16 +32,10 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): 'indianred1', 'limegreen', 'orange1', 'lightblue2', 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] - plugin_blacklist = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', 'Packages', - 'Rules', 'Account', 'Decisions', 'Deps', 'Git', 'Svn', - 'Fossil', 'Bzr', 'Bundler', 'TGenshi', 'SGenshi', - 'Base'] - - def __init__(self, cfile): - - Bcfg2.Server.Admin.MetadataCore.__init__(self, cfile, - self.__usage__, - pblacklist=self.plugin_blacklist) + __plugin_blacklist__ = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', 'Packages', + 'Rules', 'Account', 'Decisions', 'Deps', 'Git', + 'Svn', 'Fossil', 'Bzr', 'Bundler', 'TGenshi', + 'SGenshi', 'Base'] def __call__(self, args): Bcfg2.Server.Admin.MetadataCore.__call__(self, args) @@ -73,7 +67,7 @@ class Viz(Bcfg2.Server.Admin.MetadataCore): elif opt in ("-o", "--outfile"): outputfile = arg - data = self.Visualize(self.get_repo_path(), hset, bset, + data = self.Visualize(self.setup['repo'], hset, bset, kset, only_client, outputfile) if data: print(data) diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index 8faf68368..140465468 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py @@ -51,5 +51,10 @@ class Xcmd(Bcfg2.Server.Admin.Mode): return else: raise + except Bcfg2.Proxy.ProxyError: + err = sys.exc_info()[1] + print("Proxy Error: %s" % err) + return + if data != None: print(data) diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index 96d9703ba..fdb9a0972 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -1,5 +1,3 @@ -__revision__ = '$Revision$' - __all__ = [ 'Backup', 'Bundle', @@ -8,7 +6,6 @@ __all__ = [ 'Group', 'Init', 'Minestruct', - 'Mode', 'Perf', 'Pull', 'Query', @@ -37,12 +34,16 @@ class Mode(object): """Help message has not yet been added for mode.""" __shorthelp__ = 'Shorthelp not defined yet' __longhelp__ = 'Longhelp not defined yet' + __usage__ = None __args__ = [] - def __init__(self, configfile): - self.configfile = configfile + def __init__(self, setup): + self.setup = setup + self.configfile = setup['configfile'] self.__cfp = False self.log = logging.getLogger('Bcfg2.Server.Admin.Mode') + if self.__usage__ is not None: + setup.hm = self.__usage__ def getCFP(self): if not self.__cfp: @@ -59,16 +60,8 @@ class Mode(object): print(emsg) raise SystemExit(1) - def get_repo_path(self): - """Return repository path""" - try: - return self.cfp.get('server', 'repository') - except ConfigParser.NoSectionError: - self.errExit("Unable to find server section in bcfg2.conf") - def load_stats(self, client): - stats = lxml.etree.parse("%s/etc/statistics.xml" % - (self.get_repo_path())) + stats = lxml.etree.parse("%s/etc/statistics.xml" % self.setup['repo']) hostent = stats.xpath('//Node[@name="%s"]' % client) if not hostent: self.errExit("Could not find stats for client %s" % (client)) @@ -111,27 +104,30 @@ class Mode(object): class MetadataCore(Mode): """Base class for admin-modes that handle metadata.""" - def __init__(self, configfile, usage, pwhitelist=None, pblacklist=None): - Mode.__init__(self, configfile) - options = {'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'configfile': Bcfg2.Options.CFILE, - 'encoding': Bcfg2.Options.ENCODING} - setup = Bcfg2.Options.OptionParser(options) - setup.hm = usage - setup.parse(sys.argv[1:]) - if pwhitelist is not None: - setup['plugins'] = [x for x in setup['plugins'] - if x in pwhitelist] - elif pblacklist is not None: - setup['plugins'] = [x for x in setup['plugins'] - if x not in pblacklist] + __plugin_whitelist__ = None + __plugin_blacklist__ = None + + def __init__(self, setup): + Mode.__init__(self, setup) + if self.__plugin_whitelist__ is not None: + setup['plugins'] = [p for p in setup['plugins'] + if p in self.__plugin_whitelist__] + elif self.__plugin_blacklist__ is not None: + setup['plugins'] = [p for p in setup['plugins'] + if p not in self.__plugin_blacklist__] + try: - self.bcore = Bcfg2.Server.Core.Core(self.get_repo_path(), - setup['plugins'], - 'foo', setup['encoding']) + self.bcore = \ + Bcfg2.Server.Core.Core(setup['repo'], + setup['plugins'], + setup['password'], + setup['encoding'], + filemonitor=setup['filemonitor']) + if setup['event debug']: + self.bcore.fam.debug = True except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] - self.errExit("Core load failed because %s" % msg) + self.errExit("Core load failed: %s" % msg) self.bcore.fam.handle_events_in_interval(5) self.metadata = self.bcore.metadata diff --git a/src/lib/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index 05625cf22..4321c060b 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -1,5 +1,4 @@ """Bcfg2.Server.Core provides the runtime support for Bcfg2 modules.""" -__revision__ = '$Revision$' import atexit import logging @@ -7,6 +6,8 @@ import select import sys import threading import time +from traceback import format_exc + try: import lxml.etree except ImportError: @@ -37,6 +38,14 @@ except: pass +def sort_xml(node, key=None): + for child in node: + sort_xml(child, key) + + sorted_children = sorted(node, key=key) + node[:] = sorted_children + + class CoreInitError(Exception): """This error is raised when the core cannot be initialized.""" pass @@ -232,12 +241,12 @@ class Core(Component): self.Bind(entry, metadata) except PluginExecutionError, exc: if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) + entry.set('failure', 'bind error: %s' % format_exc()) logger.error("Failed to bind entry: %s %s" % \ (entry.tag, entry.get('name'))) except Exception, exc: if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) + entry.set('failure', 'bind error: %s' % format_exc()) logger.error("Unexpected failure in BindStructure: %s %s" \ % (entry.tag, entry.get('name')), exc_info=1) @@ -275,7 +284,8 @@ class Core(Component): if len(g2list) == 1: return g2list[0].HandleEntry(entry, metadata) entry.set('failure', 'no matching generator') - raise PluginExecutionError(entry.tag, entry.get('name')) + raise PluginExecutionError("No matching generator: %s:%s" % + (entry.tag, entry.get('name'))) def BuildConfiguration(self, client): """Build configuration for clients.""" @@ -315,6 +325,9 @@ class Core(Component): except: logger.error("error in BindStructure", exc_info=1) self.validate_goals(meta, config) + + sort_xml(config, key=lambda e: e.get('name')) + logger.info("Generated config for %s in %.03f seconds" % \ (client, time.time() - start)) return config @@ -376,9 +389,9 @@ class Core(Component): return lxml.etree.tostring(resp, encoding='UTF-8', xml_declaration=True) except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - warning = 'Client metadata resolution error for %s; check server log' % address[0] + warning = 'Client metadata resolution error for %s' % address[0] self.logger.warning(warning) - raise xmlrpclib.Fault(6, warning) + raise xmlrpclib.Fault(6, warning + "; check server log") except Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError: err_msg = 'Metadata system runtime failure' self.logger.error(err_msg) diff --git a/src/lib/Server/FileMonitor.py b/src/lib/Bcfg2/Server/FileMonitor.py index d6b313e6b..d6b313e6b 100644 --- a/src/lib/Server/FileMonitor.py +++ b/src/lib/Bcfg2/Server/FileMonitor.py diff --git a/src/lib/Server/Hostbase/.gitignore b/src/lib/Bcfg2/Server/Hostbase/.gitignore index 8e15b5395..8e15b5395 100644 --- a/src/lib/Server/Hostbase/.gitignore +++ b/src/lib/Bcfg2/Server/Hostbase/.gitignore diff --git a/src/lib/Server/Hostbase/__init__.py b/src/lib/Bcfg2/Server/Hostbase/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Server/Hostbase/__init__.py +++ b/src/lib/Bcfg2/Server/Hostbase/__init__.py diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Bcfg2/Server/Hostbase/backends.py index bf774f695..ecaf3c109 100644 --- a/src/lib/Server/Hostbase/backends.py +++ b/src/lib/Bcfg2/Server/Hostbase/backends.py @@ -2,8 +2,6 @@ from django.contrib.auth.models import User #from ldapauth import * from nisauth import * -__revision__ = '$Revision$' - ## class LDAPBackend(object): ## def authenticate(self,username=None,password=None): diff --git a/src/lib/Server/Hostbase/hostbase/__init__.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Server/Hostbase/hostbase/__init__.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py diff --git a/src/lib/Server/Hostbase/hostbase/admin.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py index 70a2233cc..70a2233cc 100644 --- a/src/lib/Server/Hostbase/hostbase/admin.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py diff --git a/src/lib/Server/Hostbase/hostbase/models.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/models.py index 3f08a09a0..3f08a09a0 100644 --- a/src/lib/Server/Hostbase/hostbase/models.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/models.py diff --git a/src/lib/Server/Hostbase/hostbase/sql/zone.sql b/src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql index b78187ab2..b78187ab2 100644 --- a/src/lib/Server/Hostbase/hostbase/sql/zone.sql +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql diff --git a/src/lib/Server/Hostbase/hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py index 0ee204abe..0ee204abe 100644 --- a/src/lib/Server/Hostbase/hostbase/urls.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py diff --git a/src/lib/Server/Hostbase/hostbase/views.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/views.py index ff1d4710d..57ef5eff8 100644 --- a/src/lib/Server/Hostbase/hostbase/views.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/views.py @@ -2,8 +2,6 @@ Contains all the views associated with the hostbase app Also has does form validation """ -__revision__ = "$Revision: $" - from django.http import HttpResponse, HttpResponseRedirect from django.contrib.auth.decorators import login_required diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/base.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html index 1d7c5565b..1d7c5565b 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/base.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html index ca8b0cc07..ca8b0cc07 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/copy.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html index 400ef58f2..400ef58f2 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/copy.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/dns.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html index da179e5a1..da179e5a1 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/dns.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html index 18bd7ab83..b1b71ab67 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html @@ -73,7 +73,7 @@ <td> <input name="{{ name.id }}priority" type="text" size="6"> <input name="{{ name.id }}mx" type="text"></td></tr> {% endfor %} - <tr> <td> <b>name</b></td> + <tr> <td> <b>name</b></td> <td> <input name="{{ ip.0.ip_addr }}name" type="text"> <select name="{{ ip.0.ip_addr }}dns_view"> {% for choice in DNS_CHOICES %} @@ -86,7 +86,7 @@ <td> <input name="{{ ip.0.ip_addr }}priority" type="text" size="6"> <input name="{{ ip.0.ip_addr }}mx" type="text"></td></tr> <tr><td></td></tr> - <tr><td><hr></td><td><hr></td></tr> + <tr><td><hr></td><td><hr></td></tr> {% endifequal %} {% endfor %} {% endfor %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/edit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html index 961c9d143..961c9d143 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/edit.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/errors.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html index e5429b86c..e5429b86c 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/errors.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/host.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html index d6b8873bc..d6b8873bc 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/host.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html index 551bf3254..b5d794b50 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html @@ -24,7 +24,7 @@ <col width="150"> <col width="*"> <tr> <td> <b>hostname</b></td> - <td> {{ object.hostname }}</td></tr> + <td> {{ object.hostname }}</td></tr> <tr> <td> <b>whatami</b></td> <td> {{ object.whatami }}</td></tr> <tr> <td> <b>netgroup</b></td> diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html index aa9679cbd..aa9679cbd 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/index.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html index 92258b648..92258b648 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/index.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/login.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html index ec24a0fc0..ec24a0fc0 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/login.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html index 994f631a8..994f631a8 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl index e71e90e76..e71e90e76 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html index 806ccd63d..806ccd63d 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl index 877d427d0..877d427d0 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/new.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html index 2dcd6271f..2dcd6271f 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/new.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/remove.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html index 4329200dd..4329200dd 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/remove.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/results.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html index 45b22058d..45b22058d 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/results.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/search.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html index 409d418fe..409d418fe 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/search.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html index ee355ee87..ee355ee87 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html index b59fa9e3c..b59fa9e3c 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zones.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html index c773e7922..c773e7922 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zones.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html index fa12e3ec5..fa12e3ec5 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Bcfg2/Server/Hostbase/ldapauth.py index f3db26f67..f3db26f67 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Bcfg2/Server/Hostbase/ldapauth.py diff --git a/src/lib/Server/Hostbase/manage.py b/src/lib/Bcfg2/Server/Hostbase/manage.py index 5e78ea979..5e78ea979 100755 --- a/src/lib/Server/Hostbase/manage.py +++ b/src/lib/Bcfg2/Server/Hostbase/manage.py diff --git a/src/lib/Server/Hostbase/media/base.css b/src/lib/Bcfg2/Server/Hostbase/media/base.css index ddbf02165..ddbf02165 100644 --- a/src/lib/Server/Hostbase/media/base.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/base.css diff --git a/src/lib/Server/Hostbase/media/boxypastel.css b/src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css index 7ae0684ef..7ae0684ef 100644 --- a/src/lib/Server/Hostbase/media/boxypastel.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css diff --git a/src/lib/Server/Hostbase/media/global.css b/src/lib/Bcfg2/Server/Hostbase/media/global.css index 73451e1bc..73451e1bc 100644 --- a/src/lib/Server/Hostbase/media/global.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/global.css diff --git a/src/lib/Server/Hostbase/media/layout.css b/src/lib/Bcfg2/Server/Hostbase/media/layout.css index 9085cc220..9085cc220 100644 --- a/src/lib/Server/Hostbase/media/layout.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/layout.css diff --git a/src/lib/Server/Hostbase/nisauth.py b/src/lib/Bcfg2/Server/Hostbase/nisauth.py index 9c7da8c0a..ae4c6c021 100644 --- a/src/lib/Server/Hostbase/nisauth.py +++ b/src/lib/Bcfg2/Server/Hostbase/nisauth.py @@ -1,10 +1,8 @@ +"""Checks with NIS to see if the current user is in the support group""" import os import crypt, nis from Bcfg2.Server.Hostbase.settings import AUTHORIZED_GROUP -"""Checks with NIS to see if the current user is in the support group""" - -__revision__ = "$Revision: $" class NISAUTHError(Exception): """NISAUTHError is raised when somehting goes boom.""" diff --git a/src/lib/Server/Hostbase/regex.py b/src/lib/Bcfg2/Server/Hostbase/regex.py index 41cc0f6f0..41cc0f6f0 100644 --- a/src/lib/Server/Hostbase/regex.py +++ b/src/lib/Bcfg2/Server/Hostbase/regex.py diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Bcfg2/Server/Hostbase/settings.py index 4e641f13c..4e641f13c 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Bcfg2/Server/Hostbase/settings.py diff --git a/src/lib/Server/Hostbase/templates/batchadd.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl index 74ea3c047..74ea3c047 100644 --- a/src/lib/Server/Hostbase/templates/batchadd.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl diff --git a/src/lib/Server/Hostbase/templates/dhcpd.conf.head b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head index a3d19547e..a3d19547e 100644 --- a/src/lib/Server/Hostbase/templates/dhcpd.conf.head +++ b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head diff --git a/src/lib/Server/Hostbase/templates/dhcpd.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl index 757b263cd..757b263cd 100644 --- a/src/lib/Server/Hostbase/templates/dhcpd.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl diff --git a/src/lib/Server/Hostbase/templates/hosts.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl index 251cb5a79..251cb5a79 100644 --- a/src/lib/Server/Hostbase/templates/hosts.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl diff --git a/src/lib/Server/Hostbase/templates/hostsappend.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl index 00e0d5d04..00e0d5d04 100644 --- a/src/lib/Server/Hostbase/templates/hostsappend.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl diff --git a/src/lib/Server/Hostbase/templates/named.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl index 03e054198..03e054198 100644 --- a/src/lib/Server/Hostbase/templates/named.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl diff --git a/src/lib/Server/Hostbase/templates/namedviews.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl index 52021620e..52021620e 100644 --- a/src/lib/Server/Hostbase/templates/namedviews.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl diff --git a/src/lib/Server/Hostbase/templates/reverseappend.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl index 6ed520c98..6ed520c98 100644 --- a/src/lib/Server/Hostbase/templates/reverseappend.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl diff --git a/src/lib/Server/Hostbase/templates/reversesoa.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl index d142eaf7f..d142eaf7f 100644 --- a/src/lib/Server/Hostbase/templates/reversesoa.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl diff --git a/src/lib/Server/Hostbase/templates/zone.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl index aad48d179..aad48d179 100644 --- a/src/lib/Server/Hostbase/templates/zone.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl diff --git a/src/lib/Server/Hostbase/test/harness.py b/src/lib/Bcfg2/Server/Hostbase/test/harness.py index befcff5c0..befcff5c0 100644 --- a/src/lib/Server/Hostbase/test/harness.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/harness.py diff --git a/src/lib/Server/Hostbase/test/test_environ_settings.py b/src/lib/Bcfg2/Server/Hostbase/test/test_environ_settings.py index ad35c624e..ad35c624e 100644 --- a/src/lib/Server/Hostbase/test/test_environ_settings.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/test_environ_settings.py diff --git a/src/lib/Server/Hostbase/test/test_ldapauth.py b/src/lib/Bcfg2/Server/Hostbase/test/test_ldapauth.py index 7fc009ad2..7fc009ad2 100644 --- a/src/lib/Server/Hostbase/test/test_ldapauth.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/test_ldapauth.py diff --git a/src/lib/Server/Hostbase/test/test_settings.py b/src/lib/Bcfg2/Server/Hostbase/test/test_settings.py index 0dfc30f38..0dfc30f38 100644 --- a/src/lib/Server/Hostbase/test/test_settings.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/test_settings.py diff --git a/src/lib/Server/Hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/urls.py index 01fe97d4f..01fe97d4f 100644 --- a/src/lib/Server/Hostbase/urls.py +++ b/src/lib/Bcfg2/Server/Hostbase/urls.py diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Bcfg2/Server/Lint/Bundles.py index 472915cfd..44626b462 100644 --- a/src/lib/Server/Lint/Bundles.py +++ b/src/lib/Bcfg2/Server/Lint/Bundles.py @@ -1,9 +1,8 @@ import lxml.etree import Bcfg2.Server.Lint - + class Bundles(Bcfg2.Server.Lint.ServerPlugin): """ Perform various bundle checks """ - def Run(self): """ run plugin """ if 'Bundler' in self.core.plugins: @@ -15,6 +14,10 @@ class Bundles(Bcfg2.Server.Lint.ServerPlugin): Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): self.bundle_names(bundle) + def Errors(self): + return {"bundle-not-found":"error", + "inconsistent-bundle-name":"warning"} + def missing_bundles(self): """ find bundles listed in Metadata but not implemented in Bundler """ if self.files is None: @@ -41,21 +44,10 @@ class Bundles(Bcfg2.Server.Lint.ServerPlugin): except AttributeError: # genshi template xdata = lxml.etree.parse(bundle.template.filepath).getroot() - + fname = bundle.name.split('Bundler/')[1].split('.')[0] bname = xdata.get('name') if fname != bname: self.LintError("inconsistent-bundle-name", "Inconsistent bundle name: filename is %s, bundle name is %s" % (fname, bname)) - - def sgenshi_groups(self, bundle): - """ ensure that Genshi Bundles do not include <Group> tags, - which are not supported """ - xdata = lxml.etree.parse(bundle.name) - groups = [self.RenderXML(g) - for g in xdata.getroottree().findall("//Group")] - if groups: - self.LintError("group-tag-not-allowed", - "<Group> tag is not allowed in SGenshi Bundle:\n%s" % - "\n".join(groups)) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 19fae1b08..e16469bb5 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -16,6 +16,12 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_infoxml() self.check_probes() + def Errors(self): + return {"unexpanded-keywords":"warning", + "keywords-not-found":"warning", + "comments-not-found":"warning", + "broken-xinclude-chain":"warning"} + def required_keywords(self, rtype): """ given a file type, fetch the list of required VCS keywords from the bcfg2-lint config """ @@ -31,13 +37,13 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): fetch the list of required items from the bcfg2-lint config """ if itype not in self.config_cache: self.config_cache[itype] = {} - + if rtype not in self.config_cache[itype]: rv = [] global_item = "global_%ss" % itype if global_item in self.config: rv.extend(self.config[global_item].split(",")) - + item = "%s_%ss" % (rtype.lower(), itype) if item in self.config: if self.config[item]: @@ -128,7 +134,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): # None == found but not expanded # True == found and expanded found = dict((k, False) for k in self.required_keywords(rtype)) - + for line in lines: # we check for both '$<keyword>:' and '$<keyword>$' to see # if the keyword just hasn't been expanded @@ -155,7 +161,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): # next, check for required comments. found is just # boolean found = dict((k, False) for k in self.required_comments(rtype)) - + for line in lines: for (comment, status) in found.items(): if not status: diff --git a/src/lib/Bcfg2/Server/Lint/Deltas.py b/src/lib/Bcfg2/Server/Lint/Deltas.py new file mode 100644 index 000000000..8d35d8e5a --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/Deltas.py @@ -0,0 +1,23 @@ +import Bcfg2.Server.Lint + +class Deltas(Bcfg2.Server.Lint.ServerPlugin): + """ Warn about usage of .cat and .diff files """ + def Run(self): + """ run plugin """ + if 'Cfg' in self.core.plugins: + cfg = self.core.plugins['Cfg'] + for basename, entry in list(cfg.entries.items()): + self.check_entry(basename, entry) + + def Errors(self): + return {"cat-file-used":"warning", + "diff-file-used":"warning"} + + def check_entry(self, basename, entry): + for fname in list(entry.entries.keys()): + if self.HandlesFile(fname): + match = entry.specific.delta_reg.match(fname) + if match: + self.LintError("%s-file-used" % match.group('delta'), + "%s file used on %s: %s" % + (match.group('delta'), basename, fname)) diff --git a/src/lib/Server/Lint/Duplicates.py b/src/lib/Bcfg2/Server/Lint/Duplicates.py index 75f620603..abc581c4f 100644 --- a/src/lib/Server/Lint/Duplicates.py +++ b/src/lib/Bcfg2/Server/Lint/Duplicates.py @@ -22,6 +22,13 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): if self.clients_xdata is not None: self.duplicate_clients() + def Errors(self): + return {"broken-xinclude-chain":"warning", + "duplicate-client":"error", + "duplicate-group":"error", + "duplicate-package":"error", + "multiple-default-groups":"error"} + def load_xdata(self): """ attempt to load XML data for groups and clients. only actually load data if all documents reference in XIncludes can @@ -30,7 +37,7 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): self.groups_xdata = self.metadata.clients_xml.xdata if self.has_all_xincludes("clients.xml"): self.clients_xdata = self.metadata.clients_xml.xdata - + def duplicate_groups(self): """ find duplicate groups """ self.duplicate_entries(self.clients_xdata.xpath('//Groups/Group'), diff --git a/src/lib/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index 552c495b2..45ddf8f2f 100755 --- a/src/lib/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,9 +1,8 @@ import genshi.template import Bcfg2.Server.Lint - + class Genshi(Bcfg2.Server.Lint.ServerPlugin): """ Check Genshi templates for syntax errors """ - def Run(self): """ run plugin """ loader = genshi.template.TemplateLoader() @@ -12,16 +11,21 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): self.check_files(self.core.plugins[plugin].entries, loader=loader) + def Errors(self): + return {"genshi-syntax-error":"error"} + def check_files(self, entries, loader=None): if loader is None: loader = genshi.template.TemplateLoader() - + for eset in entries.values(): for fname, sdata in list(eset.entries.items()): - if fname.endswith(".genshi") or fname.endswith(".newtxt"): + if (self.HandlesFile(fname) and + (fname.endswith(".genshi") or fname.endswith(".newtxt"))): try: loader.load(sdata.name, cls=genshi.template.NewTextTemplate) - except genshi.template.TemplateSyntaxError, err: + except genshi.template.TemplateSyntaxError: + err = sys.exc_info()[1] self.LintError("genshi-syntax-error", "Genshi syntax error: %s" % err) diff --git a/src/lib/Bcfg2/Server/Lint/GroupPatterns.py b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py new file mode 100644 index 000000000..f50118ce1 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/GroupPatterns.py @@ -0,0 +1,34 @@ +import sys +import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.GroupPatterns import PatternMap + +class GroupPatterns(Bcfg2.Server.Lint.ServerPlugin): + """ Check Genshi templates for syntax errors """ + + def Run(self): + """ run plugin """ + if 'GroupPatterns' in self.core.plugins: + cfg = self.core.plugins['GroupPatterns'].config + for entry in cfg.xdata.xpath('//GroupPattern'): + groups = [g.text for g in entry.findall('Group')] + self.check(entry, groups, ptype='NamePattern') + self.check(entry, groups, ptype='NameRange') + + def Errors(self): + return {"pattern-fails-to-initialize":"error"} + + def check(self, entry, groups, ptype="NamePattern"): + if ptype == "NamePattern": + pmap = lambda p: PatternMap(p, None, groups) + else: + pmap = lambda p: PatternMap(None, p, groups) + + for el in entry.findall(ptype): + pat = el.text + try: + pmap(pat) + except: + err = sys.exc_info()[1] + self.LintError("pattern-fails-to-initialize", + "Failed to initialize %s %s for %s: %s" % + (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 2054e23bf..20c218e6f 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -4,7 +4,6 @@ import Bcfg2.Server.Lint class InfoXML(Bcfg2.Server.Lint.ServerPlugin): """ ensure that all config files have an info.xml file""" - def Run(self): if 'Cfg' in self.core.plugins: for filename, entryset in self.core.plugins['Cfg'].entries.items(): @@ -18,6 +17,12 @@ class InfoXML(Bcfg2.Server.Lint.ServerPlugin): self.LintError("no-infoxml", "No info.xml found for %s" % filename) + def Errors(self): + return {"no-infoxml":"warning", + "paranoid-false":"warning", + "broken-xinclude-chain":"warning", + "required-infoxml-attrs-missing":"error"} + def check_infoxml(self, fname, xdata): for info in xdata.getroottree().findall("//Info"): required = [] diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index 52fea3d9b..3259aca44 100644 --- a/src/lib/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -1,18 +1,22 @@ import os -from copy import deepcopy +import copy from difflib import SequenceMatcher import Bcfg2.Server.Lint class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ find Probes or Cfg files with multiple similar files that might be merged into one """ - def Run(self): if 'Cfg' in self.core.plugins: self.check_cfg() if 'Probes' in self.core.plugins: self.check_probes() + def Errors(self): + return {"merge-cfg":"warning", + "merge-probes":"warning"} + + def check_cfg(self): for filename, entryset in self.core.plugins['Cfg'].entries.items(): for mset in self.get_similar(entryset.entries): @@ -26,7 +30,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): def check_probes(self): probes = self.core.plugins['Probes'].probes.entries for mset in self.get_similar(probes): - self.LintError("merge-cfg", + self.LintError("merge-probes", "The following probes are similar: %s. " "Consider merging them into a single probe." % ", ".join([p for p in mset])) @@ -43,7 +47,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): rv = [] elist = entries.items() while elist: - result = self._find_similar(elist.pop(0), deepcopy(elist), + result = self._find_similar(elist.pop(0), copy.copy(elist), threshold) if len(result) > 1: elist = [(fname, fdata) @@ -62,8 +66,8 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): if (sm.real_quick_ratio() > threshold and sm.quick_ratio() > threshold and sm.ratio() > threshold): - rv.extend(self._find_similar((cname, cdata), deepcopy(others), + rv.extend(self._find_similar((cname, cdata), copy.copy(others), threshold)) return rv - + diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py index 3960a8cf9..7c17d555a 100644 --- a/src/lib/Server/Lint/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Lint/Pkgmgr.py @@ -1,21 +1,18 @@ +import glob +import lxml.etree import Bcfg2.Server.Lint -class Pkgmgr(Bcfg2.Server.Lint.ServerPlugin): +class Pkgmgr(Bcfg2.Server.Lint.ServerlessPlugin): """ find duplicate Pkgmgr entries with the same priority """ - def Run(self): - if 'Pkgmgr' not in self.core.plugins: - self.logger.info("Pkgmgr server plugin is not enabled, skipping Pkgmgr lint checks") - return - pset = set() - for plist in self.core.plugins['Pkgmgr'].entries.values(): - if self.HandlesFile(plist.name): - xdata = plist.data + for pfile in glob.glob("%s/Pkgmgr/*.xml" % self.config['repo']): + if self.HandlesFile(pfile): + xdata = lxml.etree.parse(pfile).getroot() # get priority, type, group - priority = xdata.getroot().get('priority') - ptype = xdata.getroot().get('type') - for pkg in xdata.findall("//Package"): + priority = xdata.get('priority') + ptype = xdata.get('type') + for pkg in xdata.xpath("//Package"): if pkg.getparent().tag == 'Group': grp = pkg.getparent().get('name') if (type(grp) is not str and @@ -35,3 +32,6 @@ class Pkgmgr(Bcfg2.Server.Lint.ServerPlugin): (pkg.get('name'), priority, ptype)) else: pset.add(ptuple) + + def Errors(self): + return {"duplicate-packages":"error"} diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 55206d2ba..20947d50f 100644 --- a/src/lib/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -6,7 +6,6 @@ from Bcfg2.Server.Plugins.Packages import Apt, Yum class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): """ verify attributes for configuration entries (as defined in doc/server/configurationentries) """ - def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = { @@ -38,6 +37,13 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): self.check_rules() self.check_bundles() + def Errors(self): + return {"unknown-entry-type":"error", + "unknown-entry-tag":"error", + "required-attrs-missing":"error", + "extra-attrs":"warning"} + + def check_packages(self): """ check package sources for Source entries with missing attrs """ if 'Packages' in self.core.plugins: diff --git a/src/lib/Bcfg2/Server/Lint/TemplateHelper.py b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py new file mode 100644 index 000000000..f5f916e88 --- /dev/null +++ b/src/lib/Bcfg2/Server/Lint/TemplateHelper.py @@ -0,0 +1,63 @@ +import sys +import imp +import glob +import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.TemplateHelper import HelperModule + +class TemplateHelper(Bcfg2.Server.Lint.ServerlessPlugin): + """ find duplicate Pkgmgr entries with the same priority """ + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + hm = HelperModule("foo.py", None, None) + self.reserved_keywords = dir(hm) + + def Run(self): + for helper in glob.glob("%s/TemplateHelper/*.py" % self.config['repo']): + if not self.HandlesFile(helper): + continue + + match = HelperModule._module_name_re.search(helper) + if match: + module_name = match.group(1) + else: + module_name = helper + + try: + module = imp.load_source(module_name, helper) + except: + err = sys.exc_info()[1] + self.LintError("templatehelper-import-error", + "Failed to import %s: %s" % + (helper, err)) + continue + + if not hasattr(module, "__export__"): + self.LintError("templatehelper-no-export", + "%s has no __export__ list" % helper) + continue + elif not isinstance(module.__export__, list): + self.LintError("templatehelper-nonlist-export", + "__export__ is not a list in %s" % helper) + continue + + for sym in module.__export__: + if not hasattr(module, sym): + self.LintError("templatehelper-nonexistent-export", + "%s: exported symbol %s does not exist" % + (helper, sym)) + elif sym in self.reserved_keywords: + self.LintError("templatehelper-reserved-export", + "%s: exported symbol %s is reserved" % + (helper, sym)) + elif sym.startswith("_"): + self.LintError("templatehelper-underscore-export", + "%s: exported symbol %s starts with underscore" % + (helper, sym)) + + def Errors(self): + return {"templatehelper-import-error":"error", + "templatehelper-no-export":"error", + "templatehelper-nonlist-export":"error", + "templatehelper-nonexistent-export":"error", + "templatehelper-reserved-export":"error", + "templatehelper-underscore-export":"warning"} diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 952a65365..812ab3d00 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -36,7 +36,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): def Run(self): schemadir = self.config['schema'] - + for path, schemaname in self.filesets.items(): try: filelist = self.filelists[path] @@ -63,6 +63,15 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.check_properties() + def Errors(self): + return {"broken-xinclude-chain":"warning", + "schema-failed-to-parse":"warning", + "properties-schema-not-found":"warning", + "xml-failed-to-parse":"error", + "xml-failed-to-read":"error", + "xml-failed-to-verify":"error", + "input-output-error":"error"} + def check_properties(self): """ check Properties files against their schemas """ for filename in self.filelists['props']: @@ -98,7 +107,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.LintError("xml-failed-to-read", "Failed to open file %s" % filename) return False - + if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: @@ -180,7 +189,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): included = set([ent.get('href') for ent in xdata.findall('./{http://www.w3.org/2001/XInclude}include')]) rv = [] - + while included: try: filename = included.pop() diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index f47059ac4..f3991e3fc 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -1,5 +1,3 @@ -__revision__ = '$Revision$' - __all__ = ['Bundles', 'Comments', 'Duplicates', @@ -8,7 +6,8 @@ __all__ = ['Bundles', 'Pkgmgr', 'RequiredAttrs', 'Validate', - 'Genshi'] + 'Genshi', + 'Deltas'] import logging import os @@ -55,14 +54,19 @@ class Plugin (object): self.config = config self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: - self.errorHandler = ErrorHandler() + self.errorhandler = ErrorHandler() else: - self.errorHandler = errorhandler + self.errorhandler = errorhandler + self.errorhandler.RegisterErrors(self.Errors()) def Run(self): """ run the plugin. must be overloaded by child classes """ pass + def Errors(self): + """ returns a dict of errors the plugin supplies. must be + overloaded by child classes """ + def HandlesFile(self, fname): """ returns true if the given file should be handled by the plugin according to the files list, false otherwise """ @@ -74,8 +78,8 @@ class Plugin (object): fname)) in self.files) def LintError(self, err, msg): - self.errorHandler.dispatch(err, msg) - + self.errorhandler.dispatch(err, msg) + def RenderXML(self, element): """render an XML element for error output -- line number prefixed, no children""" @@ -92,34 +96,6 @@ class Plugin (object): class ErrorHandler (object): - # how to handle different errors by default - _errors = {"no-infoxml":"warning", - "paranoid-false":"warning", - "bundle-not-found":"error", - "inconsistent-bundle-name":"warning", - "group-tag-not-allowed":"error", - "unexpanded-keywords":"warning", - "keywords-not-found":"warning", - "comments-not-found":"warning", - "broken-xinclude-chain":"warning", - "duplicate-client":"error", - "duplicate-group":"error", - "duplicate-package":"error", - "multiple-default-groups":"error", - "required-infoxml-attrs-missing":"error", - "unknown-entry-type":"error", - "required-attrs-missing":"error", - "extra-attrs":"warning", - "schema-failed-to-parse":"warning", - "properties-schema-not-found":"warning", - "xml-failed-to-parse":"error", - "xml-failed-to-read":"error", - "xml-failed-to-verify":"error", - "merge-cfg":"warning", - "merge-probes":"warning", - "input-output-error": "error", - "genshi-syntax-error": "error"} - def __init__(self, config=None): self.errors = 0 self.warnings = 0 @@ -128,11 +104,12 @@ class ErrorHandler (object): termsize = get_termsize() if termsize is not None: - self._wrapper = textwrap.TextWrapper(initial_indent=" ", - subsequent_indent=" ", - width=termsize[0]) + twrap = textwrap.TextWrapper(initial_indent=" ", + subsequent_indent=" ", + width=termsize[0]) + self._wrapper = twrap.wrap else: - self._wrapper = None + self._wrapper = lambda s: [s] self._handlers = {} if config is not None: @@ -144,7 +121,8 @@ class ErrorHandler (object): else: self._handlers[err] = self.debug - for err, action in self._errors.items(): + def RegisterErrors(self, errors): + for err, action in errors.items(): if err not in self._handlers: if "warn" in action: self._handlers[err] = self.warn @@ -152,6 +130,7 @@ class ErrorHandler (object): self._handlers[err] = self.error else: self._handlers[err] = self.debug + def dispatch(self, err, msg): if err in self._handlers: @@ -187,13 +166,10 @@ class ErrorHandler (object): rawlines = msg.splitlines() firstline = True for rawline in rawlines: - if self._wrapper: - lines = self._wrapper.wrap(rawline) - else: - lines = [rawline] + lines = self._wrapper(rawline) for line in lines: if firstline: - logfunc("%s%s" % (prefix, line.lstrip())) + logfunc(prefix + line.lstrip()) firstline = False else: logfunc(line) diff --git a/src/lib/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index 3dc1cab38..11e6c5c20 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -1,5 +1,4 @@ """This module provides the baseclass for Bcfg2 Server Plugins.""" -__revision__ = '$Revision$' import copy import logging @@ -11,6 +10,7 @@ import posixpath import re import sys import threading +from Bcfg2.Bcfg2Py3k import ConfigParser from lxml.etree import XML, XMLSyntaxError @@ -62,11 +62,28 @@ class PluginExecutionError(Exception): pass -class Plugin(object): +class Debuggable(object): + __rmi__ = ['toggle_debug'] + + def __init__(self, name=None): + if name is None: + name = "%s.%s" % (self.__class__.__module__, + self.__class__.__name__) + self.debug_flag = False + self.logger = logging.getLogger(name) + + def toggle_debug(self): + self.debug_flag = not self.debug_flag + + def debug_log(self, message, flag=None): + if (flag is None and self.debug_flag) or flag: + self.logger.error(message) + + +class Plugin(Debuggable): """This is the base class for all Bcfg2 Server plugins. Several attributes must be defined in the subclass: name : the name of the plugin - __version__ : a version string __author__ : the author/contact for the plugin Plugins can provide three basic types of functionality: @@ -75,9 +92,7 @@ class Plugin(object): - Data collection (overloading GetProbes/ReceiveData) """ name = 'Plugin' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' - __rmi__ = ['toggle_debug'] experimental = False deprecated = False conflicts = [] @@ -89,24 +104,16 @@ class Plugin(object): def __init__(self, core, datastore): """Initialize the plugin. - + :param core: the Bcfg2.Server.Core initializing the plugin :param datastore: the filesystem path of Bcfg2's repository """ object.__init__(self) self.Entries = {} self.core = core - self.data = "%s/%s" % (datastore, self.name) - self.logger = logging.getLogger('Bcfg2.Plugins.%s' % (self.name)) + self.data = os.path.join(datastore, self.name) self.running = True - self.debug_flag = False - - def toggle_debug(self): - self.debug_flag = not self.debug_flag - - def debug_log(self, message, flag=None): - if (flag is None) and self.debug_flag or flag: - self.logger.error(message) + Debuggable.__init__(self, name=self.name) @classmethod def init_repo(cls, repo): @@ -282,7 +289,7 @@ class ThreadedStatistics(Statistics, def process_statistics(self, metadata, data): warned = False try: - self.work_queue.put_nowait((metadata, copy.deepcopy(data))) + self.work_queue.put_nowait((metadata, copy.copy(data))) warned = False except Full: if not warned: @@ -372,6 +379,12 @@ class FileBacked(object): """Update local data structures based on current file state""" pass + def __repr__(self): + return "%s: %s" % (self.__class__.__name__, str(self)) + + def __str__(self): + return "%s: %s" % (self.name, self.data) + class DirectoryBacked(object): """This object is a coherent cache for a filesystem hierarchy of files.""" @@ -385,7 +398,7 @@ class DirectoryBacked(object): :param data: the path to the data directory that will be monitored. :param fam: The FileMonitor object used to receive - notifications of changes. + notifications of changes. """ object.__init__(self) @@ -443,7 +456,7 @@ class DirectoryBacked(object): def HandleEvent(self, event): """Handle FAM/Gamin events. - + This method is invoked by FAM/Gamin when it detects a change to a filesystem object we have requsted to be monitored. @@ -567,6 +580,9 @@ class XMLFileBacked(FileBacked): def __iter__(self): return iter(self.entries) + def __str__(self): + return "%s: %s" % (self.name, lxml.etree.tostring(self.xdata)) + class SingleXMLFileBacked(XMLFileBacked): """This object is a coherent cache for an independent XML file.""" @@ -576,6 +592,29 @@ class SingleXMLFileBacked(XMLFileBacked): self.fam = fam self.fam.AddMonitor(filename, self) + def _follow_xincludes(self, fname=None, xdata=None): + ''' follow xincludes, adding included files to fam and to + self.extras ''' + if xdata is None: + if fname is None: + xdata = self.xdata.getroottree() + else: + xdata = lxml.etree.parse(fname) + included = [ent.get('href') + for ent in xdata.findall('//{http://www.w3.org/2001/XInclude}include')] + for name in included: + if name not in self.extras: + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(self.name), name) + self.add_monitor(fpath, name) + self._follow_xincludes(fname=fpath) + + def add_monitor(self, fpath, fname): + self.fam.AddMonitor(fpath, self) + self.extras.append(fname) + def Index(self): """Build local data structures.""" try: @@ -585,22 +624,14 @@ class SingleXMLFileBacked(XMLFileBacked): logger.error("Failed to parse %s: %s" % (self.name, err)) raise Bcfg2.Server.Plugin.PluginInitError - included = [ent.get('href') - for ent in self.xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.fam.AddMonitor(os.path.join(os.path.dirname(self.name), - name), - self) - self.extras.append(name) + self._follow_xincludes() + if self.extras: try: self.xdata.getroottree().xinclude() except lxml.etree.XIncludeError: err = sys.exc_info()[1] logger.error("XInclude failed on %s: %s" % (self.name, err)) - self.entries = self.xdata.getchildren() if self.__identifier__ is not None: self.label = self.xdata.attrib[self.__identifier__] @@ -609,7 +640,7 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" __identifier__ = None - + def __init__(self, name): XMLFileBacked.__init__(self, name) @@ -636,13 +667,13 @@ class StructFile(XMLFileBacked): rv.extend(self._match(child, metadata)) return rv else: - rv = copy.deepcopy(item) + rv = copy.copy(item) for child in rv.iterchildren(): rv.remove(child) for child in item.iterchildren(): rv.extend(self._match(child, metadata)) return [rv] - + def Match(self, metadata): """Return matching fragments of independent.""" rv = [] @@ -770,6 +801,9 @@ class XMLSrc(XMLFileBacked): self.pnode.Match(metadata, cache[1]) self.cache = cache + def __str__(self): + return str(self.items) + class InfoXML (XMLSrc): __node__ = InfoNode @@ -813,21 +847,19 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): attrs = self.get_attrs(entry, metadata) for key, val in list(attrs.items()): entry.attrib[key] = val - + def get_attrs(self, entry, metadata): """ get a list of attributes to add to the entry during the bind """ - for src in list(self.entries.values()): - cached = src.Cache(metadata) - if cached == False: - self.logger.error("Called before data loaded") - raise PluginExecutionError + for src in self.entries.values(): + src.Cache(metadata) + matching = [src for src in list(self.entries.values()) if (src.cache and entry.tag in src.cache[1] and self._matches(entry, metadata, src.cache[1][entry.tag]))] if len(matching) == 0: - raise PluginExecutionError + raise PluginExecutionError('No matching source for entry when retrieving attributes for %s(%s)' % (entry.tag, entry.attrib.get('name'))) elif len(matching) == 1: index = 0 else: @@ -849,7 +881,7 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): if '__text__' in data: entry.text = data['__text__'] if '__children__' in data: - [entry.append(copy.deepcopy(item)) for item in data['__children__']] + [entry.append(copy.copy(item)) for item in data['__children__']] return dict([(key, data[key]) for key in list(data.keys()) @@ -857,7 +889,6 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): # new unified EntrySet backend - class SpecificityError(Exception): """Thrown in case of filename parse failure.""" pass @@ -919,7 +950,7 @@ class SpecificData(object): logger.error("Failed to read file %s" % self.name) -class EntrySet: +class EntrySet(object): """Entry sets deal with the host- and group-specific entries.""" ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$") @@ -973,6 +1004,12 @@ class EntrySet: self.entry_init(event) else: if event.filename not in self.entries: + logger.warning("Got %s event for unknown file %s" % + (action, event.filename)) + if action == 'changed': + # received a bogus changed event; warn, but treat + # it like a created event + self.entry_init(event) return if action == 'changed': self.entries[event.filename].handle_event(event) @@ -989,7 +1026,8 @@ class EntrySet: spec = self.specificity_from_filename(event.filename) except SpecificityError: if not self.ignore.match(event.filename): - logger.error("Could not process filename %s; ignoring" % fpath) + logger.error("Could not process filename %s; ignoring" % + fpath) return self.entries[event.filename] = self.entry_type(fpath, spec, self.encoding) @@ -1069,7 +1107,6 @@ class EntrySet: class GroupSpool(Plugin, Generator): """Unified interface for handling group-specific data (e.g. .G## files).""" name = 'GroupSpool' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' filename_pattern = "" es_child_cls = object @@ -1086,41 +1123,63 @@ class GroupSpool(Plugin, Generator): self.AddDirectoryMonitor('') self.encoding = core.encoding + def add_entry(self, event): + epath = self.event_path(event) + ident = self.event_id(event) + if posixpath.isdir(epath): + self.AddDirectoryMonitor(epath[len(self.data):]) + if ident not in self.entries and posixpath.isfile(epath): + dirpath = "".join([self.data, ident]) + self.entries[ident] = self.es_cls(self.filename_pattern, + dirpath, + self.es_child_cls, + self.encoding) + self.Entries['Path'][ident] = self.entries[ident].bind_entry + if not posixpath.isdir(epath): + # do not pass through directory events + self.entries[ident].handle_event(event) + + def event_path(self, event): + return "".join([self.data, self.handles[event.requestID], + event.filename]) + + def event_id(self, event): + epath = self.event_path(event) + if posixpath.isdir(epath): + return self.handles[event.requestID] + event.filename + else: + return self.handles[event.requestID][:-1] + def HandleEvent(self, event): - """Unified FAM event handler for DirShadow.""" + """Unified FAM event handler for GroupSpool.""" action = event.code2str() if event.filename[0] == '/': return - epath = "".join([self.data, self.handles[event.requestID], - event.filename]) - if posixpath.isdir(epath): - ident = self.handles[event.requestID] + event.filename - else: - ident = self.handles[event.requestID][:-1] + ident = self.event_id(event) if action in ['exists', 'created']: - if posixpath.isdir(epath): - self.AddDirectoryMonitor(epath[len(self.data):]) - if ident not in self.entries and posixpath.isfile(epath): - dirpath = "".join([self.data, ident]) - self.entries[ident] = self.es_cls(self.filename_pattern, - dirpath, - self.es_child_cls, - self.encoding) - self.Entries['Path'][ident] = self.entries[ident].bind_entry - if not posixpath.isdir(epath): - # do not pass through directory events + self.add_entry(event) + if action == 'changed': + if ident in self.entries: self.entries[ident].handle_event(event) - if action == 'changed' and ident in self.entries: - self.entries[ident].handle_event(event) + else: + # got a changed event for a file we didn't know + # about. go ahead and process this as a 'created', but + # warn + self.logger.warning("Got changed event for unknown file %s" % + ident) + self.add_entry(event) elif action == 'deleted': fbase = self.handles[event.requestID] + event.filename if fbase in self.entries: # a directory was deleted del self.entries[fbase] del self.Entries['Path'][fbase] - else: + elif ident in self.entries: self.entries[ident].handle_event(event) + elif ident not in self.entries: + self.logger.warning("Got deleted event for unknown file %s" % + ident) def AddDirectoryMonitor(self, relative): """Add new directory to FAM structures.""" @@ -1133,3 +1192,56 @@ class GroupSpool(Plugin, Generator): return reqid = self.core.fam.AddMonitor(name, self) self.handles[reqid] = relative + +class SimpleConfig(FileBacked, + ConfigParser.SafeConfigParser): + ''' a simple plugin config using ConfigParser ''' + _required = True + + def __init__(self, plugin): + filename = os.path.join(plugin.data, plugin.name.lower() + ".conf") + self.plugin = plugin + self.fam = self.plugin.core.fam + Bcfg2.Server.Plugin.FileBacked.__init__(self, filename) + ConfigParser.SafeConfigParser.__init__(self) + + if (self._required or + (not self._required and os.path.exists(self.name))): + self.fam.AddMonitor(self.name, self) + + def Index(self): + """ Build local data structures """ + for section in self.sections(): + self.remove_section(section) + self.read(self.name) + + def get(self, section, option, **kwargs): + """ convenience method for getting config items """ + default = None + if 'default' in kwargs: + default = kwargs['default'] + del kwargs['default'] + try: + return ConfigParser.SafeConfigParser.get(self, section, option, + **kwargs) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + if default is not None: + return default + else: + raise + + def getboolean(self, section, option, **kwargs): + """ convenience method for getting boolean config items """ + default = None + if 'default' in kwargs: + default = kwargs['default'] + del kwargs['default'] + try: + return ConfigParser.SafeConfigParser.getboolean(self, section, + option, **kwargs) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, + ValueError): + if default is not None: + return default + else: + raise diff --git a/src/lib/Server/Plugins/Account.py b/src/lib/Bcfg2/Server/Plugins/Account.py index f67819b9d..f2703dccb 100644 --- a/src/lib/Server/Plugins/Account.py +++ b/src/lib/Bcfg2/Server/Plugins/Account.py @@ -1,5 +1,4 @@ """This handles authentication setup.""" -__revision__ = '$Revision$' import Bcfg2.Server.Plugin @@ -16,7 +15,6 @@ class Account(Bcfg2.Server.Plugin.Plugin, """ name = 'Account' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/BB.py b/src/lib/Bcfg2/Server/Plugins/BB.py index 137142b66..c015ec47c 100644 --- a/src/lib/Server/Plugins/BB.py +++ b/src/lib/Bcfg2/Server/Plugins/BB.py @@ -62,7 +62,6 @@ class BB(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): """The BB plugin maps users to machines and metadata to machines.""" name = 'BB' - version = '$Revision$' deprecated = True def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py index 5de57a87c..389ca7a95 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Bcfg2/Server/Plugins/Base.py @@ -1,5 +1,4 @@ """This module sets up a base list of configuration entries.""" -__revision__ = '$Revision$' import copy import lxml.etree @@ -18,7 +17,6 @@ class Base(Bcfg2.Server.Plugin.Plugin, needed for most actual systems. """ name = 'Base' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = Bcfg2.Server.Plugin.StructFile deprecated = True @@ -41,5 +39,5 @@ class Base(Bcfg2.Server.Plugin.Plugin, fragments = reduce(lambda x, y: x + y, [base.Match(metadata) for base in list(self.entries.values())], []) - [ret.append(copy.deepcopy(frag)) for frag in fragments] + [ret.append(copy.copy(frag)) for frag in fragments] return [ret] diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index 4b73a17d4..ccb99481e 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -1,5 +1,4 @@ """This provides bundle clauses with translation functionality.""" -__revision__ = '$Revision$' import copy import lxml.etree @@ -24,7 +23,7 @@ class BundleFile(Bcfg2.Server.Plugin.StructFile): def get_xml_value(self, metadata): bundlename = os.path.splitext(os.path.basename(self.name))[0] bundle = lxml.etree.Element('Bundle', name=bundlename) - [bundle.append(copy.deepcopy(item)) for item in self.Match(metadata)] + [bundle.append(copy.copy(item)) for item in self.Match(metadata)] return bundle @@ -35,7 +34,6 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, bundle/translation scheme from Bcfg1. """ name = 'Bundler' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$') @@ -75,23 +73,27 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, def BuildStructures(self, metadata): """Build all structures for client (metadata).""" bundleset = [] + + bundle_entries = {} + for key, item in self.entries.items(): + bundle_entries.setdefault(self.patterns.match(os.path.basename(key)).group('name'), + []).append(item) + for bundlename in metadata.bundles: - entries = [item for (key, item) in self.entries.items() if \ - self.patterns.match(os.path.basename(key)).group('name') == bundlename] - if len(entries) == 0: + try: + entries = bundle_entries[bundlename] + except KeyError: + self.logger.error("Bundler: Bundle %s does not exist" % + bundlename) continue - elif len(entries) == 1: - try: - bundleset.append(entries[0].get_xml_value(metadata)) - except genshi.template.base.TemplateError: - t = sys.exc_info()[1] - self.logger.error("Bundler: Failed to template genshi bundle %s" \ - % (bundlename)) - self.logger.error(t) - except: - self.logger.error("Bundler: Unexpected bundler error for %s" \ - % (bundlename), exc_info=1) - else: - self.logger.error("Got multiple matches for bundle %s" \ - % (bundlename)) + try: + bundleset.append(entries[0].get_xml_value(metadata)) + except genshi.template.base.TemplateError: + t = sys.exc_info()[1] + self.logger.error("Bundler: Failed to template genshi bundle %s" + % bundlename) + self.logger.error(t) + except: + self.logger.error("Bundler: Unexpected bundler error for %s" % + bundlename, exc_info=1) return bundleset diff --git a/src/lib/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index a9a5eb814..a71021cb5 100644 --- a/src/lib/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -10,7 +10,6 @@ class Bzr(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Bzr is a version plugin for dealing with Bcfg2 repos.""" name = 'Bzr' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Bcfg2/Server/Plugins/Cfg.py index 0a791f171..81904d082 100644 --- a/src/lib/Server/Plugins/Cfg.py +++ b/src/lib/Bcfg2/Server/Plugins/Cfg.py @@ -1,5 +1,4 @@ """This module implements a config file repository.""" -__revision__ = '$Revision$' import binascii import logging @@ -31,6 +30,7 @@ try: except: have_cheetah = False +# setup logging logger = logging.getLogger('Bcfg2.Plugins.Cfg') @@ -63,7 +63,7 @@ def process_delta(data, delta): basefile.write(data) basefile.close() os.close(basehandle) - + cmd = ["patch", "-u", "-f", basefile.name] patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) stderr = patch.communicate(input=delta.data)[1] @@ -100,6 +100,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): self.specific = CfgMatcher(path.split('/')[-1]) path = path + def debug_log(self, message, flag=None): + if (flag is None and self.debug_flag) or flag: + logger.error(message) + def sort_by_specific(self, one, other): return cmp(one.specific, other.specific) @@ -115,8 +119,9 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): base_files = [matching.index(m) for m in matching if not m.specific.delta] if not base_files: - logger.error("No base file found for %s" % entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "No base file found for %s" % entry.get('name') + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) base = min(base_files) used = matching[:base + 1] used.reverse() @@ -128,15 +133,16 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): basefile = used.pop(0) if entry.get('perms').lower() == 'inherit': # use on-disk permissions - fname = "%s/%s" % (self.path, entry.get('name')) + fname = os.path.join(self.path, entry.get('name')) entry.set('perms', str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) if entry.tag == 'Path': entry.set('type', 'file') if basefile.name.endswith(".genshi"): if not have_genshi: - logger.error("Cfg: Genshi is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Genshi is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: template_cls = NewTextTemplate loader = TemplateLoader() @@ -154,13 +160,15 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if data == '': entry.set('empty', 'true') except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: genshi exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: genshi exception (%s): %s" % (entry.get("name"), + sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) elif basefile.name.endswith(".cheetah"): if not have_cheetah: - logger.error("Cfg: Cheetah is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Cheetah is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: fname = entry.get('realname', entry.get('name')) s = {'useStackFrames': False} @@ -173,9 +181,10 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if data == '': entry.set('empty', 'true') except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: cheetah exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: cheetah exception (%s): %s" % (entry.get("name"), + sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) else: data = basefile.data for delta in used: @@ -186,17 +195,18 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): try: entry.text = u_str(data, self.encoding) except UnicodeDecodeError: - e = sys.exc_info()[1] - logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) + msg = "Failed to decode %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) except ValueError: - e = sys.exc_info()[1] - logger.error("Error in specification for %s" % entry.get('name')) - logger.error("%s" % e) + msg = "Error in specification for %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) logger.error("You need to specify base64 encoding for %s." % entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) if entry.text in ['', None]: entry.set('empty', 'true') @@ -223,22 +233,34 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): if 'text' in new_entry: name = self.build_filename(specific) if os.path.exists("%s.genshi" % name): - logger.error("Cfg: Unable to pull data for genshi types") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Unable to pull data for genshi types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) elif os.path.exists("%s.cheetah" % name): - logger.error("Cfg: Unable to pull data for cheetah types") - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Unable to pull data for cheetah types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) try: etext = new_entry['text'].encode(self.encoding) except: - logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) - raise Bcfg2.Server.Plugin.PluginExecutionError + msg = "Cfg: Cannot encode content of %s as %s" % (name, + self.encoding) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) open(name, 'w').write(etext) - if log: - logger.info("Wrote file %s" % name) + self.debug_log("Wrote file %s" % name, flag=log) badattr = [attr for attr in ['owner', 'group', 'perms'] if attr in new_entry] if badattr: + # check for info files and inform user of their removal + if os.path.exists(self.path + "/:info"): + logger.info("Removing :info file and replacing with " + "info.xml") + os.remove(self.path + "/:info") + if os.path.exists(self.path + "/info"): + logger.info("Removing info file and replacing with " + "info.xml") + os.remove(self.path + "/info") metadata_updates = {} metadata_updates.update(self.metadata) for attr in badattr: @@ -250,15 +272,14 @@ class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): ofile = open(self.path + "/info.xml", "w") ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) ofile.close() - if log: - logger.info("Wrote file %s" % (self.path + "/info.xml")) + self.debug_log("Wrote file %s" % (self.path + "/info.xml"), + flag=log) class Cfg(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.PullTarget): """This generator in the configuration file repository for Bcfg2.""" name = 'Cfg' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' es_cls = CfgEntrySet es_child_cls = Bcfg2.Server.Plugin.SpecificData diff --git a/src/lib/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index ea898c023..6ce72acd2 100644 --- a/src/lib/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -10,7 +10,6 @@ class Cvs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """CVS is a version plugin for dealing with Bcfg2 repository.""" name = 'Cvs' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index 8761d282d..999e078b9 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -22,7 +22,6 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ThreadedStatistics, Bcfg2.Server.Plugin.PullSource): name = 'DBStats' - __version__ = '$Id$' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -100,6 +99,8 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, ret.append(getattr(entry.reason, "current_%s" % t)) if entry.reason.is_sensitive: raise Bcfg2.Server.Plugin.PluginExecutionError + elif len(entry.reason.unpruned) != 0: + ret.append('\n'.join(entry.reason.unpruned)) elif entry.reason.current_diff != '': if entry.reason.is_binary: ret.append(binascii.a2b_base64(entry.reason.current_diff)) diff --git a/src/lib/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index eb34a52c4..9fb9ff4f1 100644 --- a/src/lib/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -10,7 +10,6 @@ class Darcs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Darcs is a version plugin for dealing with Bcfg2 repos.""" name = 'Darcs' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 556f75502..b432474f2 100644 --- a/src/lib/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -50,7 +50,6 @@ class Decisions(DecisionSet, Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Decision): name = 'Decisions' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index 23104946e..718192e2a 100644 --- a/src/lib/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -1,5 +1,4 @@ """This generator provides rule-based entry mappings.""" -__revision__ = '$Revision$' import re import Bcfg2.Server.Plugin @@ -9,7 +8,6 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, Bcfg2.Server.Plugin.StructureValidator): """Set default attributes on bound entries""" name = 'Defaults' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' # Rules is a Generator that happens to implement all of the @@ -49,3 +47,7 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, finally: if is_bound: entry.tag = "Bound" + entry.tag + + def _regex_enabled(self): + """ Defaults depends on regex matching, so force it enabled """ + return True diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Bcfg2/Server/Plugins/Deps.py index 482d457af..9b848baae 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Bcfg2/Server/Plugins/Deps.py @@ -1,5 +1,4 @@ """This plugin provides automatic dependency handling.""" -__revision__ = '$Revision$' import lxml.etree @@ -45,7 +44,6 @@ class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc): class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.StructureValidator): name = 'Deps' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = DepXMLSrc diff --git a/src/lib/Server/Plugins/Editor.py b/src/lib/Bcfg2/Server/Plugins/Editor.py index 76a03a325..c0d2cfbad 100644 --- a/src/lib/Server/Plugins/Editor.py +++ b/src/lib/Bcfg2/Server/Plugins/Editor.py @@ -59,7 +59,6 @@ class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): class Editor(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.Probing): name = 'Editor' - __version__ = '$Id$' __author__ = 'bcfg2-dev@mcs.anl.gov' filename_pattern = 'edits' es_child_cls = EditDirectives diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index 269664ef4..5beec7be0 100644 --- a/src/lib/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -3,9 +3,9 @@ added to the specification. On subsequent runs, the file will be replaced on the client if it is missing; if it has changed on the client, it can either be updated in the specification or replaced on the client """ -__revision__ = '$Revision: 1465 $' import os +import sys import errno import binascii import lxml.etree @@ -54,7 +54,6 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, name = 'FileProbes' experimental = True - __version__ = '$Id$' __author__ = 'chris.a.st.pierre@gmail.com' def __init__(self, core, datastore): @@ -89,33 +88,27 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, interpreter="/usr/bin/env python") probe.text = probecode % path self.probes[metadata.hostname].append(probe) - self.logger.debug("Adding file probe for %s to %s" % - (path, metadata.hostname)) + self.debug_log("Adding file probe for %s to %s" % + (path, metadata.hostname)) return self.probes[metadata.hostname] def ReceiveData(self, metadata, datalist): """Receive data from probe.""" - self.logger.debug("Receiving file probe data from %s" % - metadata.hostname) + self.debug_log("Receiving file probe data from %s" % metadata.hostname) for data in datalist: if data.text is None: self.logger.error("Got null response to %s file probe from %s" % (data.get('name'), metadata.hostname)) else: - self.logger.debug("%s:fileprobe:%s:%s" % - (metadata.hostname, - data.get("name"), - data.text)) try: - filedata = lxml.etree.XML(data.text) - self.write_file(filedata, metadata) + self.write_data(lxml.etree.XML(data.text), metadata) except lxml.etree.XMLSyntaxError: # if we didn't get XML back from the probe, assume # it's an error message self.logger.error(data.text) - def write_file(self, data, metadata): + def write_data(self, data, metadata): """Write the probed file data to the bcfg2 specification.""" filename = data.get("name") contents = binascii.a2b_base64(data.text) @@ -138,75 +131,81 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, else: entrydata = entry.text - if create: - self.logger.info("Writing new probed file %s" % fileloc) - try: - os.makedirs(os.path.dirname(fileloc)) - except OSError, err: - if err.errno == errno.EEXIST: - pass - else: - raise - open(fileloc, 'wb').write(contents) - - infoxml = os.path.join("%s%s" % (cfg.data, filename), - "info.xml") - if not os.path.exists(infoxml): - self.write_infoxml(infoxml, entry, data) - - # Service the FAM events queued up by the key generation - # so the data structure entries will be available for - # binding. - # - # NOTE: We wait for up to ten seconds. There is some - # potential for race condition, because if the file - # monitor doesn't get notified about the new key files in - # time, those entries won't be available for binding. In - # practice, this seems "good enough". - tries = 0 - is_bound = False - while not is_bound: - if tries >= 10: - self.logger.error("%s still not registered" % filename) - raise Bcfg2.Server.Plugin.PluginExecutionError - self.core.fam.handle_events_in_interval(1) - try: - cfg.entries[filename].bind_entry(entry, metadata) - is_bound = True - except Bcfg2.Server.Plugin.PluginExecutionError: - pass - tries += 1 + if create: + self.logger.info("Writing new probed file %s" % fileloc) + self.write_file(fileloc, contents) + self.verify_file(filename, contents, metadata) + infoxml = os.path.join("%s%s" % (cfg.data, filename), "info.xml") + self.write_infoxml(infoxml, entry, data) elif entrydata == contents: - self.logger.debug("Existing %s contents match probed contents" % - filename) + self.debug_log("Existing %s contents match probed contents" % + filename) return elif (entry.get('update', 'false').lower() == "true"): self.logger.info("Writing updated probed file %s" % fileloc) - open(fileloc, 'wb').write(contents) - - # service FAM events - tries = 0 - updated = False - while not updated: - if tries >= 10: - self.logger.error("%s still not registered" % filename) - raise Bcfg2.Server.Plugin.PluginExecutionError - self.core.fam.handle_events_in_interval(1) - cfg.entries[filename].bind_entry(entry, metadata) - # get current entry data - if entry.get("encoding") == "base64": - entrydata = binascii.a2b_base64(entry.text) - else: - entrydata = entry.text - if entrydata == contents: - updated = True - tries += 1 + self.write_file(fileloc, contents) + self.verify_file(filename, contents, metadata) else: self.logger.info("Skipping updated probed file %s" % fileloc) return - + + def write_file(self, fileloc, contents): + try: + os.makedirs(os.path.dirname(fileloc)) + except OSError: + err = sys.exc_info()[1] + if err.errno == errno.EEXIST: + pass + else: + self.logger.error("Could not create parent directories for %s: " + "%s" % (fileloc, err)) + return + + try: + open(fileloc, 'wb').write(contents) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Could not write %s: %s" % (fileloc, err)) + return + + def verify_file(self, filename, contents, metadata): + # Service the FAM events queued up by the key generation so + # the data structure entries will be available for binding. + # + # NOTE: We wait for up to ten seconds. There is some potential + # for race condition, because if the file monitor doesn't get + # notified about the new key files in time, those entries + # won't be available for binding. In practice, this seems + # "good enough". + entry = self.entries[metadata.hostname][filename] + cfg = self.core.plugins['Cfg'] + tries = 0 + updated = False + while not updated: + if tries >= 10: + self.logger.error("%s still not registered" % filename) + return + self.core.fam.handle_events_in_interval(1) + try: + cfg.entries[filename].bind_entry(entry, metadata) + except Bcfg2.Server.Plugin.PluginExecutionError: + tries += 1 + continue + + # get current entry data + if entry.get("encoding") == "base64": + entrydata = binascii.a2b_base64(entry.text) + else: + entrydata = entry.text + if entrydata == contents: + updated = True + tries += 1 + def write_infoxml(self, infoxml, entry, data): """ write an info.xml for the file """ + if os.path.exists(infoxml): + return + self.logger.info("Writing info.xml at %s for %s" % (infoxml, data.get("name"))) info = \ @@ -219,8 +218,13 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Options.MDATA_PERMS.value), encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) - + root = lxml.etree.Element("FileInfo") root.append(info) - open(infoxml, "w").write(lxml.etree.tostring(root, - pretty_print=True)) + try: + open(infoxml, "w").write(lxml.etree.tostring(root, + pretty_print=True)) + except IOError: + err = sys.exc_info()[1] + self.logger.error("Could not write %s: %s" % (fileloc, err)) + return diff --git a/src/lib/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 57d427673..1b1627688 100644 --- a/src/lib/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -10,7 +10,6 @@ class Fossil(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Fossil is a version plugin for dealing with Bcfg2 repos.""" name = 'Fossil' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index aaeac12ae..8f8ea87f1 100644 --- a/src/lib/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -13,7 +13,6 @@ class Git(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Git is a version plugin for dealing with Bcfg2 repos.""" name = 'Git' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 76a628931..58b4d4afb 100644 --- a/src/lib/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -1,9 +1,8 @@ -import lxml.etree import re - +import logging +import lxml.etree import Bcfg2.Server.Plugin - class PackedDigitRange(object): def __init__(self, digit_range): self.sparse = list() @@ -25,7 +24,7 @@ class PackedDigitRange(object): class PatternMap(object): - range_finder = '\\[\\[[\d\-,]+\\]\\]' + range_finder = r'\[\[[\d\-,]+\]\]' def __init__(self, pattern, rangestr, groups): self.pattern = pattern @@ -35,15 +34,18 @@ class PatternMap(object): self.re = re.compile(pattern) self.process = self.process_re elif rangestr != None: + if '\\' in rangestr: + raise Exception("Backslashes are not allowed in NameRanges") self.process = self.process_range - self.re = re.compile('^' + re.subn(self.range_finder, '(\d+)', - rangestr)[0]) - dmatcher = re.compile(re.subn(self.range_finder, - '\\[\\[([\d\-,]+)\\]\\]', - rangestr)[0]) - self.dranges = [PackedDigitRange(x) for x in dmatcher.match(rangestr).groups()] + self.re = re.compile('^' + re.sub(self.range_finder, '(\d+)', + rangestr)) + dmatcher = re.compile(re.sub(self.range_finder, + r'\[\[([\d\-,]+)\]\]', + rangestr)) + self.dranges = [PackedDigitRange(x) + for x in dmatcher.match(rangestr).groups()] else: - raise Exception + raise Exception("No pattern or range given") def process_range(self, name): match = self.re.match(name) @@ -75,6 +77,7 @@ class PatternFile(Bcfg2.Server.Plugin.SingleXMLFileBacked): def __init__(self, filename, fam): Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) self.patterns = [] + self.logger = logging.getLogger(self.__class__.__name__) def Index(self): Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) diff --git a/src/lib/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index b217378d6..eea92f30f 100644 --- a/src/lib/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -12,7 +12,7 @@ python -c "from guppy import hpy;hpy().monitor()" For example: # python -c "from guppy import hpy;hpy().monitor()" -<Monitor> +<Monitor> *** Connection 1 opened *** <Monitor> lc CID PID ARGV @@ -21,7 +21,7 @@ CID PID ARGV Remote connection 1. To return to Monitor, type <Ctrl-C> or .<RETURN> <Annex> int Remote interactive console. To return to Annex, type '-'. ->>> hp.heap() +>>> hp.heap() ... @@ -32,7 +32,6 @@ import Bcfg2.Server.Plugin class Guppy(Bcfg2.Server.Plugin.Plugin): """Guppy is a debugging plugin to help trace memory leaks""" name = 'Guppy' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 70e33ef1f..0c3537613 100644 --- a/src/lib/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -10,7 +10,6 @@ class Hg(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Mercurial is a version plugin for dealing with Bcfg2 repository.""" name = 'Mercurial' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Bcfg2/Server/Plugins/Hostbase.py index 4180fd716..e9c1c1cff 100644 --- a/src/lib/Server/Plugins/Hostbase.py +++ b/src/lib/Bcfg2/Server/Plugins/Hostbase.py @@ -2,7 +2,6 @@ This file provides the Hostbase plugin. It manages dns/dhcp/nis host information """ -__revision__ = '$Revision$' import os os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' @@ -23,7 +22,6 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): """The Hostbase plugin handles host/network info.""" name = 'Hostbase' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' filepath = '/my/adm/hostbase/files/bind' diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index f1e2198d2..29abf5b13 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -50,7 +50,7 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked): def Index(self): """ Reregisters the queries in the config file - + The config will take care of actually registering the queries, so we just load it once and don't keep it. """ @@ -63,24 +63,23 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): The Ldap plugin allows adding data from an LDAP server to your metadata. """ name = "Ldap" - version = "$Revision: $" experimental = True debug_flag = False - + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = ConfigFile(self.data + "/config.py", core.fam) - + def debug_log(self, message, flag = None): if (flag is None) and self.debug_flag or flag: self.logger.error(message) - + def get_additional_data(self, metadata): query = None try: data = {} - self.debug_log("LdapPlugin debug: found queries " + + self.debug_log("LdapPlugin debug: found queries " + str(LDAP_QUERIES)) for QueryClass in LDAP_QUERIES: query = QueryClass() @@ -95,14 +94,14 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): except Exception: if hasattr(query, "name"): Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "Exception during processing of query named '" + + "Exception during processing of query named '" + str(query.name) + - "', query results will be empty" + + "', query results will be empty" + " and may cause bind failures") for line in traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + line.replace("\n", "")) return {} @@ -110,23 +109,23 @@ class LdapConnection(object): """ Connection to an LDAP server. """ - def __init__(self, host = "localhost", port = 389, + def __init__(self, host = "localhost", port = 389, binddn = None, bindpw = None): self.host = host self.port = port self.binddn = binddn self.bindpw = bindpw self.conn = None - + def __del__(self): if self.conn: self.conn.unbind() - + def init_conn(self): self.conn = ldap.initialize(self.url) if self.binddn is not None and self.bindpw is not None: self.conn.simple_bind_s(self.binddn, self.bindpw) - + def run_query(self, query): result = None for attempt in range(RETRY_COUNT + 1): @@ -148,17 +147,17 @@ class LdapConnection(object): self.conn = None time.sleep(RETRY_DELAY) return result - + @property def url(self): return "ldap://" + self.host + ":" + str(self.port) - + class LdapQuery(object): """ Query referencing an LdapConnection and providing several methods for query manipulation. """ - + name = "unknown" base = "" scope = "sub" @@ -166,10 +165,10 @@ class LdapQuery(object): attrs = None connection = None result = None - + def __unicode__(self): return "LdapQuery:" + self.name - + def is_applicable(self, metadata): """ Overrideable method to determine if the query is to be executed for @@ -177,27 +176,27 @@ class LdapQuery(object): Defaults to true. """ return True - + def prepare_query(self, metadata): """ Overrideable method to alter the query based on metadata. Defaults to doing nothing. In most cases, you will do something like - + self.filter = "(cn=" + metadata.hostname + ")" - + here. """ pass - + def process_result(self, metadata): """ Overrideable method to post-process the query result. Defaults to returning the unaltered result. """ return self.result - + def get_result(self, metadata): """ Method to handle preparing, executing and processing the query. @@ -208,7 +207,7 @@ class LdapQuery(object): self.result = self.process_result(metadata) return self.result else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None @@ -224,14 +223,14 @@ class LdapSubQuery(LdapQuery): Defaults to doing nothing. """ pass - + def process_result(self, metadata, **kwargs): """ Overrideable method to post-process the query result. Defaults to returning the unaltered result. """ return self.result - + def get_result(self, metadata, **kwargs): """ Method to handle preparing, executing and processing the query. @@ -241,6 +240,6 @@ class LdapSubQuery(LdapQuery): self.result = self.connection.run_query(self) return self.process_result(metadata, **kwargs) else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 8afcd9c50..970126b80 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -2,8 +2,6 @@ This file stores persistent metadata for the Bcfg2 Configuration Repository. """ -__revision__ = '$Revision$' - import copy import fcntl import lxml.etree @@ -38,13 +36,16 @@ class MetadataRuntimeError(Exception): pass -class XMLMetadataConfig(object): +class XMLMetadataConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked): """Handles xml config files and all XInclude statements""" def __init__(self, metadata, watch_clients, basefile): + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, + os.path.join(metadata.data, + basefile), + metadata.core.fam) self.metadata = metadata self.basefile = basefile self.should_monitor = watch_clients - self.extras = [] self.data = None self.basedata = None self.basedir = metadata.data @@ -64,43 +65,40 @@ class XMLMetadataConfig(object): raise MetadataRuntimeError return self.basedata - def add_monitor(self, fname): + def add_monitor(self, fpath, fname): """Add a fam monitor for an included file""" if self.should_monitor: - self.metadata.core.fam.AddMonitor("%s/%s" % (self.basedir, fname), - self.metadata) + self.metadata.core.fam.AddMonitor(fpath, self.metadata) self.extras.append(fname) def load_xml(self): """Load changes from XML""" try: - xdata = lxml.etree.parse("%s/%s" % (self.basedir, self.basefile)) + xdata = lxml.etree.parse(os.path.join(self.basedir, self.basefile)) except lxml.etree.XMLSyntaxError: - self.logger.error('Failed to parse %s' % (self.basefile)) + self.logger.error('Failed to parse %s' % self.basefile) return - self.basedata = copy.deepcopy(xdata) - included = [ent.get('href') for ent in \ - xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.add_monitor(name) + self.extras = [] + self.basedata = copy.copy(xdata) + self._follow_xincludes(xdata=xdata) + if self.extras: try: xdata.xinclude() except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % self.basefile) + self.logger.error("Failed to process XInclude for file %s" % + self.basefile) self.data = xdata def write(self): """Write changes to xml back to disk.""" - self.write_xml("%s/%s" % (self.basedir, self.basefile), + self.write_xml(os.path.join(self.basedir, self.basefile), self.basedata) def write_xml(self, fname, xmltree): """Write changes to xml back to disk.""" tmpfile = "%s.new" % fname try: - datafile = open("%s" % tmpfile, 'w') + datafile = open(tmpfile, 'w') except IOError: e = sys.exc_info()[1] self.logger.error("Failed to write %s: %s" % (tmpfile, e)) @@ -116,40 +114,42 @@ class XMLMetadataConfig(object): datafile.write(newcontents) except: fcntl.lockf(fd, fcntl.LOCK_UN) - self.logger.error("Metadata: Failed to write new xml data to %s" % tmpfile, exc_info=1) - os.unlink("%s" % tmpfile) + self.logger.error("Metadata: Failed to write new xml data to %s" % + tmpfile, exc_info=1) + os.unlink(tmpfile) raise MetadataRuntimeError datafile.close() # check if clients.xml is a symlink - xmlfile = "%s" % fname - if os.path.islink(xmlfile): - xmlfile = os.readlink(xmlfile) + if os.path.islink(fname): + fname = os.readlink(fname) try: - os.rename("%s" % tmpfile, xmlfile) + os.rename(tmpfile, fname) except: self.logger.error("Metadata: Failed to rename %s" % tmpfile) raise MetadataRuntimeError def find_xml_for_xpath(self, xpath): - """Find and load xml data containing the xpath query""" + """Find and load xml file containing the xpath query""" if self.pseudo_monitor: # Reload xml if we don't have a real monitor self.load_xml() cli = self.basedata.xpath(xpath) if len(cli) > 0: - return {'filename': "%s/%s" % (self.basedir, self.basefile), + return {'filename': os.path.join(self.basedir, self.basefile), 'xmltree': self.basedata, 'xquery': cli} else: """Try to find the data in included files""" for included in self.extras: try: - xdata = lxml.etree.parse("%s/%s" % (self.basedir, included)) + xdata = lxml.etree.parse(os.path.join(self.basedir, + included)) cli = xdata.xpath(xpath) if len(cli) > 0: - return {'filename': "%s/%s" % (self.basedir, included), + return {'filename': os.path.join(self.basedir, + included), 'xmltree': xdata, 'xquery': cli} except lxml.etree.XMLSyntaxError: @@ -221,7 +221,6 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Metadata, Bcfg2.Server.Plugin.Statistics): """This class contains data for bcfg2 server metadata.""" - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' name = "Metadata" sort_order = 500 @@ -232,8 +231,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Statistics.__init__(self) if watch_clients: try: - core.fam.AddMonitor("%s/%s" % (self.data, "groups.xml"), self) - core.fam.AddMonitor("%s/%s" % (self.data, "clients.xml"), self) + core.fam.AddMonitor(os.path.join(self.data, "groups.xml"), self) + core.fam.AddMonitor(os.path.join(self.data, "clients.xml"), self) except: print("Unable to add file monitor for groups.xml or clients.xml") raise Bcfg2.Server.Plugin.PluginInitError @@ -274,269 +273,241 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, @classmethod def init_repo(cls, repo, groups, os_selection, clients): - path = '%s/%s' % (repo, cls.name) + path = os.path.join(repo, cls.name) os.makedirs(path) - open("%s/Metadata/groups.xml" % - repo, "w").write(groups % os_selection) - open("%s/Metadata/clients.xml" % - repo, "w").write(clients % socket.getfqdn()) + open(os.path.join(repo, "Metadata", "groups.xml"), + "w").write(groups % os_selection) + open(os.path.join(repo, "Metadata", "clients.xml"), + "w").write(clients % socket.getfqdn()) def get_groups(self): '''return groups xml tree''' - groups_tree = lxml.etree.parse(self.data + "/groups.xml") + groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml")) root = groups_tree.getroot() return root - def search_group(self, group_name, tree): - """Find a group.""" - for node in tree.findall("//Group"): - if node.get("name") == group_name: + def _search_xdata(self, tag, name, tree, alias=False): + for node in tree.findall("//%s" % tag): + if node.get("name") == name: return node - for child in node: - if child.tag == "Alias" and child.attrib["name"] == group_name: - return node + elif alias: + for child in node: + if (child.tag == "Alias" and + child.attrib["name"] == name): + return node return None - def add_group(self, group_name, attribs): - """Add group to groups.xml.""" + def search_group(self, group_name, tree): + """Find a group.""" + return self._search_xdata("Group", group_name, tree) + + def search_bundle(self, bundle_name, tree): + """Find a bundle.""" + return self._search_xdata("Bundle", bundle_name, tree) + + def search_client(self, client_name, tree): + return self._search_xdata("Client", client_name, tree, alias=True) - node = self.search_group(group_name, self.groups_xml.xdata) + def _add_xdata(self, config, tag, name, attribs=None, alias=False): + node = self._search_xdata(tag, name, config.xdata, alias=alias) if node != None: - self.logger.error("Group \"%s\" already exists" % (group_name)) + self.logger.error("%s \"%s\" already exists" % (tag, name)) raise MetadataConsistencyError + element = lxml.etree.SubElement(config.base_xdata.getroot(), + tag, name=name) + if attribs: + for key, val in list(attribs.items()): + element.set(key, val) + config.write() - element = lxml.etree.SubElement(self.groups_xml.base_xdata.getroot(), - "Group", name=group_name) - for key, val in list(attribs.items()): - element.set(key, val) - self.groups_xml.write() + def add_group(self, group_name, attribs): + """Add group to groups.xml.""" + return self._add_xdata(self.groups_xml, "Group", group_name, + attribs=attribs) - def update_group(self, group_name, attribs): - """Update a groups attributes.""" - node = self.search_group(group_name, self.groups_xml.xdata) + def add_bundle(self, bundle_name): + """Add bundle to groups.xml.""" + return self._add_xdata(self.groups_xml, "Bundle", bundle_name) + + def add_client(self, client_name, attribs): + """Add client to clients.xml.""" + return self._add_xdata(self.clients_xml, "Client", client_name, + attribs=attribs, alias=True) + + def _update_xdata(self, config, tag, name, attribs, alias=False): + node = self._search_xdata(tag, name, config.xdata, alias=alias) if node == None: - self.logger.error("Group \"%s\" does not exist" % (group_name)) + self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise MetadataConsistencyError - xdict = self.groups_xml.find_xml_for_xpath('.//Group[@name="%s"]' % (node.get('name'))) + xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % + (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding group") + self.logger.error("Unexpected error finding %s \"%s\"" % + (tag, name)) raise MetadataConsistencyError - for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) - self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) + config.write_xml(xdict['filename'], xdict['xmltree']) - def remove_group(self, group_name): - """Remove a group.""" - node = self.search_group(group_name, self.groups_xml.xdata) + def update_group(self, group_name, attribs): + """Update a groups attributes.""" + return self._update_xdata(self.groups_xml, "Group", group_name, attribs) + + def update_client(self, client_name, attribs): + """Update a clients attributes.""" + return self._update_xdata(self.clients_xml, "Client", client_name, + attribs, alias=True) + + def _remove_xdata(self, config, tag, name, alias=False): + node = self._search_xdata(tag, name, config.xdata) if node == None: - self.logger.error("Group \"%s\" does not exist" % (group_name)) + self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise MetadataConsistencyError - xdict = self.groups_xml.find_xml_for_xpath('.//Group[@name="%s"]' % (node.get('name'))) + xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % + (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding group") + self.logger.error("Unexpected error finding %s \"%s\"" % + (tag, name)) raise MetadataConsistencyError xdict['xquery'][0].getparent().remove(xdict['xquery'][0]) self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) - def add_bundle(self, bundle_name): - """Add bundle to groups.xml.""" - tree = lxml.etree.parse(self.data + "/groups.xml") - root = tree.getroot() - element = lxml.etree.Element("Bundle", name=bundle_name) - node = self.search_group(bundle_name, tree) - if node != None: - self.logger.error("Bundle \"%s\" already exists" % (bundle_name)) - raise MetadataConsistencyError - root.append(element) - group_tree = open(self.data + "/groups.xml", "w") - fd = group_tree.fileno() - while True: - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - continue - else: - break - tree.write(group_tree) - fcntl.lockf(fd, fcntl.LOCK_UN) - group_tree.close() + def remove_group(self, group_name): + """Remove a group.""" + return self._remove_xdata(self.groups_xml, "Group", group_name) def remove_bundle(self, bundle_name): """Remove a bundle.""" - tree = lxml.etree.parse(self.data + "/groups.xml") - root = tree.getroot() - node = self.search_group(bundle_name, tree) - if node == None: - self.logger.error("Bundle \"%s\" not found" % (bundle_name)) - raise MetadataConsistencyError - root.remove(node) - group_tree = open(self.data + "/groups.xml", "w") - fd = group_tree.fileno() - while True: - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - continue - else: - break - tree.write(group_tree) - fcntl.lockf(fd, fcntl.LOCK_UN) - group_tree.close() - - def search_client(self, client_name, tree): - """Find a client.""" - for node in tree.findall("//Client"): - if node.get("name") == client_name: - return node - for child in node: - if child.tag == "Alias" and child.attrib["name"] == client_name: - return node - return None - - def add_client(self, client_name, attribs): - """Add client to clients.xml.""" - node = self.search_client(client_name, self.clients_xml.xdata) - if node != None: - self.logger.error("Client \"%s\" already exists" % (client_name)) - raise MetadataConsistencyError - - element = lxml.etree.SubElement(self.clients_xml.base_xdata.getroot(), - "Client", name=client_name) - for key, val in list(attribs.items()): - element.set(key, val) - self.clients_xml.write() - - def update_client(self, client_name, attribs): - """Update a clients attributes.""" - node = self.search_client(client_name, self.clients_xml.xdata) - if node == None: - self.logger.error("Client \"%s\" does not exist" % (client_name)) - raise MetadataConsistencyError - - xdict = self.clients_xml.find_xml_for_xpath('.//Client[@name="%s"]' % (node.get('name'))) - if not xdict: - self.logger.error("Unexpected error finding client") - raise MetadataConsistencyError + return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) - node = xdict['xquery'][0] - [node.set(key, value) for key, value in list(attribs.items())] - self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) + def _handle_clients_xml_event(self, event): + xdata = self.clients_xml.xdata + self.clients = {} + self.aliases = {} + self.raliases = {} + self.bad_clients = {} + self.secure = [] + self.floating = [] + self.addresses = {} + self.raddresses = {} + for client in xdata.findall('.//Client'): + clname = client.get('name').lower() + if 'address' in client.attrib: + caddr = client.get('address') + if caddr in self.addresses: + self.addresses[caddr].append(clname) + else: + self.addresses[caddr] = [clname] + if clname not in self.raddresses: + self.raddresses[clname] = set() + self.raddresses[clname].add(caddr) + if 'auth' in client.attrib: + self.auth[client.get('name')] = client.get('auth', + 'cert+password') + if 'uuid' in client.attrib: + self.uuid[client.get('uuid')] = clname + if client.get('secure', 'false') == 'true': + self.secure.append(clname) + if client.get('location', 'fixed') == 'floating': + self.floating.append(clname) + if 'password' in client.attrib: + self.passwords[clname] = client.get('password') + + self.raliases[clname] = set() + for alias in client.findall('Alias'): + self.aliases.update({alias.get('name'): clname}) + self.raliases[clname].add(alias.get('name')) + if 'address' not in alias.attrib: + continue + if alias.get('address') in self.addresses: + self.addresses[alias.get('address')].append(clname) + else: + self.addresses[alias.get('address')] = [clname] + if clname not in self.raddresses: + self.raddresses[clname] = set() + self.raddresses[clname].add(alias.get('address')) + self.clients.update({clname: client.get('profile')}) + self.states['clients.xml'] = True + + def _handle_groups_xml_event(self, event): + xdata = self.groups_xml.xdata + self.public = [] + self.private = [] + self.profiles = [] + self.groups = {} + grouptmp = {} + self.categories = {} + groupseen = list() + for group in xdata.xpath('//Groups/Group'): + if group.get('name') not in groupseen: + groupseen.append(group.get('name')) + else: + self.logger.error("Metadata: Group %s defined multiply" % + group.get('name')) + grouptmp[group.get('name')] = \ + ([item.get('name') for item in group.findall('./Bundle')], + [item.get('name') for item in group.findall('./Group')]) + grouptmp[group.get('name')][1].append(group.get('name')) + if group.get('default', 'false') == 'true': + self.default = group.get('name') + if group.get('profile', 'false') == 'true': + self.profiles.append(group.get('name')) + if group.get('public', 'false') == 'true': + self.public.append(group.get('name')) + elif group.get('public', 'true') == 'false': + self.private.append(group.get('name')) + if 'category' in group.attrib: + self.categories[group.get('name')] = group.get('category') + + for group in grouptmp: + # self.groups[group] => (bundles, groups, categories) + self.groups[group] = (set(), set(), {}) + tocheck = [group] + group_cat = self.groups[group][2] + while tocheck: + now = tocheck.pop() + self.groups[group][1].add(now) + if now in grouptmp: + (bundles, groups) = grouptmp[now] + for ggg in groups: + if ggg in self.groups[group][1]: + continue + if (ggg not in self.categories or \ + self.categories[ggg] not in self.groups[group][2]): + self.groups[group][1].add(ggg) + tocheck.append(ggg) + if ggg in self.categories: + group_cat[self.categories[ggg]] = ggg + elif ggg in self.categories: + self.logger.info("Group %s: %s cat-suppressed %s" % \ + (group, + group_cat[self.categories[ggg]], + ggg)) + [self.groups[group][0].add(bund) for bund in bundles] + self.states['groups.xml'] = True def HandleEvent(self, event): """Handle update events for data files.""" if self.clients_xml.HandleEvent(event): - xdata = self.clients_xml.xdata - self.clients = {} - self.aliases = {} - self.raliases = {} - self.bad_clients = {} - self.secure = [] - self.floating = [] - self.addresses = {} - self.raddresses = {} - for client in xdata.findall('.//Client'): - clname = client.get('name').lower() - if 'address' in client.attrib: - caddr = client.get('address') - if caddr in self.addresses: - self.addresses[caddr].append(clname) - else: - self.addresses[caddr] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(caddr) - if 'auth' in client.attrib: - self.auth[client.get('name')] = client.get('auth', - 'cert+password') - if 'uuid' in client.attrib: - self.uuid[client.get('uuid')] = clname - if client.get('secure', 'false') == 'true': - self.secure.append(clname) - if client.get('location', 'fixed') == 'floating': - self.floating.append(clname) - if 'password' in client.attrib: - self.passwords[clname] = client.get('password') - for alias in [alias for alias in client.findall('Alias')\ - if 'address' in alias.attrib]: - if alias.get('address') in self.addresses: - self.addresses[alias.get('address')].append(clname) - else: - self.addresses[alias.get('address')] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(alias.get('address')) - self.clients.update({clname: client.get('profile')}) - [self.aliases.update({alias.get('name'): clname}) \ - for alias in client.findall('Alias')] - self.raliases[clname] = set() - [self.raliases[clname].add(alias.get('name')) for alias \ - in client.findall('Alias')] - self.states['clients.xml'] = True + self._handle_clients_xml_event(event) elif self.groups_xml.HandleEvent(event): - xdata = self.groups_xml.xdata - self.public = [] - self.private = [] - self.profiles = [] - self.groups = {} - grouptmp = {} - self.categories = {} - groupseen = list() - for group in xdata.xpath('//Groups/Group'): - if group.get('name') not in groupseen: - groupseen.append(group.get('name')) - else: - self.logger.error("Metadata: Group %s defined multiply" % (group.get('name'))) - grouptmp[group.get('name')] = tuple([[item.get('name') for item in group.findall(spec)] - for spec in ['./Bundle', './Group']]) - grouptmp[group.get('name')][1].append(group.get('name')) - if group.get('default', 'false') == 'true': - self.default = group.get('name') - if group.get('profile', 'false') == 'true': - self.profiles.append(group.get('name')) - if group.get('public', 'false') == 'true': - self.public.append(group.get('name')) - elif group.get('public', 'true') == 'false': - self.private.append(group.get('name')) - if 'category' in group.attrib: - self.categories[group.get('name')] = group.get('category') - for group in grouptmp: - # self.groups[group] => (bundles, groups, categories) - self.groups[group] = (set(), set(), {}) - tocheck = [group] - group_cat = self.groups[group][2] - while tocheck: - now = tocheck.pop() - self.groups[group][1].add(now) - if now in grouptmp: - (bundles, groups) = grouptmp[now] - for ggg in [ggg for ggg in groups if ggg not in self.groups[group][1]]: - if ggg not in self.categories or \ - self.categories[ggg] not in self.groups[group][2]: - self.groups[group][1].add(ggg) - tocheck.append(ggg) - if ggg in self.categories: - group_cat[self.categories[ggg]] = ggg - elif ggg in self.categories: - self.logger.info("Group %s: %s cat-suppressed %s" % \ - (group, - group_cat[self.categories[ggg]], - ggg)) - [self.groups[group][0].add(bund) for bund in bundles] - self.states['groups.xml'] = True + self._handle_groups_xml_event(event) + if False not in list(self.states.values()): # check that all client groups are real and complete real = list(self.groups.keys()) for client in list(self.clients.keys()): if self.clients[client] not in self.profiles: - self.logger.error("Client %s set as nonexistent or incomplete group %s" \ - % (client, self.clients[client])) - self.logger.error("Removing client mapping for %s" % (client)) + self.logger.error("Client %s set as nonexistent or " + "incomplete group %s" % + (client, self.clients[client])) + self.logger.error("Removing client mapping for %s" % client) self.bad_clients[client] = self.clients[client] del self.clients[client] for bclient in list(self.bad_clients.keys()): if self.bad_clients[bclient] in self.profiles: - self.logger.info("Restored profile mapping for client %s" % bclient) + self.logger.info("Restored profile mapping for client %s" % + bclient) self.clients[bclient] = self.bad_clients[bclient] del self.bad_clients[bclient] @@ -546,42 +517,37 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, if False in list(self.states.values()): raise MetadataRuntimeError if profile not in self.public: - self.logger.error("Failed to set client %s to private group %s" % (client, profile)) + self.logger.error("Failed to set client %s to private group %s" % + (client, profile)) raise MetadataConsistencyError if client in self.clients: - self.logger.info("Changing %s group from %s to %s" % (client, self.clients[client], profile)) - xdict = self.clients_xml.find_xml_for_xpath('.//Client[@name="%s"]' % (client)) - if not xdict: - self.logger.error("Metadata: Unable to update profile for client %s. Use of Xinclude?" % client) - raise MetadataConsistencyError - xdict['xquery'][0].set('profile', profile) - self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) + self.logger.info("Changing %s group from %s to %s" % + (client, self.clients[client], profile)) + self.update_client(client, dict(profile=profile)) else: - self.logger.info("Creating new client: %s, profile %s" % \ + self.logger.info("Creating new client: %s, profile %s" % (client, profile)) if addresspair in self.session_cache: # we are working with a uuid'd client - lxml.etree.SubElement(self.clients_xml.base_xdata.getroot(), - 'Client', - name=self.session_cache[addresspair][1], - uuid=client, profile=profile, - address=addresspair[0]) + self.add_client(self.session_cache[addresspair][1], + dict(uuid=client, profile=profile, + address=addresspair[0])) else: - lxml.etree.SubElement(self.clients_xml.base_xdata.getroot(), - 'Client', name=client, - profile=profile) + self.add_client(client, dict(profile=profile)) self.clients[client] = profile self.clients_xml.write() def resolve_client(self, addresspair, cleanup_cache=False): """Lookup address locally or in DNS to get a hostname.""" if addresspair in self.session_cache: - # client _was_ cached, so there can be some expired entries - # we need to clean them up to avoid potentially infinite memory swell + # client _was_ cached, so there can be some expired + # entries. we need to clean them up to avoid potentially + # infinite memory swell cache_ttl = 90 if cleanup_cache: - # remove entries for this client's IP address with _any_ port numbers - # - perhaps a priority queue could be faster? + # remove entries for this client's IP address with + # _any_ port numbers - perhaps a priority queue could + # be faster? curtime = time.time() for addrpair in self.session_cache.keys(): if addresspair[0] == addrpair[0]: @@ -589,13 +555,18 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, if curtime - stamp > cache_ttl: del self.session_cache[addrpair] # return the cached data - (stamp, uuid) = self.session_cache[addresspair] - if time.time() - stamp < cache_ttl: - return self.session_cache[addresspair][1] + try: + (stamp, uuid) = self.session_cache[addresspair] + if time.time() - stamp < cache_ttl: + return self.session_cache[addresspair][1] + except KeyError: + # we cleaned all cached data for this client in cleanup_cache + pass address = addresspair[0] if address in self.addresses: if len(self.addresses[address]) != 1: - self.logger.error("Address %s has multiple reverse assignments; a uuid must be used" % (address)) + self.logger.error("Address %s has multiple reverse assignments; " + "a uuid must be used" % (address)) raise MetadataConsistencyError return self.addresses[address][0] try: @@ -620,7 +591,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (bundles, groups, categories) = self.groups[profile] else: if self.default == None: - self.logger.error("Cannot set group for client %s; no default group set" % (client)) + self.logger.error("Cannot set group for client %s; " + "no default group set" % client) raise MetadataConsistencyError self.set_profile(client, self.default, (None, None)) profile = self.default @@ -635,7 +607,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, password = self.passwords[client] else: password = None - uuids = [item for item, value in list(self.uuid.items()) if value == client] + uuids = [item for item, value in list(self.uuid.items()) + if value == client] if uuids: uuid = uuids[0] else: @@ -649,7 +622,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, [newgroups.add(g) for g in ngroups if g not in newgroups] newcategories.update(ncategories) return ClientMetadata(client, profile, newgroups, newbundles, aliases, - addresses, newcategories, uuid, password, self.query) + addresses, newcategories, uuid, password, + self.query) def get_all_group_names(self): all_groups = set() @@ -673,22 +647,27 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def merge_additional_groups(self, imd, groups): for group in groups: - if group in self.categories and \ - self.categories[group] in imd.categories: + if (group in self.categories and + self.categories[group] in imd.categories): continue - nb, ng, _ = self.groups.get(group, (list(), [group], dict())) - for b in nb: - if b not in imd.bundles: - imd.bundles.add(b) - for g in ng: - if g not in imd.groups: - if g in self.categories and \ - self.categories[g] in imd.categories: + newbundles, newgroups, _ = self.groups.get(group, + (list(), + [group], + dict())) + for newbundle in newbundles: + if newbundle not in imd.bundles: + imd.bundles.add(newbundle) + for newgroup in newgroups: + if newgroup not in imd.groups: + if (newgroup in self.categories and + self.categories[newgroup] in imd.categories): continue - if g in self.private: - self.logger.error("Refusing to add dynamic membership in private group %s for client %s" % (g, imd.hostname)) + if newgroup in self.private: + self.logger.error("Refusing to add dynamic membership " + "in private group %s for client %s" % + (newgroup, imd.hostname)) continue - imd.groups.add(g) + imd.groups.add(newgroup) def merge_additional_data(self, imd, source, data): if not hasattr(imd, source): @@ -703,18 +682,19 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, return True if address in self.addresses: if client in self.addresses[address]: - self.debug_log("Client %s matches address %s" % (client, address)) + self.debug_log("Client %s matches address %s" % + (client, address)) return True else: - self.logger.error("Got request for non-float client %s from %s" \ - % (client, address)) + self.logger.error("Got request for non-float client %s from %s" % + (client, address)) return False resolved = self.resolve_client(addresspair) if resolved.lower() == client.lower(): return True else: - self.logger.error("Got request for %s from incorrect address %s" \ - % (client, address)) + self.logger.error("Got request for %s from incorrect address %s" % + (client, address)) self.logger.error("Resolved to %s" % resolved) return False @@ -732,7 +712,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, try: client = self.resolve_client(address) except MetadataConsistencyError: - self.logger.error("Client %s failed to resolve; metadata problem" % (address[0])) + self.logger.error("Client %s failed to resolve; metadata problem" + % address[0]) return False else: id_method = 'uuid' @@ -764,10 +745,12 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, if client not in self.passwords: if client in self.secure: - self.logger.error("Client %s in secure mode but has no password" % (address[0])) + self.logger.error("Client %s in secure mode but has no password" + % address[0]) return False if password != self.password: - self.logger.error("Client %s used incorrect global password" % (address[0])) + self.logger.error("Client %s used incorrect global password" % + address[0]) return False if client not in self.secure: if client in self.passwords: @@ -775,14 +758,14 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, else: plist = [self.password] if password not in plist: - self.logger.error("Client %s failed to use either allowed password" % \ - (address[0])) + self.logger.error("Client %s failed to use either allowed " + "password" % address[0]) return False else: # client in secure mode and has a client password if password != self.passwords[client]: - self.logger.error("Client %s failed to use client password in secure mode" % \ - (address[0])) + self.logger.error("Client %s failed to use client password in " + "secure mode" % address[0]) return False # populate the session cache if user.decode('utf-8') != 'root': @@ -793,13 +776,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, """Hook into statistics interface to toggle clients in bootstrap mode.""" client = meta.hostname if client in self.auth and self.auth[client] == 'bootstrap': - self.logger.info("Asserting client %s auth mode to cert" % client) - xdict = self.clients_xml.find_xml_for_xpath('.//Client[@name="%s"]' % (client)) - if not xdict: - self.logger.error("Metadata: Unable to update profile for client %s. Use of Xinclude?" % client) - raise MetadataConsistencyError - xdict['xquery'][0].set('auth', 'cert') - self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) + self.update_client(client, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): """Admin mode viz support.""" @@ -814,15 +791,16 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def include_group(group): return not only_client or group in clientmeta.groups - - groups_tree = lxml.etree.parse(self.data + "/groups.xml") + + groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml")) try: groups_tree.xinclude() except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % dest) + self.logger.error("Failed to process XInclude for file %s: %s" % + (dest, sys.exc_info()[1])) groups = groups_tree.getroot() categories = {'default': 'grey83'} - viz_str = "" + viz_str = [] egroups = groups.findall("Group") + groups.findall('.//Groups/Group') for group in egroups: if not group.get('category') in categories: @@ -842,10 +820,10 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, instances[profile] = [client] for profile, clist in list(instances.items()): clist.sort() - viz_str += '''\t"%s-instances" [ label="%s", shape="record" ];\n''' \ - % (profile, '|'.join(clist)) - viz_str += '''\t"%s-instances" -> "group-%s";\n''' \ - % (profile, profile) + viz_str.append('"%s-instances" [ label="%s", shape="record" ];' % + (profile, '|'.join(clist))) + viz_str.append('"%s-instances" -> "group-%s";' % + (profile, profile)) if bundles: bundles = [] [bundles.append(bund.get('name')) \ @@ -854,8 +832,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, and include_bundle(bund.get('name'))] bundles.sort() for bundle in bundles: - viz_str += '''\t"bundle-%s" [ label="%s", shape="septagon"];\n''' \ - % (bundle, bundle) + viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' % + (bundle, bundle)) gseen = [] for group in egroups: if group.get('profile', 'false') == 'true': @@ -864,24 +842,25 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, style = "filled" gseen.append(group.get('name')) if include_group(group.get('name')): - viz_str += '\t"group-%s" [label="%s", style="%s", fillcolor=%s];\n' % \ - (group.get('name'), group.get('name'), style, group.get('color')) + viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' % + (group.get('name'), group.get('name'), style, + group.get('color'))) if bundles: for bundle in group.findall('Bundle'): - viz_str += '\t"group-%s" -> "bundle-%s";\n' % \ - (group.get('name'), bundle.get('name')) - gfmt = '\t"group-%s" [label="%s", style="filled", fillcolor="grey83"];\n' + viz_str.append('"group-%s" -> "bundle-%s";' % + (group.get('name'), bundle.get('name'))) + gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];' for group in egroups: for parent in group.findall('Group'): if parent.get('name') not in gseen and include_group(parent.get('name')): - viz_str += gfmt % (parent.get('name'), parent.get('name')) + viz_str.append(gfmt % (parent.get('name'), + parent.get('name'))) gseen.append(parent.get("name")) if include_group(group.get('name')): - viz_str += '\t"group-%s" -> "group-%s" ;\n' % \ - (group.get('name'), parent.get('name')) + viz_str.append('"group-%s" -> "group-%s";' % + (group.get('name'), parent.get('name'))) if key: for category in categories: - viz_str += '''\t"''' + category + '''" [label="''' + category + \ - '''", shape="record", style="filled", fillcolor=''' + \ - categories[category] + '''];\n''' - return viz_str + viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' % + (category, category, categories[category])) + return "\n".join("\t" + s for s in viz_str) diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index 8a76c130d..4dbd57d16 100644 --- a/src/lib/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -20,14 +20,13 @@ class NagiosGenConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - + class NagiosGen(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): """NagiosGen is a Bcfg2 plugin that dynamically generates Nagios configuration file based on Bcfg2 data. """ name = 'NagiosGen' - __version__ = '0.7' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): @@ -131,14 +130,14 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, group_data = [] for host in host_configs: host_data.append(open(host, 'r').read()) - + for group in group_configs: group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group) if "\n".join(host_data).find(group_name) != -1: groupfile = open(group, 'r') group_data.append(groupfile.read()) groupfile.close() - + entry.text = "%s\n\n%s" % ("\n".join(group_data), "\n".join(host_data)) [entry.attrib.__setitem__(key, value) for (key, value) in list(self.server_attrib.items())] diff --git a/src/lib/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 5fff20d98..5fff20d98 100644 --- a/src/lib/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py diff --git a/src/lib/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index ed954af3b..cbe2b4f2c 100644 --- a/src/lib/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -1,12 +1,9 @@ import re import gzip -import logging from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source from Bcfg2.Bcfg2Py3k import cPickle, file -logger = logging.getLogger("Packages") - class AptCollection(Collection): def get_group(self, group): self.logger.warning("Packages: Package groups are not supported by APT") @@ -15,6 +12,7 @@ class AptCollection(Collection): class AptSource(Source): basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] ptype = 'deb' + essentialpkgs = set() def __init__(self, basepath, xsource, config): Source.__init__(self, basepath, xsource, config) @@ -53,10 +51,9 @@ class AptSource(Source): def read_files(self): bdeps = dict() bprov = dict() + depfnames = ['Depends', 'Pre-Depends'] if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] + depfnames.append('Recommends') for fname in self.files: if not self.rawurl: barch = [x @@ -72,7 +69,7 @@ class AptSource(Source): try: reader = gzip.GzipFile(fname) except: - logger.error("Packages: Failed to read file %s" % fname) + self.logger.error("Packages: Failed to read file %s" % fname) raise for line in reader.readlines(): words = str(line.strip()).split(':', 1) @@ -80,6 +77,8 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + elif words[0] == 'Essential' and self.essential: + self.essentialpkgs.add(pkgname) elif words[0] in depfnames: vindex = 0 for dep in words[1].split(','): diff --git a/src/lib/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index f8cd8e690..959dac03b 100644 --- a/src/lib/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -1,13 +1,14 @@ import copy import logging +import Bcfg2.Server.Plugin + +logger = logging.getLogger(__name__) try: from hashlib import md5 except ImportError: from md5 import md5 -logger = logging.getLogger("Packages") - # we have to cache Collection objects so that calling Packages.Refresh # or .Reload can tell the collection objects to clean up their cache, # but we don't actually use the cache to return a Collection object @@ -20,12 +21,13 @@ logger = logging.getLogger("Packages") # sources to that client.) collections = dict() -class Collection(object): - def __init__(self, metadata, sources, basepath): - """ don't call this directly; use the Factory method """ +class Collection(Bcfg2.Server.Plugin.Debuggable): + def __init__(self, metadata, sources, basepath, debug=False): + """ don't call this directly; use the factory function """ + Bcfg2.Server.Plugin.Debuggable.__init__(self) + self.debug_flag = debug self.metadata = metadata self.sources = sources - self.logger = logging.getLogger("Packages") self.basepath = basepath self.virt_pkgs = dict() @@ -95,6 +97,12 @@ class Collection(object): return source.get_deps(self.metadata, package) return [] + def get_essential(self): + essential = set() + for source in self.sources: + essential |= source.essentialpkgs + return essential + def get_provides(self, package): for source in self.sources: providers = source.get_provides(self.metadata, package) @@ -193,14 +201,14 @@ class Collection(object): # direct packages; current can be added, and all deps # should be resolved current = pkgs.pop() - self.logger.debug("Packages: handling package requirement %s" % - current) + self.debug_log("Packages: handling package requirement %s" % + current) packages.add(current) deps = self.get_deps(current) newdeps = set(deps).difference(examined) if newdeps: - self.logger.debug("Packages: Package %s added " - "requirements %s" % (current, newdeps)) + self.debug_log("Packages: Package %s added requirements %s" + % (current, newdeps)) unclassified.update(newdeps) satisfied_vpkgs = set() @@ -208,16 +216,15 @@ class Collection(object): # virtual dependencies, satisfied if one of N in the # config, or can be forced if only one provider if len(vpkg_cache[current]) == 1: - self.logger.debug("Packages: requirement %s satisfied by " - "%s" % (current, - vpkg_cache[current])) + self.debug_log("Packages: requirement %s satisfied by %s" % + (current, vpkg_cache[current])) unclassified.update(vpkg_cache[current].difference(examined)) satisfied_vpkgs.add(current) else: satisfiers = [item for item in vpkg_cache[current] if item in packages] - self.logger.debug("Packages: requirement %s satisfied by " - "%s" % (current, satisfiers)) + self.debug_log("Packages: requirement %s satisfied by %s" % + (current, satisfiers)) satisfied_vpkgs.add(current) vpkgs.difference_update(satisfied_vpkgs) @@ -230,8 +237,8 @@ class Collection(object): satisfiers = [item for item in vpkg_cache[current] if item in packages] if satisfiers: - self.logger.debug("Packages: requirement %s satisfied by " - "%s" % (current, satisfiers)) + self.debug_log("Packages: requirement %s satisfied by %s" % + (current, satisfiers)) satisfied_both.add(current) elif current in packagelist or final_pass: pkgs.add(current) @@ -287,14 +294,14 @@ def clear_cache(): global collections collections = dict() -def factory(metadata, sources, basepath): +def factory(metadata, sources, basepath, debug=False): global collections if not sources.loaded: # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object return Collection(metadata, [], basepath) - + sclasses = set() relevant = list() @@ -314,7 +321,9 @@ def factory(metadata, sources, basepath): # have multiple Bcfg2 servers and Packages-relevant groups set # by probes); and b) templates that query all or multiple # machines (e.g., with metadata.query.all_clients()) - logger.debug("Packages: No sources found for %s" % metadata.hostname) + if debug: + logger.error("Packages: No sources found for %s" % + metadata.hostname) cclass = Collection else: stype = sclasses.pop().__name__.replace("Source", "") @@ -330,10 +339,11 @@ def factory(metadata, sources, basepath): logger.warning("Packages: No collection class found for %s sources" % stype) - logger.debug("Packages: Using %s for Collection of sources for %s" % - (cclass.__name__, metadata.hostname)) + if debug: + logger.error("Packages: Using %s for Collection of sources for %s" % + (cclass.__name__, metadata.hostname)) - collection = cclass(metadata, relevant, basepath) + collection = cclass(metadata, relevant, basepath, debug=debug) collections[metadata.hostname] = collection return collection diff --git a/src/lib/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 35fb39d02..99a090739 100644 --- a/src/lib/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -1,15 +1,12 @@ import gzip import tarfile -import logging from Bcfg2.Bcfg2Py3k import cPickle, file from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import Source -logger = logging.getLogger("Packages") - class PacCollection(Collection): def get_group(self, group): - self.logger.warning("Packages: Package groups are not supported by APT") + self.logger.warning("Packages: Package groups are not supported by Pacman") return [] class PacSource(Source): @@ -54,10 +51,9 @@ class PacSource(Source): bdeps = dict() bprov = dict() + depfnames = ['Depends', 'Pre-Depends'] if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] + depfnames.append('Recommends') for fname in self.files: if not self.rawurl: @@ -66,22 +62,23 @@ class PacSource(Source): # RawURL entries assume that they only have one <Arch></Arch> # element and that it is the architecture of the source. barch = self.arches[0] - + if barch not in bdeps: bdeps[barch] = dict() bprov[barch] = dict() try: - logger.debug("Packages: try to read : " + fname) + self.debug_log("Packages: try to read %s" % fname) tar = tarfile.open(fname, "r") reader = gzip.GzipFile(fname) except: - logger.error("Packages: Failed to read file %s" % fname) + self.logger.error("Packages: Failed to read file %s" % fname) raise for tarinfo in tar: if tarinfo.isdir(): self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - logger.debug("Packages: added : " + tarinfo.name.rsplit("-", 2)[0]) + self.debug_log("Packages: added %s" % + tarinfo.name.rsplit("-", 2)[0]) tar.close() self.deps['global'] = dict() diff --git a/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py new file mode 100644 index 000000000..3846c06ce --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesConfig.py @@ -0,0 +1,15 @@ +import Bcfg2.Server.Plugin + +class PackagesConfig(Bcfg2.Server.Plugin.SimpleConfig): + _required = False + + def Index(self): + """ Build local data structures """ + Bcfg2.Server.Plugin.SimpleConfig.Index(self) + + if hasattr(self.plugin, "sources") and self.plugin.sources.loaded: + # only reload Packages plugin if sources have been loaded. + # otherwise, this is getting called on server startup, and + # we have to wait until all sources have been indexed + # before we can call Packages.Reload() + self.plugin.Reload() diff --git a/src/lib/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index 4fbccab30..8d0067b6a 100644 --- a/src/lib/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -1,17 +1,16 @@ import os import sys import lxml.etree -import logging import Bcfg2.Server.Plugin from Bcfg2.Server.Plugins.Packages.Source import SourceInitError -logger = logging.getLogger("Packages") - class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile): + Bcfg2.Server.Plugin.StructFile, + Bcfg2.Server.Plugin.Debuggable): __identifier__ = None - + def __init__(self, filename, cachepath, fam, packages, config): + Bcfg2.Server.Plugin.Debuggable.__init__(self) try: Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, @@ -21,25 +20,45 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, msg = "Packages: Failed to read configuration file: %s" % err if not os.path.exists(self.name): msg += " Have you created it?" - logger.error(msg) + self.logger.error(msg) raise Bcfg2.Server.Plugin.PluginInitError(msg) Bcfg2.Server.Plugin.StructFile.__init__(self, filename) self.cachepath = cachepath self.config = config if not os.path.exists(self.cachepath): # create cache directory if needed - os.makedirs(self.cachepath) + try: + os.makedirs(self.cachepath) + except OSError: + err = sys.exc_info()[1] + self.logger.error("Could not create Packages cache at %s: %s" % + (self.cachepath, err)) self.pkg_obj = packages self.parsed = set() self.loaded = False + def toggle_debug(self): + Bcfg2.Server.Plugin.Debuggable.toggle_debug(self) + for source in self.entries: + source.toggle_debug() + def HandleEvent(self, event=None): Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event) - if event.filename != self.name: - self.parsed.add(os.path.basename(event.filename)) + if event and event.filename != self.name: + for fname in self.extras: + fpath = None + if fname.startswith("/"): + fpath = os.path.abspath(fname) + else: + fpath = \ + os.path.abspath(os.path.join(os.path.dirname(self.name), + fname)) + if fpath == os.path.abspath(event.filename): + self.parsed.add(fname) + break if sorted(list(self.parsed)) == sorted(self.extras): - logger.info("Reloading Packages plugin") + self.logger.info("Reloading Packages plugin") self.pkg_obj.Reload() self.loaded = True @@ -56,7 +75,8 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, sources.xml """ stype = xsource.get("type") if stype is None: - logger.error("Packages: No type specified for source, skipping") + self.logger.error("Packages: No type specified for source, " + "skipping") return None try: @@ -65,14 +85,14 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, stype.title()) cls = getattr(module, "%sSource" % stype.title()) except (ImportError, AttributeError): - logger.error("Packages: Unknown source type %s" % stype) + self.logger.error("Packages: Unknown source type %s" % stype) return None try: source = cls(self.cachepath, xsource, self.config) except SourceInitError: err = sys.exc_info()[1] - logger.error("Packages: %s" % err) + self.logger.error("Packages: %s" % err) source = None return source diff --git a/src/lib/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 72c7a4bfd..a7c246940 100644 --- a/src/lib/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -2,7 +2,7 @@ import os import re import sys import base64 -import logging +import Bcfg2.Server.Plugin from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \ HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ urlopen, file, cPickle @@ -12,8 +12,6 @@ try: except ImportError: from md5 import md5 -logger = logging.getLogger('Packages') - def fetch_url(url): if '@' in url: mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) @@ -32,11 +30,14 @@ class SourceInitError(Exception): pass -class Source(object): - reponame_re = re.compile(r'.*/(?:RPMS\.)?([^/]+)') +class Source(Bcfg2.Server.Plugin.Debuggable): + mrepo_re = re.compile(r'/RPMS\.([^/]+)') + pulprepo_re = re.compile(r'pulp/repos/([^/]+)') + genericrepo_re = re.compile(r'https?://[^/]+/(.+?)/?$') basegroups = [] def __init__(self, basepath, xsource, config): + Bcfg2.Server.Plugin.Debuggable.__init__(self) self.basepath = basepath self.xsource = xsource self.config = config @@ -53,8 +54,9 @@ class Source(object): self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] + self.essential = xsource.get('essential', 'true').lower() == 'true' self.recommended = xsource.get('recommended', 'false').lower() == 'true' - + self.rawurl = xsource.get('rawurl', '') if self.rawurl and not self.rawurl.endswith("/"): self.rawurl += "/" @@ -111,16 +113,16 @@ class Source(object): if os.path.exists(self.cachefile): try: self.load_state() - should_read = False + should_read = False except: - logger.error("Packages: Cachefile %s load failed; " - "falling back to file read" % self.cachefile) + self.logger.error("Packages: Cachefile %s load failed; " + "falling back to file read" % self.cachefile) if should_read: try: self.read_files() except: - logger.error("Packages: File read failed; " - "falling back to file download") + self.logger.error("Packages: File read failed; " + "falling back to file download") should_download = True if should_download or force_update: @@ -128,23 +130,34 @@ class Source(object): self.update() self.read_files() except: - logger.error("Packages: Failed to load data for Source of %s. " - "Some Packages will be missing." - % self.urls) + self.logger.error("Packages: Failed to load data for Source " + "of %s. Some Packages will be missing." % + self.urls) def get_repo_name(self, url_map): # try to find a sensible name for a repo - match = self.reponame_re.search(url_map['url']) - if url_map['component']: - return url_map['component'] - elif match: - return match.group(1) + if url_map['components']: + # use the first component as the name + rname = url_map['components'][0] else: - # couldn't figure out the name from the URL or URL map - # (which probably means its a screwy URL), so we just - # generate a random one - name = base64.b64encode(os.urandom(16))[:-2] - return "%s-%s" % (self.groups[0], name) + name = None + for repo_re in (self.mrepo_re, + self.pulprepo_re, + self.genericrepo_re): + match = repo_re.search(url_map['url']) + if match: + break + if name is None: + # couldn't figure out the name from the URL or URL map + # (which probably means its a screwy URL), so we just + # generate a random one + name = base64.b64encode(os.urandom(16))[:-2] + rname = "%s-%s" % (self.groups[0], name) + # see yum/__init__.py in the yum source, lines 441-449, for + # the source of this regex. yum doesn't like anything but + # string.ascii_letters, string.digits, and [-_.:]. There + # doesn't seem to be a reason for this, because yum. + return re.sub(r'[^A-Za-z0-9-_.:]', '-', rname) def __str__(self): if self.rawurl: @@ -194,18 +207,18 @@ class Source(object): def update(self): for url in self.urls: - logger.info("Packages: Updating %s" % url) + self.logger.info("Packages: Updating %s" % url) fname = self.escape_url(url) try: data = fetch_url(url) file(fname, 'w').write(data) except ValueError: - logger.error("Packages: Bad url string %s" % url) + self.logger.error("Packages: Bad url string %s" % url) raise except HTTPError: err = sys.exc_info()[1] - logger.error("Packages: Failed to fetch url %s. HTTP response code=%s" % - (url, err.code)) + self.logger.error("Packages: Failed to fetch url %s. HTTP " + "response code=%s" % (url, err.code)) raise def applies(self, metadata): @@ -257,9 +270,8 @@ class Source(object): if not found_arch: return False - if (self.config.has_section("global") and - self.config.has_option("global", "magic_groups") and - self.config.getboolean("global", "magic_groups") == False): + if self.config.getboolean("global", "magic_groups", + default=True) == False: return True else: for group in self.basegroups: diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 369b7a7d2..b39b6aed2 100644 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -16,7 +16,7 @@ from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url -logger = logging.getLogger("Packages") +logger = logging.getLogger(__name__) try: from pulp.client.consumer.config import ConsumerConfig @@ -68,10 +68,10 @@ def _setup_pulp(config): logger.error("Packages: Required option not found in " "Packages/packages.conf: %s" % err) raise Bcfg2.Server.Plugin.PluginInitError - + PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server - + PULPSERVER = server.PulpServer(serveropts['host'], int(serveropts['port']), serveropts['scheme'], @@ -85,18 +85,16 @@ class YumCollection(Collection): # options that are included in the [yum] section but that should # not be included in the temporary yum.conf we write out option_blacklist = ["use_yum_libraries", "helper"] - - def __init__(self, metadata, sources, basepath): - Collection.__init__(self, metadata, sources, basepath) + + def __init__(self, metadata, sources, basepath, debug=False): + Collection.__init__(self, metadata, sources, basepath, debug=debug) self.keypath = os.path.join(self.basepath, "keys") if len(sources): config = sources[0].config - self.use_yum = has_yum - try: - self.use_yum &= config.getboolean("yum", "use_yum_libraries") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.use_yum = False + self.use_yum = has_yum and config.getboolean("yum", + "use_yum_libraries", + default=False) else: self.use_yum = False @@ -105,7 +103,7 @@ class YumCollection(Collection): "cache-%s" % self.cachekey) if not os.path.exists(self.cachefile): os.mkdir(self.cachefile) - + self.configdir = os.path.join(self.basepath, "yum") if not os.path.exists(self.configdir): os.mkdir(self.configdir) @@ -113,11 +111,8 @@ class YumCollection(Collection): "%s-yum.conf" % self.cachekey) self.write_config() - try: - self.helper = self.config.get("yum", "helper") - except ConfigParser.NoOptionError: - self.helper = "/usr/sbin/bcfg2-yum-helper" - + self.helper = self.config.get("yum", "helper", + default="/usr/sbin/bcfg2-yum-helper") if has_pulp: _setup_pulp(self.config) @@ -125,10 +120,11 @@ class YumCollection(Collection): if not os.path.exists(self.cfgfile): yumconf = self.get_config(raw=True) yumconf.add_section("main") - + mainopts = dict(cachedir=self.cachefile, keepcache="0", sslverify="0", + debuglevel="0", reposdir="/dev/null") try: for opt in self.config.options("yum"): @@ -149,8 +145,22 @@ class YumCollection(Collection): source.get_urls() for url_map in source.url_map: if url_map['arch'] in self.metadata.groups: - reponame = source.get_repo_name(url_map) - config.add_section(reponame) + basereponame = source.get_repo_name(url_map) + reponame = basereponame + + added = False + while not added: + try: + config.add_section(reponame) + added = True + except ConfigParser.DuplicateSectionError: + match = re.match("-(\d)", reponame) + if match: + rid = int(match.group(1)) + 1 + else: + rid = 1 + reponame = "%s-%d" % (basereponame, rid) + config.set(reponame, "name", reponame) config.set(reponame, "baseurl", url_map['url']) config.set(reponame, "enabled", "1") @@ -187,20 +197,30 @@ class YumCollection(Collection): needkeys.add(key) if len(needkeys): - keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey", - type=self.ptype, origin='Packages') + if has_yum: + # this must be be has_yum, not use_yum, because + # regardless of whether the user wants to use the yum + # resolver we want to include gpg key data + keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey", + type=self.ptype, origin='Packages') + else: + self.logger.warning("GPGKeys were specified for yum sources in " + "sources.xml, but no yum libraries were " + "found") + self.logger.warning("GPG key version/release data cannot be " + "determined automatically") + self.logger.warning("Install yum libraries, or manage GPG keys " + "manually") + keypkg = None for key in needkeys: # figure out the path of the key on the client - try: - keydir = self.config.get("global", "gpg_keypath") - except (ConfigParser.NoOptionError, - ConfigParser.NoSectionError): - keydir = "/etc/pki/rpm-gpg" + keydir = self.config.get("global", "gpg_keypath", + default="/etc/pki/rpm-gpg") remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() - + # copy the key to the client keypath = lxml.etree.Element("BoundPath", name=remotekey, encoding='ascii', @@ -212,7 +232,8 @@ class YumCollection(Collection): # hook to add version/release info if possible self._add_gpg_instances(keypkg, kdata, localkey, remotekey) independent.append(keypath) - independent.append(keypkg) + if keypkg is not None: + independent.append(keypkg) # see if there are any pulp sources to handle has_pulp_sources = False @@ -256,29 +277,36 @@ class YumCollection(Collection): pass except socket.error: err = sys.exc_info()[1] - logger.error("Packages: Could not contact Pulp server: %s" % err) + self.logger.error("Packages: Could not contact Pulp server: %s" % + err) except: err = sys.exc_info()[1] - logger.error("Packages: Unknown error querying Pulp server: %s" % err) + self.logger.error("Packages: Unknown error querying Pulp server: %s" + % err) return consumer def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey): """ add gpg keys to the specification to ensure they get installed """ - if self.use_yum: - try: - kinfo = yum.misc.getgpgkeyinfo(keydata) - version = yum.misc.keyIdToRPMVer(kinfo['keyid']) - release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) - - lxml.etree.SubElement(keyentry, 'Instance', - version=version, - release=release, - simplefile=remotekey) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not read GPG key %s: %s" % - (localkey, err)) + # this must be be has_yum, not use_yum, because regardless of + # whether the user wants to use the yum resolver we want to + # include gpg key data + if not has_yum: + return + + try: + kinfo = yum.misc.getgpgkeyinfo(keydata) + version = yum.misc.keyIdToRPMVer(kinfo['keyid']) + release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) + + lxml.etree.SubElement(keyentry, 'Instance', + version=version, + release=release, + simplefile=remotekey) + except ValueError: + err = sys.exc_info()[1] + self.logger.error("Packages: Could not read GPG key %s: %s" % + (localkey, err)) def is_package(self, package): if not self.use_yum: @@ -350,7 +378,7 @@ class YumCollection(Collection): unknown = set([str(p) for p in result['unknown']]) self.filter_unknown(unknown) - + return packages, unknown def call_helper(self, command, input=None): @@ -364,8 +392,11 @@ class YumCollection(Collection): # get the 'setup' variable, so we don't know how verbose # bcfg2-server is. It'd also be nice if we could tell yum to # log to syslog. So would a unicorn. - cmd = [self.helper, "-c", self.cfgfile, command] - self.logger.debug("Packages: running %s" % " ".join(cmd)) + cmd = [self.helper, "-c", self.cfgfile] + if self.debug_flag: + cmd.append("-v") + cmd.append(command) + self.debug_log("Packages: running %s" % " ".join(cmd)) try: helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) except OSError: @@ -373,7 +404,7 @@ class YumCollection(Collection): self.logger.error("Packages: Failed to execute %s: %s" % (" ".join(cmd), err)) return None - + if input: idata = json.dumps(input) (stdout, stderr) = helper.communicate(idata) @@ -383,6 +414,9 @@ class YumCollection(Collection): if rv: self.logger.error("Packages: error running bcfg2-yum-helper " "(returned %d): %s" % (rv, stderr)) + elif self.debug_flag: + self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % + stderr) try: return json.loads(stdout) except ValueError: @@ -402,7 +436,7 @@ class YumCollection(Collection): os.unlink(self.cfgfile) self.write_config() - + if force_update: self.call_helper("clean") @@ -416,24 +450,23 @@ class YumSource(Source): self.pulp_id = None if has_pulp and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") - + _setup_pulp(self.config) repoapi = RepositoryAPI() try: self.repo = repoapi.repository(self.pulp_id) - self.gpgkeys = ["%s/%s" % (PULPCONFIG.cds['keyurl'], key) + self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key) for key in repoapi.listkeys(self.pulp_id)] except server.ServerRequestError: err = sys.exc_info()[1] if err[0] == 401: msg = "Packages: Error authenticating to Pulp: %s" % err[1] elif err[0] == 404: - msg = "Packages: Pulp repo id %s not found: %s" % (self.pulp_id, - err[1]) + msg = "Packages: Pulp repo id %s not found: %s" % \ + (self.pulp_id, err[1]) else: - msg = "Packages: Error %d fetching pulp repo %s: %s" % (err[0], - self.pulp_id, - err[1]) + msg = "Packages: Error %d fetching pulp repo %s: %s" % \ + (err[0], self.pulp_id, err[1]) raise SourceInitError(msg) except socket.error: err = sys.exc_info()[1] @@ -445,7 +478,7 @@ class YumSource(Source): self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'], self.repo['relative_path']) self.arches = [self.repo['arch']] - + if not self.rawurl: self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" else: @@ -458,11 +491,8 @@ class YumSource(Source): self.needed_paths = set() self.file_to_arch = dict() - self.use_yum = has_yum - try: - self.use_yum &= config.getboolean("yum", "use_yum_libraries") - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - self.use_yum = False + self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries", + default=False) def save_state(self): if not self.use_yum: @@ -470,7 +500,7 @@ class YumSource(Source): cPickle.dump((self.packages, self.deps, self.provides, self.filemap, self.url_map), cache, 2) cache.close() - + def load_state(self): if not self.use_yum: @@ -486,7 +516,7 @@ class YumSource(Source): usettings = [{'version':self.version, 'component':comp, 'arch':arch} for comp in self.components] - else: # rawurl given + else: # rawurl given usettings = [{'version':self.version, 'component':None, 'arch':arch}] @@ -504,23 +534,23 @@ class YumSource(Source): def _get_urls_from_repodata(self, url, arch): if self.use_yum: return [url] - + rmdurl = '%srepodata/repomd.xml' % url try: repomd = fetch_url(rmdurl) xdata = lxml.etree.XML(repomd) except ValueError: - logger.error("Packages: Bad url string %s" % rmdurl) + self.logger.error("Packages: Bad url string %s" % rmdurl) return [] except HTTPError: err = sys.exc_info()[1] - logger.error("Packages: Failed to fetch url %s. code=%s" % - (rmdurl, err.code)) + self.logger.error("Packages: Failed to fetch url %s. code=%s" % + (rmdurl, err.code)) return [] except lxml.etree.XMLSyntaxError: err = sys.exc_info()[1] - logger.error("Packages: Failed to process metadata at %s: %s" % - (rmdurl, err)) + self.logger.error("Packages: Failed to process metadata at %s: %s" % + (rmdurl, err)) return [] urls = [] @@ -594,12 +624,13 @@ class YumSource(Source): self.packages[arch].add(pkgname) pdata = pkg.find(XP + 'format') - pre = pdata.find(RP + 'requires') self.deps[arch][pkgname] = set() - for entry in pre.getchildren(): - self.deps[arch][pkgname].add(entry.get('name')) - if entry.get('name').startswith('/'): - self.needed_paths.add(entry.get('name')) + pre = pdata.find(RP + 'requires') + if pre is not None: + for entry in pre.getchildren(): + self.deps[arch][pkgname].add(entry.get('name')) + if entry.get('name').startswith('/'): + self.needed_paths.add(entry.get('name')) pro = pdata.find(RP + 'provides') if pro != None: for entry in pro.getchildren(): @@ -620,7 +651,7 @@ class YumSource(Source): def get_vpkgs(self, metadata): if self.use_yum: return dict() - + rv = Source.get_vpkgs(self, metadata) for arch, fmdata in list(self.filemap.items()): if arch not in metadata.groups and arch != 'global': diff --git a/src/lib/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 9e1ccfa37..e4793a28d 100644 --- a/src/lib/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -4,7 +4,6 @@ import time import copy import glob import shutil -import logging import lxml.etree import Bcfg2.Logger import Bcfg2.Server.Plugin @@ -13,8 +12,6 @@ from Bcfg2.Server.Plugins.Packages import Collection from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig -logger = logging.getLogger('Packages') - class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, @@ -39,25 +36,40 @@ class Packages(Bcfg2.Server.Plugin.Plugin, os.makedirs(self.keypath) # set up config files - self.config = PackagesConfig(os.path.join(self.data, "packages.conf"), - core.fam, self) + self.config = PackagesConfig(self) self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, self.config) + def toggle_debug(self): + Bcfg2.Server.Plugin.Plugin.toggle_debug(self) + self.sources.toggle_debug() + @property def disableResolver(self): try: - return self.config.get("global", "resolver").lower() == "disabled" + return not self.config.getboolean("global", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False + except ValueError: + # for historical reasons we also accept "enabled" and + # "disabled", which are not handled according to the + # Python docs but appear to be handled properly by + # ConfigParser in at least some versions + return self.config.get("global", "resolver", + default="enabled").lower() == "disabled" @property def disableMetaData(self): try: - return self.config.get("global", "metadata").lower() == "disabled" + return not self.config.getboolean("global", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False + except ValueError: + # for historical reasons we also accept "enabled" and + # "disabled" + return self.config.get("global", "metadata", + default="enabled").lower() == "disabled" def create_config(self, entry, metadata): """ create yum/apt config for the specified host """ @@ -67,38 +79,41 @@ class Packages(Bcfg2.Server.Plugin.Plugin, 'type': 'file', 'perms': '0644'} - collection = Collection.factory(metadata, self.sources, self.data) + collection = self._get_collection(metadata) entry.text = collection.get_config() for (key, value) in list(attrib.items()): entry.attrib.__setitem__(key, value) def HandleEntry(self, entry, metadata): if entry.tag == 'Package': - collection = Collection.factory(metadata, self.sources, self.data) + collection = self._get_collection(metadata) entry.set('version', 'auto') + entry.set('version', self.config.get("global", + "version", + default="auto")) entry.set('type', collection.ptype) elif entry.tag == 'Path': - if (self.config.has_section("global") and - ((self.config.has_option("global", "yum_config") and - entry.get("name") == self.config.get("global", - "yum_config")) or - (self.config.has_option("global", "apt_config") and - entry.get("name") == self.config.get("global", - "apt_config")))): + if (entry.get("name") == self.config.get("global", "yum_config", + default="") or + entry.get("name") == self.config.get("global", "apt_config", + default="")): self.create_config(entry, metadata) def HandlesEntry(self, entry, metadata): if entry.tag == 'Package': - collection = Collection.factory(metadata, self.sources, self.data) - if collection.magic_groups_match(): + if self.config.getboolean("global", "magic_groups", + default=True) == True: + collection = self._get_collection(metadata) + if collection.magic_groups_match(): + return True + else: return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if ((self.config.has_option("global", "yum_config") and - entry.get("name") == self.config.get("global", - "yum_config")) or - (self.config.has_option("global", "apt_config") and - entry.get("name") == self.config.get("global", "apt_config"))): + if (entry.get("name") == self.config.get("global", "yum_config", + default="") or + entry.get("name") == self.config.get("global", "apt_config", + default="")): return True return False @@ -109,7 +124,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, metadata - client metadata instance structures - a list of structure-stage entry combinations ''' - collection = Collection.factory(metadata, self.sources, self.data) + collection = self._get_collection(metadata) indep = lxml.etree.Element('Independent') self._build_packages(metadata, indep, structures, collection=collection) @@ -125,12 +140,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return if collection is None: - collection = Collection.factory(metadata, self.sources, self.data) + collection = self._get_collection(metadata) # initial is the set of packages that are explicitly specified # in the configuration initial = set() # base is the set of initial packages with groups expanded base = set() + # essential pkgs are those marked as such by the distribution + essential = collection.get_essential() to_remove = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): @@ -152,7 +169,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, else: self.logger.error("Packages: Malformed Package: %s" % lxml.etree.tostring(pkg)) - base.update(initial) + base.update(initial | essential) for el in to_remove: el.getparent().remove(el) @@ -161,13 +178,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, self.logger.info("Packages: Got %d unknown entries" % len(unknown)) self.logger.info("Packages: %s" % list(unknown)) newpkgs = list(packages.difference(initial)) - self.logger.debug("Packages: %d initial, %d complete, %d new" % - (len(initial), len(packages), len(newpkgs))) + self.debug_log("Packages: %d initial, %d complete, %d new" % + (len(initial), len(packages), len(newpkgs))) newpkgs.sort() for pkg in newpkgs: lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, - version='auto', type=collection.ptype, - origin='Packages') + version=self.config.get("global", "version", + default="auto"), + type=collection.ptype, origin='Packages') def Refresh(self): '''Packages.Refresh() => True|False\nReload configuration @@ -218,8 +236,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, os.unlink(cfile) except OSError: err = sys.exc_info()[1] - logger.error("Packages: Could not remove cache file %s: %s" - % (cfile, err)) + self.logger.error("Packages: Could not remove cache file " + "%s: %s" % (cfile, err)) def _load_gpg_keys(self, force_update): """ Load gpg keys from the config """ @@ -242,6 +260,10 @@ class Packages(Bcfg2.Server.Plugin.Plugin, if kfile not in keyfiles: os.unlink(kfile) + def _get_collection(self, metadata): + return Collection.factory(metadata, self.sources, self.data, + debug=self.debug_flag) + def get_additional_data(self, metadata): - collection = Collection.factory(metadata, self.sources, self.data) + collection = self._get_collection(metadata) return dict(sources=collection.get_additional_data()) diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index bf674d0d0..e9254cdcc 100644 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -1,10 +1,13 @@ '''This module implements a package management scheme for all images''' -__revision__ = '$Revision$' import logging import re import Bcfg2.Server.Plugin import lxml +try: + set +except NameError: + from sets import Set as set logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr') @@ -63,8 +66,8 @@ class PNode(Bcfg2.Server.Plugin.INode): if 'Package' not in pdict: pdict['Package'] = set() for child in data.getchildren(): - for attr in [key for key in list(data.attrib.keys()) - if key != 'name' and key not in child.attrib]: + attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name']) + for attr in attrs: try: child.set(attr, data.get(attr)) except: @@ -131,7 +134,6 @@ class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles package assignments.""" name = 'Pkgmgr' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = PkgSrc __element__ = 'Package' diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 5ab92c011..af908eee8 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -51,10 +51,10 @@ class ProbeData(object): def __str__(self): return str(self.data) - + def __repr__(self): return repr(self.data) - + def __getattr__(self, name): """ make ProbeData act like a str object """ return getattr(self.data, name) @@ -137,6 +137,16 @@ class ProbeSet(Bcfg2.Server.Plugin.EntrySet): def HandleEvent(self, event): if event.filename != self.path: + if (event.code2str == 'changed' and + event.filename.endswith("probed.xml") and + event.filename not in self.entries): + # for some reason, probed.xml is particularly prone to + # getting changed events before created events, + # because gamin is the worst ever. anyhow, we + # specifically handle it here to avoid a warning on + # every single server startup. + self.entry_init(event) + return return self.handle_event(event) def get_probe_data(self, metadata): @@ -171,7 +181,6 @@ class Probes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): """A plugin to gather information from a client machine.""" name = 'Probes' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 58f7215c9..680881858 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -1,4 +1,5 @@ import os +import re import sys import copy import logging @@ -49,6 +50,7 @@ class PropertyFile(Bcfg2.Server.Plugin.StructFile): class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile + patterns = re.compile(r'.*\.xml$') class Properties(Bcfg2.Server.Plugin.Plugin, @@ -58,7 +60,6 @@ class Properties(Bcfg2.Server.Plugin.Plugin, files into client metadata instances. """ name = 'Properties' - version = '$Revision$' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -72,4 +73,4 @@ class Properties(Bcfg2.Server.Plugin.Plugin, raise Bcfg2.Server.Plugin.PluginInitError def get_additional_data(self, _): - return copy.deepcopy(self.store.entries) + return copy.copy(self.store.entries) diff --git a/src/lib/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index a146dca6a..b80ef351a 100644 --- a/src/lib/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -1,15 +1,21 @@ """This generator provides rule-based entry mappings.""" -__revision__ = '$Revision$' import re import Bcfg2.Server.Plugin +class RulesConfig(Bcfg2.Server.Plugin.SimpleConfig): + _required = False + class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" name = 'Rules' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) + self.config = RulesConfig(self) + self._regex_cache = dict() + def HandlesEntry(self, entry, metadata): if entry.tag in self.Entries: return self._matches(entry, metadata, @@ -28,10 +34,22 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): def _matches(self, entry, metadata, rules): if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, rules): return True - else: + elif (entry.tag == "Path" and + ((entry.get('name').endswith("/") and + entry.get('name').rstrip("/") in rules) or + (not entry.get('name').endswith("/") and + entry.get('name') + '/' in rules))): + # special case for Path tags: + # http://trac.mcs.anl.gov/projects/bcfg2/ticket/967 + return True + elif self._regex_enabled: # attempt regular expression matching for rule in rules: - if re.match("%s$" % rule, entry.get('name')): + if rule not in self._regex_cache: + self._regex_cache[rule] = re.compile("%s$" % rule) + if self._regex_cache[rule].match(entry.get('name')): return True return False - + + def _regex_enabled(self): + return self.config.getboolean("rules", "regex", default=False) diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Bcfg2/Server/Plugins/SGenshi.py index f6a98c141..0ba08125e 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Bcfg2/Server/Plugins/SGenshi.py @@ -1,5 +1,4 @@ '''This module implements a templating generator based on Genshi''' -__revision__ = '$Revision$' import genshi.input import genshi.template @@ -84,7 +83,6 @@ class SGenshi(SGenshiEntrySet, Bcfg2.Server.Plugin.Structure): """The SGenshi plugin provides templated structures.""" name = 'SGenshi' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' deprecated = True diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index 724d169f5..ac281ad1a 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -1,5 +1,4 @@ '''This module manages ssh key files for bcfg2''' -__revision__ = '$Revision$' import binascii import re @@ -23,7 +22,7 @@ class KeyData(Bcfg2.Server.Plugin.SpecificData): Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, encoding) self.encoding = encoding - + def bind_entry(self, entry, metadata): entry.set('type', 'file') if entry.get('encoding') == 'base64': @@ -99,7 +98,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, """ name = 'SSHbase' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' keypatterns = ["ssh_host_dsa_key", @@ -148,7 +146,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for s in list(self.static.values())] mquery = self.core.metadata.query - + # build hostname cache names = dict() for cmeta in mquery.all(): @@ -206,13 +204,13 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, # have no clients yet. don't warn about # this. continue - + if key not in self.badnames: self.badnames[key] = True self.logger.info("Ignoring key for unknown %s %s" % (ktype, key)) continue - + skn.append("%s %s" % (','.join(hostnames), entry.data.decode().rstrip())) @@ -230,11 +228,13 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, action = event.code2str() if action == "endExist" or event.filename == self.data: return - + for entry in list(self.entries.values()): if entry.specific.match(event.filename): entry.handle_event(event) if event.filename.endswith(".pub"): + self.logger.info("New public key %s; invalidating " + "ssh_known_hosts cache" % event.filename) self.skn = False return @@ -244,6 +244,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, return if event.filename.endswith('.static'): + self.logger.info("Static key %s %s; invalidating ssh_known_hosts " + "cache" % (event.filename, action)) if action == "deleted" and event.filename in self.static: del self.static[event.filename] self.skn = False @@ -367,7 +369,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: self.logger.error("Unknown key filename: %s" % filename) return - + fileloc = "%s/%s" % (self.data, hostkey) publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub', "H_%s" % client]) diff --git a/src/lib/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index 7b4a08ae1..0072dc62d 100644 --- a/src/lib/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -16,7 +16,6 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): management of ssl certificates and their keys. """ name = 'SSLCA' - __version__ = '$Id:$' __author__ = 'g.hagger@gmail.com' __child__ = Bcfg2.Server.Plugin.FileBacked key_specs = {} diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py index aeb3b9f74..aeb3b9f74 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Bcfg2/Server/Plugins/Snapshots.py diff --git a/src/lib/Server/Plugins/Statistics.py b/src/lib/Bcfg2/Server/Plugins/Statistics.py index 7251ab1b5..265ef95a8 100644 --- a/src/lib/Server/Plugins/Statistics.py +++ b/src/lib/Bcfg2/Server/Plugins/Statistics.py @@ -1,5 +1,4 @@ '''This file manages the statistics collected by the BCFG2 Server''' -__revision__ = '$Revision$' import binascii import copy @@ -98,7 +97,7 @@ class StatisticsStore(object): newstat.set('time', asctime(localtime())) # Add statistic - node.append(copy.deepcopy(newstat)) + node.append(copy.copy(newstat)) # Set dirty self.dirty = 1 @@ -117,7 +116,6 @@ class Statistics(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ThreadedStatistics, Bcfg2.Server.Plugin.PullSource): name = 'Statistics' - __version__ = '$Id$' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) diff --git a/src/lib/Server/Plugins/Svcmgr.py b/src/lib/Bcfg2/Server/Plugins/Svcmgr.py index 6d25c1a6d..f4232ad5c 100644 --- a/src/lib/Server/Plugins/Svcmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Svcmgr.py @@ -1,5 +1,4 @@ """This generator provides service mappings.""" -__revision__ = '$Revision$' import Bcfg2.Server.Plugin @@ -7,6 +6,5 @@ import Bcfg2.Server.Plugin class Svcmgr(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" name = 'Svcmgr' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' deprecated = True diff --git a/src/lib/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index a127d0273..ae43388ea 100644 --- a/src/lib/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -1,4 +1,5 @@ import os +import pipes from subprocess import Popen, PIPE import Bcfg2.Server.Plugin @@ -11,7 +12,6 @@ class Svn(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" name = 'Svn' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Bcfg2/Server/Plugins/Svn2.py index b8a8e6b7e..e4df9574f 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn2.py @@ -9,7 +9,6 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" name = 'Svn2' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' conflicts = ['Svn'] @@ -72,7 +71,8 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, self.revision = self.client.update(self.datastore, recurse=True)[0] self.logger.info("Svn2: Commited changes. At %s" % self.revision.number) - except Exception, err: + except Exception: + err = sys.exc_info()[1] # try to be smart about the error we got back details = None if "callback_ssl_server_trust_prompt" in str(err): diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Bcfg2/Server/Plugins/TCheetah.py index 49be88881..8879fdef1 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Bcfg2/Server/Plugins/TCheetah.py @@ -1,5 +1,4 @@ '''This module implements a templating generator based on Cheetah''' -__revision__ = '$Revision$' import binascii import logging @@ -76,7 +75,6 @@ class TemplateFile: class TCheetah(Bcfg2.Server.Plugin.GroupSpool): """The TCheetah generator implements a templating mechanism for configuration files.""" name = 'TCheetah' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' filename_pattern = 'template' es_child_cls = TemplateFile diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Bcfg2/Server/Plugins/TGenshi.py index bc5e00400..c4dd40614 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Bcfg2/Server/Plugins/TGenshi.py @@ -1,5 +1,4 @@ """This module implements a templating generator based on Genshi.""" -__revision__ = '$Revision$' import binascii import logging @@ -115,13 +114,13 @@ class TemplateFile: if entry.text == '': entry.set('empty', 'true') except TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - raise Bcfg2.Server.Plugin.PluginExecutionError + err = sys.exc_info()[1] + logger.exception('Genshi template error') + raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template error: %s' % err) except AttributeError: err = sys.exc_info()[1] - logger.error('Genshi template loading error: %s' % err) - raise Bcfg2.Server.Plugin.PluginExecutionError + logger.exception('Genshi template loading error') + raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template loading error: %s' % err) class TGenshi(Bcfg2.Server.Plugin.GroupSpool): @@ -131,7 +130,6 @@ class TGenshi(Bcfg2.Server.Plugin.GroupSpool): """ name = 'TGenshi' - __version__ = '$Id$' __author__ = 'jeff@ocjtech.us' filename_pattern = 'template\.(txt|newtxt|xml)' es_child_cls = TemplateFile diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py new file mode 100644 index 000000000..2c0ee03e0 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -0,0 +1,83 @@ +import re +import imp +import sys +import logging +import Bcfg2.Server.Plugin + +logger = logging.getLogger(__name__) + +class HelperModule(Bcfg2.Server.Plugin.SpecificData): + _module_name_re = re.compile(r'([^/]+?)\.py') + + def __init__(self, name, specific, encoding): + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, + encoding) + match = self._module_name_re.search(self.name) + if match: + self._module_name = match.group(1) + else: + self._module_name = name + self._attrs = [] + + def handle_event(self, event): + Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) + try: + module = imp.load_source(self._module_name, self.name) + except: + err = sys.exc_info()[1] + logger.error("TemplateHelper: Failed to import %s: %s" % + (self.name, err)) + return + + if not hasattr(module, "__export__"): + logger.error("TemplateHelper: %s has no __export__ list" % + self.name) + return + + for sym in module.__export__: + if sym not in self._attrs and hasattr(self, sym): + logger.warning("TemplateHelper: %s: %s is a reserved keyword, " + "skipping export" % (self.name, sym)) + setattr(self, sym, getattr(module, sym)) + # remove old exports + for sym in set(self._attrs) - set(module.__export__): + delattr(self, sym) + + self._attrs = module.__export__ + + +class HelperSet(Bcfg2.Server.Plugin.EntrySet): + ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") + + def __init__(self, path, fam, encoding, plugin_name): + fpattern = '[0-9A-Za-z_\-]+\.py' + self.plugin_name = plugin_name + Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path, + HelperModule, encoding) + fam.AddMonitor(path, self) + + def HandleEvent(self, event): + if (event.filename != self.path and + not self.ignore.match(event.filename)): + return self.handle_event(event) + + +class TemplateHelper(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + """ A plugin to provide helper classes and functions to templates """ + name = 'TemplateHelper' + __author__ = 'chris.a.st.pierre@gmail.com' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + + try: + self.helpers = HelperSet(self.data, core.fam, core.encoding, + self.name) + except: + raise Bcfg2.Server.Plugin.PluginInitError + + def get_additional_data(self, metadata): + return dict([(h._module_name, h) + for h in list(self.helpers.entries.values())]) diff --git a/src/lib/Server/Plugins/Trigger.py b/src/lib/Bcfg2/Server/Plugins/Trigger.py index f6dd47e12..b0d21545c 100644 --- a/src/lib/Server/Plugins/Trigger.py +++ b/src/lib/Bcfg2/Server/Plugins/Trigger.py @@ -17,7 +17,6 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Statistics): """Trigger is a plugin that calls external scripts (on the server).""" name = 'Trigger' - __version__ = '$Id' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): @@ -26,12 +25,19 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, try: os.stat(self.data) except: - self.logger.error("Trigger: spool directory %s does not exist; unloading" % self.data) + self.logger.error("Trigger: spool directory %s does not exist; " + "unloading" % self.data) raise Bcfg2.Server.Plugin.PluginInitError def process_statistics(self, metadata, _): args = [metadata.hostname, '-p', metadata.profile, '-g', ':'.join([g for g in metadata.groups])] for notifier in os.listdir(self.data): - n = self.data + '/' + notifier - async_run(n, args) + if ((notifier[-1] == '~') or + (notifier[:2] == '.#') or + (notifier[-4:] == '.swp') or + (notifier in ['SCCS', '.svn', '4913'])): + continue + npath = self.data + '/' + notifier + self.logger.debug("Running %s %s" % (npath, " ".join(args))) + async_run(npath, args) diff --git a/src/lib/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py index c69c37452..f9f1b4e52 100644 --- a/src/lib/Server/Plugins/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/__init__.py @@ -1,5 +1,4 @@ """Imports for Bcfg2.Server.Plugins.""" -__revision__ = '$Revision$' __all__ = [ 'Account', @@ -8,7 +7,7 @@ __all__ = [ 'Bzr', 'Cfg', 'Cvs', - 'Darcs', + 'Darcs', 'Decisions', 'Fossil', 'Git', diff --git a/src/lib/Server/Reports/__init__.py b/src/lib/Bcfg2/Server/Reports/__init__.py index bdf908f4a..bdf908f4a 100644 --- a/src/lib/Server/Reports/__init__.py +++ b/src/lib/Bcfg2/Server/Reports/__init__.py diff --git a/src/lib/Server/Reports/backends.py b/src/lib/Bcfg2/Server/Reports/backends.py index 85241932f..85241932f 100644 --- a/src/lib/Server/Reports/backends.py +++ b/src/lib/Bcfg2/Server/Reports/backends.py diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py index 7dfac6fae..16df86a9b 100755 --- a/src/lib/Server/Reports/importscript.py +++ b/src/lib/Bcfg2/Server/Reports/importscript.py @@ -3,7 +3,6 @@ Imports statistics.xml and clients.xml files in to database backend for new statistics engine """ -__revision__ = '$Revision$' import binascii import os @@ -41,6 +40,7 @@ from Bcfg2.Bcfg2Py3k import ConfigParser def build_reason_kwargs(r_ent, encoding, logger): binary_file = False sensitive_file = False + unpruned_entries = '' if r_ent.get('sensitive') in ['true', 'True']: sensitive_file = True rc_diff = '' @@ -58,6 +58,10 @@ def build_reason_kwargs(r_ent, encoding, logger): rc_diff = r_ent.get('current_diff') else: rc_diff = '' + # detect unmanaged entries in pruned directories + if r_ent.get('prune', 'false') == 'true' and r_ent.get('qtest'): + unpruned_elist = [e.get('path') for e in r_ent.findall('Prune')] + unpruned_entries = "\n".join(unpruned_elist) if not binary_file: try: rc_diff = rc_diff.decode(encoding) @@ -79,7 +83,8 @@ def build_reason_kwargs(r_ent, encoding, logger): current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", current_diff=rc_diff, is_binary=binary_file, - is_sensitive=sensitive_file) + is_sensitive=sensitive_file, + unpruned=unpruned_entries) def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): @@ -118,8 +123,6 @@ def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location='') default="unknown"), repo_rev_code=statistics.get('revision', default="unknown"), - client_version=statistics.get('client_version', - default="unknown"), goodcount=statistics.get('good', default="0"), totalcount=statistics.get('total', diff --git a/src/lib/Server/Reports/manage.py b/src/lib/Bcfg2/Server/Reports/manage.py index 858bddeca..858bddeca 100755 --- a/src/lib/Server/Reports/manage.py +++ b/src/lib/Bcfg2/Server/Reports/manage.py diff --git a/src/lib/Server/Reports/nisauth.py b/src/lib/Bcfg2/Server/Reports/nisauth.py index 6fc346f1e..b3e37113b 100644 --- a/src/lib/Server/Reports/nisauth.py +++ b/src/lib/Bcfg2/Server/Reports/nisauth.py @@ -4,8 +4,6 @@ from Bcfg2.Server.Reports.settings import AUTHORIZED_GROUP """Checks with NIS to see if the current user is in the support group""" -__revision__ = "$Revision: $" - class NISAUTHError(Exception): """NISAUTHError is raised when somehting goes boom.""" diff --git a/src/lib/Server/Reports/reports/__init__.py b/src/lib/Bcfg2/Server/Reports/reports/__init__.py index ccdce8943..ccdce8943 100644 --- a/src/lib/Server/Reports/reports/__init__.py +++ b/src/lib/Bcfg2/Server/Reports/reports/__init__.py diff --git a/src/lib/Server/Reports/reports/fixtures/initial_version.xml b/src/lib/Bcfg2/Server/Reports/reports/fixtures/initial_version.xml index 919265d48..bde236989 100644 --- a/src/lib/Server/Reports/reports/fixtures/initial_version.xml +++ b/src/lib/Bcfg2/Server/Reports/reports/fixtures/initial_version.xml @@ -36,4 +36,8 @@ <field type='IntegerField' name='version'>18</field> <field type='DateTimeField' name='updated'>2011-06-30 00:00:00</field> </object> + <object pk="8" model="reports.internaldatabaseversion"> + <field type='IntegerField' name='version'>19</field> + <field type='DateTimeField' name='updated'>2012-03-28 00:00:00</field> + </object> </django-objects> diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py index 870239641..0438ea133 100644 --- a/src/lib/Server/Reports/reports/models.py +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -156,7 +156,6 @@ class Interaction(models.Model): timestamp = models.DateTimeField() # Timestamp for this record state = models.CharField(max_length=32) # good/bad/modified/etc repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction - client_version = models.CharField(max_length=32) # Client Version goodcount = models.IntegerField() # of good config-items totalcount = models.IntegerField() # of total config-items server = models.CharField(max_length=256) # Name of the server used for the interaction @@ -278,6 +277,7 @@ class Reason(models.Model): current_diff = models.TextField(max_length=1280, blank=True) is_binary = models.BooleanField(default=False) is_sensitive = models.BooleanField(default=False) + unpruned = models.TextField(max_length=1280, blank=True) def _str_(self): return "Reason" diff --git a/src/lib/Server/Reports/reports/sql/client.sql b/src/lib/Bcfg2/Server/Reports/reports/sql/client.sql index 8c63754c9..8c63754c9 100644 --- a/src/lib/Server/Reports/reports/sql/client.sql +++ b/src/lib/Bcfg2/Server/Reports/reports/sql/client.sql diff --git a/src/lib/Server/Reports/reports/templates/404.html b/src/lib/Bcfg2/Server/Reports/reports/templates/404.html index 168bd9fec..168bd9fec 100644 --- a/src/lib/Server/Reports/reports/templates/404.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/404.html diff --git a/src/lib/Server/Reports/reports/templates/base-timeview.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html index 842de36f0..842de36f0 100644 --- a/src/lib/Server/Reports/reports/templates/base-timeview.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base.html index 75f46c426..f541c0d2b 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/base.html @@ -87,7 +87,7 @@ <div style='clear:both'></div> </div><!-- document --> <div id="footer"> - <span>Bcfg2 Version 1.2.0</span> + <span>Bcfg2 Version 1.2.2</span> </div> <div id="calendar_div" style='position:absolute; visibility:hidden; background-color:white; layer-background-color:white;'></div> diff --git a/src/lib/Server/Reports/reports/templates/clients/detail.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html index dd4295f21..dd4295f21 100644 --- a/src/lib/Server/Reports/reports/templates/clients/detail.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html diff --git a/src/lib/Server/Reports/reports/templates/clients/detailed-list.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html index 0c1fae8d5..0c1fae8d5 100644 --- a/src/lib/Server/Reports/reports/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html diff --git a/src/lib/Server/Reports/reports/templates/clients/history.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html index 01d4ec2f4..01d4ec2f4 100644 --- a/src/lib/Server/Reports/reports/templates/clients/history.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html diff --git a/src/lib/Server/Reports/reports/templates/clients/index.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html index e0c0d2d7a..e0c0d2d7a 100644 --- a/src/lib/Server/Reports/reports/templates/clients/index.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html diff --git a/src/lib/Server/Reports/reports/templates/clients/manage.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html index 5725ae577..5725ae577 100644 --- a/src/lib/Server/Reports/reports/templates/clients/manage.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html diff --git a/src/lib/Server/Reports/reports/templates/config_items/item.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html index cc99ef503..cadc178a7 100644 --- a/src/lib/Server/Reports/reports/templates/config_items/item.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% load split %} {% load syntax_coloring %} @@ -91,6 +92,20 @@ div.entry_list h3 { </div> {% endif %} + <!-- display extra directory entries --> + {% if item.reason.unpruned %} + <div class='entry_list'> + <div class='entry_list_head'> + <h3>Extra entries found</h3> + </div> + <table class='entry_list' cellpadding='3'> + {% for unpruned_item in item.reason.unpruned|split %} + <tr><td>{{ unpruned_item }}</td></tr> + {% endfor %} + </table> + </div> + {% endif %} + <div class='entry_list'> <div class='entry_list_head'> diff --git a/src/lib/Server/Reports/reports/templates/config_items/listing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html index 9b1026a08..9b1026a08 100644 --- a/src/lib/Server/Reports/reports/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html diff --git a/src/lib/Server/Reports/reports/templates/displays/summary.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html index b9847cf96..b9847cf96 100644 --- a/src/lib/Server/Reports/reports/templates/displays/summary.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html diff --git a/src/lib/Server/Reports/reports/templates/displays/timing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html index 47accb2cb..ff775ded5 100644 --- a/src/lib/Server/Reports/reports/templates/displays/timing.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html @@ -20,7 +20,7 @@ <td>Install</td> <td>Config</td> <td>Total</td> - </tr> + </tr> {% for metric in metrics|dictsort:"name" %} <tr class='{% cycle listview,listview_alt %}'> <td><a style='font-size: 100%' diff --git a/src/lib/Server/Reports/reports/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html index 6b57baf6a..6fbe585ab 100644 --- a/src/lib/Server/Reports/reports/templates/widgets/filter_bar.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html @@ -2,7 +2,7 @@ {% if filters %} {% for filter, filter_url in filters %} {% if forloop.first %} - <div class="filter_bar">Active filters (click to remove): + <div class="filter_bar">Active filters (click to remove): {% endif %} <a href='{{ filter_url }}'>{{ filter|capfirst }}</a>{% if not forloop.last %}, {% endif %} {% if forloop.last %} diff --git a/src/lib/Server/Reports/reports/templates/widgets/interaction_list.inc b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc index 8f2dec1dc..8f2dec1dc 100644 --- a/src/lib/Server/Reports/reports/templates/widgets/interaction_list.inc +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc diff --git a/src/lib/Server/Reports/reports/templates/widgets/page_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html index aa0def83e..aa0def83e 100644 --- a/src/lib/Server/Reports/reports/templates/widgets/page_bar.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html diff --git a/src/lib/Server/Reports/reports/templatetags/__init__.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Server/Reports/reports/templatetags/__init__.py +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py diff --git a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py index 629984f26..f738f7bdd 100644 --- a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py @@ -81,7 +81,7 @@ def page_navigator(context): fragment['pager'] = pager fragment['page_limits'] = page_limits - + except Resolver404: path = "404" except NoReverseMatch: @@ -155,7 +155,7 @@ def isstale(timestamp, entry_max=None): """ Check for a stale timestamp - Compares two timestamps and returns True if the + Compares two timestamps and returns True if the difference is greater then 24 hours. """ if not entry_max: diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py new file mode 100644 index 000000000..a9b4f0371 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py @@ -0,0 +1,8 @@ +from django import template +register = template.Library() + + +@register.filter +def split(s): + """split by newlines""" + return s.split('\n') diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py index 2e30125f9..2e30125f9 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py diff --git a/src/lib/Server/Reports/reports/urls.py b/src/lib/Bcfg2/Server/Reports/reports/urls.py index 434ce07b7..434ce07b7 100644 --- a/src/lib/Server/Reports/reports/urls.py +++ b/src/lib/Bcfg2/Server/Reports/reports/urls.py diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Bcfg2/Server/Reports/reports/views.py index ccd71a60e..ccd71a60e 100644 --- a/src/lib/Server/Reports/reports/views.py +++ b/src/lib/Bcfg2/Server/Reports/reports/views.py diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Bcfg2/Server/Reports/settings.py index 128658ff1..4d567f1a2 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Bcfg2/Server/Reports/settings.py @@ -6,20 +6,18 @@ from Bcfg2.Bcfg2Py3k import ConfigParser # Django settings for bcfg2 reports project. c = ConfigParser.ConfigParser() if len(c.read(['/etc/bcfg2.conf', '/etc/bcfg2-web.conf'])) == 0: - print("Please check that bcfg2.conf or bcfg2-web.conf exists " - "and is readable by your web server.") - sys.exit(1) + raise ImportError("Please check that bcfg2.conf or bcfg2-web.conf exists " + "and is readable by your web server.") try: - dset = c.get('statistics', 'web_debug') + DEBUG = c.getboolean('statistics', 'web_debug') except: - dset = 'false' - -if dset == "True": - DEBUG = True -else: DEBUG = False - + +if DEBUG: + print("Warning: Setting web_debug to True causes extraordinary memory " + "leaks. Only use this setting if you know what you're doing.") + TEMPLATE_DEBUG = DEBUG ADMINS = ( @@ -31,8 +29,7 @@ try: db_engine = c.get('statistics', 'database_engine') except ConfigParser.NoSectionError: e = sys.exc_info()[1] - print("Failed to determine database engine: %s" % e) - sys.exit(1) + raise ImportError("Failed to determine database engine: %s" % e) db_name = '' if c.has_option('statistics', 'database_name'): db_name = c.get('statistics', 'database_name') @@ -67,10 +64,10 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 2: # Local time zone for this installation. All choices can be found here: # http://docs.djangoproject.com/en/dev/ref/settings/#time-zone -try: - TIME_ZONE = c.get('statistics', 'time_zone') -except: - if django.VERSION[0] == 1 and django.VERSION[1] > 2: +if django.VERSION[0] == 1 and django.VERSION[1] > 2: + try: + TIME_ZONE = c.get('statistics', 'time_zone') + except: TIME_ZONE = None # Language code for this installation. All choices can be found here: @@ -123,12 +120,12 @@ AUTHORIZED_GROUP = '' try: import django.contrib.auth except ImportError: - print('Import of Django module failed. Is Django installed?') + raise ImportError('Import of Django module failed. Is Django installed?') django.contrib.auth.LOGIN_URL = '/login' SESSION_EXPIRE_AT_BROWSER_CLOSE = True - - + + TEMPLATE_DIRS = ( # Put strings here, like "/home/html/django_templates". diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Bcfg2/Server/Reports/updatefix.py index 7cebaaca9..39fc10b56 100644 --- a/src/lib/Server/Reports/updatefix.py +++ b/src/lib/Bcfg2/Server/Reports/updatefix.py @@ -1,8 +1,9 @@ import Bcfg2.Server.Reports.settings -from django.db import connection +from django.db import connection, DatabaseError import django.core.management import logging +import sys import traceback from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion, \ TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA @@ -16,9 +17,9 @@ def _merge_database_table_entries(): find_cursor = connection.cursor() cursor.execute(""" Select name, kind from reports_bad - union + union select name, kind from reports_modified - union + union select name, kind from reports_extra """) # this fetch could be better done @@ -51,7 +52,7 @@ def _merge_database_table_entries(): def _interactions_constraint_or_idx(): - '''sqlite doesn't support alter tables.. or constraints''' + """sqlite doesn't support alter tables.. or constraints""" cursor = connection.cursor() try: cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') @@ -59,6 +60,72 @@ def _interactions_constraint_or_idx(): cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') +def _remove_table_column(tbl, col): + """sqlite doesn't support deleting a column via alter table""" + cursor = connection.cursor() + try: + cursor.execute('alter table %s ' + 'drop column %s;' % (tbl, col)) + except DatabaseError: + # sqlite wants us to create a new table containing the columns we want + # and copy into it http://www.sqlite.org/faq.html#q11 + + tmptbl_name = "t_backup" + _tmptbl_create = \ +"""create temporary table "%s" ( + "id" integer NOT NULL PRIMARY KEY, + "client_id" integer NOT NULL REFERENCES "reports_client" ("id"), + "timestamp" datetime NOT NULL, + "state" varchar(32) NOT NULL, + "repo_rev_code" varchar(64) NOT NULL, + "goodcount" integer NOT NULL, + "totalcount" integer NOT NULL, + "server" varchar(256) NOT NULL, + "bad_entries" integer NOT NULL, + "modified_entries" integer NOT NULL, + "extra_entries" integer NOT NULL, + UNIQUE ("client_id", "timestamp") +);""" % tmptbl_name + _newtbl_create = \ +"""create table "%s" ( + "id" integer NOT NULL PRIMARY KEY, + "client_id" integer NOT NULL REFERENCES "reports_client" ("id"), + "timestamp" datetime NOT NULL, + "state" varchar(32) NOT NULL, + "repo_rev_code" varchar(64) NOT NULL, + "goodcount" integer NOT NULL, + "totalcount" integer NOT NULL, + "server" varchar(256) NOT NULL, + "bad_entries" integer NOT NULL, + "modified_entries" integer NOT NULL, + "extra_entries" integer NOT NULL, + UNIQUE ("client_id", "timestamp") +);""" % tbl + new_cols = "id,\ + client_id,\ + timestamp,\ + state,\ + repo_rev_code,\ + goodcount,\ + totalcount,\ + server,\ + bad_entries,\ + modified_entries,\ + extra_entries" + + delete_col = [_tmptbl_create, + "insert into %s select %s from %s;" % (tmptbl_name, new_cols, tbl), + "drop table %s" % tbl, + _newtbl_create, + "create index reports_interaction_client_id on %s (client_id);" % tbl, + "insert into %s select %s from %s;" % (tbl, new_cols, + tmptbl_name), + "drop table %s;" % tmptbl_name] + + for sql in delete_col: + cursor.execute(sql) + + def _populate_interaction_entry_counts(): '''Populate up the type totals for the interaction table''' cursor = connection.cursor() @@ -103,6 +170,8 @@ _fixes = [_merge_database_table_entries, _interactions_constraint_or_idx, 'alter table reports_reason add is_binary bool NOT NULL default False;', 'alter table reports_reason add is_sensitive bool NOT NULL default False;', + _remove_table_column('reports_interaction', 'client_version'), + "alter table reports_reason add unpruned varchar(1280) not null default 'N/A';", ] # this will calculate the last possible version of the database @@ -110,7 +179,7 @@ lastversion = len(_fixes) def rollupdate(current_version): - """ function responsible to coordinates all the updates + """function responsible to coordinates all the updates need current_version as integer """ ret = None @@ -122,8 +191,10 @@ def rollupdate(current_version): else: _fixes[i]() except: - logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) - # since array start at 0 but version start at 1 we add 1 to the normal count + logger.error("Failed to perform db update %s" % (_fixes[i]), + exc_info=1) + # since the array starts at 0 but version + # starts at 1 we add 1 to the normal count ret = InternalDatabaseVersion.objects.create(version=i + 1) return ret else: @@ -135,16 +206,19 @@ def dosync(): # try to detect if it's a fresh new database try: cursor = connection.cursor() - # If this table goes missing then don't forget to change it to the new one + # If this table goes missing, + # don't forget to change it to the new one cursor.execute("Select * from reports_client") # if we get here with no error then the database has existing tables fresh = False except: - logger.debug("there was an error while detecting the freshness of the database") + logger.debug("there was an error while detecting " + "the freshness of the database") #we should get here if the database is new fresh = True - # ensure database connection are close, so that the management can do it's job right + # ensure database connections are closed + # so that the management can do its job right try: cursor.close() connection.close() @@ -169,7 +243,8 @@ def dosync(): def update_database(): - ''' methode to search where we are in the revision of the database models and update them ''' + """method to search where we are in the revision + of the database models and update them""" try: logger.debug("Running upgrade of models to the new one") dosync() diff --git a/src/lib/Server/Reports/urls.py b/src/lib/Bcfg2/Server/Reports/urls.py index d7ff1eee5..d7ff1eee5 100644 --- a/src/lib/Server/Reports/urls.py +++ b/src/lib/Bcfg2/Server/Reports/urls.py diff --git a/src/lib/Server/Reports/utils.py b/src/lib/Bcfg2/Server/Reports/utils.py index e0b6ead59..e0b6ead59 100755 --- a/src/lib/Server/Reports/utils.py +++ b/src/lib/Bcfg2/Server/Reports/utils.py diff --git a/src/lib/Server/Snapshots/__init__.py b/src/lib/Bcfg2/Server/Snapshots/__init__.py index 7c901adb2..7c901adb2 100644 --- a/src/lib/Server/Snapshots/__init__.py +++ b/src/lib/Bcfg2/Server/Snapshots/__init__.py diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Bcfg2/Server/Snapshots/model.py index f30c38a05..5d7973c16 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Bcfg2/Server/Snapshots/model.py @@ -211,7 +211,7 @@ class File(Base, Uniquer): type = Column(Unicode(12)) owner = Column(Unicode(12)) group = Column(Unicode(16)) - perms = Column(Integer(5)) + perms = Column(Integer) contents = Column(UnicodeText) diff --git a/src/lib/Server/__init__.py b/src/lib/Bcfg2/Server/__init__.py index 25f397565..96777b0bf 100644 --- a/src/lib/Server/__init__.py +++ b/src/lib/Bcfg2/Server/__init__.py @@ -1,6 +1,4 @@ -# $Id$ """This is the set of modules for Bcfg2.Server.""" -__revision__ = '$Revision$' __all__ = ["Admin", "Core", "FileMonitor", "Plugin", "Plugins", "Hostbase", "Reports", "Snapshots"] diff --git a/src/lib/Statistics.py b/src/lib/Bcfg2/Statistics.py index a0cb8f39b..a0cb8f39b 100644 --- a/src/lib/Statistics.py +++ b/src/lib/Bcfg2/Statistics.py diff --git a/src/lib/__init__.py b/src/lib/Bcfg2/__init__.py index d36c0a00a..357f66f6d 100644 --- a/src/lib/__init__.py +++ b/src/lib/Bcfg2/__init__.py @@ -1,4 +1,3 @@ """Base modules definition.""" -__revision__ = '$Revision$' __all__ = ['Server', 'Client', 'Component', 'Logger', 'Options', 'Proxy', 'Statistics'] diff --git a/src/lib/Bcfg2Py3Incompat.py b/src/lib/Bcfg2Py3Incompat.py deleted file mode 100644 index 6b66e72b0..000000000 --- a/src/lib/Bcfg2Py3Incompat.py +++ /dev/null @@ -1,2 +0,0 @@ -def fprint(s, f): - print(s, file=f) diff --git a/src/lib/Client/Tools/Portage.py b/src/lib/Client/Tools/Portage.py deleted file mode 100644 index 17163afa9..000000000 --- a/src/lib/Client/Tools/Portage.py +++ /dev/null @@ -1,72 +0,0 @@ -"""This is the Bcfg2 tool for the Gentoo Portage system.""" -__revision__ = '$Revision$' - -import re -import Bcfg2.Client.Tools - - -class Portage(Bcfg2.Client.Tools.PkgTool): - """The Gentoo toolset implements package and service operations and - inherits the rest from Toolset.Toolset.""" - name = 'Portage' - __execs__ = ['/usr/bin/emerge', '/usr/bin/equery'] - __handles__ = [('Package', 'ebuild')] - __req__ = {'Package': ['name', 'version']} - pkgtype = 'ebuild' - # requires a working PORTAGE_BINHOST in make.conf - pkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', ['name', 'version'])) - - def __init__(self, logger, cfg, setup): - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, cfg, setup) - self.__important__ = self.__important__ + ['/etc/make.conf'] - self.cfg = cfg - self.installed = {} - self.RefreshPackages() - - def RefreshPackages(self): - """Refresh memory hashes of packages.""" - ret, cache = self.cmd.run("equery -q list '*'") - if ret == 2: - cache = self.cmd.run("equery -q list '*'")[1] - pattern = re.compile('(.*)-(\d.*)') - self.installed = {} - for pkg in cache: - if pattern.match(pkg): - name = pattern.match(pkg).group(1) - version = pattern.match(pkg).group(2) - self.installed[name] = version - else: - self.logger.info("Failed to parse pkg name %s" % pkg) - - def VerifyPackage(self, entry, modlist): - """Verify package for entry.""" - if not 'version' in entry.attrib: - self.logger.info("Cannot verify unversioned package %s" % - (entry.attrib['name'])) - return False - if entry.attrib['name'] in self.installed: - if self.installed[entry.attrib['name']] == entry.attrib['version']: - if not self.setup['quick'] and \ - entry.get('verify', 'true') == 'true': - output = self.cmd.run("/usr/bin/equery -N check '=%s-%s' 2>&1 " - "| grep '!!!' | awk '{print $2}'" \ - % (entry.get('name'), entry.get('version')))[1] - if [filename for filename in output \ - if filename not in modlist]: - return False - return True - else: - entry.set('current_version', self.installed[entry.get('name')]) - return False - entry.set('current_exists', 'false') - return False - - def RemovePackages(self, packages): - """Deal with extra configuration detected.""" - pkgnames = " ".join([pkg.get('name') for pkg in packages]) - if len(packages) > 0: - self.logger.info('Removing packages:') - self.logger.info(pkgnames) - self.cmd.run("emerge --unmerge --quiet %s" % " ".join(pkgnames.split(' '))) - self.RefreshPackages() - self.extra = self.FindExtraPackages() diff --git a/src/lib/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Server/Plugins/Packages/PackagesConfig.py deleted file mode 100644 index d3732bf96..000000000 --- a/src/lib/Server/Plugins/Packages/PackagesConfig.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import logging -from Bcfg2.Bcfg2Py3k import ConfigParser -from Bcfg2.Server.Plugins.Packages import * - -logger = logging.getLogger('Packages') - -class PackagesConfig(Bcfg2.Server.Plugin.FileBacked, - ConfigParser.SafeConfigParser): - def __init__(self, filename, fam, packages): - Bcfg2.Server.Plugin.FileBacked.__init__(self, filename) - ConfigParser.SafeConfigParser.__init__(self) - - self.fam = fam - # packages.conf isn't strictly necessary, so only set a - # monitor if it exists. if it gets added, that will require a - # server restart - if os.path.exists(self.name): - self.fam.AddMonitor(self.name, self) - - self.pkg_obj = packages - - def Index(self): - """ Build local data structures """ - for section in self.sections(): - self.remove_section(section) - self.read(self.name) - if self.pkg_obj.sources.loaded: - # only reload Packages plugin if sources have been loaded. - # otherwise, this is getting called on server startup, and - # we have to wait until all sources have been indexed - # before we can call Packages.Reload() - self.pkg_obj.Reload() |