diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/lib/Bcfg2/Bcfg2Py3k.py (renamed from src/lib/Bcfg2Py3k.py) | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Frame.py (renamed from src/lib/Client/Frame.py) | 60 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/APK.py (renamed from src/lib/Client/Tools/APK.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/APT.py (renamed from src/lib/Client/Tools/APT.py) | 46 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Action.py (renamed from src/lib/Client/Tools/Action.py) | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Blast.py (renamed from src/lib/Client/Tools/Blast.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Chkconfig.py (renamed from src/lib/Client/Tools/Chkconfig.py) | 48 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/DebInit.py (renamed from src/lib/Client/Tools/DebInit.py) | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Encap.py (renamed from src/lib/Client/Tools/Encap.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/FreeBSDInit.py (renamed from src/lib/Client/Tools/FreeBSDInit.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py (renamed from src/lib/Client/Tools/FreeBSDPackage.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/IPS.py (renamed from src/lib/Client/Tools/IPS.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/MacPorts.py (renamed from src/lib/Client/Tools/MacPorts.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/OpenCSW.py | 33 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/POSIX.py (renamed from src/lib/Client/Tools/POSIX.py) | 28 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Pacman.py (renamed from src/lib/Client/Tools/Pacman.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Portage.py | 104 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/RPMng.py (renamed from src/lib/Client/Tools/RPMng.py) | 132 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/RcUpdate.py (renamed from src/lib/Client/Tools/RcUpdate.py) | 41 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/SMF.py (renamed from src/lib/Client/Tools/SMF.py) | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/SYSV.py (renamed from src/lib/Client/Tools/SYSV.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Systemd.py (renamed from src/lib/Client/Tools/Systemd.py) | 17 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/Upstart.py (renamed from src/lib/Client/Tools/Upstart.py) | 6 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/VCS.py (renamed from src/lib/Client/Tools/VCS.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/YUM24.py (renamed from src/lib/Client/Tools/YUM24.py) | 28 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/YUMng.py (renamed from src/lib/Client/Tools/YUMng.py) | 61 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/__init__.py (renamed from src/lib/Client/Tools/__init__.py) | 40 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/Tools/launchd.py (renamed from src/lib/Client/Tools/launchd.py) | 6 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Client/Tools/rpmtools.py (renamed from src/lib/Client/Tools/rpmtools.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/XML.py (renamed from src/lib/Client/XML.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Client/__init__.py (renamed from src/lib/Client/__init__.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Component.py (renamed from src/lib/Component.py) | 6 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Encryption.py | 75 | ||||
-rw-r--r-- | src/lib/Bcfg2/Logger.py (renamed from src/lib/Logger.py) | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Options.py | 880 | ||||
-rw-r--r-- | src/lib/Bcfg2/Proxy.py (renamed from src/lib/Proxy.py) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/SSLServer.py (renamed from src/lib/SSLServer.py) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Backup.py (renamed from src/lib/Server/Admin/Backup.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Bundle.py (renamed from src/lib/Server/Admin/Bundle.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Client.py (renamed from src/lib/Server/Admin/Client.py) | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Compare.py (renamed from src/lib/Server/Admin/Compare.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Group.py (renamed from src/lib/Server/Admin/Group.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Init.py (renamed from src/lib/Server/Admin/Init.py) | 56 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Minestruct.py (renamed from src/lib/Server/Admin/Minestruct.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Perf.py (renamed from src/lib/Server/Admin/Perf.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Pull.py (renamed from src/lib/Server/Admin/Pull.py) | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Query.py (renamed from src/lib/Server/Admin/Query.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Reports.py (renamed from src/lib/Server/Admin/Reports.py) | 69 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Snapshots.py (renamed from src/lib/Server/Admin/Snapshots.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Tidy.py (renamed from src/lib/Server/Admin/Tidy.py) | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Viz.py (renamed from src/lib/Server/Admin/Viz.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/Xcmd.py (renamed from src/lib/Server/Admin/Xcmd.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Admin/__init__.py (renamed from src/lib/Server/Admin/__init__.py) | 10 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py (renamed from src/lib/Server/Core.py) | 61 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Fam.py | 82 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Gamin.py | 64 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 64 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Pseudo.py | 27 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/__init__.py | 141 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/.gitignore (renamed from src/lib/Server/Hostbase/.gitignore) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/__init__.py (renamed from src/lib/Server/Hostbase/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/backends.py (renamed from src/lib/Server/Hostbase/backends.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py (renamed from src/lib/Server/Hostbase/hostbase/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py (renamed from src/lib/Server/Hostbase/hostbase/admin.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/models.py (renamed from src/lib/Server/Hostbase/hostbase/models.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql (renamed from src/lib/Server/Hostbase/hostbase/sql/zone.sql) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py (renamed from src/lib/Server/Hostbase/hostbase/urls.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/views.py (renamed from src/lib/Server/Hostbase/hostbase/views.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/base.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/copy.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/dns.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/edit.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/errors.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/host.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/index.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/login.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/logout.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/new.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/remove.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/results.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/search.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zones.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html (renamed from src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/ldapauth.py (renamed from src/lib/Server/Hostbase/ldapauth.py) | 0 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Hostbase/manage.py (renamed from src/lib/Server/Hostbase/manage.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/base.css (renamed from src/lib/Server/Hostbase/media/base.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css (renamed from src/lib/Server/Hostbase/media/boxypastel.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/global.css (renamed from src/lib/Server/Hostbase/media/global.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/media/layout.css (renamed from src/lib/Server/Hostbase/media/layout.css) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/nisauth.py (renamed from src/lib/Server/Hostbase/nisauth.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/regex.py (renamed from src/lib/Server/Hostbase/regex.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/settings.py (renamed from src/lib/Server/Hostbase/settings.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl (renamed from src/lib/Server/Hostbase/templates/batchadd.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head (renamed from src/lib/Server/Hostbase/templates/dhcpd.conf.head) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl (renamed from src/lib/Server/Hostbase/templates/dhcpd.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl (renamed from src/lib/Server/Hostbase/templates/hosts.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl (renamed from src/lib/Server/Hostbase/templates/hostsappend.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl (renamed from src/lib/Server/Hostbase/templates/named.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl (renamed from src/lib/Server/Hostbase/templates/namedviews.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl (renamed from src/lib/Server/Hostbase/templates/reverseappend.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl (renamed from src/lib/Server/Hostbase/templates/reversesoa.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl (renamed from src/lib/Server/Hostbase/templates/zone.tmpl) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/harness.py (renamed from src/lib/Server/Hostbase/test/harness.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/test_environ_settings.py (renamed from src/lib/Server/Hostbase/test/test_environ_settings.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/test_ldapauth.py (renamed from src/lib/Server/Hostbase/test/test_ldapauth.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/test/test_settings.py (renamed from src/lib/Server/Hostbase/test/test_settings.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Hostbase/urls.py (renamed from src/lib/Server/Hostbase/urls.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Comments.py (renamed from src/lib/Server/Lint/Comments.py) | 47 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Duplicates.py (renamed from src/lib/Server/Lint/Duplicates.py) | 10 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Lint/Genshi.py (renamed from src/lib/Server/Lint/Genshi.py) | 12 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/InfoXML.py (renamed from src/lib/Server/Lint/InfoXML.py) | 27 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/MergeFiles.py (renamed from src/lib/Server/Lint/MergeFiles.py) | 16 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/RequiredAttrs.py (renamed from src/lib/Server/Lint/RequiredAttrs.py) | 9 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/Validate.py (renamed from src/lib/Server/Lint/Validate.py) | 16 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Lint/__init__.py (renamed from src/lib/Server/Lint/__init__.py) | 52 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin.py (renamed from src/lib/Server/Plugin.py) | 231 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Account.py (renamed from src/lib/Server/Plugins/Account.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/BB.py (renamed from src/lib/Server/Plugins/BB.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Base.py (renamed from src/lib/Server/Plugins/Base.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Bundler.py (renamed from src/lib/Server/Plugins/Bundler.py) | 77 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Bzr.py (renamed from src/lib/Server/Plugins/Bzr.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py | 20 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py | 31 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py | 27 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py | 14 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py | 63 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py | 26 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py | 33 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py | 67 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py | 24 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py | 32 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py | 435 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Cvs.py (renamed from src/lib/Server/Plugins/Cvs.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/DBStats.py (renamed from src/lib/Server/Plugins/DBStats.py) | 39 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Darcs.py (renamed from src/lib/Server/Plugins/Darcs.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Decisions.py (renamed from src/lib/Server/Plugins/Decisions.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Defaults.py (renamed from src/lib/Server/Plugins/Defaults.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Deps.py (renamed from src/lib/Server/Plugins/Deps.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Editor.py (renamed from src/lib/Server/Plugins/Editor.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/FileProbes.py (renamed from src/lib/Server/Plugins/FileProbes.py) | 12 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Fossil.py (renamed from src/lib/Server/Plugins/Fossil.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Git.py (renamed from src/lib/Server/Plugins/Git.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/GroupPatterns.py (renamed from src/lib/Server/Plugins/GroupPatterns.py) | 32 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Guppy.py (renamed from src/lib/Server/Plugins/Guppy.py) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Hg.py (renamed from src/lib/Server/Plugins/Hg.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Hostbase.py (renamed from src/lib/Server/Plugins/Hostbase.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Ldap.py (renamed from src/lib/Server/Plugins/Ldap.py) | 53 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py (renamed from src/lib/Server/Plugins/Metadata.py) | 637 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/NagiosGen.py (renamed from src/lib/Server/Plugins/NagiosGen.py) | 23 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Ohai.py (renamed from src/lib/Server/Plugins/Ohai.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Apt.py (renamed from src/lib/Server/Plugins/Packages/Apt.py) | 7 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Collection.py (renamed from src/lib/Server/Plugins/Packages/Collection.py) | 98 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Pac.py (renamed from src/lib/Server/Plugins/Packages/Pac.py) | 9 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py (renamed from src/lib/Server/Plugins/Packages/PackagesSources.py) | 29 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Source.py (renamed from src/lib/Server/Plugins/Packages/Source.py) | 17 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/Yum.py (renamed from src/lib/Server/Plugins/Packages/Yum.py) | 74 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Packages/__init__.py (renamed from src/lib/Server/Plugins/Packages/__init__.py) | 84 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Pkgmgr.py (renamed from src/lib/Server/Plugins/Pkgmgr.py) | 46 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Probes.py (renamed from src/lib/Server/Plugins/Probes.py) | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Properties.py (renamed from src/lib/Server/Plugins/Properties.py) | 58 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/PuppetENC.py | 120 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Rules.py (renamed from src/lib/Server/Plugins/Rules.py) | 10 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SGenshi.py (renamed from src/lib/Server/Plugins/SGenshi.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SSHbase.py (renamed from src/lib/Server/Plugins/SSHbase.py) | 29 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/SSLCA.py (renamed from src/lib/Server/Plugins/SSLCA.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Snapshots.py (renamed from src/lib/Server/Plugins/Snapshots.py) | 9 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Statistics.py (renamed from src/lib/Server/Plugins/Statistics.py) | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Svcmgr.py (renamed from src/lib/Server/Plugins/Svcmgr.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Svn.py (renamed from src/lib/Server/Plugins/Svn.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Svn2.py (renamed from src/lib/Server/Plugins/Svn2.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TCheetah.py (renamed from src/lib/Server/Plugins/TCheetah.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TGenshi.py (renamed from src/lib/Server/Plugins/TGenshi.py) | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/TemplateHelper.py | 149 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Trigger.py (renamed from src/lib/Server/Plugins/Trigger.py) | 32 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/__init__.py (renamed from src/lib/Server/Plugins/__init__.py) | 3 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py | 11 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py | 59 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py | 27 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py (renamed from src/lib/Server/Reports/reports/templatetags/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/Routines.py | 279 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/Updater/__init__.py | 239 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/__init__.py (renamed from src/lib/Server/Reports/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/backends.py (renamed from src/lib/Server/Reports/backends.py) | 0 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Reports/importscript.py | 334 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Reports/manage.py (renamed from src/lib/Server/Reports/manage.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/nisauth.py (renamed from src/lib/Server/Reports/nisauth.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/__init__.py (renamed from src/lib/Server/Reports/reports/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/models.py (renamed from src/lib/Server/Reports/reports/models.py) | 150 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/404.html (renamed from src/lib/Server/Reports/reports/templates/404.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html (renamed from src/lib/Server/Reports/reports/templates/base-timeview.html) | 5 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/base.html (renamed from src/lib/Server/Reports/reports/templates/base.html) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html (renamed from src/lib/Server/Reports/reports/templates/clients/detail.html) | 56 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html (renamed from src/lib/Server/Reports/reports/templates/clients/detailed-list.html) | 20 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html (renamed from src/lib/Server/Reports/reports/templates/clients/history.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html | 35 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html (renamed from src/lib/Server/Reports/reports/templates/clients/manage.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html | 42 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html | 30 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html (renamed from src/lib/Server/Reports/reports/templates/config_items/item.html) | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html (renamed from src/lib/Server/Reports/reports/templates/config_items/listing.html) | 22 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html (renamed from src/lib/Server/Reports/reports/templates/displays/summary.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html (renamed from src/lib/Server/Reports/reports/templates/displays/timing.html) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html | 25 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc (renamed from src/lib/Server/Reports/reports/templates/widgets/interaction_list.inc) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html (renamed from src/lib/Server/Reports/reports/templates/widgets/page_bar.html) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py | 415 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py (renamed from src/lib/Server/Reports/reports/templatetags/syntax_coloring.py) | 17 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/urls.py (renamed from src/lib/Server/Reports/reports/urls.py) | 9 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/reports/views.py (renamed from src/lib/Server/Reports/reports/views.py) | 266 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/settings.py (renamed from src/lib/Server/Reports/settings.py) | 37 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Reports/urls.py (renamed from src/lib/Server/Reports/urls.py) | 0 | ||||
-rwxr-xr-x | src/lib/Bcfg2/Server/Reports/utils.py (renamed from src/lib/Server/Reports/utils.py) | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Snapshots/__init__.py (renamed from src/lib/Server/Snapshots/__init__.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Snapshots/model.py (renamed from src/lib/Server/Snapshots/model.py) | 8 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/__init__.py (renamed from src/lib/Server/__init__.py) | 2 | ||||
-rw-r--r-- | src/lib/Bcfg2/Statistics.py (renamed from src/lib/Statistics.py) | 0 | ||||
-rw-r--r-- | src/lib/Bcfg2/__init__.py (renamed from src/lib/__init__.py) | 1 | ||||
-rw-r--r-- | src/lib/Bcfg2Py3Incompat.py | 2 | ||||
-rw-r--r-- | src/lib/Client/Tools/Portage.py | 72 | ||||
-rw-r--r-- | src/lib/Options.py | 392 | ||||
-rw-r--r-- | src/lib/Server/FileMonitor.py | 315 | ||||
-rw-r--r-- | src/lib/Server/Lint/Bundles.py | 61 | ||||
-rw-r--r-- | src/lib/Server/Lint/Deltas.py | 20 | ||||
-rw-r--r-- | src/lib/Server/Lint/GroupPatterns.py | 31 | ||||
-rw-r--r-- | src/lib/Server/Lint/Pkgmgr.py | 35 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Cfg.py | 295 | ||||
-rw-r--r-- | src/lib/Server/Plugins/Packages/PackagesConfig.py | 15 | ||||
-rwxr-xr-x | src/lib/Server/Reports/importscript.py | 311 | ||||
-rw-r--r-- | src/lib/Server/Reports/reports/fixtures/initial_version.xml | 39 | ||||
-rw-r--r-- | src/lib/Server/Reports/reports/sql/client.sql | 9 | ||||
-rw-r--r-- | src/lib/Server/Reports/reports/templates/clients/index.html | 34 | ||||
-rw-r--r-- | src/lib/Server/Reports/reports/templates/widgets/filter_bar.html | 13 | ||||
-rw-r--r-- | src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py | 276 | ||||
-rw-r--r-- | src/lib/Server/Reports/updatefix.py | 190 | ||||
-rwxr-xr-x | src/sbin/bcfg2 | 118 | ||||
-rwxr-xr-x | src/sbin/bcfg2-admin | 31 | ||||
-rwxr-xr-x | src/sbin/bcfg2-build-reports | 2 | ||||
-rwxr-xr-x | src/sbin/bcfg2-crypt | 356 | ||||
-rwxr-xr-x | src/sbin/bcfg2-info | 255 | ||||
-rwxr-xr-x | src/sbin/bcfg2-lint | 125 | ||||
-rwxr-xr-x | src/sbin/bcfg2-ping-sweep | 2 | ||||
-rwxr-xr-x | src/sbin/bcfg2-reports | 576 | ||||
-rwxr-xr-x | src/sbin/bcfg2-server | 50 | ||||
-rwxr-xr-x | src/sbin/bcfg2-test | 21 | ||||
-rwxr-xr-x | src/sbin/bcfg2-yum-helper | 2 |
258 files changed, 6696 insertions, 4268 deletions
diff --git a/src/lib/Bcfg2Py3k.py b/src/lib/Bcfg2/Bcfg2Py3k.py index 5d32c79a0..30feaf8c5 100644 --- a/src/lib/Bcfg2Py3k.py +++ b/src/lib/Bcfg2/Bcfg2Py3k.py @@ -77,16 +77,10 @@ def u_str(string, encoding=None): else: return unicode(string) -""" -In order to use the new syntax for printing to a file, we need to do -a conditional import because there is a syntax incompatibility between -the two versions of python. -""" -if sys.hexversion >= 0x03000000: - from Bcfg2.Bcfg2Py3Incompat import fprint -else: - def fprint(s, f): - print >> f, s +try: + input = raw_input +except: + input = input if sys.hexversion >= 0x03000000: from io import FileIO as file diff --git a/src/lib/Client/Frame.py b/src/lib/Bcfg2/Client/Frame.py index d17f70f1b..a8bcb69bf 100644 --- a/src/lib/Client/Frame.py +++ b/src/lib/Bcfg2/Client/Frame.py @@ -2,7 +2,6 @@ Frame is the Client Framework that verifies and installs entries, and generates statistics. """ -__revision__ = '$Revision$' import logging import sys @@ -191,14 +190,23 @@ class Frame: self.whitelist = [x for x in self.whitelist if x not in b_to_rem] # take care of important entries first - if not self.dryrun and not self.setup['bundle']: - for cfile in [cfl for cfl in self.config.findall(".//Path") \ - if cfl.get('name') in self.__important__ and \ - cfl.get('type') == 'file']: - if cfile not in self.whitelist: + if not self.dryrun: + for cfile in self.config.findall(".//Path"): + if (cfile.get('name') not in self.__important__ or + cfile.get('type') != 'file' or + cfile not in self.whitelist): continue - tl = [t for t in self.tools if t.handlesEntry(cfile) \ - and t.canVerify(cfile)] + parent = cfile.getparent() + if ((parent.tag == "Bundle" and + ((self.setup['bundle'] and + parent.get("name") not in self.setup['bundle']) or + (self.setup['skipbundle'] and + parent.get("name") in self.setup['skipbundle']))) or + (parent.tag == "Independent" and + (self.setup['bundle'] or self.setup['skipindep']))): + continue + tl = [t for t in self.tools + if t.handlesEntry(cfile) and t.canVerify(cfile)] if tl: if self.setup['interactive'] and not \ promptFilter("Install %s: %s? (y/N):", [cfile]): @@ -263,22 +271,31 @@ class Frame: return # Here is where most of the work goes # first perform bundle filtering + all_bundle_names = [b.get('name') + for b in self.config.findall('./Bundle')] + bundles = self.config.getchildren() if self.setup['bundle']: - all_bundle_names = [b.get('name') for b in - self.config.findall('./Bundle')] # warn if non-existent bundle given for bundle in self.setup['bundle']: if bundle not in all_bundle_names: self.logger.info("Warning: Bundle %s not found" % bundle) - bundles = [b for b in self.config.findall('./Bundle') - if b.get('name') in self.setup['bundle']] - self.whitelist = [e for e in self.whitelist - if True in [e in b for b in bundles]] + bundles = filter(lambda b: b.get('name') in self.setup['bundle'], + bundles) elif self.setup['indep']: - bundles = [nb for nb in self.config.getchildren() - if nb.tag != 'Bundle'] - else: - bundles = self.config.getchildren() + bundles = filter(lambda b: b.tag != 'Bundle', bundles) + if self.setup['skipbundle']: + # warn if non-existent bundle given + for bundle in self.setup['skipbundle']: + if bundle not in all_bundle_names: + self.logger.info("Warning: Bundle %s not found" % bundle) + bundles = filter(lambda b: \ + b.get('name') not in self.setup['skipbundle'], + bundles) + if self.setup['skipindep']: + bundles = filter(lambda b: b.tag == 'Bundle', bundles) + + self.whitelist = [e for e in self.whitelist + if True in [e in b for b in bundles]] # first process prereq actions for bundle in bundles[:]: @@ -292,7 +309,7 @@ class Frame: if self.setup['interactive']: promptFilter(prompt, actions) self.DispatchInstallCalls(actions) - + # need to test to fail entries in whitelist if False in [self.states[a] for a in actions]: # then display bundles forced off with entries @@ -427,10 +444,10 @@ class Frame: stats = Bcfg2.Client.XML.SubElement(feedback, 'Statistics', total=str(len(self.states)), - client_version=__revision__, version='2.0', revision=self.config.get('revision', '-1')) - good = len([key for key, val in list(self.states.items()) if val]) + good_entries = [key for key, val in list(self.states.items()) if val] + good = len(good_entries) stats.set('good', str(good)) if len([key for key, val in list(self.states.items()) if not val]) == 0: stats.set('state', 'clean') @@ -439,6 +456,7 @@ class Frame: # List bad elements of the configuration for (data, ename) in [(self.modified, 'Modified'), (self.extra, "Extra"), \ + (good_entries, "Good"), ([entry for entry in self.states if not \ self.states[entry]], "Bad")]: container = Bcfg2.Client.XML.SubElement(stats, ename) diff --git a/src/lib/Client/Tools/APK.py b/src/lib/Bcfg2/Client/Tools/APK.py index 6a6fd51b3..aaaf2472f 100644 --- a/src/lib/Client/Tools/APK.py +++ b/src/lib/Bcfg2/Client/Tools/APK.py @@ -1,5 +1,4 @@ """This provides Bcfg2 support for Alpine Linux APK packages.""" -__revision__ = '$Revision$' import Bcfg2.Client.Tools diff --git a/src/lib/Client/Tools/APT.py b/src/lib/Bcfg2/Client/Tools/APT.py index db8cd56e7..0ad42748c 100644 --- a/src/lib/Client/Tools/APT.py +++ b/src/lib/Bcfg2/Client/Tools/APT.py @@ -1,5 +1,4 @@ """This is the Bcfg2 support for apt-get.""" -__revision__ = '$Revision$' # suppress apt API warnings import warnings @@ -7,22 +6,7 @@ warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import apt.cache import os - import Bcfg2.Client.Tools -import Bcfg2.Options - -# Options for tool locations -opts = {'install_path': Bcfg2.Options.CLIENT_APT_TOOLS_INSTALL_PATH, - 'var_path': Bcfg2.Options.CLIENT_APT_TOOLS_VAR_PATH, - 'etc_path': Bcfg2.Options.CLIENT_SYSTEM_ETC_PATH} -setup = Bcfg2.Options.OptionParser(opts) -setup.parse([]) -install_path = setup['install_path'] -var_path = setup['var_path'] -etc_path = setup['etc_path'] -DEBSUMS = '%s/bin/debsums' % install_path -APTGET = '%s/bin/apt-get' % install_path -DPKG = '%s/bin/dpkg' % install_path class APT(Bcfg2.Client.Tools.Tool): """The Debian toolset implements package and service operations and inherits @@ -30,17 +14,26 @@ class APT(Bcfg2.Client.Tools.Tool): """ name = 'APT' - __execs__ = [DEBSUMS, APTGET, DPKG] + __execs__ = [] __handles__ = [('Package', 'deb'), ('Path', 'ignore')] __req__ = {'Package': ['name', 'version'], 'Path': ['type']} def __init__(self, logger, setup, config): Bcfg2.Client.Tools.Tool.__init__(self, logger, setup, config) + + self.install_path = setup.get('apt_install_path', '/usr') + self.var_path = setup.get('apt_var_path', '/var') + self.etc_path = setup.get('apt_etc_path', '/etc') + self.debsums = '%s/bin/debsums' % self.install_path + self.aptget = '%s/bin/apt-get' % self.install_path + self.dpkg = '%s/bin/dpkg' % self.install_path + self.__execs__ = [self.debsums, self.aptget, self.dpkg] + path_entries = os.environ['PATH'].split(':') for reqdir in ['/sbin', '/usr/sbin']: if reqdir not in path_entries: os.environ['PATH'] = os.environ['PATH'] + ':' + reqdir - self.pkgcmd = '%s ' % APTGET + \ + self.pkgcmd = '%s ' % self.aptget + \ '-o DPkg::Options::=--force-overwrite ' + \ '-o DPkg::Options::=--force-confold ' + \ '-o DPkg::Options::=--force-confmiss ' + \ @@ -54,21 +47,21 @@ class APT(Bcfg2.Client.Tools.Tool): if entry.tag == 'Path' and \ entry.get('type') == 'ignore'] self.__important__ = self.__important__ + \ - ["%s/cache/debconf/config.dat" % var_path, - "%s/cache/debconf/templates.dat" % var_path, + ["%s/cache/debconf/config.dat" % self.var_path, + "%s/cache/debconf/templates.dat" % self.var_path, '/etc/passwd', '/etc/group', - '%s/apt/apt.conf' % etc_path, - '%s/dpkg/dpkg.cfg' % etc_path] + \ + '%s/apt/apt.conf' % self.etc_path, + '%s/dpkg/dpkg.cfg' % self.etc_path] + \ [entry.get('name') for struct in config for entry in struct \ if entry.tag == 'Path' and \ - entry.get('name').startswith('%s/apt/sources.list' % etc_path)] + entry.get('name').startswith('%s/apt/sources.list' % self.etc_path)] self.nonexistent = [entry.get('name') for struct in config for entry in struct \ if entry.tag == 'Path' and entry.get('type') == 'nonexistent'] os.environ["DEBIAN_FRONTEND"] = 'noninteractive' self.actions = {} if self.setup['kevlar'] and not self.setup['dryrun']: - self.cmd.run("%s --force-confold --configure --pending" % DPKG) - self.cmd.run("%s clean" % APTGET) + self.cmd.run("%s --force-confold --configure --pending" % self.dpkg) + self.cmd.run("%s clean" % self.aptget) try: self.pkg_cache = apt.cache.Cache() except SystemError: @@ -96,7 +89,8 @@ class APT(Bcfg2.Client.Tools.Tool): for (name, version) in extras] def VerifyDebsums(self, entry, modlist): - output = self.cmd.run("%s -as %s" % (DEBSUMS, entry.get('name')))[1] + output = self.cmd.run("%s -as %s" % (self.debsums, + entry.get('name')))[1] if len(output) == 1 and "no md5sums for" in output[0]: self.logger.info("Package %s has no md5sums. Cannot verify" % \ entry.get('name')) diff --git a/src/lib/Client/Tools/Action.py b/src/lib/Bcfg2/Client/Tools/Action.py index c089cde1d..e13134e1f 100644 --- a/src/lib/Client/Tools/Action.py +++ b/src/lib/Bcfg2/Client/Tools/Action.py @@ -1,8 +1,8 @@ """Action driver""" -__revision__ = '$Revision$' import Bcfg2.Client.Tools from Bcfg2.Client.Frame import matches_white_list, passes_black_list +from Bcfg2.Bcfg2Py3k import input """ <Action timing='pre|post|both' @@ -45,11 +45,7 @@ class Action(Bcfg2.Client.Tools.Tool): if self.setup['interactive']: prompt = ('Run Action %s, %s: (y/N): ' % (entry.get('name'), entry.get('command'))) - # py3k compatibility - try: - ans = raw_input(prompt) - except NameError: - ans = input(prompt) + ans = input(prompt) if ans not in ['y', 'Y']: return False if self.setup['servicemode'] == 'build': diff --git a/src/lib/Client/Tools/Blast.py b/src/lib/Bcfg2/Client/Tools/Blast.py index 29cdfa116..5d5e74ab2 100644 --- a/src/lib/Client/Tools/Blast.py +++ b/src/lib/Bcfg2/Client/Tools/Blast.py @@ -1,6 +1,4 @@ -# This is the bcfg2 support for blastwave packages (pkg-get) """This provides Bcfg2 support for Blastwave.""" -__revision__ = '$Revision$' import tempfile import Bcfg2.Client.Tools.SYSV diff --git a/src/lib/Client/Tools/Chkconfig.py b/src/lib/Bcfg2/Client/Tools/Chkconfig.py index 17e8bf09b..0169b12da 100644 --- a/src/lib/Client/Tools/Chkconfig.py +++ b/src/lib/Bcfg2/Client/Tools/Chkconfig.py @@ -1,8 +1,6 @@ # This is the bcfg2 support for chkconfig -# $Id$ """This is chkconfig support.""" -__revision__ = '$Revision$' import os @@ -47,30 +45,14 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): except IndexError: onlevels = [] + pstatus = self.check_service(entry) if entry.get('status') == 'on': - status = (len(onlevels) > 0) + status = (len(onlevels) > 0 and pstatus) command = 'start' else: - status = (len(onlevels) == 0) + status = (len(onlevels) == 0 and not pstatus) command = 'stop' - if entry.get('mode', 'default') == 'supervised': - # turn on or off the service in supervised mode - pstatus = self.cmd.run('/sbin/service %s status' % \ - entry.get('name'))[0] - needs_modification = ((command == 'start' and pstatus) or \ - (command == 'stop' and not pstatus)) - if (not self.setup.get('dryrun') and - self.setup['servicemode'] != 'disabled' and - needs_modification): - self.cmd.run(self.get_svc_command(entry, command)) - # service was modified, so it failed - pstatus = False - - # chkconfig/init.d service - if entry.get('status') == 'on': - status = status and not pstatus - if not status: if entry.get('status') == 'on': entry.set('current_status', 'off') @@ -80,22 +62,22 @@ class Chkconfig(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service entry.""" - # don't take any actions for mode='manual' - if entry.get('mode', 'default') == 'manual': - self.logger.info("Service %s mode set to manual. Skipping " - "installation." % (entry.get('name'))) - return False rcmd = "/sbin/chkconfig %s %s" self.cmd.run("/sbin/chkconfig --add %s" % (entry.attrib['name'])) self.logger.info("Installing Service %s" % (entry.get('name'))) - pass1 = True + rv = True if entry.get('status') == 'off': - rc = self.cmd.run(rcmd % (entry.get('name'), - entry.get('status')) + \ - " --level 0123456")[0] - pass1 = rc == 0 - rc = self.cmd.run(rcmd % (entry.get('name'), entry.get('status')))[0] - return pass1 and rc == 0 + rv &= self.cmd.run(rcmd + " --level 0123456" % + (entry.get('name'), + entry.get('status')))[0] == 0 + if entry.get("current_status") == "on": + rv &= self.stop_service(entry) + else: + rv &= self.cmd.run(rcmd % (entry.get('name'), + entry.get('status')))[0] == 0 + if entry.get("current_status") == "off": + rv &= self.start_service(entry) + return rv def FindExtra(self): """Locate extra chkconfig Services.""" diff --git a/src/lib/Client/Tools/DebInit.py b/src/lib/Bcfg2/Client/Tools/DebInit.py index 022332602..7d5af1127 100644 --- a/src/lib/Client/Tools/DebInit.py +++ b/src/lib/Bcfg2/Client/Tools/DebInit.py @@ -1,5 +1,4 @@ """Debian Init Support for Bcfg2""" -__revision__ = '$Revision$' import glob import os @@ -77,11 +76,6 @@ class DebInit(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service for entry.""" - # don't take any actions for mode='manual' - if entry.get('mode', 'default') == 'manual': - self.logger.info("Service %s mode set to manual. Skipping " - "installation." % (entry.get('name'))) - return False self.logger.info("Installing Service %s" % (entry.get('name'))) try: os.stat('/etc/init.d/%s' % entry.get('name')) diff --git a/src/lib/Client/Tools/Encap.py b/src/lib/Bcfg2/Client/Tools/Encap.py index 92062a750..fa09c3ec7 100644 --- a/src/lib/Client/Tools/Encap.py +++ b/src/lib/Bcfg2/Client/Tools/Encap.py @@ -1,7 +1,5 @@ """Bcfg2 Support for Encap Packages""" -__revision__ = '$Revision$' - import glob import re import Bcfg2.Client.Tools diff --git a/src/lib/Client/Tools/FreeBSDInit.py b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py index 10f0f2e93..10f0f2e93 100644 --- a/src/lib/Client/Tools/FreeBSDInit.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDInit.py diff --git a/src/lib/Client/Tools/FreeBSDPackage.py b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py index 04c05adaa..3e6f2b6bb 100644 --- a/src/lib/Client/Tools/FreeBSDPackage.py +++ b/src/lib/Bcfg2/Client/Tools/FreeBSDPackage.py @@ -1,5 +1,4 @@ """This is the Bcfg2 tool for the FreeBSD package system.""" -__revision__ = '$Rev$' # TODO # - actual package installation diff --git a/src/lib/Client/Tools/IPS.py b/src/lib/Bcfg2/Client/Tools/IPS.py index 9afd23143..e30bbd2a4 100644 --- a/src/lib/Client/Tools/IPS.py +++ b/src/lib/Bcfg2/Client/Tools/IPS.py @@ -1,5 +1,4 @@ """This is the Bcfg2 support for OpenSolaris packages.""" -__revision__ = '$Revision$' import pkg.client.image as image import pkg.client.progress as progress diff --git a/src/lib/Client/Tools/MacPorts.py b/src/lib/Bcfg2/Client/Tools/MacPorts.py index 2a7ba9eb9..9724fab57 100644 --- a/src/lib/Client/Tools/MacPorts.py +++ b/src/lib/Bcfg2/Client/Tools/MacPorts.py @@ -1,5 +1,4 @@ """This provides Bcfg2 support for macports packages.""" -__revision__ = '$Revision$' import Bcfg2.Client.Tools diff --git a/src/lib/Bcfg2/Client/Tools/OpenCSW.py b/src/lib/Bcfg2/Client/Tools/OpenCSW.py new file mode 100644 index 000000000..6aafe316f --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/OpenCSW.py @@ -0,0 +1,33 @@ +# This is the bcfg2 support for opencsw packages (pkgutil) +"""This provides Bcfg2 support for OpenCSW packages.""" + +import tempfile +import Bcfg2.Client.Tools.SYSV + + +class OpenCSW(Bcfg2.Client.Tools.SYSV.SYSV): + """Support for OpenCSW packages.""" + pkgtype = 'opencsw' + pkgtool = ("/opt/csw/bin/pkgutil -y -i %s", ("%s", ["bname"])) + name = 'OpenCSW' + __execs__ = ['/opt/csw/bin/pkgutil', "/usr/bin/pkginfo"] + __handles__ = [('Package', 'opencsw')] + __ireq__ = {'Package': ['name', 'version', 'bname']} + + def __init__(self, logger, setup, config): + # dont use the sysv constructor + Bcfg2.Client.Tools.PkgTool.__init__(self, logger, setup, config) + noaskfile = tempfile.NamedTemporaryFile() + self.noaskname = noaskfile.name + try: + noaskfile.write(Bcfg2.Client.Tools.SYSV.noask) + except: + pass + + # VerifyPackage comes from Bcfg2.Client.Tools.SYSV + # Install comes from Bcfg2.Client.Tools.PkgTool + # Extra comes from Bcfg2.Client.Tools.Tool + # Remove comes from Bcfg2.Client.Tools.SYSV + def FindExtraPackages(self): + """Pass through to null FindExtra call.""" + return [] diff --git a/src/lib/Client/Tools/POSIX.py b/src/lib/Bcfg2/Client/Tools/POSIX.py index 049b90d32..995d82356 100644 --- a/src/lib/Client/Tools/POSIX.py +++ b/src/lib/Bcfg2/Client/Tools/POSIX.py @@ -1,5 +1,4 @@ """All POSIX Type client support for Bcfg2.""" -__revision__ = '$Revision$' import binascii from datetime import datetime @@ -21,7 +20,7 @@ import Bcfg2.Client.Tools import Bcfg2.Options from Bcfg2.Client import XML -log = logging.getLogger('posix') +log = logging.getLogger('POSIX') # map between dev_type attribute and stat constants device_map = {'block': stat.S_IFBLK, @@ -265,8 +264,8 @@ class POSIX(Bcfg2.Client.Tools.Tool): if entry.get('perms') == None or \ entry.get('owner') == None or \ entry.get('group') == None: - self.logger.error('Entry %s not completely specified. ' - 'Try running bcfg2-lint.' % (entry.get('name'))) + self.logger.error("POSIX: Entry %s not completely specified. " + "Try running bcfg2-lint." % (entry.get('name'))) return False while len(entry.get('perms', '')) < 4: entry.set('perms', '0' + entry.get('perms', '')) @@ -274,15 +273,15 @@ class POSIX(Bcfg2.Client.Tools.Tool): ondisk = os.stat(entry.get('name')) except OSError: entry.set('current_exists', 'false') - self.logger.debug("%s %s does not exist" % + self.logger.info("POSIX: %s %s does not exist" % (entry.tag, entry.get('name'))) return False try: owner = str(ondisk[stat.ST_UID]) group = str(ondisk[stat.ST_GID]) except (OSError, KeyError): - self.logger.error('User/Group resolution failed for path %s' % \ - entry.get('name')) + self.logger.info("POSIX: User/Group resolution failed " + "for path %s" % entry.get('name')) owner = 'root' group = '0' finfo = os.stat(entry.get('name')) @@ -307,11 +306,11 @@ class POSIX(Bcfg2.Client.Tools.Tool): ex_ents = [e for e in entries if e not in modlist] if ex_ents: pruneTrue = False - self.logger.debug("Directory %s contains extra entries:" % \ - entry.get('name')) - self.logger.debug(ex_ents) + self.logger.info("POSIX: Directory %s contains " + "extra entries:" % entry.get('name')) + self.logger.info(ex_ents) nqtext = entry.get('qtext', '') + '\n' - nqtext += "Directory %s contains extra entries:" % \ + nqtext += "Directory %s contains extra entries: " % \ entry.get('name') nqtext += ":".join(ex_ents) entry.set('qtest', nqtext) @@ -502,7 +501,7 @@ class POSIX(Bcfg2.Client.Tools.Tool): (err.filename, err)) return False different = content != tempdata - + if different: if self.setup['interactive']: prompt = [entry.get('qtext', '')] @@ -767,7 +766,10 @@ class POSIX(Bcfg2.Client.Tools.Tool): self.logger.error('Failed to remove %s: %s' % (ename, e.strerror)) else: - if os.path.isdir(ename): + if os.path.islink(ename): + os.remove(ename) + return True + elif os.path.isdir(ename): try: os.rmdir(ename) return True diff --git a/src/lib/Client/Tools/Pacman.py b/src/lib/Bcfg2/Client/Tools/Pacman.py index c8c05061c..c8c05061c 100644 --- a/src/lib/Client/Tools/Pacman.py +++ b/src/lib/Bcfg2/Client/Tools/Pacman.py diff --git a/src/lib/Bcfg2/Client/Tools/Portage.py b/src/lib/Bcfg2/Client/Tools/Portage.py new file mode 100644 index 000000000..36d48b8d3 --- /dev/null +++ b/src/lib/Bcfg2/Client/Tools/Portage.py @@ -0,0 +1,104 @@ +"""This is the Bcfg2 tool for the Gentoo Portage system.""" + +import re +import Bcfg2.Client.Tools + +class Portage(Bcfg2.Client.Tools.PkgTool): + """The Gentoo toolset implements package and service operations and + inherits the rest from Toolset.Toolset.""" + name = 'Portage' + __execs__ = ['/usr/bin/emerge', '/usr/bin/equery'] + __handles__ = [('Package', 'ebuild')] + __req__ = {'Package': ['name', 'version']} + pkgtype = 'ebuild' + # requires a working PORTAGE_BINHOST in make.conf + _binpkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', \ + ['name', 'version'])) + pkgtool = ('emerge %s', ('=%s-%s', ['name', 'version'])) + + def __init__(self, logger, cfg, setup): + self._initialised = False + Bcfg2.Client.Tools.PkgTool.__init__(self, logger, cfg, setup) + self._initialised = True + self.__important__ = self.__important__ + ['/etc/make.conf'] + self._pkg_pattern = re.compile('(.*)-(\d.*)') + self._ebuild_pattern = re.compile('(ebuild|binary)') + self.cfg = cfg + self.installed = {} + self._binpkgonly = self.setup.get('portage_binpkgonly', False) + if self._binpkgonly: + self.pkgtool = self._binpkgtool + self.RefreshPackages() + + def RefreshPackages(self): + """Refresh memory hashes of packages.""" + if not self._initialised: + return + self.logger.info('Getting list of installed packages') + cache = self.cmd.run("equery -q list '*'")[1] + self.installed = {} + for pkg in cache: + if self._pkg_pattern.match(pkg): + name = self._pkg_pattern.match(pkg).group(1) + version = self._pkg_pattern.match(pkg).group(2) + self.installed[name] = version + else: + self.logger.info("Failed to parse pkg name %s" % pkg) + + def VerifyPackage(self, entry, modlist): + """Verify package for entry.""" + if not 'version' in entry.attrib: + self.logger.info("Cannot verify unversioned package %s" % + (entry.get('name'))) + return False + + if not (entry.get('name') in self.installed): + # Can't verify package that isn't installed + entry.set('current_exists', 'false') + return False + + # get the installed version + version = self.installed[entry.get('name')] + entry.set('current_version', version) + + if not self.setup['quick']: + if ('verify' not in entry.attrib or + entry.get('verify').lower == 'true'): + + # Check the package if: + # - Not running in quick mode + # - No verify option is specified in the literal configuration + # OR + # - Verify option is specified and is true + + self.logger.debug('Running equery check on %s' % + entry.get('name')) + output = self.cmd.run("/usr/bin/equery -N check '=%s-%s' " + "2>&1 | grep '!!!' | awk '{print $2}'" + % ((entry.get('name'), version)))[1] + if [filename for filename in output \ + if filename not in modlist]: + return False + + # By now the package must be in one of the following states: + # - Not require checking + # - Have no files modified at all + # - Have modified files in the modlist only + if self.installed[entry.get('name')] == version: + # Specified package version is installed + # Specified package version may be any in literal configuration + return True + + # Something got skipped. Indicates a bug + return False + + def RemovePackages(self, packages): + """Deal with extra configuration detected.""" + pkgnames = " ".join([pkg.get('name') for pkg in packages]) + if len(packages) > 0: + self.logger.info('Removing packages:') + self.logger.info(pkgnames) + self.cmd.run("emerge --unmerge --quiet %s" % + " ".join(pkgnames.split(' '))) + self.RefreshPackages() + self.extra = self.FindExtraPackages() diff --git a/src/lib/Client/Tools/RPMng.py b/src/lib/Bcfg2/Client/Tools/RPMng.py index 5376118c2..d6e5e82c5 100644 --- a/src/lib/Client/Tools/RPMng.py +++ b/src/lib/Bcfg2/Client/Tools/RPMng.py @@ -1,19 +1,9 @@ """Bcfg2 Support for RPMS""" -__revision__ = '$Revision$' - import os.path import rpm import rpmtools import Bcfg2.Client.Tools -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - -# Fix for python2.3 -try: - set -except NameError: - from sets import Set as set class RPMng(Bcfg2.Client.Tools.PkgTool): """Support for RPM packages.""" @@ -52,82 +42,42 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): self.modlists = {} self.gpg_keyids = self.getinstalledgpg() - # Process thee RPMng section from the config file. - RPMng_CP = ConfigParser.ConfigParser() - RPMng_CP.read(self.setup.get('setup')) - - # installonlypackages - self.installOnlyPkgs = [] - if RPMng_CP.has_option(self.name, 'installonlypackages'): - for i in RPMng_CP.get(self.name, 'installonlypackages').split(','): - self.installOnlyPkgs.append(i.strip()) - if self.installOnlyPkgs == []: - self.installOnlyPkgs = ['kernel', 'kernel-bigmem', 'kernel-enterprise', 'kernel-smp', - 'kernel-modules', 'kernel-debug', 'kernel-unsupported', - 'kernel-source', 'kernel-devel', 'kernel-default', - 'kernel-largesmp-devel', 'kernel-largesmp', 'kernel-xen', - 'gpg-pubkey'] + opt_prefix = self.name.lower() + self.installOnlyPkgs = self.setup["%s_installonly" % opt_prefix] if 'gpg-pubkey' not in self.installOnlyPkgs: self.installOnlyPkgs.append('gpg-pubkey') - self.logger.debug('installOnlyPackages = %s' % self.installOnlyPkgs) - - # erase_flags - self.erase_flags = [] - if RPMng_CP.has_option(self.name, 'erase_flags'): - for i in RPMng_CP.get(self.name, 'erase_flags').split(','): - self.erase_flags.append(i.strip()) - if self.erase_flags == []: - self.erase_flags = ['allmatches'] - self.logger.debug('erase_flags = %s' % self.erase_flags) - - # pkg_checks - if RPMng_CP.has_option(self.name, 'pkg_checks'): - self.pkg_checks = RPMng_CP.get(self.name, 'pkg_checks').lower() - else: - self.pkg_checks = 'true' - self.logger.debug('pkg_checks = %s' % self.pkg_checks) - - # pkg_verify - if RPMng_CP.has_option(self.name, 'pkg_verify'): - self.pkg_verify = RPMng_CP.get(self.name, 'pkg_verify').lower() - else: - self.pkg_verify = 'true' - self.logger.debug('pkg_verify = %s' % self.pkg_verify) - - # installed_action - if RPMng_CP.has_option(self.name, 'installed_action'): - self.installed_action = RPMng_CP.get(self.name, 'installed_action').lower() - else: - self.installed_action = 'install' - self.logger.debug('installed_action = %s' % self.installed_action) - - # version_fail_action - if RPMng_CP.has_option(self.name, 'version_fail_action'): - self.version_fail_action = RPMng_CP.get(self.name, 'version_fail_action').lower() - else: - self.version_fail_action = 'upgrade' - self.logger.debug('version_fail_action = %s' % self.version_fail_action) - - # verify_fail_action - if self.name == "RPMng": - if RPMng_CP.has_option(self.name, 'verify_fail_action'): - self.verify_fail_action = RPMng_CP.get(self.name, 'verify_fail_action').lower() - else: - self.verify_fail_action = 'reinstall' - else: # yum can't reinstall packages. - self.verify_fail_action = 'none' - self.logger.debug('verify_fail_action = %s' % self.verify_fail_action) - - # version_fail_action - if RPMng_CP.has_option(self.name, 'verify_flags'): - self.verify_flags = RPMng_CP.get(self.name, 'verify_flags').lower().split(',') - else: - self.verify_flags = [] + self.erase_flags = self.setup['%s_erase_flags' % opt_prefix] + self.pkg_checks = self.setup['%s_pkg_checks' % opt_prefix] + self.pkg_verify = self.setup['%s_pkg_verify' % opt_prefix] + self.installed_action = self.setup['%s_installed_action' % opt_prefix] + self.version_fail_action = self.setup['%s_version_fail_action' % + opt_prefix] + self.verify_fail_action = self.setup['%s_verify_fail_action' % + opt_prefix] + self.verify_flags = self.setup['%s_verify_flags' % opt_prefix] if '' in self.verify_flags: self.verify_flags.remove('') - self.logger.debug('version_fail_action = %s' % self.version_fail_action) + + self.logger.debug('%s: installOnlyPackages = %s' % + (self.name, self.installOnlyPkgs)) + self.logger.debug('%s: erase_flags = %s' % + (self.name, self.erase_flags)) + self.logger.debug('%s: pkg_checks = %s' % + (self.name, self.pkg_checks)) + self.logger.debug('%s: pkg_verify = %s' % + (self.name, self.pkg_verify)) + self.logger.debug('%s: installed_action = %s' % + (self.name, self.installed_action)) + self.logger.debug('%s: version_fail_action = %s' % + (self.name, self.version_fail_action)) + self.logger.debug('%s: verify_fail_action = %s' % + (self.name, self.verify_fail_action)) + self.logger.debug('%s: verify_flags = %s' % + (self.name, self.verify_flags)) + # Force a re- prelink of all packages if prelink exists. - # Many, if not most package verifies can be caused by out of date prelinking. + # Many, if not most package verifies can be caused by out of + # date prelinking. if os.path.isfile('/usr/sbin/prelink') and not self.setup['dryrun']: cmdrc, output = self.cmd.run('/usr/sbin/prelink -a -mR') if cmdrc == 0: @@ -201,7 +151,8 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): instance = Bcfg2.Client.XML.SubElement(entry, 'Package') for attrib in list(entry.attrib.keys()): instance.attrib[attrib] = entry.attrib[attrib] - if self.pkg_checks == 'true' and entry.get('pkg_checks', 'true') == 'true': + if (self.pkg_checks and + entry.get('pkg_checks', 'true').lower() == 'true'): if 'any' in [entry.get('version'), pinned_version]: version, release = 'any', 'any' elif entry.get('version') == 'auto': @@ -223,7 +174,8 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): if entry.get('name') in self.installed: # There is at least one instance installed. - if self.pkg_checks == 'true' and entry.get('pkg_checks', 'true') == 'true': + if (self.pkg_checks and + entry.get('pkg_checks', 'true').lower() == 'true'): rpmTs = rpm.TransactionSet() rpmHeader = None for h in rpmTs.dbMatch(rpm.RPMTAG_NAME, entry.get('name')): @@ -251,8 +203,8 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): self.logger.debug(" %s" % self.str_evra(inst)) self.instance_status[inst]['installed'] = True - if self.pkg_verify == 'true' and \ - inst.get('pkg_verify', 'true') == 'true': + if (self.pkg_verify and + inst.get('pkg_verify', 'true').lower() == 'true'): flags = inst.get('verify_flags', '').split(',') + self.verify_flags if pkg.get('gpgkeyid', '')[-8:] not in self.gpg_keyids and \ entry.get('name') != 'gpg-pubkey': @@ -310,8 +262,8 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): self.logger.debug(" %s" % self.str_evra(inst)) self.instance_status[inst]['installed'] = True - if self.pkg_verify == 'true' and \ - inst.get('pkg_verify', 'true') == 'true': + if (self.pkg_verify and + inst.get('pkg_verify', 'true').lower() == 'true'): flags = inst.get('verify_flags', '').split(',') + self.verify_flags if pkg.get('gpgkeyid', '')[-8:] not in self.gpg_keyids and \ 'nosignature' not in flags: @@ -461,7 +413,7 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): packages is a list of Package Entries with Instances generated by FindExtraPackages(). - + """ self.logger.debug('Running RPMng.RemovePackages()') @@ -832,8 +784,8 @@ class RPMng(Bcfg2.Client.Tools.PkgTool): return False # We don't want to do any checks so we don't care what the entry has in it. - if self.pkg_checks == 'false' or \ - entry.get('pkg_checks', 'true').lower() == 'false': + if (not self.pkg_checks or + entry.get('pkg_checks', 'true').lower() == 'false'): return True instances = entry.findall('Instance') diff --git a/src/lib/Client/Tools/RcUpdate.py b/src/lib/Bcfg2/Client/Tools/RcUpdate.py index d832d98a8..ddf9c1f2d 100644 --- a/src/lib/Client/Tools/RcUpdate.py +++ b/src/lib/Bcfg2/Client/Tools/RcUpdate.py @@ -1,5 +1,4 @@ """This is rc-update support.""" -__revision__ = '$Revision$' import os import Bcfg2.Client.Tools @@ -24,22 +23,18 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): rc = self.cmd.run(cmd % entry.get('name'))[0] is_enabled = (rc == 0) - if entry.get('mode', 'default') == 'supervised': - # check if init script exists - try: - os.stat('/etc/init.d/%s' % entry.get('name')) - except OSError: - self.logger.debug('Init script for service %s does not exist' % - entry.get('name')) - return False + # check if init script exists + try: + os.stat('/etc/init.d/%s' % entry.get('name')) + except OSError: + self.logger.debug('Init script for service %s does not exist' % + entry.get('name')) + return False - # check if service is enabled - cmd = '/etc/init.d/%s status | grep started' - rc = self.cmd.run(cmd % entry.attrib['name'])[0] - is_running = (rc == 0) - else: - # we don't care - is_running = is_enabled + # check if service is enabled + cmd = '/etc/init.d/%s status | grep started' + rc = self.cmd.run(cmd % entry.attrib['name'])[0] + is_running = (rc == 0) if entry.get('status') == 'on' and not (is_enabled and is_running): entry.set('current_status', 'off') @@ -54,19 +49,11 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """ Install Service entry - In supervised mode we also take care it's (not) running. """ - # don't take any actions for mode='manual' - if entry.get('mode', 'default') == 'manual': - self.logger.info("Service %s mode set to manual. Skipping " - "installation." % (entry.get('name'))) - return False self.logger.info('Installing Service %s' % entry.get('name')) if entry.get('status') == 'on': - # make sure it's running if in supervised mode - if entry.get('mode', 'default') == 'supervised' \ - and entry.get('current_status') == 'off': + if entry.get('current_status') == 'off': self.start_service(entry) # make sure it's enabled cmd = '/sbin/rc-update add %s default' @@ -74,9 +61,7 @@ class RcUpdate(Bcfg2.Client.Tools.SvcTool): return (rc == 0) elif entry.get('status') == 'off': - # make sure it's not running if in supervised mode - if entry.get('mode', 'default') == 'supervised' \ - and entry.get('current_status') == 'on': + if entry.get('current_status') == 'on': self.stop_service(entry) # make sure it's disabled cmd = '/sbin/rc-update del %s default' diff --git a/src/lib/Client/Tools/SMF.py b/src/lib/Bcfg2/Client/Tools/SMF.py index 944408326..3e0a9da13 100644 --- a/src/lib/Client/Tools/SMF.py +++ b/src/lib/Bcfg2/Client/Tools/SMF.py @@ -1,5 +1,4 @@ """SMF support for Bcfg2""" -__revision__ = '$Revision$' import glob import os @@ -74,11 +73,6 @@ class SMF(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install SMF Service entry.""" - # don't take any actions for mode='manual' - if entry.get('mode', 'default') == 'manual': - self.logger.info("Service %s mode set to manual. Skipping " - "installation." % (entry.get('name'))) - return False self.logger.info("Installing Service %s" % (entry.get('name'))) if entry.get('status') == 'off': if entry.get("FMRI").startswith('lrc'): diff --git a/src/lib/Client/Tools/SYSV.py b/src/lib/Bcfg2/Client/Tools/SYSV.py index b5e1f1c59..eb4a13dfb 100644 --- a/src/lib/Client/Tools/SYSV.py +++ b/src/lib/Bcfg2/Client/Tools/SYSV.py @@ -1,6 +1,4 @@ -# This is the bcfg2 support for solaris sysv packages """This provides bcfg2 support for Solaris SYSV packages.""" -__revision__ = '$Revision$' import tempfile diff --git a/src/lib/Client/Tools/Systemd.py b/src/lib/Bcfg2/Client/Tools/Systemd.py index e3f6a4169..a295bc608 100644 --- a/src/lib/Client/Tools/Systemd.py +++ b/src/lib/Bcfg2/Client/Tools/Systemd.py @@ -42,18 +42,11 @@ class Systemd(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service entry.""" - # don't take any actions for mode = 'manual' - if entry.get('mode', 'default') == 'manual': - self.logger.info("Service %s mode set to manual. Skipping " - "installation." % (entry.get('name'))) - return True - if entry.get('status') == 'on': - pstatus = self.cmd.run(self.get_svc_command(entry, 'enable'))[0] - pstatus = self.cmd.run(self.get_svc_command(entry, 'start'))[0] - + rv = self.cmd.run(self.get_svc_command(entry, 'enable'))[0] == 0 + rv &= self.cmd.run(self.get_svc_command(entry, 'start'))[0] == 0 else: - pstatus = self.cmd.run(self.get_svc_command(entry, 'stop'))[0] - pstatus = self.cmd.run(self.get_svc_command(entry, 'disable'))[0] + rv = self.cmd.run(self.get_svc_command(entry, 'stop'))[0] == 0 + rv &= self.cmd.run(self.get_svc_command(entry, 'disable'))[0] == 0 - return not pstatus + return rv diff --git a/src/lib/Client/Tools/Upstart.py b/src/lib/Bcfg2/Client/Tools/Upstart.py index 41a585c23..aa5a921a6 100644 --- a/src/lib/Client/Tools/Upstart.py +++ b/src/lib/Bcfg2/Client/Tools/Upstart.py @@ -1,5 +1,4 @@ """Upstart support for Bcfg2.""" -__revision__ = '$Revision$' import glob import re @@ -70,11 +69,6 @@ class Upstart(Bcfg2.Client.Tools.SvcTool): def InstallService(self, entry): """Install Service for entry.""" - # don't take any actions for mode='manual' - if entry.get('mode', 'default') == 'manual': - self.logger.info("Service %s mode set to manual. Skipping " - "installation." % (entry.get('name'))) - return False if entry.get('status') == 'on': pstatus = self.cmd.run(self.get_svc_command(entry, 'start'))[0] elif entry.get('status') == 'off': diff --git a/src/lib/Client/Tools/VCS.py b/src/lib/Bcfg2/Client/Tools/VCS.py index e6081dc1c..e6081dc1c 100644 --- a/src/lib/Client/Tools/VCS.py +++ b/src/lib/Bcfg2/Client/Tools/VCS.py diff --git a/src/lib/Client/Tools/YUM24.py b/src/lib/Bcfg2/Client/Tools/YUM24.py index 66768fb34..2bc821db3 100644 --- a/src/lib/Client/Tools/YUM24.py +++ b/src/lib/Bcfg2/Client/Tools/YUM24.py @@ -1,5 +1,4 @@ """This provides bcfg2 support for yum.""" -__revision__ = '$Revision: $' import copy import os.path @@ -7,26 +6,6 @@ import sys import yum import Bcfg2.Client.XML import Bcfg2.Client.Tools.RPMng -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - -# Fix for python2.3 -try: - set -except NameError: - from sets import Set as set - -YAD = True -CP = ConfigParser.ConfigParser() -try: - if '-C' in sys.argv: - CP.read([sys.argv[sys.argv.index('-C') + 1]]) - else: - CP.read(['/etc/bcfg2.conf']) - if CP.get('YUMng', 'autodep').lower() == 'false': - YAD = False -except: - pass if not hasattr(Bcfg2.Client.Tools.RPMng, 'RPMng'): raise ImportError @@ -86,6 +65,7 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): (entry.get('name').startswith('/etc/yum.d') \ or entry.get('name').startswith('/etc/yum.repos.d')) \ or entry.get('name') == '/etc/yum.conf'] + self.autodep = setup.get("yum24_autodep") self.yum_avail = dict() self.yum_installed = dict() self.yb = yum.YumBase() @@ -280,7 +260,7 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): if len(install_pkgs) > 0: self.logger.info("Attempting to install packages") - if YAD: + if self.autodep: pkgtool = "/usr/bin/yum -d0 -y install %s" else: pkgtool = "/usr/bin/yum -d0 install %s" @@ -316,7 +296,7 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): if len(upgrade_pkgs) > 0: self.logger.info("Attempting to upgrade packages") - if YAD: + if self.autodep: pkgtool = "/usr/bin/yum -d0 -y update %s" else: pkgtool = "/usr/bin/yum -d0 update %s" @@ -366,7 +346,7 @@ class YUM24(Bcfg2.Client.Tools.RPMng.RPMng): """ self.logger.debug('Running YUMng.RemovePackages()') - if YAD: + if self.autodep: pkgtool = "/usr/bin/yum -d0 -y erase %s" else: pkgtool = "/usr/bin/yum -d0 erase %s" diff --git a/src/lib/Client/Tools/YUMng.py b/src/lib/Bcfg2/Client/Tools/YUMng.py index febe6b9a0..d58e08f57 100644 --- a/src/lib/Client/Tools/YUMng.py +++ b/src/lib/Bcfg2/Client/Tools/YUMng.py @@ -1,5 +1,4 @@ """This provides bcfg2 support for yum.""" -__revision__ = '$Revision$' import copy import os.path @@ -13,15 +12,6 @@ import yum.misc import rpmUtils.arch import Bcfg2.Client.XML import Bcfg2.Client.Tools -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - -# Fix for python2.3 -try: - set -except NameError: - from sets import Set as set - def build_yname(pkgname, inst): """Build yum appropriate package name.""" @@ -65,20 +55,6 @@ def nevraString(p): return ret -class Parser(ConfigParser.ConfigParser): - - def get(self, section, option, default): - """ - Override ConfigParser.get: If the request option is not in the - config file then return the value of default rather than raise - an exception. We still raise exceptions on missing sections. - """ - try: - return ConfigParser.ConfigParser.get(self, section, option) - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - return default - - class RPMDisplay(yum.rpmtrans.RPMBaseCallback): """We subclass the default RPM transaction callback so that we can control Yum's verbosity and pipe it through the right logger.""" @@ -231,38 +207,24 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): def _loadConfig(self): # Process the YUMng section from the config file. - CP = Parser() - CP.read(self.setup.get('setup')) - truth = ['true', 'yes', '1'] - # These are all boolean flags, either we do stuff or we don't - self.pkg_checks = CP.get(self.name, "pkg_checks", "true").lower() \ - in truth - self.pkg_verify = CP.get(self.name, "pkg_verify", "true").lower() \ - in truth - self.doInstall = CP.get(self.name, "installed_action", - "install").lower() == "install" - self.doUpgrade = CP.get(self.name, - "version_fail_action", "upgrade").lower() == "upgrade" - self.doReinst = CP.get(self.name, "verify_fail_action", - "reinstall").lower() == "reinstall" - self.verifyFlags = CP.get(self.name, "verify_flags", - "").lower().replace(' ', ',') + self.pkg_checks = self.setup["yumng_pkg_checks"] + self.pkg_verify = self.setup["yumng_pkg_verify"] + self.doInstall = self.setup["yumng_installed_action"] == "install" + self.doUpgrade = self.setup["yumng_version_fail_action"] == "upgrade" + self.doReinst = self.setup["yumng_verify_fail_action"] == "reinstall" + self.verifyFlags = self.setup["yumng_verify_flags"] self.installOnlyPkgs = self.yb.conf.installonlypkgs if 'gpg-pubkey' not in self.installOnlyPkgs: self.installOnlyPkgs.append('gpg-pubkey') - self.logger.debug("YUMng: Install missing: %s" \ - % self.doInstall) + self.logger.debug("YUMng: Install missing: %s" % self.doInstall) self.logger.debug("YUMng: pkg_checks: %s" % self.pkg_checks) self.logger.debug("YUMng: pkg_verify: %s" % self.pkg_verify) - self.logger.debug("YUMng: Upgrade on version fail: %s" \ - % self.doUpgrade) - self.logger.debug("YUMng: Reinstall on verify fail: %s" \ - % self.doReinst) - self.logger.debug("YUMng: installOnlyPkgs: %s" \ - % str(self.installOnlyPkgs)) + self.logger.debug("YUMng: Upgrade on version fail: %s" % self.doUpgrade) + self.logger.debug("YUMng: Reinstall on verify fail: %s" % self.doReinst) + self.logger.debug("YUMng: installOnlyPkgs: %s" % self.installOnlyPkgs) self.logger.debug("YUMng: verify_flags: %s" % self.verifyFlags) def _fixAutoVersion(self, entry): @@ -767,7 +729,8 @@ class YUMng(Bcfg2.Client.Tools.PkgTool): else: self.logger.error("Second pass yum install failed.") self.logger.debug(" %s" % restring) - except yum.Errors.YumBaseError, e: + except yum.Errors.YumBaseError: + e = sys.exc_info()[1] self.logger.error("Yum transaction error: %s" % str(e)) self.yb.conf.skip_broken = skipBroken diff --git a/src/lib/Client/Tools/__init__.py b/src/lib/Bcfg2/Client/Tools/__init__.py index e879d7dd6..d423b6380 100644 --- a/src/lib/Client/Tools/__init__.py +++ b/src/lib/Bcfg2/Client/Tools/__init__.py @@ -1,8 +1,4 @@ """This contains all Bcfg2 Tool modules""" -# suppress popen2 warnings for python 2.3 -import warnings -warnings.filterwarnings("ignore", "The popen2 module is deprecated.*", - DeprecationWarning) import os import stat import sys @@ -10,7 +6,7 @@ from subprocess import Popen, PIPE import time import Bcfg2.Client.XML -__revision__ = '$Revision$' +from Bcfg2.Bcfg2Py3k import input __all__ = [tool.split('.')[0] \ for tool in os.listdir(os.path.dirname(__file__)) \ @@ -310,8 +306,7 @@ class SvcTool(Tool): return self.cmd.run(self.get_svc_command(service, restart_target))[0] def check_service(self, service): - # not supported for this driver - return 0 + return self.cmd.run(self.get_svc_command(service, 'status'))[0] == 0 def Remove(self, services): """ Dummy implementation of service removal method """ @@ -326,13 +321,12 @@ class SvcTool(Tool): return for entry in [ent for ent in bundle if self.handlesEntry(ent)]: - mode = entry.get('mode', 'default') - if (mode == 'manual' or - (mode == 'interactive_only' and + restart = entry.get("restart", "true") + if (restart.lower() == "false" or + (restart.lower == "interactive" and not self.setup['interactive'])): continue - # need to handle servicemode = (build|default) - # need to handle mode = (default|supervised) + rc = None if entry.get('status') == 'on': if self.setup['servicemode'] == 'build': @@ -341,11 +335,7 @@ class SvcTool(Tool): if self.setup['interactive']: prompt = ('Restart service %s?: (y/N): ' % entry.get('name')) - # py3k compatibility - try: - ans = raw_input(prompt) - except NameError: - ans = input(prompt) + ans = input(prompt) if ans not in ['y', 'Y']: continue rc = self.restart_service(entry) @@ -356,3 +346,19 @@ class SvcTool(Tool): if rc: self.logger.error("Failed to manipulate service %s" % (entry.get('name'))) + + def Install(self, entries, states): + """Install all entries in sublist.""" + for entry in entries: + if entry.get('install', 'true').lower() == 'false': + self.logger.info("Service %s installation is false. Skipping " + "installation." % (entry.get('name'))) + continue + try: + func = getattr(self, "Install%s" % (entry.tag)) + states[entry] = func(entry) + if states[entry]: + self.modified.append(entry) + except: + self.logger.error("Unexpected failure of install method for entry type %s" + % (entry.tag), exc_info=1) diff --git a/src/lib/Client/Tools/launchd.py b/src/lib/Bcfg2/Client/Tools/launchd.py index d7cbfa07f..6f08559a2 100644 --- a/src/lib/Client/Tools/launchd.py +++ b/src/lib/Bcfg2/Client/Tools/launchd.py @@ -1,5 +1,4 @@ """launchd support for Bcfg2.""" -__revision__ = '$Revision$' import os @@ -89,11 +88,6 @@ class launchd(Bcfg2.Client.Tools.Tool): def InstallService(self, entry): """Enable or disable launchd item.""" - # don't take any actions for mode='manual' - if entry.get('mode', 'default') == 'manual': - self.logger.info("Service %s mode set to manual. Skipping " - "installation." % (entry.get('name'))) - return False name = entry.get('name') if entry.get('status') == 'on': self.logger.error("Installing service %s" % name) diff --git a/src/lib/Client/Tools/rpmtools.py b/src/lib/Bcfg2/Client/Tools/rpmtools.py index 3cd2b7014..7441b2c06 100755 --- a/src/lib/Client/Tools/rpmtools.py +++ b/src/lib/Bcfg2/Client/Tools/rpmtools.py @@ -18,7 +18,6 @@ Run 'rpmtools' -h for the options. """ -__revision__ = '$Revision$' import grp import optparse diff --git a/src/lib/Client/XML.py b/src/lib/Bcfg2/Client/XML.py index 42b1017ac..858479611 100644 --- a/src/lib/Client/XML.py +++ b/src/lib/Bcfg2/Client/XML.py @@ -1,5 +1,4 @@ '''XML lib compatibility layer for the Bcfg2 client''' -__revision__ = '$Revision$' # library will use lxml, then builtin xml.etree, then ElementTree diff --git a/src/lib/Client/__init__.py b/src/lib/Bcfg2/Client/__init__.py index ea60a4259..6ed37b257 100644 --- a/src/lib/Client/__init__.py +++ b/src/lib/Bcfg2/Client/__init__.py @@ -1,4 +1,3 @@ """This contains all Bcfg2 Client modules""" -__revision__ = '$Revision$' __all__ = ["Frame", "Tools", "XML"] diff --git a/src/lib/Component.py b/src/lib/Bcfg2/Component.py index caea3eda9..eb9ea166a 100644 --- a/src/lib/Component.py +++ b/src/lib/Bcfg2/Component.py @@ -1,7 +1,5 @@ """Cobalt component base.""" -__revision__ = '$Revision$' - __all__ = ["Component", "exposed", "automatic", "run_component"] import inspect @@ -16,7 +14,7 @@ import Bcfg2.Logger from Bcfg2.Statistics import Statistics from Bcfg2.SSLServer import XMLRPCServer # Compatibility import -from Bcfg2.Bcfg2Py3k import xmlrpclib, urlparse, fprint +from Bcfg2.Bcfg2Py3k import xmlrpclib, urlparse logger = logging.getLogger() @@ -57,7 +55,7 @@ def run_component(component_cls, listen_all, location, daemon, pidfile_name, os.chdir(os.sep) pidfile = open(pidfile_name or "/dev/null", "w") - fprint(os.getpid(), pidfile) + pidfile.write("%s\n" % os.getpid()) pidfile.close() component = component_cls(cfile=cfile, **cls_kwargs) diff --git a/src/lib/Bcfg2/Encryption.py b/src/lib/Bcfg2/Encryption.py new file mode 100755 index 000000000..62b22d7de --- /dev/null +++ b/src/lib/Bcfg2/Encryption.py @@ -0,0 +1,75 @@ +#!/usr/bin/python -Ott + +import os +import base64 +from M2Crypto import Rand +from M2Crypto.EVP import Cipher, EVPError +from Bcfg2.Bcfg2Py3k import StringIO + +try: + from hashlib import md5 +except ImportError: + from md5 import md5 + +ENCRYPT = 1 +DECRYPT = 0 +ALGORITHM = "aes_256_cbc" +IV = '\0' * 16 + +Rand.rand_seed(os.urandom(1024)) + +def _cipher_filter(cipher, instr): + inbuf = StringIO(instr) + outbuf = StringIO() + while 1: + buf = inbuf.read() + if not buf: + break + outbuf.write(cipher.update(buf)) + outbuf.write(cipher.final()) + rv = outbuf.getvalue() + inbuf.close() + outbuf.close() + return rv + +def str_encrypt(plaintext, key, iv=IV, algorithm=ALGORITHM, salt=None): + """ encrypt a string """ + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=ENCRYPT, salt=salt) + return _cipher_filter(cipher, plaintext) + +def str_decrypt(crypted, key, iv=IV, algorithm=ALGORITHM): + """ decrypt a string """ + cipher = Cipher(alg=algorithm, key=key, iv=iv, op=DECRYPT) + return _cipher_filter(cipher, crypted) + +def ssl_decrypt(data, passwd, algorithm=ALGORITHM): + """ decrypt openssl-encrypted data """ + # base64-decode the data if necessary + try: + data = base64.b64decode(data) + except TypeError: + # already decoded + pass + + salt = data[8:16] + hashes = [md5(passwd + salt).digest()] + for i in range(1,3): + hashes.append(md5(hashes[i-1] + passwd + salt).digest()) + key = hashes[0] + hashes[1] + iv = hashes[2] + + return str_decrypt(data[16:], key=key, iv=iv) + +def ssl_encrypt(plaintext, passwd, algorithm=ALGORITHM, salt=None): + """ encrypt data in a format that is openssl compatible """ + if salt is None: + salt = Rand.rand_bytes(8) + + hashes = [md5(passwd + salt).digest()] + for i in range(1,3): + hashes.append(md5(hashes[i-1] + passwd + salt).digest()) + key = hashes[0] + hashes[1] + iv = hashes[2] + + crypted = str_encrypt(plaintext, key=key, salt=salt, iv=iv) + return base64.b64encode("Salted__" + salt + crypted) + "\n" diff --git a/src/lib/Logger.py b/src/lib/Bcfg2/Logger.py index e8f9ecd47..81b45550f 100644 --- a/src/lib/Logger.py +++ b/src/lib/Bcfg2/Logger.py @@ -1,5 +1,4 @@ """Bcfg2 logging support""" -__revision__ = '$Revision$' import copy import fcntl @@ -96,9 +95,9 @@ class FragmentingSysLogHandler(logging.handlers.SysLogHandler): else: msgs = [record] for newrec in msgs: - msg = self.log_format_string % (self.encodePriority(self.facility, - newrec.levelname.lower()), - self.format(newrec)) + msg = '<%d>%s\000' % (self.encodePriority(self.facility, + newrec.levelname.lower()), + self.format(newrec)) try: self.socket.send(msg.encode('ascii')) except socket.error: diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py new file mode 100644 index 000000000..bbbbec343 --- /dev/null +++ b/src/lib/Bcfg2/Options.py @@ -0,0 +1,880 @@ +"""Option parsing library for utilities.""" + +import re +import os +import sys +import copy +import shlex +import getopt +import Bcfg2.Client.Tools +# Compatibility imports +from Bcfg2.Bcfg2Py3k import ConfigParser + + +class OptionFailure(Exception): + pass + +DEFAULT_CONFIG_LOCATION = '/etc/bcfg2.conf' +DEFAULT_INSTALL_PREFIX = '/usr' + + +class DefaultConfigParser(ConfigParser.ConfigParser): + def get(self, section, option, **kwargs): + """ convenience method for getting config items """ + default = None + if 'default' in kwargs: + default = kwargs['default'] + del kwargs['default'] + try: + return ConfigParser.ConfigParser.get(self, section, option, + **kwargs) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + if default is not None: + return default + else: + raise + + def getboolean(self, section, option, **kwargs): + """ convenience method for getting boolean config items """ + default = None + if 'default' in kwargs: + default = kwargs['default'] + del kwargs['default'] + try: + return ConfigParser.ConfigParser.getboolean(self, section, + option, **kwargs) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, + ValueError): + if default is not None: + return default + else: + raise + + +class Option(object): + def get_cooked_value(self, value): + if self.boolean: + return True + if self.cook: + return self.cook(value) + else: + return value + + def __init__(self, desc, default, cmd=False, odesc=False, + env=False, cf=False, cook=False, long_arg=False): + self.desc = desc + self.default = default + self.cmd = cmd + self.long = long_arg + if not self.long: + if cmd and (cmd[0] != '-' or len(cmd) != 2): + raise OptionFailure("Poorly formed command %s" % cmd) + else: + if cmd and (not cmd.startswith('--')): + raise OptionFailure("Poorly formed command %s" % cmd) + self.odesc = odesc + self.env = env + self.cf = cf + self.boolean = False + if not odesc and not cook and isinstance(self.default, bool): + self.boolean = True + self.cook = cook + + def buildHelpMessage(self): + vals = [] + if not self.cmd: + return '' + if self.odesc: + if self.long: + vals.append("%s=%s" % (self.cmd, self.odesc)) + else: + vals.append("%s %s" % (self.cmd, self.odesc)) + else: + vals.append(self.cmd) + vals.append(self.desc) + return " %-28s %s\n" % tuple(vals) + + def buildGetopt(self): + gstr = '' + if self.long: + return gstr + if self.cmd: + gstr = self.cmd[1] + if self.odesc: + gstr += ':' + return gstr + + def buildLongGetopt(self): + if self.odesc: + return self.cmd[2:] + '=' + else: + return self.cmd[2:] + + def parse(self, opts, rawopts, configparser=None): + if self.cmd and opts: + # Processing getopted data + optinfo = [opt[1] for opt in opts if opt[0] == self.cmd] + if optinfo: + if optinfo[0]: + self.value = self.get_cooked_value(optinfo[0]) + else: + self.value = True + return + if self.cmd and self.cmd in rawopts: + data = rawopts[rawopts.index(self.cmd) + 1] + self.value = self.get_cooked_value(data) + return + # No command line option found + if self.env and self.env in os.environ: + self.value = self.get_cooked_value(os.environ[self.env]) + return + if self.cf and configparser: + try: + self.value = self.get_cooked_value(configparser.get(*self.cf)) + return + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + pass + # Default value not cooked + self.value = self.default + + +class OptionSet(dict): + def __init__(self, *args, **kwargs): + dict.__init__(self, *args) + self.hm = self.buildHelpMessage() + if 'configfile' in kwargs: + self.cfile = kwargs['configfile'] + else: + self.cfile = DEFAULT_CONFIG_LOCATION + self.cfp = DefaultConfigParser() + if (len(self.cfp.read(self.cfile)) == 0 and + ('quiet' not in kwargs or not kwargs['quiet'])): + print("Warning! Unable to read specified configuration file: %s" % + self.cfile) + + def buildGetopt(self): + return ''.join([opt.buildGetopt() for opt in list(self.values())]) + + def buildLongGetopt(self): + return [opt.buildLongGetopt() for opt in list(self.values()) + if opt.long] + + def buildHelpMessage(self): + if hasattr(self, 'hm'): + return self.hm + hlist = [] # list of _non-empty_ help messages + for opt in list(self.values()): + hm = opt.buildHelpMessage() + if hm: + hlist.append(hm) + return ''.join(hlist) + + def helpExit(self, msg='', code=1): + if msg: + print(msg) + print("Usage:") + print(self.buildHelpMessage()) + raise SystemExit(code) + + def parse(self, argv, do_getopt=True): + '''Parse options from command line.''' + if do_getopt: + try: + opts, args = getopt.getopt(argv, self.buildGetopt(), + self.buildLongGetopt()) + except getopt.GetoptError: + err = sys.exc_info()[1] + self.helpExit(err) + if '-h' in argv: + self.helpExit('', 0) + self['args'] = args + for key in list(self.keys()): + if key == 'args': + continue + option = self[key] + if do_getopt: + option.parse(opts, [], configparser=self.cfp) + else: + option.parse([], argv, configparser=self.cfp) + if hasattr(option, 'value'): + val = option.value + self[key] = val + + +def list_split(c_string): + if c_string: + return re.split("\s*,\s*", c_string) + return [] + + +def colon_split(c_string): + if c_string: + return c_string.split(':') + return [] + + +def get_bool(s): + # these values copied from ConfigParser.RawConfigParser.getboolean + # with the addition of True and False + truelist = ["1", "yes", "True", "true", "on"] + falselist = ["0", "no", "False", "false", "off"] + if s in truelist: + return True + elif s in falselist: + return False + else: + raise ValueError + +""" +Options: + + Accepts keyword argument list with the following values: + + default: default value for the option + cmd: command line switch + odesc: option description + cf: tuple containing section/option + cook: method for parsing option + long_arg: (True|False) specifies whether cmd is a long argument +""" +# General options +CFILE = \ + Option('Specify configuration file', + default=DEFAULT_CONFIG_LOCATION, + cmd='-C', + odesc='<conffile>') +LOCKFILE = \ + Option('Specify lockfile', + default='/var/lock/bcfg2.run', + odesc='<Path to lockfile>', + cf=('components', 'lockfile')) +HELP = \ + Option('Print this usage message', + default=False, + cmd='-h') +DEBUG = \ + Option("Enable debugging output", + default=False, + cmd='-d') +VERBOSE = \ + Option("Enable verbose output", + default=False, + cmd='-v') +DAEMON = \ + Option("Daemonize process, storing pid", + default=None, + cmd='-D', + odesc='<pidfile>') +INSTALL_PREFIX = \ + Option('Installation location', + default=DEFAULT_INSTALL_PREFIX, + odesc='</path>', + cf=('server', 'prefix')) +SENDMAIL_PATH = \ + Option('Path to sendmail', + default='/usr/lib/sendmail', + cf=('reports', 'sendmailpath')) +INTERACTIVE = \ + Option('Run interactively, prompting the user for each change', + default=False, + cmd='-I', ) +ENCODING = \ + Option('Encoding of cfg files', + default='UTF-8', + cmd='-E', + odesc='<encoding>', + cf=('components', 'encoding')) +PARANOID_PATH = \ + Option('Specify path for paranoid file backups', + default='/var/cache/bcfg2', + odesc='<paranoid backup path>', + cf=('paranoid', 'path')) +PARANOID_MAX_COPIES = \ + Option('Specify the number of paranoid copies you want', + default=1, + odesc='<max paranoid copies>', + cf=('paranoid', 'max_copies')) +OMIT_LOCK_CHECK = \ + Option('Omit lock check', + default=False, + cmd='-O') +CORE_PROFILE = \ + Option('profile', + default=False, + cmd='-p') +SCHEMA_PATH = \ + Option('Path to XML Schema files', + default='%s/share/bcfg2/schemas' % DEFAULT_INSTALL_PREFIX, + cmd='--schema', + odesc='<schema path>', + long_arg=True) + +# Metadata options (mdata section) +MDATA_OWNER = \ + Option('Default Path owner', + default='root', + odesc='owner permissions', + cf=('mdata', 'owner')) +MDATA_GROUP = \ + Option('Default Path group', + default='root', + odesc='group permissions', + cf=('mdata', 'group')) +MDATA_IMPORTANT = \ + Option('Default Path priority (importance)', + default='False', + odesc='Important entries are installed first', + cf=('mdata', 'important')) +MDATA_PERMS = \ + Option('Default Path permissions', + default='644', + odesc='octal permissions', + cf=('mdata', 'perms')) +MDATA_PARANOID = \ + Option('Default Path paranoid setting', + default='true', + odesc='Path paranoid setting', + cf=('mdata', 'paranoid')) +MDATA_SENSITIVE = \ + Option('Default Path sensitive setting', + default='false', + odesc='Path sensitive setting', + cf=('mdata', 'sensitive')) + +# Server options +SERVER_REPOSITORY = \ + Option('Server repository path', + default='/var/lib/bcfg2', + cmd='-Q', + odesc='<repository path>', + cf=('server', 'repository')) +SERVER_PLUGINS = \ + Option('Server plugin list', + # default server plugins + default=['Bundler', 'Cfg', 'Metadata', 'Pkgmgr', 'Rules', 'SSHbase'], + cf=('server', 'plugins'), + cook=list_split) +SERVER_MCONNECT = \ + Option('Server Metadata Connector list', + default=['Probes'], + cf=('server', 'connectors'), + cook=list_split) +SERVER_FILEMONITOR = \ + Option('Server file monitor', + default='default', + odesc='File monitoring driver', + cf=('server', 'filemonitor')) +SERVER_FAM_IGNORE = \ + Option('File globs to ignore', + default=['*~', '*#', '.#*', '*.swp', '.*.swx', 'SCCS', '.svn', + '4913', '.gitignore',], + cf=('server', 'ignore_files'), + cook=list_split) +SERVER_LISTEN_ALL = \ + Option('Listen on all interfaces', + default=False, + cmd='--listen-all', + cf=('server', 'listen_all'), + cook=get_bool, + long_arg=True) +SERVER_LOCATION = \ + Option('Server Location', + default='https://localhost:6789', + cmd='-S', + odesc='https://server:port', + cf=('components', 'bcfg2')) +SERVER_STATIC = \ + Option('Server runs on static port', + default=False, + cf=('components', 'bcfg2')) +SERVER_KEY = \ + Option('Path to SSL key', + default=None, + cmd='--ssl-key', + odesc='<ssl key>', + cf=('communication', 'key'), + long_arg=True) +SERVER_CERT = \ + Option('Path to SSL certificate', + default='/etc/bcfg2.key', + odesc='<ssl cert>', + cf=('communication', 'certificate')) +SERVER_CA = \ + Option('Path to SSL CA Cert', + default=None, + odesc='<ca cert>', + cf=('communication', 'ca')) +SERVER_PASSWORD = \ + Option('Communication Password', + default=None, + cmd='-x', + odesc='<password>', + cf=('communication', 'password')) +SERVER_PROTOCOL = \ + Option('Server Protocol', + default='xmlrpc/ssl', + cf=('communication', 'procotol')) + +# Client options +CLIENT_KEY = \ + Option('Path to SSL key', + default=None, + cmd='--ssl-key', + odesc='<ssl key>', + cf=('communication', 'key'), + long_arg=True) +CLIENT_CERT = \ + Option('Path to SSL certificate', + default=None, + cmd='--ssl-cert', + odesc='<ssl cert>', + cf=('communication', 'certificate'), + long_arg=True) +CLIENT_CA = \ + Option('Path to SSL CA Cert', + default=None, + cmd='--ca-cert', + odesc='<ca cert>', + cf=('communication', 'ca'), + long_arg=True) +CLIENT_SCNS = \ + Option('List of server commonNames', + default=None, + cmd='--ssl-cns', + odesc='<CN1:CN2>', + cf=('communication', 'serverCommonNames'), + cook=list_split, + long_arg=True) +CLIENT_PROFILE = \ + Option('Assert the given profile for the host', + default=None, + cmd='-p', + odesc='<profile>') +CLIENT_RETRIES = \ + Option('The number of times to retry network communication', + default='3', + cmd='-R', + odesc='<retry count>', + cf=('communication', 'retries')) +CLIENT_DRYRUN = \ + Option('Do not actually change the system', + default=False, + cmd='-n') +CLIENT_EXTRA_DISPLAY = \ + Option('enable extra entry output', + default=False, + cmd='-e') +CLIENT_PARANOID = \ + Option('Make automatic backups of config files', + default=False, + cmd='-P', + cf=('client', 'paranoid'), + cook=get_bool) +CLIENT_DRIVERS = \ + Option('Specify tool driver set', + default=Bcfg2.Client.Tools.default, + cmd='-D', + odesc='<driver1,driver2>', + cf=('client', 'drivers'), + cook=list_split) +CLIENT_CACHE = \ + Option('Store the configuration in a file', + default=None, + cmd='-c', + odesc='<cache path>') +CLIENT_REMOVE = \ + Option('Force removal of additional configuration items', + default=None, + cmd='-r', + odesc='<entry type|all>') +CLIENT_BUNDLE = \ + Option('Only configure the given bundle(s)', + default=[], + cmd='-b', + odesc='<bundle:bundle>', + cook=colon_split) +CLIENT_SKIPBUNDLE = \ + Option('Configure everything except the given bundle(s)', + default=[], + cmd='-B', + odesc='<bundle:bundle>', + cook=colon_split) +CLIENT_BUNDLEQUICK = \ + Option('Only verify/configure the given bundle(s)', + default=False, + cmd='-Q') +CLIENT_INDEP = \ + Option('Only configure independent entries, ignore bundles', + default=False, + cmd='-z') +CLIENT_SKIPINDEP = \ + Option('Do not configure independent entries', + default=False, + cmd='-Z') +CLIENT_KEVLAR = \ + Option('Run in kevlar (bulletproof) mode', + default=False, + cmd='-k', ) +CLIENT_FILE = \ + Option('Configure from a file rather than querying the server', + default=None, + cmd='-f', + odesc='<specification path>') +CLIENT_QUICK = \ + Option('Disable some checksum verification', + default=False, + cmd='-q') +CLIENT_USER = \ + Option('The user to provide for authentication', + default='root', + cmd='-u', + odesc='<user>', + cf=('communication', 'user')) +CLIENT_SERVICE_MODE = \ + Option('Set client service mode', + default='default', + cmd='-s', + odesc='<default|disabled|build>') +CLIENT_TIMEOUT = \ + Option('Set the client XML-RPC timeout', + default=90, + cmd='-t', + odesc='<timeout>', + cf=('communication', 'timeout')) +CLIENT_DLIST = \ + Option('Run client in server decision list mode', + default='none', + cmd='-l', + odesc='<whitelist|blacklist|none>', + cf=('client', 'decision')) +CLIENT_DECISION_LIST = \ + Option('Decision List', + default=False, + cmd='--decision-list', + odesc='<file>', + long_arg=True) + +# bcfg2-test and bcfg2-lint options +TEST_NOSEOPTS = \ + Option('Options to pass to nosetests', + default=[], + cmd='--nose-options', + odesc='<opts>', + cf=('bcfg2_test', 'nose_options'), + cook=shlex.split, + long_arg=True) +TEST_IGNORE = \ + Option('Ignore these entries if they fail to build.', + default=[], + cmd='--ignore', + odesc='<Type>:<name>,<Type>:<name>', + cf=('bcfg2_test', 'ignore_entries'), + cook=list_split, + long_arg=True) +LINT_CONFIG = \ + Option('Specify bcfg2-lint configuration file', + default='/etc/bcfg2-lint.conf', + cmd='--lint-config', + odesc='<conffile>', + long_arg=True) +LINT_SHOW_ERRORS = \ + Option('Show error handling', + default=False, + cmd='--list-errors', + long_arg=True) +LINT_FILES_ON_STDIN = \ + Option('Operate on a list of files supplied on stdin', + default=False, + cmd='--stdin', + long_arg=True) + +# individual client tool options +CLIENT_APT_TOOLS_INSTALL_PATH = \ + Option('Apt tools install path', + default='/usr', + cf=('APT', 'install_path')) +CLIENT_APT_TOOLS_VAR_PATH = \ + Option('Apt tools var path', + default='/var', + cf=('APT', 'var_path')) +CLIENT_SYSTEM_ETC_PATH = \ + Option('System etc path', + default='/etc', + cf=('APT', 'etc_path')) +CLIENT_PORTAGE_BINPKGONLY = \ + Option('Portage binary packages only', + default=False, + cf=('Portage', 'binpkgonly'), + cook=get_bool) +CLIENT_RPMNG_INSTALLONLY = \ + Option('RPMng install-only packages', + default=['kernel', 'kernel-bigmem', 'kernel-enterprise', + 'kernel-smp', 'kernel-modules', 'kernel-debug', + 'kernel-unsupported', 'kernel-devel', 'kernel-source', + 'kernel-default', 'kernel-largesmp-devel', + 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], + cf=('RPMng', 'installonlypackages'), + cook=list_split) +CLIENT_RPMNG_PKG_CHECKS = \ + Option("Perform RPMng package checks", + default=True, + cf=('RPMng', 'pkg_checks'), + cook=get_bool) +CLIENT_RPMNG_PKG_VERIFY = \ + Option("Perform RPMng package verify", + default=True, + cf=('RPMng', 'pkg_verify'), + cook=get_bool) +CLIENT_RPMNG_INSTALLED_ACTION = \ + Option("RPMng installed action", + default="install", + cf=('RPMng', 'installed_action')) +CLIENT_RPMNG_ERASE_FLAGS = \ + Option("RPMng erase flags", + default=["allmatches"], + cf=('RPMng', 'erase_flags'), + cook=list_split) +CLIENT_RPMNG_VERSION_FAIL_ACTION = \ + Option("RPMng version fail action", + default="upgrade", + cf=('RPMng', 'version_fail_action')) +CLIENT_RPMNG_VERIFY_FAIL_ACTION = \ + Option("RPMng verify fail action", + default="reinstall", + cf=('RPMng', 'verify_fail_action')) +CLIENT_RPMNG_VERIFY_FLAGS = \ + Option("RPMng verify flags", + default=[], + cf=('RPMng', 'verify_flags'), + cook=list_split) +CLIENT_YUM24_INSTALLONLY = \ + Option('RPMng install-only packages', + default=['kernel', 'kernel-bigmem', 'kernel-enterprise', + 'kernel-smp', 'kernel-modules', 'kernel-debug', + 'kernel-unsupported', 'kernel-devel', 'kernel-source', + 'kernel-default', 'kernel-largesmp-devel', + 'kernel-largesmp', 'kernel-xen', 'gpg-pubkey'], + cf=('RPMng', 'installonlypackages'), + cook=list_split) +CLIENT_YUM24_PKG_CHECKS = \ + Option("Perform YUM24 package checks", + default=True, + cf=('YUM24', 'pkg_checks'), + cook=get_bool) +CLIENT_YUM24_PKG_VERIFY = \ + Option("Perform YUM24 package verify", + default=True, + cf=('YUM24', 'pkg_verify'), + cook=get_bool) +CLIENT_YUM24_INSTALLED_ACTION = \ + Option("YUM24 installed action", + default="install", + cf=('YUM24', 'installed_action')) +CLIENT_YUM24_ERASE_FLAGS = \ + Option("YUM24 erase flags", + default=["allmatches"], + cf=('YUM24', 'erase_flags'), + cook=list_split) +CLIENT_YUM24_VERSION_FAIL_ACTION = \ + Option("YUM24 version fail action", + cf=('YUM24', 'version_fail_action'), + default="upgrade") +CLIENT_YUM24_VERIFY_FAIL_ACTION = \ + Option("YUM24 verify fail action", + default="reinstall", + cf=('YUM24', 'verify_fail_action')) +CLIENT_YUM24_VERIFY_FLAGS = \ + Option("YUM24 verify flags", + default=[], + cf=('YUM24', 'verify_flags'), + cook=list_split) +CLIENT_YUM24_AUTODEP = \ + Option("YUM24 autodependency processing", + default=True, + cf=('YUM24', 'autodep'), + cook=get_bool) +CLIENT_YUMNG_PKG_CHECKS = \ + Option("Perform YUMng package checks", + default=True, + cf=('YUMng', 'pkg_checks'), + cook=get_bool) +CLIENT_YUMNG_PKG_VERIFY = \ + Option("Perform YUMng package verify", + default=True, + cf=('YUMng', 'pkg_verify'), + cook=get_bool) +CLIENT_YUMNG_INSTALLED_ACTION = \ + Option("YUMng installed action", + default="install", + cf=('YUMng', 'installed_action')) +CLIENT_YUMNG_VERSION_FAIL_ACTION = \ + Option("YUMng version fail action", + default="upgrade", + cf=('YUMng', 'version_fail_action')) +CLIENT_YUMNG_VERIFY_FAIL_ACTION = \ + Option("YUMng verify fail action", + default="reinstall", + cf=('YUMng', 'verify_fail_action')) +CLIENT_YUMNG_VERIFY_FLAGS = \ + Option("YUMng verify flags", + default=[], + cf=('YUMng', 'verify_flags'), + cook=list_split) + +# Logging options +LOGGING_FILE_PATH = \ + Option('Set path of file log', + default=None, + cmd='-o', + odesc='<path>', + cf=('logging', 'path')) + +# Plugin-specific options +CFG_VALIDATION = \ + Option('Run validation on Cfg files', + default=True, + cmd='--cfg-validation', + cf=('cfg', 'validation'), + long_arg=True, + cook=get_bool) + +# bcfg2-crypt options +ENCRYPT = \ + Option('Encrypt the specified file', + default=False, + cmd='--encrypt', + long_arg=True) +DECRYPT = \ + Option('Decrypt the specified file', + default=False, + cmd='--decrypt', + long_arg=True) +CRYPT_PASSPHRASE = \ + Option('Encryption passphrase (name or passphrase)', + default=None, + cmd='-p', + odesc='<passphrase>') +CRYPT_XPATH = \ + Option('XPath expression to select elements to encrypt', + default=None, + cmd='--xpath', + odesc='<xpath>', + long_arg=True) +CRYPT_PROPERTIES = \ + Option('Encrypt the specified file as a Properties file', + default=False, + cmd="--properties", + long_arg=True) +CRYPT_CFG = \ + Option('Encrypt the specified file as a Cfg file', + default=False, + cmd="--cfg", + long_arg=True) +CRYPT_REMOVE = \ + Option('Remove the plaintext file after encrypting', + default=False, + cmd="--remove", + long_arg=True) + +# Option groups +CLI_COMMON_OPTIONS = dict(configfile=CFILE, + debug=DEBUG, + help=HELP, + verbose=VERBOSE, + encoding=ENCODING, + logging=LOGGING_FILE_PATH) + +DAEMON_COMMON_OPTIONS = dict(daemon=DAEMON, + listen_all=SERVER_LISTEN_ALL) + +SERVER_COMMON_OPTIONS = dict(repo=SERVER_REPOSITORY, + plugins=SERVER_PLUGINS, + password=SERVER_PASSWORD, + filemonitor=SERVER_FILEMONITOR, + ignore=SERVER_FAM_IGNORE, + location=SERVER_LOCATION, + static=SERVER_STATIC, + key=SERVER_KEY, + cert=SERVER_CERT, + ca=SERVER_CA, + protocol=SERVER_PROTOCOL) + +CRYPT_OPTIONS = dict(encrypt=ENCRYPT, + decrypt=DECRYPT, + passphrase=CRYPT_PASSPHRASE, + xpath=CRYPT_XPATH, + properties=CRYPT_PROPERTIES, + cfg=CRYPT_CFG, + remove=CRYPT_REMOVE) + +DRIVER_OPTIONS = \ + dict(apt_install_path=CLIENT_APT_TOOLS_INSTALL_PATH, + apt_var_path=CLIENT_APT_TOOLS_VAR_PATH, + apt_etc_path=CLIENT_SYSTEM_ETC_PATH, + portage_binpkgonly=CLIENT_PORTAGE_BINPKGONLY, + rpmng_installonly=CLIENT_RPMNG_INSTALLONLY, + rpmng_pkg_checks=CLIENT_RPMNG_PKG_CHECKS, + rpmng_pkg_verify=CLIENT_RPMNG_PKG_VERIFY, + rpmng_installed_action=CLIENT_RPMNG_INSTALLED_ACTION, + rpmng_erase_flags=CLIENT_RPMNG_ERASE_FLAGS, + rpmng_version_fail_action=CLIENT_RPMNG_VERSION_FAIL_ACTION, + rpmng_verify_fail_action=CLIENT_RPMNG_VERIFY_FAIL_ACTION, + rpmng_verify_flags=CLIENT_RPMNG_VERIFY_FLAGS, + yum24_installonly=CLIENT_YUM24_INSTALLONLY, + yum24_pkg_checks=CLIENT_YUM24_PKG_CHECKS, + yum24_pkg_verify=CLIENT_YUM24_PKG_VERIFY, + yum24_installed_action=CLIENT_YUM24_INSTALLED_ACTION, + yum24_erase_flags=CLIENT_YUM24_ERASE_FLAGS, + yum24_version_fail_action=CLIENT_YUM24_VERSION_FAIL_ACTION, + yum24_verify_fail_action=CLIENT_YUM24_VERIFY_FAIL_ACTION, + yum24_verify_flags=CLIENT_YUM24_VERIFY_FLAGS, + yum24_autodep=CLIENT_YUM24_AUTODEP, + yumng_pkg_checks=CLIENT_YUMNG_PKG_CHECKS, + yumng_pkg_verify=CLIENT_YUMNG_PKG_VERIFY, + yumng_installed_action=CLIENT_YUMNG_INSTALLED_ACTION, + yumng_version_fail_action=CLIENT_YUMNG_VERSION_FAIL_ACTION, + yumng_verify_fail_action=CLIENT_YUMNG_VERIFY_FAIL_ACTION, + yumng_verify_flags=CLIENT_YUMNG_VERIFY_FLAGS) + +class OptionParser(OptionSet): + """ + OptionParser bootstraps option parsing, + getting the value of the config file + """ + def __init__(self, args): + self.Bootstrap = OptionSet([('configfile', CFILE)], quiet=True) + self.Bootstrap.parse(sys.argv[1:], do_getopt=False) + OptionSet.__init__(self, args, configfile=self.Bootstrap['configfile']) + self.optinfo = copy.copy(args) + + def HandleEvent(self, event): + if 'configfile' not in self or not isinstance(self['configfile'], str): + # we haven't parsed options yet, or CFILE wasn't included + # in the options + return + if event.filename != self['configfile']: + print("Got event for unknown file: %s" % event.filename) + return + if event.code2str() == 'deleted': + return + self.reparse() + + def reparse(self): + for key, opt in self.optinfo.items(): + self[key] = opt + if "args" not in self.optinfo: + del self['args'] + self.parse(self.argv, self.do_getopt) + + def parse(self, argv, do_getopt=True): + self.argv = argv + self.do_getopt = do_getopt + OptionSet.parse(self, self.argv, do_getopt=self.do_getopt) + + def add_option(self, name, opt): + self[name] = opt + self.optinfo[name] = opt + + def update(self, optdict): + dict.update(self, optdict) + self.optinfo.update(optdict) diff --git a/src/lib/Proxy.py b/src/lib/Bcfg2/Proxy.py index fe2b18de8..422d642db 100644 --- a/src/lib/Proxy.py +++ b/src/lib/Bcfg2/Proxy.py @@ -8,9 +8,6 @@ load_config -- read configuration files """ -__revision__ = '$Revision: $' - - import logging import re import socket @@ -315,7 +312,7 @@ class XMLRPCTransport(xmlrpclib.Transport): 408, str(err), self._extra_headers)) - + if errcode != 200: raise ProxyError(xmlrpclib.ProtocolError(host + handler, errcode, diff --git a/src/lib/SSLServer.py b/src/lib/Bcfg2/SSLServer.py index 32ab9933b..33707eda8 100644 --- a/src/lib/SSLServer.py +++ b/src/lib/Bcfg2/SSLServer.py @@ -1,7 +1,5 @@ """Bcfg2 SSL server.""" -__revision__ = '$Revision$' - __all__ = [ "SSLServer", "XMLRPCRequestHandler", "XMLRPCServer", ] @@ -16,7 +14,6 @@ import logging import ssl import threading import time -import types # Compatibility imports from Bcfg2.Bcfg2Py3k import xmlrpclib, SimpleXMLRPCServer, SocketServer @@ -48,7 +45,7 @@ class XMLRPCDispatcher (SimpleXMLRPCServer.SimpleXMLRPCDispatcher): params = (address, ) + params response = self.instance._dispatch(method, params, self.funcs) # py3k compatibility - if type(response) not in [bool, str, list, dict, types.NoneType]: + if type(response) not in [bool, str, list, dict]: response = (response.decode('utf-8'), ) else: response = (response, ) diff --git a/src/lib/Server/Admin/Backup.py b/src/lib/Bcfg2/Server/Admin/Backup.py index 3744abca3..3744abca3 100644 --- a/src/lib/Server/Admin/Backup.py +++ b/src/lib/Bcfg2/Server/Admin/Backup.py diff --git a/src/lib/Server/Admin/Bundle.py b/src/lib/Bcfg2/Server/Admin/Bundle.py index 89c099602..89c099602 100644 --- a/src/lib/Server/Admin/Bundle.py +++ b/src/lib/Bcfg2/Server/Admin/Bundle.py diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Bcfg2/Server/Admin/Client.py index 4d580c54c..734e9573d 100644 --- a/src/lib/Server/Admin/Client.py +++ b/src/lib/Bcfg2/Server/Admin/Client.py @@ -59,6 +59,3 @@ class Client(Bcfg2.Server.Admin.MetadataCore): tree.xinclude() for node in tree.findall("//Client"): print(node.attrib["name"]) - else: - print("No command specified") - raise SystemExit(1) diff --git a/src/lib/Server/Admin/Compare.py b/src/lib/Bcfg2/Server/Admin/Compare.py index 050dd69f8..050dd69f8 100644 --- a/src/lib/Server/Admin/Compare.py +++ b/src/lib/Bcfg2/Server/Admin/Compare.py diff --git a/src/lib/Server/Admin/Group.py b/src/lib/Bcfg2/Server/Admin/Group.py index 16a773d6f..16a773d6f 100644 --- a/src/lib/Server/Admin/Group.py +++ b/src/lib/Bcfg2/Server/Admin/Group.py diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Bcfg2/Server/Admin/Init.py index 832190b7d..8d0c2a4a9 100644 --- a/src/lib/Server/Admin/Init.py +++ b/src/lib/Bcfg2/Server/Admin/Init.py @@ -6,9 +6,11 @@ import stat import string import sys import subprocess + import Bcfg2.Server.Admin import Bcfg2.Server.Plugin import Bcfg2.Options +from Bcfg2.Bcfg2Py3k import input # default config file config = ''' @@ -55,12 +57,13 @@ groups = '''<Groups version='3.0'> <Group name='suse'/> <Group name='mandrake'/> <Group name='solaris'/> + <Group name='arch'/> </Groups> ''' # Default contents of clients.xml clients = '''<Clients version="3.0"> - <Client profile="basic" pingable="Y" pingtime="0" name="%s"/> + <Client profile="basic" name="%s"/> </Clients> ''' @@ -71,7 +74,8 @@ os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), ('Debian', 'debian'), ('Ubuntu', 'ubuntu'), ('Gentoo', 'gentoo'), - ('FreeBSD', 'freebsd')] + ('FreeBSD', 'freebsd'), + ('Arch', 'arch')] # Complete list of plugins plugin_list = ['Account', @@ -104,14 +108,6 @@ plugin_list = ['Account', default_plugins = Bcfg2.Options.SERVER_PLUGINS.default -def get_input(prompt): - """py3k compatible function to get input""" - try: - return raw_input(prompt) - except NameError: - return input(prompt) - - def gen_password(length): """Generates a random alphanumeric password with length characters.""" chars = string.letters + string.digits @@ -145,8 +141,8 @@ def create_key(hostname, keypath, certpath, country, state, location): def create_conf(confpath, confdata, keypath): # Don't overwrite existing bcfg2.conf file if os.path.exists(confpath): - result = get_input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) + result = input("\nWarning: %s already exists. " + "Overwrite? [y/N]: " % confpath) if result not in ['Y', 'y']: print("Leaving %s unchanged" % confpath) return @@ -204,8 +200,8 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_hostname(self): """Ask for the server hostname.""" - data = get_input("What is the server's hostname [%s]: " % - socket.getfqdn()) + data = input("What is the server's hostname [%s]: " % + socket.getfqdn()) if data != '': self.shostname = data else: @@ -213,21 +209,21 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_config(self): """Ask for the configuration file path.""" - newconfig = get_input("Store Bcfg2 configuration in [%s]: " % - self.configfile) + newconfig = input("Store Bcfg2 configuration in [%s]: " % + self.configfile) if newconfig != '': self.configfile = os.path.abspath(newconfig) def _prompt_repopath(self): """Ask for the repository path.""" while True: - newrepo = get_input("Location of Bcfg2 repository [%s]: " % - self.repopath) + newrepo = input("Location of Bcfg2 repository [%s]: " % + self.repopath) if newrepo != '': self.repopath = os.path.abspath(newrepo) if os.path.isdir(self.repopath): - response = get_input("Directory %s exists. Overwrite? [y/N]:" \ - % self.repopath) + response = input("Directory %s exists. Overwrite? [y/N]:" \ + % self.repopath) if response.lower().strip() == 'y': break else: @@ -243,8 +239,8 @@ class Init(Bcfg2.Server.Admin.Mode): def _prompt_server(self): """Ask for the server name.""" - newserver = get_input("Input the server location [%s]: " % - self.server_uri) + newserver = input("Input the server location [%s]: " % + self.server_uri) if newserver != '': self.server_uri = newserver @@ -256,19 +252,19 @@ class Init(Bcfg2.Server.Admin.Mode): prompt += ': ' while True: try: - osidx = int(get_input(prompt)) + osidx = int(input(prompt)) self.os_sel = os_list[osidx - 1][1] break except ValueError: continue def _prompt_plugins(self): - default = get_input("Use default plugins? (%s) [Y/n]: " % - ''.join(default_plugins)).lower() + default = input("Use default plugins? (%s) [Y/n]: " % + ''.join(default_plugins)).lower() if default != 'y' or default != '': while True: plugins_are_valid = True - plug_str = get_input("Specify plugins: ") + plug_str = input("Specify plugins: ") plugins = plug_str.split(',') for plugin in plugins: plugin = plugin.strip() @@ -282,26 +278,26 @@ class Init(Bcfg2.Server.Admin.Mode): """Ask for the key details (country, state, and location).""" print("The following questions affect SSL certificate generation.") print("If no data is provided, the default values are used.") - newcountry = get_input("Country name (2 letter code) for certificate: ") + newcountry = input("Country name (2 letter code) for certificate: ") if newcountry != '': if len(newcountry) == 2: self.country = newcountry else: while len(newcountry) != 2: - newcountry = get_input("2 letter country code (eg. US): ") + newcountry = input("2 letter country code (eg. US): ") if len(newcountry) == 2: self.country = newcountry break else: self.country = 'US' - newstate = get_input("State or Province Name (full name) for certificate: ") + newstate = input("State or Province Name (full name) for certificate: ") if newstate != '': self.state = newstate else: self.state = 'Illinois' - newlocation = get_input("Locality Name (eg, city) for certificate: ") + newlocation = input("Locality Name (eg, city) for certificate: ") if newlocation != '': self.location = newlocation else: diff --git a/src/lib/Server/Admin/Minestruct.py b/src/lib/Bcfg2/Server/Admin/Minestruct.py index b929a9a8c..b929a9a8c 100644 --- a/src/lib/Server/Admin/Minestruct.py +++ b/src/lib/Bcfg2/Server/Admin/Minestruct.py diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Bcfg2/Server/Admin/Perf.py index 411442698..411442698 100644 --- a/src/lib/Server/Admin/Perf.py +++ b/src/lib/Bcfg2/Server/Admin/Perf.py diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Bcfg2/Server/Admin/Pull.py index daf353107..816fd2ca7 100644 --- a/src/lib/Server/Admin/Pull.py +++ b/src/lib/Bcfg2/Server/Admin/Pull.py @@ -2,6 +2,7 @@ import getopt import sys import Bcfg2.Server.Admin +from Bcfg2.Bcfg2Py3k import input class Pull(Bcfg2.Server.Admin.MetadataCore): @@ -109,11 +110,8 @@ class Pull(Bcfg2.Server.Admin.MetadataCore): (choice.group, choice.prio)) else: print(" => host entry: %s" % (choice.hostname)) - # py3k compatibility - try: - ans = raw_input("Use this entry? [yN]: ") in ['y', 'Y'] - except NameError: - ans = input("Use this entry? [yN]: ") in ['y', 'Y'] + + ans = input("Use this entry? [yN]: ") in ['y', 'Y'] if ans: return choice return False diff --git a/src/lib/Server/Admin/Query.py b/src/lib/Bcfg2/Server/Admin/Query.py index 3dd326645..3dd326645 100644 --- a/src/lib/Server/Admin/Query.py +++ b/src/lib/Bcfg2/Server/Admin/Query.py diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Bcfg2/Server/Admin/Reports.py index 974cdff9d..17e1e1e4d 100644 --- a/src/lib/Server/Admin/Reports.py +++ b/src/lib/Bcfg2/Server/Admin/Reports.py @@ -26,7 +26,7 @@ import Bcfg2.Server.Reports.settings # Load django and reports stuff _after_ we know we can load settings import django.core.management from Bcfg2.Server.Reports.importscript import load_stats -from Bcfg2.Server.Reports.updatefix import update_database +from Bcfg2.Server.Reports.Updater import update_database, UpdaterError from Bcfg2.Server.Reports.utils import * project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) @@ -41,7 +41,7 @@ from django.db import connection, transaction from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \ Entries_interactions, Performance, \ - Reason, Ping + Reason def printStats(fn): @@ -55,7 +55,6 @@ def printStats(fn): start_i = Interaction.objects.count() start_ei = Entries_interactions.objects.count() start_perf = Performance.objects.count() - start_ping = Ping.objects.count() fn(self, *data) @@ -67,8 +66,6 @@ def printStats(fn): (start_ei - Entries_interactions.objects.count())) self.log.info("Metrics removed: %s" % (start_perf - Performance.objects.count())) - self.log.info("Ping metrics removed: %s" % - (start_ping - Ping.objects.count())) return print_stats @@ -77,16 +74,13 @@ class Reports(Bcfg2.Server.Admin.Mode): '''Admin interface for dynamic reports''' __shorthelp__ = "Manage dynamic reports" __longhelp__ = (__shorthelp__) - django_commands = ['syncdb', 'sqlall', 'validate'] + django_commands = ['dbshell', 'shell', 'syncdb', 'sqlall', 'validate'] __usage__ = ("bcfg2-admin reports [command] [options]\n" - " -v|--verbose Be verbose\n" - " -q|--quiet Print only errors\n" "\n" " Commands:\n" " init Initialize the database\n" " load_stats Load statistics data\n" " -s|--stats Path to statistics.xml file\n" - " -c|--clients-file Path to clients.xml file\n" " -O3 Fast mode. Duplicates data!\n" " purge Purge records\n" " --client [n] Client to operate on\n" @@ -95,12 +89,11 @@ class Reports(Bcfg2.Server.Admin.Mode): " scrub Scrub the database for duplicate reasons and orphaned entries\n" " update Apply any updates to the reporting database\n" "\n" - " Django commands:\n " - "\n ".join(django_commands)) + " Django commands:\n " \ + + "\n ".join(django_commands)) def __init__(self, setup): Bcfg2.Server.Admin.Mode.__init__(self, setup) - self.log.setLevel(logging.INFO) def __call__(self, args): Bcfg2.Server.Admin.Mode.__call__(self, args) @@ -108,28 +101,21 @@ class Reports(Bcfg2.Server.Admin.Mode): print(self.__usage__) raise SystemExit(0) - verb = 0 - - if '-v' in args or '--verbose' in args: - self.log.setLevel(logging.DEBUG) - verb = 1 - if '-q' in args or '--quiet' in args: - self.log.setLevel(logging.WARNING) - # FIXME - dry run if args[0] in self.django_commands: self.django_command_proxy(args[0]) elif args[0] == 'scrub': self.scrub() - elif args[0] == 'init': - update_database() - elif args[0] == 'update': - update_database() + elif args[0] in ['init', 'update']: + try: + update_database() + except UpdaterError: + print "Update failed" + raise SystemExit(-1) elif args[0] == 'load_stats': quick = '-O3' in args stats_file = None - clients_file = None i = 1 while i < len(args): if args[i] == '-s' or args[i] == '--stats': @@ -137,11 +123,9 @@ class Reports(Bcfg2.Server.Admin.Mode): if stats_file[0] == '-': self.errExit("Invalid statistics file: %s" % stats_file) elif args[i] == '-c' or args[i] == '--clients-file': - clients_file = args[i + 1] - if clients_file[0] == '-': - self.errExit("Invalid clients file: %s" % clients_file) + print "DeprecationWarning: %s is no longer used" % args[i] i = i + 1 - self.load_stats(stats_file, clients_file, verb, quick) + self.load_stats(stats_file, self.log.getEffectiveLevel() > logging.WARNING, quick) elif args[0] == 'purge': expired = False client = None @@ -239,7 +223,7 @@ class Reports(Bcfg2.Server.Admin.Mode): else: django.core.management.call_command(command) - def load_stats(self, stats_file=None, clientspath=None, verb=0, quick=False): + def load_stats(self, stats_file=None, verb=0, quick=False): '''Load statistics data into the database''' location = '' @@ -258,27 +242,18 @@ class Reports(Bcfg2.Server.Admin.Mode): except: encoding = 'UTF-8' - if not clientspath: - try: - clientspath = "%s/Metadata/clients.xml" % \ - self.cfp.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.errExit("Could not read bcfg2.conf; exiting") try: - clientsdata = XML(open(clientspath).read()) - except (IOError, XMLSyntaxError): - self.errExit("StatReports: Failed to parse %s" % (clientspath)) - - try: - load_stats(clientsdata, - statsdata, + load_stats(statsdata, encoding, verb, self.log, quick=quick, location=platform.node()) + except UpdaterError: + self.errExit("StatReports: Database updater failed") except: - pass + self.errExit("failed to import stats: %s" + % traceback.format_exc().splitlines()[-1]) @printStats def purge(self, client=None, maxdate=None, state=None): @@ -306,12 +281,6 @@ class Reports(Bcfg2.Server.Admin.Mode): self.log.debug("Filtering by maxdate: %s" % maxdate) ipurge = ipurge.filter(timestamp__lt=maxdate) - # Handle ping data as well - ping = Ping.objects.filter(endtime__lt=maxdate) - if client: - ping = ping.filter(client=cobj) - ping.delete() - if state: filtered = True if state not in ('dirty', 'clean', 'modified'): diff --git a/src/lib/Server/Admin/Snapshots.py b/src/lib/Bcfg2/Server/Admin/Snapshots.py index 8bc56f1f1..8bc56f1f1 100644 --- a/src/lib/Server/Admin/Snapshots.py +++ b/src/lib/Bcfg2/Server/Admin/Snapshots.py diff --git a/src/lib/Server/Admin/Tidy.py b/src/lib/Bcfg2/Server/Admin/Tidy.py index 82319b93e..65aa955b4 100644 --- a/src/lib/Server/Admin/Tidy.py +++ b/src/lib/Bcfg2/Server/Admin/Tidy.py @@ -3,6 +3,7 @@ import re import socket import Bcfg2.Server.Admin +from Bcfg2.Bcfg2Py3k import input class Tidy(Bcfg2.Server.Admin.Mode): @@ -22,11 +23,7 @@ class Tidy(Bcfg2.Server.Admin.Mode): if '-f' in args or '-I' in args: if '-I' in args: for name in badfiles[:]: - # py3k compatibility - try: - answer = raw_input("Unlink file %s? [yN] " % name) - except NameError: - answer = input("Unlink file %s? [yN] " % name) + answer = input("Unlink file %s? [yN] " % name) if answer not in ['y', 'Y']: badfiles.remove(name) for name in badfiles: diff --git a/src/lib/Server/Admin/Viz.py b/src/lib/Bcfg2/Server/Admin/Viz.py index 2faa423c1..2faa423c1 100644 --- a/src/lib/Server/Admin/Viz.py +++ b/src/lib/Bcfg2/Server/Admin/Viz.py diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Bcfg2/Server/Admin/Xcmd.py index 140465468..140465468 100644 --- a/src/lib/Server/Admin/Xcmd.py +++ b/src/lib/Bcfg2/Server/Admin/Xcmd.py diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Bcfg2/Server/Admin/__init__.py index a9f0b8cd6..a7269a289 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Bcfg2/Server/Admin/__init__.py @@ -1,5 +1,3 @@ -__revision__ = '$Revision$' - __all__ = [ 'Backup', 'Bundle', @@ -108,7 +106,7 @@ class MetadataCore(Mode): """Base class for admin-modes that handle metadata.""" __plugin_whitelist__ = None __plugin_blacklist__ = None - + def __init__(self, setup): Mode.__init__(self, setup) if self.__plugin_whitelist__ is not None: @@ -124,9 +122,9 @@ class MetadataCore(Mode): setup['plugins'], setup['password'], setup['encoding'], - filemonitor=setup['filemonitor']) - if setup['event debug']: - self.bcore.fam.debug = True + filemonitor=setup['filemonitor'], + cfile=setup['configfile'], + setup=setup) except Bcfg2.Server.Core.CoreInitError: msg = sys.exc_info()[1] self.errExit("Core load failed: %s" % msg) diff --git a/src/lib/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index bbeceb3f2..e8ebd511d 100644 --- a/src/lib/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -1,5 +1,4 @@ """Bcfg2.Server.Core provides the runtime support for Bcfg2 modules.""" -__revision__ = '$Revision$' import atexit import logging @@ -44,7 +43,10 @@ def sort_xml(node, key=None): for child in node: sort_xml(child, key) - sorted_children = sorted(node, key=key) + try: + sorted_children = sorted(node, key=key) + except TypeError: + sorted_children = node node[:] = sorted_children @@ -61,16 +63,24 @@ class Core(Component): implementation = 'bcfg2-server' def __init__(self, repo, plugins, password, encoding, - cfile='/etc/bcfg2.conf', ca=None, + cfile='/etc/bcfg2.conf', ca=None, setup=None, filemonitor='default', start_fam_thread=False): Component.__init__(self) self.datastore = repo - if filemonitor not in Bcfg2.Server.FileMonitor.available: + + try: + fm = Bcfg2.Server.FileMonitor.available[filemonitor] + except KeyError: logger.error("File monitor driver %s not available; " "forcing to default" % filemonitor) - filemonitor = 'default' + fm = Bcfg2.Server.FileMonitor.available['default'] + famargs = dict(ignore=[], debug=False) + if 'ignore' in setup: + famargs['ignore'] = setup['ignore'] + if 'debug' in setup: + famargs['debug'] = setup['debug'] try: - self.fam = Bcfg2.Server.FileMonitor.available[filemonitor]() + self.fam = fm(**famargs) except IOError: msg = "Failed to instantiate fam driver %s" % filemonitor logger.error(msg, exc_info=1) @@ -83,6 +93,7 @@ class Core(Component): self.revision = '-1' self.password = password self.encoding = encoding + self.setup = setup atexit.register(self.shutdown) # Create an event to signal worker threads to shutdown self.terminate = threading.Event() @@ -129,6 +140,11 @@ class Core(Component): self.fam_thread = threading.Thread(target=self._file_monitor_thread) if start_fam_thread: self.fam_thread.start() + self.monitor_cfile() + + def monitor_cfile(self): + if self.setup: + self.fam.AddMonitor(self.cfile, self.setup) def plugins_by_type(self, base_cls): """Return a list of loaded plugins that match the passed type. @@ -189,9 +205,24 @@ class Core(Component): """Shutting down the plugins.""" if not self.terminate.isSet(): self.terminate.set() + self.fam.shutdown() for plugin in list(self.plugins.values()): plugin.shutdown() + def client_run_hook(self, hook, metadata): + """Checks the data structure.""" + for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.ClientRunHooks): + try: + getattr(plugin, hook)(metadata) + except AttributeError: + err = sys.exc_info()[1] + logger.error("Unknown attribute: %s" % err) + raise + except: + err = sys.exc_info()[1] + logger.error("%s: Error invoking hook %s: %s" % (plugin, hook, + err)) + def validate_structures(self, metadata, data): """Checks the data structure.""" for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): @@ -240,12 +271,14 @@ class Core(Component): continue try: self.Bind(entry, metadata) - except PluginExecutionError, exc: + except PluginExecutionError: + exc = sys.exc_info()[1] if 'failure' not in entry.attrib: entry.set('failure', 'bind error: %s' % format_exc()) - logger.error("Failed to bind entry: %s %s" % \ - (entry.tag, entry.get('name'))) - except Exception, exc: + logger.error("Failed to bind entry %s:%s: %s" % + (entry.tag, entry.get('name'), exc)) + except Exception: + exc = sys.exc_info()[1] if 'failure' not in entry.attrib: entry.set('failure', 'bind error: %s' % format_exc()) logger.error("Unexpected failure in BindStructure: %s %s" \ @@ -299,6 +332,8 @@ class Core(Component): logger.error("Metadata consistency error for client %s" % client) return lxml.etree.Element("error", type='metadata error') + self.client_run_hook("start_client_run", meta) + try: structures = self.GetStructures(meta) except: @@ -326,7 +361,9 @@ class Core(Component): except: logger.error("error in BindStructure", exc_info=1) self.validate_goals(meta, config) - + + self.client_run_hook("end_client_run", meta) + sort_xml(config, key=lambda e: e.get('name')) logger.info("Generated config for %s in %.03f seconds" % \ @@ -374,6 +411,8 @@ class Core(Component): logger.info("Client %s reported state %s" % (client_name, state.get('state'))) + self.client_run_hook("end_statistics", meta) + # XMLRPC handlers start here @exposed diff --git a/src/lib/Bcfg2/Server/FileMonitor/Fam.py b/src/lib/Bcfg2/Server/FileMonitor/Fam.py new file mode 100644 index 000000000..1a00fffa0 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Fam.py @@ -0,0 +1,82 @@ +""" Fam provides FAM support for file alteration events """ + +import os +import _fam +import stat +import logging +from time import time +from Bcfg2.Server.FileMonitor import FileMonitor + +logger = logging.getLogger(__name__) + +class Fam(FileMonitor): + __priority__ = 90 + + def __init__(self, ignore=None, debug=False): + FileMonitor.__init__(self, ignore=ignore, debug=debug) + self.fm = _fam.open() + self.users = {} + + def fileno(self): + """Return fam file handle number.""" + return self.fm.fileno() + + def handle_event_set(self, _): + self.Service() + + def handle_events_in_interval(self, interval): + now = time() + while (time() - now) < interval: + if self.Service(): + now = time() + + def AddMonitor(self, path, obj): + """Add a monitor to path, installing a callback to obj.HandleEvent.""" + mode = os.stat(path)[stat.ST_MODE] + if stat.S_ISDIR(mode): + handle = self.fm.monitorDirectory(path, None) + else: + handle = self.fm.monitorFile(path, None) + self.handles[handle.requestID()] = handle + if obj != None: + self.users[handle.requestID()] = obj + return handle.requestID() + + def Service(self, interval=0.50): + """Handle all fam work.""" + count = 0 + collapsed = 0 + rawevents = [] + start = time() + now = time() + while (time() - now) < interval: + if self.fm.pending(): + while self.fm.pending(): + count += 1 + rawevents.append(self.fm.nextEvent()) + now = time() + unique = [] + bookkeeping = [] + for event in rawevents: + if self.should_ignore(event): + continue + if event.code2str() != 'changed': + # process all non-change events + unique.append(event) + else: + if (event.filename, event.requestID) not in bookkeeping: + bookkeeping.append((event.filename, event.requestID)) + unique.append(event) + else: + collapsed += 1 + for event in unique: + if event.requestID in self.users: + try: + self.users[event.requestID].HandleEvent(event) + except: + logger.error("Handling event for file %s" % event.filename, + exc_info=1) + end = time() + logger.info("Processed %s fam events in %03.03f seconds. %s coalesced" % + (count, (end - start), collapsed)) + return count diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py new file mode 100644 index 000000000..60f80c9c3 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -0,0 +1,64 @@ +""" Gamin driver for file alteration events """ + +import os +import stat +import logging +from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ + GAMChanged, GAMDeleted +from Bcfg2.Server.FileMonitor import Event, FileMonitor + +logger = logging.getLogger(__name__) + +class GaminEvent(Event): + """ + This class provides an event analogous to + python-fam events based on gamin sources. + """ + action_map = {GAMCreated: 'created', GAMExists: 'exists', + GAMChanged: 'changed', GAMDeleted: 'deleted', + GAMEndExist: 'endExist'} + + def __init__(self, request_id, filename, code): + Event.__init__(self, request_id, filename, code) + if code in self.action_map: + self.action = self.action_map[code] + +class Gamin(FileMonitor): + __priority__ = 10 + + def __init__(self, ignore=None, debug=False): + FileMonitor.__init__(self, ignore=ignore, debug=debug) + self.mon = WatchMonitor() + self.counter = 0 + + def fileno(self): + return self.mon.get_fd() + + def queue(self, path, action, request_id): + """queue up the event for later handling""" + self.events.append(GaminEvent(request_id, path, action)) + + def AddMonitor(self, path, obj): + """Add a monitor to path, installing a callback to obj.""" + handle = self.counter + self.counter += 1 + mode = os.stat(path)[stat.ST_MODE] + + # Flush queued gamin events + while self.mon.event_pending(): + self.mon.handle_one_event() + + if stat.S_ISDIR(mode): + self.mon.watch_directory(path, self.queue, handle) + else: + self.mon.watch_file(path, self.queue, handle) + self.handles[handle] = obj + return handle + + def pending(self): + return FileMonitor.pending(self) or self.mon.event_pending() + + def get_event(self): + if self.mon.event_pending(): + self.mon.handle_one_event() + return FileMonitor.get_event(self) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py new file mode 100644 index 000000000..50c724279 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -0,0 +1,64 @@ +""" Inotify driver for file alteration events """ + +import os +import stat +import logging +import operator +import pyinotify +from Bcfg2.Server.FileMonitor import Event +from Bcfg2.Server.FileMonitor.Pseudo import Pseudo + +logger = logging.getLogger(__name__) + +class Inotify(Pseudo, pyinotify.ProcessEvent): + __priority__ = 1 + mask = pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY + action_map = {pyinotify.IN_CREATE: 'created', + pyinotify.IN_DELETE: 'deleted', + pyinotify.IN_MODIFY: 'changed'} + + def __init__(self, ignore=None, debug=False): + Pseudo.__init__(self, ignore=ignore, debug=debug) + self.wm = pyinotify.WatchManager() + self.notifier = pyinotify.ThreadedNotifier(self.wm, self) + self.notifier.start() + + def fileno(self): + return self.wm.get_fd() + + def process_default(self, ievent): + action = ievent.maskname + for amask, aname in self.action_map.items(): + if ievent.mask & amask: + action = aname + break + # FAM-style file monitors return the full path to the parent + # directory that is being watched, relative paths to anything + # contained within the directory + watch = self.wm.watches[ievent.wd] + if watch.path == ievent.pathname: + path = ievent.pathname + else: + # relative path + path = os.path.basename(ievent.pathname) + evt = Event(ievent.wd, path, action) + self.events.append(evt) + + def AddMonitor(self, path, obj): + res = self.wm.add_watch(path, self.mask, quiet=False) + if not res: + # if we didn't get a return, but we also didn't get an + # exception, we're already watching this directory, so we + # need to find the watch descriptor for it + for wd, watch in self.wm.watches.items(): + if watch.path == path: + wd = watch.wd + else: + wd = res[path] + + # inotify doesn't produce initial 'exists' events, so we + # inherit from Pseudo to produce those + return Pseudo.AddMonitor(self, path, obj, handleID=wd) + + def shutdown(self): + self.notifier.stop() diff --git a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py new file mode 100644 index 000000000..baff871d0 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py @@ -0,0 +1,27 @@ +""" Pseudo provides static monitor support for file alteration events """ + +import os +import stat +import logging +from Bcfg2.Server.FileMonitor import FileMonitor, Event + +logger = logging.getLogger(__name__) + +class Pseudo(FileMonitor): + __priority__ = 99 + + def AddMonitor(self, path, obj, handleID=None): + """add a monitor to path, installing a callback to obj.HandleEvent""" + if handleID is None: + handleID = len(list(self.handles.keys())) + mode = os.stat(path)[stat.ST_MODE] + self.events.append(Event(handleID, path, 'exists')) + if stat.S_ISDIR(mode): + dirList = os.listdir(path) + for includedFile in dirList: + self.events.append(Event(handleID, includedFile, 'exists')) + self.events.append(Event(handleID, path, 'endExist')) + + if obj != None: + self.handles[handleID] = obj + return handleID diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py new file mode 100644 index 000000000..784384c65 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -0,0 +1,141 @@ +"""Bcfg2.Server.FileMonitor provides the support for monitoring files.""" + +import os +import sys +import fnmatch +import logging +import pkgutil +from time import sleep, time + +logger = logging.getLogger(__name__) + +class Event(object): + def __init__(self, request_id, filename, code): + self.requestID = request_id + self.filename = filename + self.action = code + + def code2str(self): + """return static code for event""" + return self.action + + def __str__(self): + return "%s: %s %s" % (self.__class__.__name__, + self.filename, self.action) + + def __repr__(self): + return "%s (request ID %s)" % (str(self), self.requestID) + + +class FileMonitor(object): + """File Monitor baseclass.""" + def __init__(self, ignore=None, debug=False): + object.__init__(self) + self.debug = debug + self.handles = dict() + self.events = [] + if ignore is None: + ignore = [] + self.ignore = ignore + + def __str__(self): + return "%s: %s" % (__name__, self.__class__.__name__) + + def __repr__(self): + return "%s (%s events, fd %s)" % (str(self), len(events), self.fileno) + + def should_ignore(self, event): + for pattern in self.ignore: + if (fnmatch.fnmatch(event.filename, pattern) or + fnmatch.fnmatch(os.path.split(event.filename)[-1], pattern)): + if self.debug: + logger.info("Ignoring %s" % event) + return True + return False + + def pending(self): + return bool(self.events) + + def get_event(self): + return self.events.pop(0) + + def fileno(self): + return 0 + + def handle_one_event(self, event): + if self.should_ignore(event): + return + if event.requestID not in self.handles: + logger.info("Got event for unexpected id %s, file %s" % + (event.requestID, event.filename)) + return + if self.debug: + logger.info("Dispatching event %s %s to obj %s" % + (event.code2str(), event.filename, + self.handles[event.requestID])) + try: + self.handles[event.requestID].HandleEvent(event) + except: + err = sys.exc_info()[1] + logger.error("Error in handling of event for %s: %s" % + (event.filename, err)) + + def handle_event_set(self, lock=None): + count = 1 + event = self.get_event() + start = time() + if lock: + lock.acquire() + try: + self.handle_one_event(event) + while self.pending(): + self.handle_one_event(self.get_event()) + count += 1 + except: + pass + if lock: + lock.release() + end = time() + logger.info("Handled %d events in %.03fs" % (count, (end - start))) + + def handle_events_in_interval(self, interval): + end = time() + interval + while time() < end: + if self.pending(): + self.handle_event_set() + end = time() + interval + else: + sleep(0.5) + + def shutdown(self): + pass + + +available = dict() + +# todo: loading the monitor drivers should be automatic +from Bcfg2.Server.FileMonitor.Pseudo import Pseudo +available['pseudo'] = Pseudo + +try: + from Bcfg2.Server.FileMonitor.Fam import Fam + available['fam'] = Fam +except ImportError: + pass + +try: + from Bcfg2.Server.FileMonitor.Gamin import Gamin + available['gamin'] = Gamin +except ImportError: + pass + +try: + from Bcfg2.Server.FileMonitor.Inotify import Inotify + available['inotify'] = Inotify +except ImportError: + pass + +for fdrv in sorted(available.keys(), key=lambda k: available[k].__priority__): + if fdrv in available: + available['default'] = available[fdrv] + break diff --git a/src/lib/Server/Hostbase/.gitignore b/src/lib/Bcfg2/Server/Hostbase/.gitignore index 8e15b5395..8e15b5395 100644 --- a/src/lib/Server/Hostbase/.gitignore +++ b/src/lib/Bcfg2/Server/Hostbase/.gitignore diff --git a/src/lib/Server/Hostbase/__init__.py b/src/lib/Bcfg2/Server/Hostbase/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Server/Hostbase/__init__.py +++ b/src/lib/Bcfg2/Server/Hostbase/__init__.py diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Bcfg2/Server/Hostbase/backends.py index bf774f695..ecaf3c109 100644 --- a/src/lib/Server/Hostbase/backends.py +++ b/src/lib/Bcfg2/Server/Hostbase/backends.py @@ -2,8 +2,6 @@ from django.contrib.auth.models import User #from ldapauth import * from nisauth import * -__revision__ = '$Revision$' - ## class LDAPBackend(object): ## def authenticate(self,username=None,password=None): diff --git a/src/lib/Server/Hostbase/hostbase/__init__.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Server/Hostbase/hostbase/__init__.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/__init__.py diff --git a/src/lib/Server/Hostbase/hostbase/admin.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py index 70a2233cc..70a2233cc 100644 --- a/src/lib/Server/Hostbase/hostbase/admin.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/admin.py diff --git a/src/lib/Server/Hostbase/hostbase/models.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/models.py index 3f08a09a0..3f08a09a0 100644 --- a/src/lib/Server/Hostbase/hostbase/models.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/models.py diff --git a/src/lib/Server/Hostbase/hostbase/sql/zone.sql b/src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql index b78187ab2..b78187ab2 100644 --- a/src/lib/Server/Hostbase/hostbase/sql/zone.sql +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/sql/zone.sql diff --git a/src/lib/Server/Hostbase/hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py index 0ee204abe..0ee204abe 100644 --- a/src/lib/Server/Hostbase/hostbase/urls.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/urls.py diff --git a/src/lib/Server/Hostbase/hostbase/views.py b/src/lib/Bcfg2/Server/Hostbase/hostbase/views.py index ff1d4710d..57ef5eff8 100644 --- a/src/lib/Server/Hostbase/hostbase/views.py +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/views.py @@ -2,8 +2,6 @@ Contains all the views associated with the hostbase app Also has does form validation """ -__revision__ = "$Revision: $" - from django.http import HttpResponse, HttpResponseRedirect from django.contrib.auth.decorators import login_required diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/base.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html index 1d7c5565b..1d7c5565b 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/base.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/base.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html index ca8b0cc07..ca8b0cc07 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/confirm.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/copy.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html index 400ef58f2..400ef58f2 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/copy.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/copy.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/dns.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html index da179e5a1..da179e5a1 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/dns.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dns.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html index 18bd7ab83..b1b71ab67 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/dnsedit.html @@ -73,7 +73,7 @@ <td> <input name="{{ name.id }}priority" type="text" size="6"> <input name="{{ name.id }}mx" type="text"></td></tr> {% endfor %} - <tr> <td> <b>name</b></td> + <tr> <td> <b>name</b></td> <td> <input name="{{ ip.0.ip_addr }}name" type="text"> <select name="{{ ip.0.ip_addr }}dns_view"> {% for choice in DNS_CHOICES %} @@ -86,7 +86,7 @@ <td> <input name="{{ ip.0.ip_addr }}priority" type="text" size="6"> <input name="{{ ip.0.ip_addr }}mx" type="text"></td></tr> <tr><td></td></tr> - <tr><td><hr></td><td><hr></td></tr> + <tr><td><hr></td><td><hr></td></tr> {% endifequal %} {% endfor %} {% endfor %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/edit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html index 961c9d143..961c9d143 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/edit.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/edit.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/errors.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html index e5429b86c..e5429b86c 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/errors.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/errors.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/host.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html index d6b8873bc..d6b8873bc 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/host.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/host.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html index 551bf3254..b5d794b50 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html @@ -24,7 +24,7 @@ <col width="150"> <col width="*"> <tr> <td> <b>hostname</b></td> - <td> {{ object.hostname }}</td></tr> + <td> {{ object.hostname }}</td></tr> <tr> <td> <b>whatami</b></td> <td> {{ object.whatami }}</td></tr> <tr> <td> <b>netgroup</b></td> diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html index aa9679cbd..aa9679cbd 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/index.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html index 92258b648..92258b648 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/index.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/index.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/login.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html index ec24a0fc0..ec24a0fc0 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/login.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/login.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html index 994f631a8..994f631a8 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl index e71e90e76..e71e90e76 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logout.tmpl diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html index 806ccd63d..806ccd63d 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/logviewer.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl index 877d427d0..877d427d0 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/navbar.tmpl diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/new.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html index 2dcd6271f..2dcd6271f 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/new.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/new.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/remove.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html index 4329200dd..4329200dd 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/remove.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/remove.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/results.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html index 45b22058d..45b22058d 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/results.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/results.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/search.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html index 409d418fe..409d418fe 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/search.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/search.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html index ee355ee87..ee355ee87 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneedit.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html index b59fa9e3c..b59fa9e3c 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zonenew.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zones.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html index c773e7922..c773e7922 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zones.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zones.html diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html index fa12e3ec5..fa12e3ec5 100644 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html +++ b/src/lib/Bcfg2/Server/Hostbase/hostbase/webtemplates/zoneview.html diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Bcfg2/Server/Hostbase/ldapauth.py index f3db26f67..f3db26f67 100644 --- a/src/lib/Server/Hostbase/ldapauth.py +++ b/src/lib/Bcfg2/Server/Hostbase/ldapauth.py diff --git a/src/lib/Server/Hostbase/manage.py b/src/lib/Bcfg2/Server/Hostbase/manage.py index 5e78ea979..5e78ea979 100755 --- a/src/lib/Server/Hostbase/manage.py +++ b/src/lib/Bcfg2/Server/Hostbase/manage.py diff --git a/src/lib/Server/Hostbase/media/base.css b/src/lib/Bcfg2/Server/Hostbase/media/base.css index ddbf02165..ddbf02165 100644 --- a/src/lib/Server/Hostbase/media/base.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/base.css diff --git a/src/lib/Server/Hostbase/media/boxypastel.css b/src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css index 7ae0684ef..7ae0684ef 100644 --- a/src/lib/Server/Hostbase/media/boxypastel.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/boxypastel.css diff --git a/src/lib/Server/Hostbase/media/global.css b/src/lib/Bcfg2/Server/Hostbase/media/global.css index 73451e1bc..73451e1bc 100644 --- a/src/lib/Server/Hostbase/media/global.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/global.css diff --git a/src/lib/Server/Hostbase/media/layout.css b/src/lib/Bcfg2/Server/Hostbase/media/layout.css index 9085cc220..9085cc220 100644 --- a/src/lib/Server/Hostbase/media/layout.css +++ b/src/lib/Bcfg2/Server/Hostbase/media/layout.css diff --git a/src/lib/Server/Hostbase/nisauth.py b/src/lib/Bcfg2/Server/Hostbase/nisauth.py index 9c7da8c0a..ae4c6c021 100644 --- a/src/lib/Server/Hostbase/nisauth.py +++ b/src/lib/Bcfg2/Server/Hostbase/nisauth.py @@ -1,10 +1,8 @@ +"""Checks with NIS to see if the current user is in the support group""" import os import crypt, nis from Bcfg2.Server.Hostbase.settings import AUTHORIZED_GROUP -"""Checks with NIS to see if the current user is in the support group""" - -__revision__ = "$Revision: $" class NISAUTHError(Exception): """NISAUTHError is raised when somehting goes boom.""" diff --git a/src/lib/Server/Hostbase/regex.py b/src/lib/Bcfg2/Server/Hostbase/regex.py index 41cc0f6f0..41cc0f6f0 100644 --- a/src/lib/Server/Hostbase/regex.py +++ b/src/lib/Bcfg2/Server/Hostbase/regex.py diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Bcfg2/Server/Hostbase/settings.py index 4e641f13c..4e641f13c 100644 --- a/src/lib/Server/Hostbase/settings.py +++ b/src/lib/Bcfg2/Server/Hostbase/settings.py diff --git a/src/lib/Server/Hostbase/templates/batchadd.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl index 74ea3c047..74ea3c047 100644 --- a/src/lib/Server/Hostbase/templates/batchadd.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/batchadd.tmpl diff --git a/src/lib/Server/Hostbase/templates/dhcpd.conf.head b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head index a3d19547e..a3d19547e 100644 --- a/src/lib/Server/Hostbase/templates/dhcpd.conf.head +++ b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.conf.head diff --git a/src/lib/Server/Hostbase/templates/dhcpd.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl index 757b263cd..757b263cd 100644 --- a/src/lib/Server/Hostbase/templates/dhcpd.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/dhcpd.tmpl diff --git a/src/lib/Server/Hostbase/templates/hosts.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl index 251cb5a79..251cb5a79 100644 --- a/src/lib/Server/Hostbase/templates/hosts.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/hosts.tmpl diff --git a/src/lib/Server/Hostbase/templates/hostsappend.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl index 00e0d5d04..00e0d5d04 100644 --- a/src/lib/Server/Hostbase/templates/hostsappend.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/hostsappend.tmpl diff --git a/src/lib/Server/Hostbase/templates/named.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl index 03e054198..03e054198 100644 --- a/src/lib/Server/Hostbase/templates/named.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/named.tmpl diff --git a/src/lib/Server/Hostbase/templates/namedviews.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl index 52021620e..52021620e 100644 --- a/src/lib/Server/Hostbase/templates/namedviews.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/namedviews.tmpl diff --git a/src/lib/Server/Hostbase/templates/reverseappend.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl index 6ed520c98..6ed520c98 100644 --- a/src/lib/Server/Hostbase/templates/reverseappend.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/reverseappend.tmpl diff --git a/src/lib/Server/Hostbase/templates/reversesoa.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl index d142eaf7f..d142eaf7f 100644 --- a/src/lib/Server/Hostbase/templates/reversesoa.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/reversesoa.tmpl diff --git a/src/lib/Server/Hostbase/templates/zone.tmpl b/src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl index aad48d179..aad48d179 100644 --- a/src/lib/Server/Hostbase/templates/zone.tmpl +++ b/src/lib/Bcfg2/Server/Hostbase/templates/zone.tmpl diff --git a/src/lib/Server/Hostbase/test/harness.py b/src/lib/Bcfg2/Server/Hostbase/test/harness.py index befcff5c0..befcff5c0 100644 --- a/src/lib/Server/Hostbase/test/harness.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/harness.py diff --git a/src/lib/Server/Hostbase/test/test_environ_settings.py b/src/lib/Bcfg2/Server/Hostbase/test/test_environ_settings.py index ad35c624e..ad35c624e 100644 --- a/src/lib/Server/Hostbase/test/test_environ_settings.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/test_environ_settings.py diff --git a/src/lib/Server/Hostbase/test/test_ldapauth.py b/src/lib/Bcfg2/Server/Hostbase/test/test_ldapauth.py index 7fc009ad2..7fc009ad2 100644 --- a/src/lib/Server/Hostbase/test/test_ldapauth.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/test_ldapauth.py diff --git a/src/lib/Server/Hostbase/test/test_settings.py b/src/lib/Bcfg2/Server/Hostbase/test/test_settings.py index 0dfc30f38..0dfc30f38 100644 --- a/src/lib/Server/Hostbase/test/test_settings.py +++ b/src/lib/Bcfg2/Server/Hostbase/test/test_settings.py diff --git a/src/lib/Server/Hostbase/urls.py b/src/lib/Bcfg2/Server/Hostbase/urls.py index 01fe97d4f..01fe97d4f 100644 --- a/src/lib/Server/Hostbase/urls.py +++ b/src/lib/Bcfg2/Server/Hostbase/urls.py diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Bcfg2/Server/Lint/Comments.py index 19fae1b08..f5d0e265f 100644 --- a/src/lib/Server/Lint/Comments.py +++ b/src/lib/Bcfg2/Server/Lint/Comments.py @@ -1,6 +1,10 @@ import os.path import lxml.etree import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.Cfg.CfgPlaintextGenerator import CfgPlaintextGenerator +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML class Comments(Bcfg2.Server.Lint.ServerPlugin): """ check files for various required headers """ @@ -13,9 +17,15 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): self.check_properties() self.check_metadata() self.check_cfg() - self.check_infoxml() self.check_probes() + @classmethod + def Errors(cls): + return {"unexpanded-keywords":"warning", + "keywords-not-found":"warning", + "comments-not-found":"warning", + "broken-xinclude-chain":"warning"} + def required_keywords(self, rtype): """ given a file type, fetch the list of required VCS keywords from the bcfg2-lint config """ @@ -31,13 +41,13 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): fetch the list of required items from the bcfg2-lint config """ if itype not in self.config_cache: self.config_cache[itype] = {} - + if rtype not in self.config_cache[itype]: rv = [] global_item = "global_%ss" % itype if global_item in self.config: rv.extend(self.config[global_item].split(",")) - + item = "%s_%ss" % (rtype.lower(), itype) if item in self.config: if self.config[item]: @@ -83,25 +93,24 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): "metadata") def check_cfg(self): - """ check Cfg files for required headers """ + """ check Cfg files and info.xml files for required headers """ if 'Cfg' in self.core.plugins: for entryset in self.core.plugins['Cfg'].entries.values(): for entry in entryset.entries.values(): - if entry.name.endswith(".genshi"): + rtype = None + if isinstance(entry, CfgGenshiGenerator): rtype = "tgenshi" - else: + elif isinstance(entry, CfgPlaintextGenerator): rtype = "cfg" - self.check_plaintext(entry.name, entry.data, rtype) - - def check_infoxml(self): - """ check info.xml files for required headers """ - if 'Cfg' in self.core.plugins: - for entryset in self.core.plugins['Cfg'].entries.items(): - if (hasattr(entryset, "infoxml") and - entryset.infoxml is not None): - self.check_xml(entryset.infoxml.name, - entryset.infoxml.pnode.data, - "infoxml") + elif isinstance(entry, CfgCheetahGenerator): + rtype = "tcheetah" + elif isinstance(entry, CfgInfoXML): + self.check_xml(entry.infoxml.name, + entry.infoxml.pnode.data, + "infoxml") + continue + if rtype: + self.check_plaintext(entry.name, entry.data, rtype) def check_probes(self): """ check probes for required headers """ @@ -128,7 +137,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): # None == found but not expanded # True == found and expanded found = dict((k, False) for k in self.required_keywords(rtype)) - + for line in lines: # we check for both '$<keyword>:' and '$<keyword>$' to see # if the keyword just hasn't been expanded @@ -155,7 +164,7 @@ class Comments(Bcfg2.Server.Lint.ServerPlugin): # next, check for required comments. found is just # boolean found = dict((k, False) for k in self.required_comments(rtype)) - + for line in lines: for (comment, status) in found.items(): if not status: diff --git a/src/lib/Server/Lint/Duplicates.py b/src/lib/Bcfg2/Server/Lint/Duplicates.py index 75f620603..ee6b7a2e6 100644 --- a/src/lib/Server/Lint/Duplicates.py +++ b/src/lib/Bcfg2/Server/Lint/Duplicates.py @@ -22,6 +22,14 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): if self.clients_xdata is not None: self.duplicate_clients() + @classmethod + def Errors(cls): + return {"broken-xinclude-chain":"warning", + "duplicate-client":"error", + "duplicate-group":"error", + "duplicate-package":"error", + "multiple-default-groups":"error"} + def load_xdata(self): """ attempt to load XML data for groups and clients. only actually load data if all documents reference in XIncludes can @@ -30,7 +38,7 @@ class Duplicates(Bcfg2.Server.Lint.ServerPlugin): self.groups_xdata = self.metadata.clients_xml.xdata if self.has_all_xincludes("clients.xml"): self.clients_xdata = self.metadata.clients_xml.xdata - + def duplicate_groups(self): """ find duplicate groups """ self.duplicate_entries(self.clients_xdata.xpath('//Groups/Group'), diff --git a/src/lib/Server/Lint/Genshi.py b/src/lib/Bcfg2/Server/Lint/Genshi.py index 56803246c..b6007161e 100755 --- a/src/lib/Server/Lint/Genshi.py +++ b/src/lib/Bcfg2/Server/Lint/Genshi.py @@ -1,9 +1,8 @@ import genshi.template import Bcfg2.Server.Lint - + class Genshi(Bcfg2.Server.Lint.ServerPlugin): """ Check Genshi templates for syntax errors """ - def Run(self): """ run plugin """ loader = genshi.template.TemplateLoader() @@ -12,10 +11,14 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): self.check_files(self.core.plugins[plugin].entries, loader=loader) + @classmethod + def Errors(cls): + return {"genshi-syntax-error":"error"} + def check_files(self, entries, loader=None): if loader is None: loader = genshi.template.TemplateLoader() - + for eset in entries.values(): for fname, sdata in list(eset.entries.items()): if (self.HandlesFile(fname) and @@ -23,6 +26,7 @@ class Genshi(Bcfg2.Server.Lint.ServerPlugin): try: loader.load(sdata.name, cls=genshi.template.NewTextTemplate) - except genshi.template.TemplateSyntaxError, err: + except genshi.template.TemplateSyntaxError: + err = sys.exc_info()[1] self.LintError("genshi-syntax-error", "Genshi syntax error: %s" % err) diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Bcfg2/Server/Lint/InfoXML.py index 2054e23bf..3884c1ed4 100644 --- a/src/lib/Server/Lint/InfoXML.py +++ b/src/lib/Bcfg2/Server/Lint/InfoXML.py @@ -1,23 +1,34 @@ import os.path import Bcfg2.Options import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.Cfg.CfgInfoXML import CfgInfoXML class InfoXML(Bcfg2.Server.Lint.ServerPlugin): """ ensure that all config files have an info.xml file""" - def Run(self): - if 'Cfg' in self.core.plugins: - for filename, entryset in self.core.plugins['Cfg'].entries.items(): + for plugin in ['Cfg', 'TCheetah', 'TGenshi']: + if plugin not in self.core.plugins: + continue + for filename, entryset in self.core.plugins[plugin].entries.items(): infoxml_fname = os.path.join(entryset.path, "info.xml") if self.HandlesFile(infoxml_fname): - if (hasattr(entryset, "infoxml") and - entryset.infoxml is not None): - self.check_infoxml(infoxml_fname, - entryset.infoxml.pnode.data) - else: + found = False + for entry in entryset.entries.values(): + if isinstance(entry, CfgInfoXML): + self.check_infoxml(infoxml_fname, + entry.infoxml.pnode.data) + found = True + if not found: self.LintError("no-infoxml", "No info.xml found for %s" % filename) + @classmethod + def Errors(cls): + return {"no-infoxml":"warning", + "paranoid-false":"warning", + "broken-xinclude-chain":"warning", + "required-infoxml-attrs-missing":"error"} + def check_infoxml(self, fname, xdata): for info in xdata.getroottree().findall("//Info"): required = [] diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Bcfg2/Server/Lint/MergeFiles.py index ff6e3449a..68d010316 100644 --- a/src/lib/Server/Lint/MergeFiles.py +++ b/src/lib/Bcfg2/Server/Lint/MergeFiles.py @@ -2,20 +2,28 @@ import os import copy from difflib import SequenceMatcher import Bcfg2.Server.Lint +from Bcfg2.Server.Plugins.Cfg import CfgGenerator class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): """ find Probes or Cfg files with multiple similar files that might be merged into one """ - def Run(self): if 'Cfg' in self.core.plugins: self.check_cfg() if 'Probes' in self.core.plugins: self.check_probes() + @classmethod + def Errors(cls): + return {"merge-cfg":"warning", + "merge-probes":"warning"} + + def check_cfg(self): for filename, entryset in self.core.plugins['Cfg'].entries.items(): - for mset in self.get_similar(entryset.entries): + candidates = dict([(f, e) for f, e in entryset.entries.items() + if isinstance(e, CfgGenerator)]) + for mset in self.get_similar(candidates): self.LintError("merge-cfg", "The following files are similar: %s. " "Consider merging them into a single Genshi " @@ -26,7 +34,7 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): def check_probes(self): probes = self.core.plugins['Probes'].probes.entries for mset in self.get_similar(probes): - self.LintError("merge-cfg", + self.LintError("merge-probes", "The following probes are similar: %s. " "Consider merging them into a single probe." % ", ".join([p for p in mset])) @@ -66,4 +74,4 @@ class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): threshold)) return rv - + diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py index 55206d2ba..6f76cf2db 100644 --- a/src/lib/Server/Lint/RequiredAttrs.py +++ b/src/lib/Bcfg2/Server/Lint/RequiredAttrs.py @@ -6,7 +6,6 @@ from Bcfg2.Server.Plugins.Packages import Apt, Yum class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): """ verify attributes for configuration entries (as defined in doc/server/configurationentries) """ - def __init__(self, *args, **kwargs): Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) self.required_attrs = { @@ -38,6 +37,14 @@ class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): self.check_rules() self.check_bundles() + @classmethod + def Errors(cls): + return {"unknown-entry-type":"error", + "unknown-entry-tag":"error", + "required-attrs-missing":"error", + "extra-attrs":"warning"} + + def check_packages(self): """ check package sources for Source entries with missing attrs """ if 'Packages' in self.core.plugins: diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Bcfg2/Server/Lint/Validate.py index 952a65365..05fedc313 100644 --- a/src/lib/Server/Lint/Validate.py +++ b/src/lib/Bcfg2/Server/Lint/Validate.py @@ -36,7 +36,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): def Run(self): schemadir = self.config['schema'] - + for path, schemaname in self.filesets.items(): try: filelist = self.filelists[path] @@ -63,6 +63,16 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.check_properties() + @classmethod + def Errors(cls): + return {"broken-xinclude-chain":"warning", + "schema-failed-to-parse":"warning", + "properties-schema-not-found":"warning", + "xml-failed-to-parse":"error", + "xml-failed-to-read":"error", + "xml-failed-to-verify":"error", + "input-output-error":"error"} + def check_properties(self): """ check Properties files against their schemas """ for filename in self.filelists['props']: @@ -98,7 +108,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): self.LintError("xml-failed-to-read", "Failed to open file %s" % filename) return False - + if not schema.validate(datafile): cmd = ["xmllint"] if self.files is None: @@ -180,7 +190,7 @@ class Validate(Bcfg2.Server.Lint.ServerlessPlugin): included = set([ent.get('href') for ent in xdata.findall('./{http://www.w3.org/2001/XInclude}include')]) rv = [] - + while included: try: filename = included.pop() diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Bcfg2/Server/Lint/__init__.py index 4d6df8c8f..5d7dd707b 100644 --- a/src/lib/Server/Lint/__init__.py +++ b/src/lib/Bcfg2/Server/Lint/__init__.py @@ -1,5 +1,3 @@ -__revision__ = '$Revision$' - __all__ = ['Bundles', 'Comments', 'Duplicates', @@ -56,14 +54,20 @@ class Plugin (object): self.config = config self.logger = logging.getLogger('bcfg2-lint') if errorhandler is None: - self.errorHandler = ErrorHandler() + self.errorhandler = ErrorHandler() else: - self.errorHandler = errorhandler + self.errorhandler = errorhandler + self.errorhandler.RegisterErrors(self.Errors()) def Run(self): """ run the plugin. must be overloaded by child classes """ pass + @classmethod + def Errors(cls): + """ returns a dict of errors the plugin supplies. must be + overloaded by child classes """ + def HandlesFile(self, fname): """ returns true if the given file should be handled by the plugin according to the files list, false otherwise """ @@ -75,8 +79,8 @@ class Plugin (object): fname)) in self.files) def LintError(self, err, msg): - self.errorHandler.dispatch(err, msg) - + self.errorhandler.dispatch(err, msg) + def RenderXML(self, element): """render an XML element for error output -- line number prefixed, no children""" @@ -93,37 +97,6 @@ class Plugin (object): class ErrorHandler (object): - # how to handle different errors by default - _errors = {"no-infoxml":"warning", - "paranoid-false":"warning", - "bundle-not-found":"error", - "inconsistent-bundle-name":"warning", - "group-tag-not-allowed":"error", - "unexpanded-keywords":"warning", - "keywords-not-found":"warning", - "comments-not-found":"warning", - "broken-xinclude-chain":"warning", - "duplicate-client":"error", - "duplicate-group":"error", - "duplicate-package":"error", - "multiple-default-groups":"error", - "required-infoxml-attrs-missing":"error", - "unknown-entry-type":"error", - "required-attrs-missing":"error", - "extra-attrs":"warning", - "schema-failed-to-parse":"warning", - "properties-schema-not-found":"warning", - "xml-failed-to-parse":"error", - "xml-failed-to-read":"error", - "xml-failed-to-verify":"error", - "merge-cfg":"warning", - "merge-probes":"warning", - "input-output-error":"error", - "genshi-syntax-error":"error", - "pattern-fails-to-initialize":"error", - "cat-file-used":"warning", - "diff-file-used":"warning"} - def __init__(self, config=None): self.errors = 0 self.warnings = 0 @@ -149,7 +122,8 @@ class ErrorHandler (object): else: self._handlers[err] = self.debug - for err, action in self._errors.items(): + def RegisterErrors(self, errors): + for err, action in errors.items(): if err not in self._handlers: if "warn" in action: self._handlers[err] = self.warn @@ -157,7 +131,7 @@ class ErrorHandler (object): self._handlers[err] = self.error else: self._handlers[err] = self.debug - + def dispatch(self, err, msg): if err in self._handlers: self._handlers[err](msg) diff --git a/src/lib/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index 28299d8c7..fb5e115a9 100644 --- a/src/lib/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -1,5 +1,4 @@ """This module provides the baseclass for Bcfg2 Server Plugins.""" -__revision__ = '$Revision$' import copy import logging @@ -25,6 +24,11 @@ from Bcfg2.Bcfg2Py3k import Queue from Bcfg2.Bcfg2Py3k import Empty from Bcfg2.Bcfg2Py3k import Full +# make encoding available +encparse = Bcfg2.Options.OptionParser({'encoding': Bcfg2.Options.ENCODING}) +encparse.parse([]) +encoding = encparse['encoding'] + # grab default metadata info from bcfg2.conf opts = {'owner': Bcfg2.Options.MDATA_OWNER, 'group': Bcfg2.Options.MDATA_GROUP, @@ -83,9 +87,13 @@ class Debuggable(object): self.__class__.__name__) self.debug_flag = False self.logger = logging.getLogger(name) - + def toggle_debug(self): self.debug_flag = not self.debug_flag + self.debug_log("%s: debug_flag = %s" % (self.__class__.__name__, + self.debug_flag), + flag=True) + return self.debug_flag def debug_log(self, message, flag=None): if (flag is None and self.debug_flag) or flag: @@ -96,7 +104,6 @@ class Plugin(Debuggable): """This is the base class for all Bcfg2 Server plugins. Several attributes must be defined in the subclass: name : the name of the plugin - __version__ : a version string __author__ : the author/contact for the plugin Plugins can provide three basic types of functionality: @@ -105,7 +112,6 @@ class Plugin(Debuggable): - Data collection (overloading GetProbes/ReceiveData) """ name = 'Plugin' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = False deprecated = False @@ -118,14 +124,14 @@ class Plugin(Debuggable): def __init__(self, core, datastore): """Initialize the plugin. - + :param core: the Bcfg2.Server.Core initializing the plugin :param datastore: the filesystem path of Bcfg2's repository """ object.__init__(self) self.Entries = {} self.core = core - self.data = "%s/%s" % (datastore, self.name) + self.data = os.path.join(datastore, self.name) self.running = True Debuggable.__init__(self, name=self.name) @@ -137,6 +143,9 @@ class Plugin(Debuggable): def shutdown(self): self.running = False + def __str__(self): + return "%s Plugin" % self.__class__.__name__ + class Generator(object): """Generator plugins contribute to literal client configurations.""" @@ -215,7 +224,7 @@ class ThreadedStatistics(Statistics, self.terminate = core.terminate self.work_queue = Queue(100000) self.pending_file = "%s/etc/%s.pending" % (datastore, self.__class__.__name__) - self.daemon = True + self.daemon = False self.start() def save(self): @@ -366,6 +375,17 @@ class Version(object): pass +class ClientRunHooks(object): + """ Provides hooks to interact with client runs """ + def start_client_run(self, metadata): + pass + + def end_client_run(self, metadata): + pass + + def end_statistics(self, metadata): + pass + # the rest of the file contains classes for coherent file caching class FileBacked(object): @@ -379,7 +399,7 @@ class FileBacked(object): object.__init__(self) self.data = '' self.name = name - + def HandleEvent(self, event=None): """Read file upon update.""" if event and event.code2str() not in ['exists', 'changed', 'created']: @@ -414,7 +434,7 @@ class DirectoryBacked(object): :param data: the path to the data directory that will be monitored. :param fam: The FileMonitor object used to receive - notifications of changes. + notifications of changes. """ object.__init__(self) @@ -472,7 +492,7 @@ class DirectoryBacked(object): def HandleEvent(self, event): """Handle FAM/Gamin events. - + This method is invoked by FAM/Gamin when it detects a change to a filesystem object we have requsted to be monitored. @@ -494,8 +514,8 @@ class DirectoryBacked(object): return if event.requestID not in self.handles: - logger.warn("Got %s event with unknown handle (%s) for %s" - % (action, event.requestID, abspath)) + logger.warn("Got %s event with unknown handle (%s) for %s" % + (action, event.requestID, event.filename)) return # Calculate the absolute and relative paths this event refers to @@ -536,21 +556,13 @@ class DirectoryBacked(object): # didn't know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - logger.warn("Got %s event for unexpected dir %s" % (action, - abspath)) + logger.warn("Got %s event for unexpected dir %s" % + (action, abspath)) self.add_directory_monitor(relpath) else: - logger.warn("Got unknown dir event %s %s %s" % (event.requestID, - event.code2str(), - abspath)) + logger.warn("Got unknown dir event %s %s %s" % + (event.requestID, event.code2str(), abspath)) else: - # Deal with events for non-directories - if ((event.filename[-1] == '~') or - (event.filename[:2] == '.#') or - (event.filename[-4:] == '.swp') or - (event.filename in ['SCCS', '.svn', '4913']) or - (not self.patterns.match(event.filename))): - return if action in ['exists', 'created']: self.add_entry(relpath, event) elif action == 'changed': @@ -561,13 +573,13 @@ class DirectoryBacked(object): # know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - logger.warn("Got %s event for unexpected file %s" % (action, - abspath)) + logger.warn("Got %s event for unexpected file %s" % + (action, + abspath)) self.add_entry(relpath, event) else: - logger.warn("Got unknown file event %s %s %s" % (event.requestID, - event.code2str(), - abspath)) + logger.warn("Got unknown file event %s %s %s" % + (event.requestID, event.code2str(), abspath)) class XMLFileBacked(FileBacked): @@ -598,16 +610,42 @@ class XMLFileBacked(FileBacked): return iter(self.entries) def __str__(self): - return "%s: %s" % (self.name, lxml.etree.tostring(self.xdata)) + return "%s at %s" % (self.__class__.__name__, self.name) class SingleXMLFileBacked(XMLFileBacked): """This object is a coherent cache for an independent XML file.""" - def __init__(self, filename, fam): + def __init__(self, filename, fam, should_monitor=True): XMLFileBacked.__init__(self, filename) self.extras = [] self.fam = fam - self.fam.AddMonitor(filename, self) + self.should_monitor = should_monitor + if should_monitor: + self.fam.AddMonitor(filename, self) + + def _follow_xincludes(self, fname=None, xdata=None): + ''' follow xincludes, adding included files to fam and to + self.extras ''' + if xdata is None: + if fname is None: + xdata = self.xdata.getroottree() + else: + xdata = lxml.etree.parse(fname) + included = [ent.get('href') + for ent in xdata.findall('//{http://www.w3.org/2001/XInclude}include')] + for name in included: + if name not in self.extras: + if name.startswith("/"): + fpath = name + else: + fpath = os.path.join(os.path.dirname(self.name), name) + self.add_monitor(fpath, name) + self._follow_xincludes(fname=fpath) + + def add_monitor(self, fpath, fname): + if self.should_monitor: + self.fam.AddMonitor(fpath, self) + self.extras.append(fname) def Index(self): """Build local data structures.""" @@ -619,22 +657,14 @@ class SingleXMLFileBacked(XMLFileBacked): logger.error("Failed to parse %s: %s" % (self.name, err)) raise Bcfg2.Server.Plugin.PluginInitError - included = [ent.get('href') - for ent in self.xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.fam.AddMonitor(os.path.join(os.path.dirname(self.name), - name), - self) - self.extras.append(name) + self._follow_xincludes() + if self.extras: try: self.xdata.getroottree().xinclude() except lxml.etree.XIncludeError: err = sys.exc_info()[1] logger.error("XInclude failed on %s: %s" % (self.name, err)) - self.entries = self.xdata.getchildren() if self.__identifier__ is not None: self.label = self.xdata.attrib[self.__identifier__] @@ -643,7 +673,7 @@ class SingleXMLFileBacked(XMLFileBacked): class StructFile(XMLFileBacked): """This file contains a set of structure file formatting logic.""" __identifier__ = None - + def __init__(self, name): XMLFileBacked.__init__(self, name) @@ -676,7 +706,7 @@ class StructFile(XMLFileBacked): for child in item.iterchildren(): rv.extend(self._match(child, metadata)) return [rv] - + def Match(self, metadata): """Return matching fragments of independent.""" rv = [] @@ -808,7 +838,7 @@ class XMLSrc(XMLFileBacked): return str(self.items) -class InfoXML (XMLSrc): +class InfoXML(XMLSrc): __node__ = InfoNode @@ -850,7 +880,7 @@ class PrioDir(Plugin, Generator, XMLDirectoryBacked): attrs = self.get_attrs(entry, metadata) for key, val in list(attrs.items()): entry.attrib[key] = val - + def get_attrs(self, entry, metadata): """ get a list of attributes to add to the entry during the bind """ for src in self.entries.values(): @@ -952,15 +982,18 @@ class SpecificData(object): return try: self.data = open(self.name).read() + except UnicodeDecodeError: + self.data = open(self.name, mode='rb').read() except: logger.error("Failed to read file %s" % self.name) -class EntrySet: +class EntrySet(Debuggable): """Entry sets deal with the host- and group-specific entries.""" ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$") def __init__(self, basename, path, entry_type, encoding): + Debuggable.__init__(self, name=basename) self.path = path self.entry_type = entry_type self.entries = {} @@ -971,14 +1004,18 @@ class EntrySet: pattern += '(G(?P<prio>\d+)_(?P<group>\S+))))?$' self.specific = re.compile(pattern) + def sort_by_specific(self, one, other): + return cmp(one.specific, other.specific) + def get_matching(self, metadata): return [item for item in list(self.entries.values()) if item.specific.matches(metadata)] - def best_matching(self, metadata): + def best_matching(self, metadata, matching=None): """ Return the appropriate interpreted template from the set of available templates. """ - matching = self.get_matching(metadata) + if matching is None: + matching = self.get_matching(metadata) hspec = [ent for ent in matching if ent.specific.hostname] if hspec: @@ -1022,25 +1059,32 @@ class EntrySet: elif action == 'deleted': del self.entries[event.filename] - def entry_init(self, event): + def entry_init(self, event, entry_type=None, specific=None): """Handle template and info file creation.""" + if entry_type is None: + entry_type = self.entry_type + if event.filename in self.entries: logger.warn("Got duplicate add for %s" % event.filename) else: - fpath = "%s/%s" % (self.path, event.filename) + fpath = os.path.join(self.path, event.filename) try: - spec = self.specificity_from_filename(event.filename) + spec = self.specificity_from_filename(event.filename, + specific=specific) except SpecificityError: if not self.ignore.match(event.filename): - logger.error("Could not process filename %s; ignoring" % fpath) + logger.error("Could not process filename %s; ignoring" % + fpath) return - self.entries[event.filename] = self.entry_type(fpath, - spec, self.encoding) + self.entries[event.filename] = entry_type(fpath, spec, + self.encoding) self.entries[event.filename].handle_event(event) - def specificity_from_filename(self, fname): + def specificity_from_filename(self, fname, specific=None): """Construct a specificity instance from a filename and regex.""" - data = self.specific.match(fname) + if specific is None: + specific = self.specific + data = specific.match(fname) if not data: raise SpecificityError(fname) kwargs = {} @@ -1057,7 +1101,7 @@ class EntrySet: def update_metadata(self, event): """Process info and info.xml files for the templates.""" - fpath = "%s/%s" % (self.path, event.filename) + fpath = os.path.join(self.path, event.filename) if event.filename == 'info.xml': if not self.infoxml: self.infoxml = InfoXML(fpath, True) @@ -1101,7 +1145,6 @@ class EntrySet: class GroupSpool(Plugin, Generator): """Unified interface for handling group-specific data (e.g. .G## files).""" name = 'GroupSpool' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' filename_pattern = "" es_child_cls = object @@ -1145,6 +1188,12 @@ class GroupSpool(Plugin, Generator): else: return self.handles[event.requestID][:-1] + def toggle_debug(self): + for entry in self.entries.values(): + if hasattr(entry, "toggle_debug"): + entry.toggle_debug() + return Plugin.toggle_debug() + def HandleEvent(self, event): """Unified FAM event handler for GroupSpool.""" action = event.code2str() @@ -1183,69 +1232,7 @@ class GroupSpool(Plugin, Generator): name = self.data + relative if relative not in list(self.handles.values()): if not posixpath.isdir(name): - print("Failed to open directory %s" % (name)) + self.logger.error("Failed to open directory %s" % name) return reqid = self.core.fam.AddMonitor(name, self) self.handles[reqid] = relative - -class SimpleConfig(FileBacked, - ConfigParser.SafeConfigParser): - ''' a simple plugin config using ConfigParser ''' - _required = True - - def __init__(self, plugin): - filename = os.path.join(plugin.data, plugin.name.lower() + ".conf") - self.plugin = plugin - self.fam = self.plugin.core.fam - self.read_files = set() - Bcfg2.Server.Plugin.FileBacked.__init__(self, filename) - ConfigParser.SafeConfigParser.__init__(self) - - if (self._required or - (not self._required and os.path.exists(self.name))): - self.fam.AddMonitor(self.name, self) - - def Index(self): - """ Build local data structures """ - for section in self.sections(): - self.remove_section(section) - self.read_files.update(self.read(self.name)) - - def get(self, section, option, **kwargs): - """ convenience method for getting config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.SafeConfigParser.get(self, section, option, - **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - if default is not None: - return default - else: - raise - - def getboolean(self, section, option, **kwargs): - """ convenience method for getting boolean config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.SafeConfigParser.getboolean(self, section, - option, **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, - ValueError): - if default is not None: - return default - else: - raise - - @property - def loaded(self): - if os.path.exists(self.name): - return self.name in self.read_files - else: - return True - diff --git a/src/lib/Server/Plugins/Account.py b/src/lib/Bcfg2/Server/Plugins/Account.py index f67819b9d..f2703dccb 100644 --- a/src/lib/Server/Plugins/Account.py +++ b/src/lib/Bcfg2/Server/Plugins/Account.py @@ -1,5 +1,4 @@ """This handles authentication setup.""" -__revision__ = '$Revision$' import Bcfg2.Server.Plugin @@ -16,7 +15,6 @@ class Account(Bcfg2.Server.Plugin.Plugin, """ name = 'Account' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/BB.py b/src/lib/Bcfg2/Server/Plugins/BB.py index 0df6be4e8..bd518ad19 100644 --- a/src/lib/Server/Plugins/BB.py +++ b/src/lib/Bcfg2/Server/Plugins/BB.py @@ -64,7 +64,6 @@ class BB(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): """The BB plugin maps users to machines and metadata to machines.""" name = 'BB' - version = '$Revision$' deprecated = True def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Bcfg2/Server/Plugins/Base.py index e8d798ae4..389ca7a95 100644 --- a/src/lib/Server/Plugins/Base.py +++ b/src/lib/Bcfg2/Server/Plugins/Base.py @@ -1,5 +1,4 @@ """This module sets up a base list of configuration entries.""" -__revision__ = '$Revision$' import copy import lxml.etree @@ -18,7 +17,6 @@ class Base(Bcfg2.Server.Plugin.Plugin, needed for most actual systems. """ name = 'Base' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = Bcfg2.Server.Plugin.StructFile deprecated = True diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Bcfg2/Server/Plugins/Bundler.py index c795d7f90..6289439c6 100644 --- a/src/lib/Server/Plugins/Bundler.py +++ b/src/lib/Bcfg2/Server/Plugins/Bundler.py @@ -1,5 +1,4 @@ """This provides bundle clauses with translation functionality.""" -__revision__ = '$Revision$' import copy import lxml.etree @@ -9,11 +8,11 @@ import re import sys import Bcfg2.Server import Bcfg2.Server.Plugin +import Bcfg2.Server.Lint try: - import genshi.template import genshi.template.base - import Bcfg2.Server.Plugins.SGenshi + from Bcfg2.Server.Plugins.SGenshi import SGenshiTemplateFile have_genshi = True except: have_genshi = False @@ -35,7 +34,6 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, bundle/translation scheme from Bcfg1. """ name = 'Bundler' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' patterns = re.compile('^(?P<name>.*)\.(xml|genshi)$') @@ -56,22 +54,16 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, bundle = lxml.etree.parse(name, parser=Bcfg2.Server.XMLParser) nsmap = bundle.getroot().nsmap - if name.endswith('.xml'): - if have_genshi and \ - (nsmap == {'py': 'http://genshi.edgewall.org/'}): - # allow for genshi bundles with .xml extensions - spec = Bcfg2.Server.Plugin.Specificity() - return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name, - spec, - self.encoding) - else: - return BundleFile(name) - elif name.endswith('.genshi'): + if (name.endswith('.genshi') or + ('py' in nsmap and + nsmap['py'] == 'http://genshi.edgewall.org/')): if have_genshi: spec = Bcfg2.Server.Plugin.Specificity() - return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name, - spec, - self.encoding) + return SGenshiTemplateFile(name, spec, self.encoding) + else: + raise Bcfg2.Server.Plugin.PluginExecutionError("Genshi not available: %s" % name) + else: + return BundleFile(name) def BuildStructures(self, metadata): """Build all structures for client (metadata).""" @@ -100,3 +92,52 @@ class Bundler(Bcfg2.Server.Plugin.Plugin, self.logger.error("Bundler: Unexpected bundler error for %s" % bundlename, exc_info=1) return bundleset + +class BundlerLint(Bcfg2.Server.Lint.ServerPlugin): + """ Perform various bundle checks """ + def Run(self): + """ run plugin """ + self.missing_bundles() + for bundle in self.core.plugins['Bundler'].entries.values(): + if (self.HandlesFile(bundle.name) and + (not have_genshi or type(bundle) is not SGenshiTemplateFile)): + self.bundle_names(bundle) + + @classmethod + def Errors(cls): + return {"bundle-not-found":"error", + "inconsistent-bundle-name":"warning"} + + def missing_bundles(self): + """ find bundles listed in Metadata but not implemented in Bundler """ + if self.files is None: + # when given a list of files on stdin, this check is + # useless, so skip it + groupdata = self.metadata.groups_xml.xdata + ref_bundles = set([b.get("name") + for b in groupdata.findall("//Bundle")]) + + allbundles = self.core.plugins['Bundler'].entries.keys() + for bundle in ref_bundles: + xmlbundle = "%s.xml" % bundle + genshibundle = "%s.genshi" % bundle + if (xmlbundle not in allbundles and + genshibundle not in allbundles): + self.LintError("bundle-not-found", + "Bundle %s referenced, but does not exist" % + bundle) + + def bundle_names(self, bundle): + """ verify bundle name attribute matches filename """ + try: + xdata = lxml.etree.XML(bundle.data) + except AttributeError: + # genshi template + xdata = lxml.etree.parse(bundle.template.filepath).getroot() + + fname = bundle.name.split('Bundler/')[1].split('.')[0] + bname = xdata.get('name') + if fname != bname: + self.LintError("inconsistent-bundle-name", + "Inconsistent bundle name: filename is %s, " + "bundle name is %s" % (fname, bname)) diff --git a/src/lib/Server/Plugins/Bzr.py b/src/lib/Bcfg2/Server/Plugins/Bzr.py index a9a5eb814..a71021cb5 100644 --- a/src/lib/Server/Plugins/Bzr.py +++ b/src/lib/Bcfg2/Server/Plugins/Bzr.py @@ -10,7 +10,6 @@ class Bzr(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Bzr is a version plugin for dealing with Bcfg2 repos.""" name = 'Bzr' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py new file mode 100644 index 000000000..f6b175832 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCatFilter.py @@ -0,0 +1,20 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgFilter + +logger = logging.getLogger(__name__) + +class CfgCatFilter(CfgFilter): + __extensions__ = ['cat'] + + def modify_data(self, entry, metadata, data): + datalines = data.strip().split('\n') + for line in self.data.split('\n'): + if not line: + continue + if line.startswith('+'): + datalines.append(line[1:]) + elif line.startswith('-'): + if line[1:] in datalines: + datalines.remove(line[1:]) + return "\n".join(datalines) + "\n" diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py new file mode 100644 index 000000000..e74b77e83 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgCheetahGenerator.py @@ -0,0 +1,31 @@ +import copy +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +logger = logging.getLogger(__name__) + +try: + from Cheetah.Template import Template + have_cheetah = True +except ImportError: + have_cheetah = False + + +class CfgCheetahGenerator(CfgGenerator): + __extensions__ = ['cheetah'] + settings = dict(useStackFrames=False) + + def __init__(self, fname, spec, encoding): + CfgGenerator.__init__(self, fname, spec, encoding) + if not have_cheetah: + msg = "Cfg: Cheetah is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + def get_data(self, entry, metadata): + template = Template(self.data, compilerSettings=self.settings) + template.metadata = metadata + template.path = entry.get('realname', entry.get('name')) + template.source_path = self.name + return template.respond() diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py new file mode 100644 index 000000000..906666c21 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgDiffFilter.py @@ -0,0 +1,27 @@ +import os +import logging +import tempfile +import Bcfg2.Server.Plugin +from subprocess import Popen, PIPE +from Bcfg2.Server.Plugins.Cfg import CfgFilter + +logger = logging.getLogger(__name__) + +class CfgDiffFilter(CfgFilter): + __extensions__ = ['diff'] + + def modify_data(self, entry, metadata, data): + basehandle, basename = tempfile.mkstemp() + open(basename, 'w').write(data) + os.close(basehandle) + + cmd = ["patch", "-u", "-f", basefile.name] + patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + stderr = patch.communicate(input=self.data)[1] + ret = patch.wait() + output = open(basefile.name, 'r').read() + os.unlink(basefile.name) + if ret != 0: + logger.error("Error applying diff %s: %s" % (delta.name, stderr)) + raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) + return output diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py new file mode 100644 index 000000000..a75329d2a --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedCheetahGenerator.py @@ -0,0 +1,14 @@ +import logging +from Bcfg2.Server.Plugins.Cfg.CfgCheetahGenerator import CfgCheetahGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import CfgEncryptedGenerator + +logger = logging.getLogger(__name__) + +class CfgEncryptedCheetahGenerator(CfgCheetahGenerator, CfgEncryptedGenerator): + __extensions__ = ['cheetah.crypt', 'crypt.cheetah'] + + def handle_event(self, event): + CfgEncryptedGenerator.handle_event(self, event) + + def get_data(self, entry, metadata): + return CfgCheetahGenerator.get_data(self, entry, metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py new file mode 100644 index 000000000..2c926fae7 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenerator.py @@ -0,0 +1,63 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator, SETUP +try: + from Bcfg2.Encryption import ssl_decrypt, EVPError + have_crypto = True +except ImportError: + have_crypto = False + +logger = logging.getLogger(__name__) + +def passphrases(): + section = "encryption" + if SETUP.cfp.has_section(section): + return dict([(o, SETUP.cfp.get(section, o)) + for o in SETUP.cfp.options(section)]) + else: + return dict() + +def decrypt(crypted): + if not have_crypto: + msg = "Cfg: M2Crypto is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + for passwd in passphrases().values(): + try: + return ssl_decrypt(crypted, passwd) + except EVPError: + pass + raise EVPError("Failed to decrypt") + +class CfgEncryptedGenerator(CfgGenerator): + __extensions__ = ["crypt"] + + def __init__(self, fname, spec, encoding): + CfgGenerator.__init__(self, fname, spec, encoding) + if not have_crypto: + msg = "Cfg: M2Crypto is not available: %s" % entry.get("name") + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + def handle_event(self, event): + if event.code2str() == 'deleted': + return + try: + crypted = open(self.name).read() + except UnicodeDecodeError: + crypted = open(self.name, mode='rb').read() + except: + logger.error("Failed to read %s" % self.name) + return + # todo: let the user specify a passphrase by name + try: + self.data = decrypt(crypted) + except EVPError: + msg = "Failed to decrypt %s" % self.name + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + def get_data(self, entry, metadata): + if self.data is None: + raise Bcfg2.Server.Plugin.PluginExecutionError("Failed to decrypt %s" % self.name) + return CfgGenerator.get_data(self, entry, metadata) diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py new file mode 100644 index 000000000..6605cca7c --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgEncryptedGenshiGenerator.py @@ -0,0 +1,26 @@ +import logging +from Bcfg2.Bcfg2Py3k import StringIO +from Bcfg2.Server.Plugins.Cfg.CfgGenshiGenerator import CfgGenshiGenerator +from Bcfg2.Server.Plugins.Cfg.CfgEncryptedGenerator import decrypt, \ + CfgEncryptedGenerator + +logger = logging.getLogger(__name__) + +try: + from genshi.template import TemplateLoader +except ImportError: + # CfgGenshiGenerator will raise errors if genshi doesn't exist + TemplateLoader = object + + +class EncryptedTemplateLoader(TemplateLoader): + def _instantiate(self, cls, fileobj, filepath, filename, encoding=None): + plaintext = StringIO(decrypt(fileobj.read())) + return TemplateLoader._instantiate(self, cls, plaintext, filepath, + filename, encoding=encoding) + + +class CfgEncryptedGenshiGenerator(CfgGenshiGenerator): + __extensions__ = ['genshi.crypt', 'crypt.genshi'] + __loader_cls__ = EncryptedTemplateLoader + diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py new file mode 100644 index 000000000..f0c1109ec --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgExternalCommandVerifier.py @@ -0,0 +1,33 @@ +import os +import shlex +import logging +import Bcfg2.Server.Plugin +from subprocess import Popen, PIPE +from Bcfg2.Server.Plugins.Cfg import CfgVerifier, CfgVerificationError + +logger = logging.getLogger(__name__) + +class CfgExternalCommandVerifier(CfgVerifier): + __basenames__ = [':test'] + + def verify_entry(self, entry, metadata, data): + proc = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + err = proc.communicate(input=data)[1] + rv = proc.wait() + if rv != 0: + raise CfgVerificationError(err) + + def handle_event(self, event): + if event.code2str() == 'deleted': + return + self.cmd = [] + if not os.access(self.name, os.X_OK): + bangpath = open(self.name).readline().strip() + if bangpath.startswith("#!"): + self.cmd.extend(shlex.split(bangpath[2:].strip())) + else: + msg = "Cannot execute %s" % self.name + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.cmd.append(self.name) + diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py new file mode 100644 index 000000000..5447717d8 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgGenshiGenerator.py @@ -0,0 +1,67 @@ +import sys +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +logger = logging.getLogger(__name__) + +try: + import genshi.core + from genshi.template import TemplateLoader, NewTextTemplate + have_genshi = True +except ImportError: + TemplateLoader = None + have_genshi = False + +# snipped from TGenshi +def removecomment(stream): + """A genshi filter that removes comments from the stream.""" + for kind, data, pos in stream: + if kind is genshi.core.COMMENT: + continue + yield kind, data, pos + + +class CfgGenshiGenerator(CfgGenerator): + __extensions__ = ['genshi'] + __loader_cls__ = TemplateLoader + + def __init__(self, fname, spec, encoding): + CfgGenerator.__init__(self, fname, spec, encoding) + if not have_genshi: + msg = "Cfg: Genshi is not available: %s" % fname + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + self.loader = self.__loader_cls__() + self.template = None + + @classmethod + def ignore(cls, event, basename=None): + return (event.filename.endswith(".genshi_include") or + CfgGenerator.ignore(event, basename=basename)) + + def get_data(self, entry, metadata): + fname = entry.get('realname', entry.get('name')) + stream = \ + self.template.generate(name=fname, + metadata=metadata, + path=self.name).filter(removecomment) + try: + return stream.render('text', encoding=self.encoding, + strip_whitespace=False) + except TypeError: + return stream.render('text', encoding=self.encoding) + + def handle_event(self, event): + if event.code2str() == 'deleted': + return + CfgGenerator.handle_event(self, event) + try: + self.template = self.loader.load(self.name, cls=NewTextTemplate, + encoding=self.encoding) + except Exception: + msg = "Cfg: Could not load template %s: %s" % (self.name, + sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py new file mode 100644 index 000000000..8e962efb4 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgInfoXML.py @@ -0,0 +1,24 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgInfo + +logger = logging.getLogger(__name__) + +class CfgInfoXML(CfgInfo): + __basenames__ = ['info.xml'] + + def __init__(self, path): + CfgInfo.__init__(self, path) + self.infoxml = Bcfg2.Server.Plugin.InfoXML(path, noprio=True) + + def bind_info_to_entry(self, entry, metadata): + mdata = dict() + self.infoxml.pnode.Match(metadata, mdata, entry=entry) + if 'Info' not in mdata: + logger.error("Failed to set metadata for file %s" % + entry.get('name')) + raise PluginExecutionError + self._set_info(entry, mdata['Info'][None]) + + def handle_event(self, event): + self.infoxml.HandleEvent() diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py new file mode 100644 index 000000000..85c13c1ac --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgLegacyInfo.py @@ -0,0 +1,32 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgInfo + +logger = logging.getLogger(__name__) + +class CfgLegacyInfo(CfgInfo): + __basenames__ = ['info', ':info'] + + def __init__(self, path): + CfgInfo.__init__(self, path) + self.path = path + + def bind_info_to_entry(self, entry, metadata): + self._set_info(entry, self.metadata) + + def handle_event(self, event): + if event.code2str() == 'deleted': + return + for line in open(self.path).readlines(): + match = Bcfg2.Server.Plugin.info_regex.match(line) + if not match: + logger.warning("Failed to parse line in %s: %s" % (fpath, line)) + continue + else: + self.metadata = \ + dict([(key, value) + for key, value in list(match.groupdict().items()) + if value]) + if ('perms' in self.metadata and + len(self.metadata['perms']) == 3): + self.metadata['perms'] = "0%s" % self.metadata['perms'] diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py new file mode 100644 index 000000000..8e9aab465 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/CfgPlaintextGenerator.py @@ -0,0 +1,8 @@ +import logging +import Bcfg2.Server.Plugin +from Bcfg2.Server.Plugins.Cfg import CfgGenerator + +logger = logging.getLogger(__name__) + +class CfgPlaintextGenerator(CfgGenerator): + pass diff --git a/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py new file mode 100644 index 000000000..15fd71644 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/Cfg/__init__.py @@ -0,0 +1,435 @@ +"""This module implements a config file repository.""" + +import re +import os +import sys +import stat +import pkgutil +import logging +import binascii +import lxml.etree +import Bcfg2.Options +import Bcfg2.Server.Plugin +from Bcfg2.Bcfg2Py3k import u_str +import Bcfg2.Server.Lint + +logger = logging.getLogger(__name__) + +PROCESSORS = None +SETUP = None + +class CfgBaseFileMatcher(Bcfg2.Server.Plugin.SpecificData): + __basenames__ = [] + __extensions__ = [] + __ignore__ = [] + __specific__ = True + + def __init__(self, fname, spec, encoding): + Bcfg2.Server.Plugin.SpecificData.__init__(self, fname, spec, encoding) + self.encoding = encoding + self.regex = self.__class__.get_regex(fname) + + @classmethod + def get_regex(cls, fname=None, extensions=None): + if extensions is None: + extensions = cls.__extensions__ + if cls.__basenames__: + fname = '|'.join(cls.__basenames__) + + components = ['^(?P<basename>%s)' % fname] + if cls.__specific__: + components.append('(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))') + if extensions: + components.append('\\.(?P<extension>%s)' % '|'.join(extensions)) + components.append('$') + return re.compile("".join(components)) + + @classmethod + def handles(cls, event, basename=None): + if cls.__basenames__: + basenames = cls.__basenames__ + else: + basenames = [basename] + + # do simple non-regex matching first + match = False + for bname in basenames: + if event.filename.startswith(os.path.basename(bname)): + match = True + break + return (match and + cls.get_regex(fname=os.path.basename(basename)).match(event.filename)) + + @classmethod + def ignore(cls, event, basename=None): + if not cls.__ignore__: + return False + + if cls.__basenames__: + basenames = cls.__basenames__ + else: + basenames = [basename] + + # do simple non-regex matching first + match = False + for bname in basenames: + if event.filename.startswith(os.path.basename(bname)): + match = True + break + return (match and + cls.get_regex(fname=os.path.basename(basename), + extensions=cls.__ignore__).match(event.filename)) + + + def __str__(self): + return "%s(%s)" % (self.__class__.__name__, self.name) + + def match(self, fname): + return self.regex.match(fname) + + +class CfgGenerator(CfgBaseFileMatcher): + """ CfgGenerators generate the initial content of a file """ + def get_data(self, entry, metadata): + return self.data + + +class CfgFilter(CfgBaseFileMatcher): + """ CfgFilters modify the initial content of a file after it's + been generated """ + def modify_data(self, entry, metadata, data): + raise NotImplementedError + + +class CfgInfo(CfgBaseFileMatcher): + """ CfgInfos provide metadata (owner, group, paranoid, etc.) for a + file entry """ + __specific__ = False + + def __init__(self, fname): + CfgBaseFileMatcher.__init__(self, fname, None, None) + + def bind_info_to_entry(self, entry, metadata): + raise NotImplementedError + + def _set_info(self, entry, info): + for key, value in list(info.items()): + entry.attrib.__setitem__(key, value) + + +class CfgVerifier(CfgBaseFileMatcher): + """ Verifiers validate entries """ + def verify_entry(self, entry, metadata, data): + raise NotImplementedError + + +class CfgVerificationError(Exception): + pass + + +class CfgDefaultInfo(CfgInfo): + def __init__(self, defaults): + CfgInfo.__init__(self, '') + self.defaults = defaults + + def bind_info_to_entry(self, entry, metadata): + self._set_info(entry, self.defaults) + +DEFAULT_INFO = CfgDefaultInfo(Bcfg2.Server.Plugin.default_file_metadata) + +class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): + def __init__(self, basename, path, entry_type, encoding): + Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, + entry_type, encoding) + self.specific = None + self.load_processors() + + def load_processors(self): + """ load Cfg file processors. this must be done at run-time, + not at compile-time, or we get a circular import and things + don't work. but finding the right way to do this at runtime + was ... problematic. so here it is, writing to a global + variable. Sorry 'bout that. """ + global PROCESSORS + if PROCESSORS is None: + PROCESSORS = [] + if hasattr(pkgutil, 'walk_packages'): + submodules = pkgutil.walk_packages(path=__path__) + else: + #python 2.4 + import glob + submodules = [] + for path in __path__: + for submodule in glob.glob("%s/*.py" % path): + mod = '.'.join(submodule.split("/")[-1].split('.')[:-1]) + if mod != '__init__': + submodules.append((None, mod, True)) + + for submodule in submodules: + module = getattr(__import__("%s.%s" % + (__name__, + submodule[1])).Server.Plugins.Cfg, + submodule[1]) + proc = getattr(module, submodule[1]) + if set(proc.__mro__).intersection([CfgInfo, CfgFilter, + CfgGenerator, CfgVerifier]): + PROCESSORS.append(proc) + + def handle_event(self, event): + action = event.code2str() + + if event.filename not in self.entries: + if action not in ['exists', 'created', 'changed']: + # process a bogus changed event like a created + return + + for proc in PROCESSORS: + if proc.handles(event, basename=self.path): + if action == 'changed': + # warn about a bogus 'changed' event, but + # handle it like a 'created' + logger.warning("Got %s event for unknown file %s" % + (action, event.filename)) + self.debug_log("%s handling %s event on %s" % + (proc.__name__, action, event.filename)) + self.entry_init(event, proc) + return + elif proc.ignore(event, basename=self.path): + return + elif action == 'changed': + self.entries[event.filename].handle_event(event) + elif action == 'deleted': + del self.entries[event.filename] + return + + logger.error("Could not process event %s for %s; ignoring" % + (action, event.filename)) + + def entry_init(self, event, proc): + if proc.__specific__: + Bcfg2.Server.Plugin.EntrySet.entry_init( + self, event, entry_type=proc, + specific=proc.get_regex(os.path.basename(self.path))) + else: + if event.filename in self.entries: + logger.warn("Got duplicate add for %s" % event.filename) + else: + fpath = os.path.join(self.path, event.filename) + self.entries[event.filename] = proc(fpath) + self.entries[event.filename].handle_event(event) + + def bind_entry(self, entry, metadata): + info_handlers = [] + generators = [] + filters = [] + verifiers = [] + for ent in self.entries.values(): + if ent.__specific__ and not ent.specific.matches(metadata): + continue + if isinstance(ent, CfgInfo): + info_handlers.append(ent) + elif isinstance(ent, CfgGenerator): + generators.append(ent) + elif isinstance(ent, CfgFilter): + filters.append(ent) + elif isinstance(ent, CfgVerifier): + verifiers.append(ent) + + DEFAULT_INFO.bind_info_to_entry(entry, metadata) + if len(info_handlers) > 1: + logger.error("More than one info supplier found for %s: %s" % + (self.name, info_handlers)) + if len(info_handlers): + info_handlers[0].bind_info_to_entry(entry, metadata) + if entry.tag == 'Path': + entry.set('type', 'file') + + generator = self.best_matching(metadata, generators) + if entry.get('perms').lower() == 'inherit': + # use on-disk permissions + fname = os.path.join(self.path, generator.name) + entry.set('perms', + str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) + try: + data = generator.get_data(entry, metadata) + except: + msg = "Cfg: exception rendering %s with %s: %s" % \ + (entry.get("name"), generator, sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + for fltr in filters: + data = fltr.modify_data(entry, metadata, data) + + if SETUP['validate']: + # we can have multiple verifiers, but we only want to use the + # best matching verifier of each class + verifiers_by_class = dict() + for verifier in verifiers: + cls = verifier.__class__.__name__ + if cls not in verifiers_by_class: + verifiers_by_class[cls] = [verifier] + else: + verifiers_by_class[cls].append(verifier) + for verifiers in verifiers_by_class.values(): + verifier = self.best_matching(metadata, verifiers) + try: + verifier.verify_entry(entry, metadata, data) + except CfgVerificationError: + msg = "Data for %s for %s failed to verify: %s" % \ + (entry.get('name'), metadata.hostname, + sys.exc_info()[1]) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + if entry.get('encoding') == 'base64': + data = binascii.b2a_base64(data) + else: + try: + data = u_str(data, self.encoding) + except UnicodeDecodeError: + msg = "Failed to decode %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) + logger.error("Please verify you are using the proper encoding.") + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + except ValueError: + msg = "Error in specification for %s: %s" % (entry.get('name'), + sys.exc_info()[1]) + logger.error(msg) + logger.error("You need to specify base64 encoding for %s." % + entry.get('name')) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + except TypeError: + # data is already unicode; newer versions of Cheetah + # seem to return unicode + pass + + if data: + entry.text = data + else: + entry.set('empty', 'true') + + def list_accept_choices(self, entry, metadata): + '''return a list of candidate pull locations''' + generators = [ent for ent in list(self.entries.values()) + if (isinstance(ent, CfgGenerator) and + ent.specific.matches(metadata))] + if not generators: + msg = "No base file found for %s" % entry.get('name') + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + rv = [] + try: + best = self.best_matching(metadata, generators) + rv.append(best.specific) + except: + pass + + if not rv or not rv[0].hostname: + rv.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) + return rv + + def build_filename(self, specific): + bfname = self.path + '/' + self.path.split('/')[-1] + if specific.all: + return bfname + elif specific.group: + return "%s.G%02d_%s" % (bfname, specific.prio, specific.group) + elif specific.hostname: + return "%s.H_%s" % (bfname, specific.hostname) + + def write_update(self, specific, new_entry, log): + if 'text' in new_entry: + name = self.build_filename(specific) + if os.path.exists("%s.genshi" % name): + msg = "Cfg: Unable to pull data for genshi types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + elif os.path.exists("%s.cheetah" % name): + msg = "Cfg: Unable to pull data for cheetah types" + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + try: + etext = new_entry['text'].encode(self.encoding) + except: + msg = "Cfg: Cannot encode content of %s as %s" % (name, + self.encoding) + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + open(name, 'w').write(etext) + self.debug_log("Wrote file %s" % name, flag=log) + badattr = [attr for attr in ['owner', 'group', 'perms'] + if attr in new_entry] + if badattr: + # check for info files and inform user of their removal + if os.path.exists(self.path + "/:info"): + logger.info("Removing :info file and replacing with " + "info.xml") + os.remove(self.path + "/:info") + if os.path.exists(self.path + "/info"): + logger.info("Removing info file and replacing with " + "info.xml") + os.remove(self.path + "/info") + metadata_updates = {} + metadata_updates.update(self.metadata) + for attr in badattr: + metadata_updates[attr] = new_entry.get(attr) + infoxml = lxml.etree.Element('FileInfo') + infotag = lxml.etree.SubElement(infoxml, 'Info') + [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ + for attr in metadata_updates] + ofile = open(self.path + "/info.xml", "w") + ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) + ofile.close() + self.debug_log("Wrote file %s" % (self.path + "/info.xml"), + flag=log) + + +class Cfg(Bcfg2.Server.Plugin.GroupSpool, + Bcfg2.Server.Plugin.PullTarget): + """This generator in the configuration file repository for Bcfg2.""" + name = 'Cfg' + __author__ = 'bcfg-dev@mcs.anl.gov' + es_cls = CfgEntrySet + es_child_cls = Bcfg2.Server.Plugin.SpecificData + + def __init__(self, core, datastore): + global SETUP + Bcfg2.Server.Plugin.GroupSpool.__init__(self, core, datastore) + Bcfg2.Server.Plugin.PullTarget.__init__(self) + + SETUP = core.setup + if 'validate' not in SETUP: + SETUP.add_option('validate', Bcfg2.Options.CFG_VALIDATION) + SETUP.reparse() + + def AcceptChoices(self, entry, metadata): + return self.entries[entry.get('name')].list_accept_choices(entry, + metadata) + + def AcceptPullData(self, specific, new_entry, log): + return self.entries[new_entry.get('name')].write_update(specific, + new_entry, + log) + +class CfgLint(Bcfg2.Server.Lint.ServerPlugin): + def Run(self): + # about usage of .cat and .diff files + self.check_deltas() + + @classmethod + def Errors(cls): + return {"cat-file-used":"warning", + "diff-file-used":"warning"} + + def check_entry(self, basename, entry): + cfg = self.core.plugins['Cfg'] + for basename, entry in list(cfg.entries.items()): + for fname, processor in entry.entries.items(): + if self.HandlesFile(fname) and isinstance(processor, CfgFilter): + extension = fname.split(".")[-1] + self.LintError("%s-file-used" % extension, + "%s file used on %s: %s" % + (extension, basename, fname)) diff --git a/src/lib/Server/Plugins/Cvs.py b/src/lib/Bcfg2/Server/Plugins/Cvs.py index ea898c023..6ce72acd2 100644 --- a/src/lib/Server/Plugins/Cvs.py +++ b/src/lib/Bcfg2/Server/Plugins/Cvs.py @@ -10,7 +10,6 @@ class Cvs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """CVS is a version plugin for dealing with Bcfg2 repository.""" name = 'Cvs' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Bcfg2/Server/Plugins/DBStats.py index 56420f646..ca948aabd 100644 --- a/src/lib/Server/Plugins/DBStats.py +++ b/src/lib/Bcfg2/Server/Plugins/DBStats.py @@ -12,18 +12,18 @@ except ImportError: pass import Bcfg2.Server.Plugin -import Bcfg2.Server.Reports.importscript +from Bcfg2.Server.Reports.importscript import load_stat from Bcfg2.Server.Reports.reports.models import Client import Bcfg2.Server.Reports.settings -from Bcfg2.Server.Reports.updatefix import update_database +from Bcfg2.Server.Reports.Updater import update_database, UpdaterError # for debugging output only logger = logging.getLogger('Bcfg2.Plugins.DBStats') + class DBStats(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ThreadedStatistics, Bcfg2.Server.Plugin.PullSource): name = 'DBStats' - __version__ = '$Id$' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) @@ -31,9 +31,12 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.PullSource.__init__(self) self.cpath = "%s/Metadata/clients.xml" % datastore self.core = core - logger.debug("Searching for new models to add to the statistics database") + logger.debug("Searching for new models to " + "add to the statistics database") try: update_database() + except UpdaterError: + raise Bcfg2.Server.Plugin.PluginInitError except Exception: inst = sys.exc_info()[1] logger.debug(str(inst)) @@ -42,32 +45,24 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, def handle_statistic(self, metadata, data): newstats = data.find("Statistics") newstats.set('time', time.asctime(time.localtime())) - # ick - data = lxml.etree.tostring(newstats) - ndx = lxml.etree.XML(data) - e = lxml.etree.Element('Node', name=metadata.hostname) - e.append(ndx) - container = lxml.etree.Element("ConfigStatistics") - container.append(e) - # FIXME need to build a metadata interface to expose a list of clients start = time.time() for i in [1, 2, 3]: try: - Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, - container, - self.core.encoding, - 0, - logger, - True, - platform.node()) + load_stat(metadata, + newstats, + self.core.encoding, + 0, + logger, + True, + platform.node()) logger.info("Imported data for %s in %s seconds" \ % (metadata.hostname, time.time() - start)) return except MultipleObjectsReturned: e = sys.exc_info()[1] - logger.error("DBStats: MultipleObjectsReturned while handling %s: %s" % \ - (metadata.hostname, e)) + logger.error("DBStats: MultipleObjectsReturned while " + "handling %s: %s" % (metadata.hostname, e)) logger.error("DBStats: Data is inconsistent") break except: @@ -101,6 +96,8 @@ class DBStats(Bcfg2.Server.Plugin.Plugin, ret.append(getattr(entry.reason, "current_%s" % t)) if entry.reason.is_sensitive: raise Bcfg2.Server.Plugin.PluginExecutionError + elif len(entry.reason.unpruned) != 0: + ret.append('\n'.join(entry.reason.unpruned)) elif entry.reason.current_diff != '': if entry.reason.is_binary: ret.append(binascii.a2b_base64(entry.reason.current_diff)) diff --git a/src/lib/Server/Plugins/Darcs.py b/src/lib/Bcfg2/Server/Plugins/Darcs.py index eb34a52c4..9fb9ff4f1 100644 --- a/src/lib/Server/Plugins/Darcs.py +++ b/src/lib/Bcfg2/Server/Plugins/Darcs.py @@ -10,7 +10,6 @@ class Darcs(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Darcs is a version plugin for dealing with Bcfg2 repos.""" name = 'Darcs' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/Decisions.py b/src/lib/Bcfg2/Server/Plugins/Decisions.py index 556f75502..b432474f2 100644 --- a/src/lib/Server/Plugins/Decisions.py +++ b/src/lib/Bcfg2/Server/Plugins/Decisions.py @@ -50,7 +50,6 @@ class Decisions(DecisionSet, Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Decision): name = 'Decisions' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Defaults.py b/src/lib/Bcfg2/Server/Plugins/Defaults.py index b208c1126..718192e2a 100644 --- a/src/lib/Server/Plugins/Defaults.py +++ b/src/lib/Bcfg2/Server/Plugins/Defaults.py @@ -1,5 +1,4 @@ """This generator provides rule-based entry mappings.""" -__revision__ = '$Revision$' import re import Bcfg2.Server.Plugin @@ -9,7 +8,6 @@ class Defaults(Bcfg2.Server.Plugins.Rules.Rules, Bcfg2.Server.Plugin.StructureValidator): """Set default attributes on bound entries""" name = 'Defaults' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' # Rules is a Generator that happens to implement all of the diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Bcfg2/Server/Plugins/Deps.py index 482d457af..9b848baae 100644 --- a/src/lib/Server/Plugins/Deps.py +++ b/src/lib/Bcfg2/Server/Plugins/Deps.py @@ -1,5 +1,4 @@ """This plugin provides automatic dependency handling.""" -__revision__ = '$Revision$' import lxml.etree @@ -45,7 +44,6 @@ class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc): class Deps(Bcfg2.Server.Plugin.PrioDir, Bcfg2.Server.Plugin.StructureValidator): name = 'Deps' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = DepXMLSrc diff --git a/src/lib/Server/Plugins/Editor.py b/src/lib/Bcfg2/Server/Plugins/Editor.py index 76a03a325..c0d2cfbad 100644 --- a/src/lib/Server/Plugins/Editor.py +++ b/src/lib/Bcfg2/Server/Plugins/Editor.py @@ -59,7 +59,6 @@ class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): class Editor(Bcfg2.Server.Plugin.GroupSpool, Bcfg2.Server.Plugin.Probing): name = 'Editor' - __version__ = '$Id$' __author__ = 'bcfg2-dev@mcs.anl.gov' filename_pattern = 'edits' es_child_cls = EditDirectives diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Bcfg2/Server/Plugins/FileProbes.py index 2332a1d40..556965fca 100644 --- a/src/lib/Server/Plugins/FileProbes.py +++ b/src/lib/Bcfg2/Server/Plugins/FileProbes.py @@ -3,7 +3,6 @@ added to the specification. On subsequent runs, the file will be replaced on the client if it is missing; if it has changed on the client, it can either be updated in the specification or replaced on the client """ -__revision__ = '$Revision: 1465 $' import os import sys @@ -56,7 +55,6 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, name = 'FileProbes' experimental = True - __version__ = '$Id$' __author__ = 'chris.a.st.pierre@gmail.com' def __init__(self, core, datastore): @@ -136,7 +134,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, else: entrydata = entry.text - if create: + if create: self.logger.info("Writing new probed file %s" % fileloc) self.write_file(fileloc, contents) self.verify_file(filename, contents, metadata) @@ -196,7 +194,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, except Bcfg2.Server.Plugin.PluginExecutionError: tries += 1 continue - + # get current entry data if entry.get("encoding") == "base64": entrydata = binascii.a2b_base64(entry.text) @@ -205,12 +203,12 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, if entrydata == contents: updated = True tries += 1 - + def write_infoxml(self, infoxml, entry, data): """ write an info.xml for the file """ if os.path.exists(infoxml): return - + self.logger.info("Writing info.xml at %s for %s" % (infoxml, data.get("name"))) info = \ @@ -223,7 +221,7 @@ class FileProbes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Options.MDATA_PERMS.value), encoding=entry.get("encoding", Bcfg2.Options.ENCODING.value)) - + root = lxml.etree.Element("FileInfo") root.append(info) try: diff --git a/src/lib/Server/Plugins/Fossil.py b/src/lib/Bcfg2/Server/Plugins/Fossil.py index 57d427673..1b1627688 100644 --- a/src/lib/Server/Plugins/Fossil.py +++ b/src/lib/Bcfg2/Server/Plugins/Fossil.py @@ -10,7 +10,6 @@ class Fossil(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Fossil is a version plugin for dealing with Bcfg2 repos.""" name = 'Fossil' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Git.py b/src/lib/Bcfg2/Server/Plugins/Git.py index aaeac12ae..8f8ea87f1 100644 --- a/src/lib/Server/Plugins/Git.py +++ b/src/lib/Bcfg2/Server/Plugins/Git.py @@ -13,7 +13,6 @@ class Git(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Git is a version plugin for dealing with Bcfg2 repos.""" name = 'Git' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/GroupPatterns.py b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py index 58b4d4afb..e883143ec 100644 --- a/src/lib/Server/Plugins/GroupPatterns.py +++ b/src/lib/Bcfg2/Server/Plugins/GroupPatterns.py @@ -1,6 +1,8 @@ import re +import sys import logging import lxml.etree +import Bcfg2.Server.Lint import Bcfg2.Server.Plugin class PackedDigitRange(object): @@ -122,3 +124,33 @@ class GroupPatterns(Bcfg2.Server.Plugin.Plugin, def get_additional_groups(self, metadata): return self.config.process_patterns(metadata.hostname) + + +class GroupPatternsLint(Bcfg2.Server.Lint.ServerPlugin): + def Run(self): + """ run plugin """ + cfg = self.core.plugins['GroupPatterns'].config + for entry in cfg.xdata.xpath('//GroupPattern'): + groups = [g.text for g in entry.findall('Group')] + self.check(entry, groups, ptype='NamePattern') + self.check(entry, groups, ptype='NameRange') + + @classmethod + def Errors(cls): + return {"pattern-fails-to-initialize":"error"} + + def check(self, entry, groups, ptype="NamePattern"): + if ptype == "NamePattern": + pmap = lambda p: PatternMap(p, None, groups) + else: + pmap = lambda p: PatternMap(None, p, groups) + + for el in entry.findall(ptype): + pat = el.text + try: + pmap(pat) + except: + err = sys.exc_info()[1] + self.LintError("pattern-fails-to-initialize", + "Failed to initialize %s %s for %s: %s" % + (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Server/Plugins/Guppy.py b/src/lib/Bcfg2/Server/Plugins/Guppy.py index b217378d6..eea92f30f 100644 --- a/src/lib/Server/Plugins/Guppy.py +++ b/src/lib/Bcfg2/Server/Plugins/Guppy.py @@ -12,7 +12,7 @@ python -c "from guppy import hpy;hpy().monitor()" For example: # python -c "from guppy import hpy;hpy().monitor()" -<Monitor> +<Monitor> *** Connection 1 opened *** <Monitor> lc CID PID ARGV @@ -21,7 +21,7 @@ CID PID ARGV Remote connection 1. To return to Monitor, type <Ctrl-C> or .<RETURN> <Annex> int Remote interactive console. To return to Annex, type '-'. ->>> hp.heap() +>>> hp.heap() ... @@ -32,7 +32,6 @@ import Bcfg2.Server.Plugin class Guppy(Bcfg2.Server.Plugin.Plugin): """Guppy is a debugging plugin to help trace memory leaks""" name = 'Guppy' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/Hg.py b/src/lib/Bcfg2/Server/Plugins/Hg.py index 70e33ef1f..0c3537613 100644 --- a/src/lib/Server/Plugins/Hg.py +++ b/src/lib/Bcfg2/Server/Plugins/Hg.py @@ -10,7 +10,6 @@ class Hg(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Mercurial is a version plugin for dealing with Bcfg2 repository.""" name = 'Mercurial' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' experimental = True diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Bcfg2/Server/Plugins/Hostbase.py index 4180fd716..e9c1c1cff 100644 --- a/src/lib/Server/Plugins/Hostbase.py +++ b/src/lib/Bcfg2/Server/Plugins/Hostbase.py @@ -2,7 +2,6 @@ This file provides the Hostbase plugin. It manages dns/dhcp/nis host information """ -__revision__ = '$Revision$' import os os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' @@ -23,7 +22,6 @@ class Hostbase(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): """The Hostbase plugin handles host/network info.""" name = 'Hostbase' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' filepath = '/my/adm/hostbase/files/bind' diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Bcfg2/Server/Plugins/Ldap.py index f1e2198d2..29abf5b13 100644 --- a/src/lib/Server/Plugins/Ldap.py +++ b/src/lib/Bcfg2/Server/Plugins/Ldap.py @@ -50,7 +50,7 @@ class ConfigFile(Bcfg2.Server.Plugin.FileBacked): def Index(self): """ Reregisters the queries in the config file - + The config will take care of actually registering the queries, so we just load it once and don't keep it. """ @@ -63,24 +63,23 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): The Ldap plugin allows adding data from an LDAP server to your metadata. """ name = "Ldap" - version = "$Revision: $" experimental = True debug_flag = False - + def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) self.config = ConfigFile(self.data + "/config.py", core.fam) - + def debug_log(self, message, flag = None): if (flag is None) and self.debug_flag or flag: self.logger.error(message) - + def get_additional_data(self, metadata): query = None try: data = {} - self.debug_log("LdapPlugin debug: found queries " + + self.debug_log("LdapPlugin debug: found queries " + str(LDAP_QUERIES)) for QueryClass in LDAP_QUERIES: query = QueryClass() @@ -95,14 +94,14 @@ class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): except Exception: if hasattr(query, "name"): Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "Exception during processing of query named '" + + "Exception during processing of query named '" + str(query.name) + - "', query results will be empty" + + "', query results will be empty" + " and may cause bind failures") for line in traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + line.replace("\n", "")) return {} @@ -110,23 +109,23 @@ class LdapConnection(object): """ Connection to an LDAP server. """ - def __init__(self, host = "localhost", port = 389, + def __init__(self, host = "localhost", port = 389, binddn = None, bindpw = None): self.host = host self.port = port self.binddn = binddn self.bindpw = bindpw self.conn = None - + def __del__(self): if self.conn: self.conn.unbind() - + def init_conn(self): self.conn = ldap.initialize(self.url) if self.binddn is not None and self.bindpw is not None: self.conn.simple_bind_s(self.binddn, self.bindpw) - + def run_query(self, query): result = None for attempt in range(RETRY_COUNT + 1): @@ -148,17 +147,17 @@ class LdapConnection(object): self.conn = None time.sleep(RETRY_DELAY) return result - + @property def url(self): return "ldap://" + self.host + ":" + str(self.port) - + class LdapQuery(object): """ Query referencing an LdapConnection and providing several methods for query manipulation. """ - + name = "unknown" base = "" scope = "sub" @@ -166,10 +165,10 @@ class LdapQuery(object): attrs = None connection = None result = None - + def __unicode__(self): return "LdapQuery:" + self.name - + def is_applicable(self, metadata): """ Overrideable method to determine if the query is to be executed for @@ -177,27 +176,27 @@ class LdapQuery(object): Defaults to true. """ return True - + def prepare_query(self, metadata): """ Overrideable method to alter the query based on metadata. Defaults to doing nothing. In most cases, you will do something like - + self.filter = "(cn=" + metadata.hostname + ")" - + here. """ pass - + def process_result(self, metadata): """ Overrideable method to post-process the query result. Defaults to returning the unaltered result. """ return self.result - + def get_result(self, metadata): """ Method to handle preparing, executing and processing the query. @@ -208,7 +207,7 @@ class LdapQuery(object): self.result = self.process_result(metadata) return self.result else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None @@ -224,14 +223,14 @@ class LdapSubQuery(LdapQuery): Defaults to doing nothing. """ pass - + def process_result(self, metadata, **kwargs): """ Overrideable method to post-process the query result. Defaults to returning the unaltered result. """ return self.result - + def get_result(self, metadata, **kwargs): """ Method to handle preparing, executing and processing the query. @@ -241,6 +240,6 @@ class LdapSubQuery(LdapQuery): self.result = self.connection.run_query(self) return self.process_result(metadata, **kwargs) else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + + Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + "No valid connection defined for query " + str(self)) return None diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 4f0ca9686..50604f5cb 100644 --- a/src/lib/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -2,8 +2,6 @@ This file stores persistent metadata for the Bcfg2 Configuration Repository. """ -__revision__ = '$Revision$' - import copy import fcntl import lxml.etree @@ -37,13 +35,21 @@ class MetadataRuntimeError(Exception): pass -class XMLMetadataConfig(object): +class XMLMetadataConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked): """Handles xml config files and all XInclude statements""" def __init__(self, metadata, watch_clients, basefile): + # we tell SingleXMLFileBacked _not_ to add a monitor for this + # file, because the main Metadata plugin has already added + # one. then we immediately set should_monitor to the proper + # value, so that XIinclude'd files get properly watched + fpath = os.path.join(metadata.data, basefile) + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, fpath, + metadata.core.fam, + should_monitor=False) + self.should_monitor = watch_clients self.metadata = metadata + self.fam = metadata.core.fam self.basefile = basefile - self.should_monitor = watch_clients - self.extras = [] self.data = None self.basedata = None self.basedir = metadata.data @@ -63,44 +69,35 @@ class XMLMetadataConfig(object): raise MetadataRuntimeError return self.basedata - def add_monitor(self, fname): - """Add a fam monitor for an included file""" - if self.should_monitor: - self.metadata.core.fam.AddMonitor("%s/%s" % (self.basedir, fname), - self.metadata) - self.extras.append(fname) - def load_xml(self): """Load changes from XML""" try: xdata = lxml.etree.parse(os.path.join(self.basedir, self.basefile), parser=Bcfg2.Server.XMLParser) except lxml.etree.XMLSyntaxError: - self.logger.error('Failed to parse %s' % (self.basefile)) + self.logger.error('Failed to parse %s' % self.basefile) return + self.extras = [] self.basedata = copy.copy(xdata) - included = [ent.get('href') for ent in \ - xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.add_monitor(name) + self._follow_xincludes(xdata=xdata) + if self.extras: try: xdata.xinclude() except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % self.basefile) + self.logger.error("Failed to process XInclude for file %s" % + self.basefile) self.data = xdata def write(self): """Write changes to xml back to disk.""" - self.write_xml("%s/%s" % (self.basedir, self.basefile), + self.write_xml(os.path.join(self.basedir, self.basefile), self.basedata) def write_xml(self, fname, xmltree): """Write changes to xml back to disk.""" tmpfile = "%s.new" % fname try: - datafile = open("%s" % tmpfile, 'w') + datafile = open(tmpfile, 'w') except IOError: e = sys.exc_info()[1] self.logger.error("Failed to write %s: %s" % (tmpfile, e)) @@ -116,30 +113,30 @@ class XMLMetadataConfig(object): datafile.write(newcontents) except: fcntl.lockf(fd, fcntl.LOCK_UN) - self.logger.error("Metadata: Failed to write new xml data to %s" % tmpfile, exc_info=1) - os.unlink("%s" % tmpfile) + self.logger.error("Metadata: Failed to write new xml data to %s" % + tmpfile, exc_info=1) + os.unlink(tmpfile) raise MetadataRuntimeError datafile.close() # check if clients.xml is a symlink - xmlfile = "%s" % fname - if os.path.islink(xmlfile): - xmlfile = os.readlink(xmlfile) + if os.path.islink(fname): + fname = os.readlink(fname) try: - os.rename("%s" % tmpfile, xmlfile) + os.rename(tmpfile, fname) except: self.logger.error("Metadata: Failed to rename %s" % tmpfile) raise MetadataRuntimeError def find_xml_for_xpath(self, xpath): - """Find and load xml data containing the xpath query""" + """Find and load xml file containing the xpath query""" if self.pseudo_monitor: # Reload xml if we don't have a real monitor self.load_xml() cli = self.basedata.xpath(xpath) if len(cli) > 0: - return {'filename': "%s/%s" % (self.basedir, self.basefile), + return {'filename': os.path.join(self.basedir, self.basefile), 'xmltree': self.basedata, 'xquery': cli} else: @@ -224,7 +221,6 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Metadata, Bcfg2.Server.Plugin.Statistics): """This class contains data for bcfg2 server metadata.""" - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' name = "Metadata" sort_order = 500 @@ -235,8 +231,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Statistics.__init__(self) if watch_clients: try: - core.fam.AddMonitor("%s/%s" % (self.data, "groups.xml"), self) - core.fam.AddMonitor("%s/%s" % (self.data, "clients.xml"), self) + core.fam.AddMonitor(os.path.join(self.data, "groups.xml"), self) + core.fam.AddMonitor(os.path.join(self.data, "clients.xml"), self) except: print("Unable to add file monitor for groups.xml or clients.xml") raise Bcfg2.Server.Plugin.PluginInitError @@ -277,272 +273,242 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, @classmethod def init_repo(cls, repo, groups, os_selection, clients): - path = '%s/%s' % (repo, cls.name) + path = os.path.join(repo, cls.name) os.makedirs(path) - open("%s/Metadata/groups.xml" % - repo, "w").write(groups % os_selection) - open("%s/Metadata/clients.xml" % - repo, "w").write(clients % socket.getfqdn()) + open(os.path.join(repo, "Metadata", "groups.xml"), + "w").write(groups % os_selection) + open(os.path.join(repo, "Metadata", "clients.xml"), + "w").write(clients % socket.getfqdn()) def get_groups(self): '''return groups xml tree''' - groups_tree = lxml.etree.parse(self.data + "/groups.xml", + groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"), parser=Bcfg2.Server.XMLParser) root = groups_tree.getroot() return root - def search_group(self, group_name, tree): - """Find a group.""" - for node in tree.findall("//Group"): - if node.get("name") == group_name: + def _search_xdata(self, tag, name, tree, alias=False): + for node in tree.findall("//%s" % tag): + if node.get("name") == name: return node - for child in node: - if child.tag == "Alias" and child.attrib["name"] == group_name: - return node + elif alias: + for child in node: + if (child.tag == "Alias" and + child.attrib["name"] == name): + return node return None - def add_group(self, group_name, attribs): - """Add group to groups.xml.""" + def search_group(self, group_name, tree): + """Find a group.""" + return self._search_xdata("Group", group_name, tree) - node = self.search_group(group_name, self.groups_xml.xdata) + def search_bundle(self, bundle_name, tree): + """Find a bundle.""" + return self._search_xdata("Bundle", bundle_name, tree) + + def search_client(self, client_name, tree): + return self._search_xdata("Client", client_name, tree, alias=True) + + def _add_xdata(self, config, tag, name, attribs=None, alias=False): + node = self._search_xdata(tag, name, config.xdata, alias=alias) if node != None: - self.logger.error("Group \"%s\" already exists" % (group_name)) + self.logger.error("%s \"%s\" already exists" % (tag, name)) raise MetadataConsistencyError + element = lxml.etree.SubElement(config.base_xdata.getroot(), + tag, name=name) + if attribs: + for key, val in list(attribs.items()): + element.set(key, val) + config.write() - element = lxml.etree.SubElement(self.groups_xml.base_xdata.getroot(), - "Group", name=group_name) - for key, val in list(attribs.items()): - element.set(key, val) - self.groups_xml.write() + def add_group(self, group_name, attribs): + """Add group to groups.xml.""" + return self._add_xdata(self.groups_xml, "Group", group_name, + attribs=attribs) - def update_group(self, group_name, attribs): - """Update a groups attributes.""" - node = self.search_group(group_name, self.groups_xml.xdata) + def add_bundle(self, bundle_name): + """Add bundle to groups.xml.""" + return self._add_xdata(self.groups_xml, "Bundle", bundle_name) + + def add_client(self, client_name, attribs): + """Add client to clients.xml.""" + return self._add_xdata(self.clients_xml, "Client", client_name, + attribs=attribs, alias=True) + + def _update_xdata(self, config, tag, name, attribs, alias=False): + node = self._search_xdata(tag, name, config.xdata, alias=alias) if node == None: - self.logger.error("Group \"%s\" does not exist" % (group_name)) + self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise MetadataConsistencyError - xdict = self.groups_xml.find_xml_for_xpath('.//Group[@name="%s"]' % (node.get('name'))) + xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % + (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding group") + self.logger.error("Unexpected error finding %s \"%s\"" % + (tag, name)) raise MetadataConsistencyError - for key, val in list(attribs.items()): xdict['xquery'][0].set(key, val) - self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) + config.write_xml(xdict['filename'], xdict['xmltree']) - def remove_group(self, group_name): - """Remove a group.""" - node = self.search_group(group_name, self.groups_xml.xdata) + def update_group(self, group_name, attribs): + """Update a groups attributes.""" + return self._update_xdata(self.groups_xml, "Group", group_name, attribs) + + def update_client(self, client_name, attribs): + """Update a clients attributes.""" + return self._update_xdata(self.clients_xml, "Client", client_name, + attribs, alias=True) + + def _remove_xdata(self, config, tag, name, alias=False): + node = self._search_xdata(tag, name, config.xdata) if node == None: - self.logger.error("Group \"%s\" does not exist" % (group_name)) + self.logger.error("%s \"%s\" does not exist" % (tag, name)) raise MetadataConsistencyError - xdict = self.groups_xml.find_xml_for_xpath('.//Group[@name="%s"]' % (node.get('name'))) + xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % + (tag, node.get('name'))) if not xdict: - self.logger.error("Unexpected error finding group") + self.logger.error("Unexpected error finding %s \"%s\"" % + (tag, name)) raise MetadataConsistencyError xdict['xquery'][0].getparent().remove(xdict['xquery'][0]) self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) - def add_bundle(self, bundle_name): - """Add bundle to groups.xml.""" - tree = lxml.etree.parse(self.data + "/groups.xml", - parser=Bcfg2.Server.XMLParser) - root = tree.getroot() - element = lxml.etree.Element("Bundle", name=bundle_name) - node = self.search_group(bundle_name, tree) - if node != None: - self.logger.error("Bundle \"%s\" already exists" % (bundle_name)) - raise MetadataConsistencyError - root.append(element) - group_tree = open(self.data + "/groups.xml", "w") - fd = group_tree.fileno() - while True: - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - continue - else: - break - tree.write(group_tree) - fcntl.lockf(fd, fcntl.LOCK_UN) - group_tree.close() + def remove_group(self, group_name): + """Remove a group.""" + return self._remove_xdata(self.groups_xml, "Group", group_name) def remove_bundle(self, bundle_name): """Remove a bundle.""" - tree = lxml.etree.parse(self.data + "/groups.xml", - parser=Bcfg2.Server.XMLParser) - root = tree.getroot() - node = self.search_group(bundle_name, tree) - if node == None: - self.logger.error("Bundle \"%s\" not found" % (bundle_name)) - raise MetadataConsistencyError - root.remove(node) - group_tree = open(self.data + "/groups.xml", "w") - fd = group_tree.fileno() - while True: - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - continue - else: - break - tree.write(group_tree) - fcntl.lockf(fd, fcntl.LOCK_UN) - group_tree.close() - - def search_client(self, client_name, tree): - """Find a client.""" - for node in tree.findall("//Client"): - if node.get("name") == client_name: - return node - for child in node: - if child.tag == "Alias" and child.attrib["name"] == client_name: - return node - return None - - def add_client(self, client_name, attribs): - """Add client to clients.xml.""" - node = self.search_client(client_name, self.clients_xml.xdata) - if node != None: - self.logger.error("Client \"%s\" already exists" % (client_name)) - raise MetadataConsistencyError - - element = lxml.etree.SubElement(self.clients_xml.base_xdata.getroot(), - "Client", name=client_name) - for key, val in list(attribs.items()): - element.set(key, val) - self.clients_xml.write() - - def update_client(self, client_name, attribs): - """Update a clients attributes.""" - node = self.search_client(client_name, self.clients_xml.xdata) - if node == None: - self.logger.error("Client \"%s\" does not exist" % (client_name)) - raise MetadataConsistencyError - - xdict = self.clients_xml.find_xml_for_xpath('.//Client[@name="%s"]' % (node.get('name'))) - if not xdict: - self.logger.error("Unexpected error finding client") - raise MetadataConsistencyError + return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) - node = xdict['xquery'][0] - [node.set(key, value) for key, value in list(attribs.items())] - self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) + def _handle_clients_xml_event(self, event): + xdata = self.clients_xml.xdata + self.clients = {} + self.aliases = {} + self.raliases = {} + self.bad_clients = {} + self.secure = [] + self.floating = [] + self.addresses = {} + self.raddresses = {} + for client in xdata.findall('.//Client'): + clname = client.get('name').lower() + if 'address' in client.attrib: + caddr = client.get('address') + if caddr in self.addresses: + self.addresses[caddr].append(clname) + else: + self.addresses[caddr] = [clname] + if clname not in self.raddresses: + self.raddresses[clname] = set() + self.raddresses[clname].add(caddr) + if 'auth' in client.attrib: + self.auth[client.get('name')] = client.get('auth', + 'cert+password') + if 'uuid' in client.attrib: + self.uuid[client.get('uuid')] = clname + if client.get('secure', 'false') == 'true': + self.secure.append(clname) + if client.get('location', 'fixed') == 'floating': + self.floating.append(clname) + if 'password' in client.attrib: + self.passwords[clname] = client.get('password') + + self.raliases[clname] = set() + for alias in client.findall('Alias'): + self.aliases.update({alias.get('name'): clname}) + self.raliases[clname].add(alias.get('name')) + if 'address' not in alias.attrib: + continue + if alias.get('address') in self.addresses: + self.addresses[alias.get('address')].append(clname) + else: + self.addresses[alias.get('address')] = [clname] + if clname not in self.raddresses: + self.raddresses[clname] = set() + self.raddresses[clname].add(alias.get('address')) + self.clients.update({clname: client.get('profile')}) + self.states['clients.xml'] = True + + def _handle_groups_xml_event(self, event): + xdata = self.groups_xml.xdata + self.public = [] + self.private = [] + self.profiles = [] + self.groups = {} + grouptmp = {} + self.categories = {} + groupseen = list() + for group in xdata.xpath('//Groups/Group'): + if group.get('name') not in groupseen: + groupseen.append(group.get('name')) + else: + self.logger.error("Metadata: Group %s defined multiply" % + group.get('name')) + grouptmp[group.get('name')] = \ + ([item.get('name') for item in group.findall('./Bundle')], + [item.get('name') for item in group.findall('./Group')]) + grouptmp[group.get('name')][1].append(group.get('name')) + if group.get('default', 'false') == 'true': + self.default = group.get('name') + if group.get('profile', 'false') == 'true': + self.profiles.append(group.get('name')) + if group.get('public', 'false') == 'true': + self.public.append(group.get('name')) + elif group.get('public', 'true') == 'false': + self.private.append(group.get('name')) + if 'category' in group.attrib: + self.categories[group.get('name')] = group.get('category') + + for group in grouptmp: + # self.groups[group] => (bundles, groups, categories) + self.groups[group] = (set(), set(), {}) + tocheck = [group] + group_cat = self.groups[group][2] + while tocheck: + now = tocheck.pop() + self.groups[group][1].add(now) + if now in grouptmp: + (bundles, groups) = grouptmp[now] + for ggg in groups: + if ggg in self.groups[group][1]: + continue + if (ggg not in self.categories or \ + self.categories[ggg] not in self.groups[group][2]): + self.groups[group][1].add(ggg) + tocheck.append(ggg) + if ggg in self.categories: + group_cat[self.categories[ggg]] = ggg + elif ggg in self.categories: + self.logger.info("Group %s: %s cat-suppressed %s" % \ + (group, + group_cat[self.categories[ggg]], + ggg)) + [self.groups[group][0].add(bund) for bund in bundles] + self.states['groups.xml'] = True def HandleEvent(self, event): """Handle update events for data files.""" if self.clients_xml.HandleEvent(event): - xdata = self.clients_xml.xdata - self.clients = {} - self.aliases = {} - self.raliases = {} - self.bad_clients = {} - self.secure = [] - self.floating = [] - self.addresses = {} - self.raddresses = {} - for client in xdata.findall('.//Client'): - clname = client.get('name').lower() - if 'address' in client.attrib: - caddr = client.get('address') - if caddr in self.addresses: - self.addresses[caddr].append(clname) - else: - self.addresses[caddr] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(caddr) - if 'auth' in client.attrib: - self.auth[client.get('name')] = client.get('auth', - 'cert+password') - if 'uuid' in client.attrib: - self.uuid[client.get('uuid')] = clname - if client.get('secure', 'false') == 'true': - self.secure.append(clname) - if client.get('location', 'fixed') == 'floating': - self.floating.append(clname) - if 'password' in client.attrib: - self.passwords[clname] = client.get('password') - for alias in [alias for alias in client.findall('Alias')\ - if 'address' in alias.attrib]: - if alias.get('address') in self.addresses: - self.addresses[alias.get('address')].append(clname) - else: - self.addresses[alias.get('address')] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(alias.get('address')) - self.clients.update({clname: client.get('profile')}) - [self.aliases.update({alias.get('name'): clname}) \ - for alias in client.findall('Alias')] - self.raliases[clname] = set() - [self.raliases[clname].add(alias.get('name')) for alias \ - in client.findall('Alias')] - self.states['clients.xml'] = True + self._handle_clients_xml_event(event) elif self.groups_xml.HandleEvent(event): - xdata = self.groups_xml.xdata - self.public = [] - self.private = [] - self.profiles = [] - self.groups = {} - grouptmp = {} - self.categories = {} - groupseen = list() - for group in xdata.xpath('//Groups/Group'): - if group.get('name') not in groupseen: - groupseen.append(group.get('name')) - else: - self.logger.error("Metadata: Group %s defined multiply" % (group.get('name'))) - grouptmp[group.get('name')] = tuple([[item.get('name') for item in group.findall(spec)] - for spec in ['./Bundle', './Group']]) - grouptmp[group.get('name')][1].append(group.get('name')) - if group.get('default', 'false') == 'true': - self.default = group.get('name') - if group.get('profile', 'false') == 'true': - self.profiles.append(group.get('name')) - if group.get('public', 'false') == 'true': - self.public.append(group.get('name')) - elif group.get('public', 'true') == 'false': - self.private.append(group.get('name')) - if 'category' in group.attrib: - self.categories[group.get('name')] = group.get('category') - for group in grouptmp: - # self.groups[group] => (bundles, groups, categories) - self.groups[group] = (set(), set(), {}) - tocheck = [group] - group_cat = self.groups[group][2] - while tocheck: - now = tocheck.pop() - self.groups[group][1].add(now) - if now in grouptmp: - (bundles, groups) = grouptmp[now] - for ggg in [ggg for ggg in groups if ggg not in self.groups[group][1]]: - if ggg not in self.categories or \ - self.categories[ggg] not in self.groups[group][2]: - self.groups[group][1].add(ggg) - tocheck.append(ggg) - if ggg in self.categories: - group_cat[self.categories[ggg]] = ggg - elif ggg in self.categories: - self.logger.info("Group %s: %s cat-suppressed %s" % \ - (group, - group_cat[self.categories[ggg]], - ggg)) - [self.groups[group][0].add(bund) for bund in bundles] - self.states['groups.xml'] = True + self._handle_groups_xml_event(event) + if False not in list(self.states.values()): # check that all client groups are real and complete real = list(self.groups.keys()) for client in list(self.clients.keys()): if self.clients[client] not in self.profiles: - self.logger.error("Client %s set as nonexistent or incomplete group %s" \ - % (client, self.clients[client])) - self.logger.error("Removing client mapping for %s" % (client)) + self.logger.error("Client %s set as nonexistent or " + "incomplete group %s" % + (client, self.clients[client])) + self.logger.error("Removing client mapping for %s" % client) self.bad_clients[client] = self.clients[client] del self.clients[client] for bclient in list(self.bad_clients.keys()): if self.bad_clients[bclient] in self.profiles: - self.logger.info("Restored profile mapping for client %s" % bclient) + self.logger.info("Restored profile mapping for client %s" % + bclient) self.clients[bclient] = self.bad_clients[bclient] del self.bad_clients[bclient] @@ -552,42 +518,37 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, if False in list(self.states.values()): raise MetadataRuntimeError if profile not in self.public: - self.logger.error("Failed to set client %s to private group %s" % (client, profile)) + self.logger.error("Failed to set client %s to private group %s" % + (client, profile)) raise MetadataConsistencyError if client in self.clients: - self.logger.info("Changing %s group from %s to %s" % (client, self.clients[client], profile)) - xdict = self.clients_xml.find_xml_for_xpath('.//Client[@name="%s"]' % (client)) - if not xdict: - self.logger.error("Metadata: Unable to update profile for client %s. Use of Xinclude?" % client) - raise MetadataConsistencyError - xdict['xquery'][0].set('profile', profile) - self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) + self.logger.info("Changing %s group from %s to %s" % + (client, self.clients[client], profile)) + self.update_client(client, dict(profile=profile)) else: - self.logger.info("Creating new client: %s, profile %s" % \ + self.logger.info("Creating new client: %s, profile %s" % (client, profile)) if addresspair in self.session_cache: # we are working with a uuid'd client - lxml.etree.SubElement(self.clients_xml.base_xdata.getroot(), - 'Client', - name=self.session_cache[addresspair][1], - uuid=client, profile=profile, - address=addresspair[0]) + self.add_client(self.session_cache[addresspair][1], + dict(uuid=client, profile=profile, + address=addresspair[0])) else: - lxml.etree.SubElement(self.clients_xml.base_xdata.getroot(), - 'Client', name=client, - profile=profile) + self.add_client(client, dict(profile=profile)) self.clients[client] = profile self.clients_xml.write() def resolve_client(self, addresspair, cleanup_cache=False): """Lookup address locally or in DNS to get a hostname.""" if addresspair in self.session_cache: - # client _was_ cached, so there can be some expired entries - # we need to clean them up to avoid potentially infinite memory swell + # client _was_ cached, so there can be some expired + # entries. we need to clean them up to avoid potentially + # infinite memory swell cache_ttl = 90 if cleanup_cache: - # remove entries for this client's IP address with _any_ port numbers - # - perhaps a priority queue could be faster? + # remove entries for this client's IP address with + # _any_ port numbers - perhaps a priority queue could + # be faster? curtime = time.time() for addrpair in self.session_cache.keys(): if addresspair[0] == addrpair[0]: @@ -595,13 +556,18 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, if curtime - stamp > cache_ttl: del self.session_cache[addrpair] # return the cached data - (stamp, uuid) = self.session_cache[addresspair] - if time.time() - stamp < cache_ttl: - return self.session_cache[addresspair][1] + try: + (stamp, uuid) = self.session_cache[addresspair] + if time.time() - stamp < cache_ttl: + return self.session_cache[addresspair][1] + except KeyError: + # we cleaned all cached data for this client in cleanup_cache + pass address = addresspair[0] if address in self.addresses: if len(self.addresses[address]) != 1: - self.logger.error("Address %s has multiple reverse assignments; a uuid must be used" % (address)) + self.logger.error("Address %s has multiple reverse assignments; " + "a uuid must be used" % (address)) raise MetadataConsistencyError return self.addresses[address][0] try: @@ -626,7 +592,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (bundles, groups, categories) = self.groups[profile] else: if self.default == None: - self.logger.error("Cannot set group for client %s; no default group set" % (client)) + self.logger.error("Cannot set group for client %s; " + "no default group set" % client) raise MetadataConsistencyError self.set_profile(client, self.default, (None, None)) profile = self.default @@ -641,7 +608,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, password = self.passwords[client] else: password = None - uuids = [item for item, value in list(self.uuid.items()) if value == client] + uuids = [item for item, value in list(self.uuid.items()) + if value == client] if uuids: uuid = uuids[0] else: @@ -655,7 +623,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, [newgroups.add(g) for g in ngroups if g not in newgroups] newcategories.update(ncategories) return ClientMetadata(client, profile, newgroups, newbundles, aliases, - addresses, newcategories, uuid, password, self.query) + addresses, newcategories, uuid, password, + self.query) def get_all_group_names(self): all_groups = set() @@ -679,22 +648,27 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def merge_additional_groups(self, imd, groups): for group in groups: - if group in self.categories and \ - self.categories[group] in imd.categories: + if (group in self.categories and + self.categories[group] in imd.categories): continue - nb, ng, _ = self.groups.get(group, (list(), [group], dict())) - for b in nb: - if b not in imd.bundles: - imd.bundles.add(b) - for g in ng: - if g not in imd.groups: - if g in self.categories and \ - self.categories[g] in imd.categories: + newbundles, newgroups, _ = self.groups.get(group, + (list(), + [group], + dict())) + for newbundle in newbundles: + if newbundle not in imd.bundles: + imd.bundles.add(newbundle) + for newgroup in newgroups: + if newgroup not in imd.groups: + if (newgroup in self.categories and + self.categories[newgroup] in imd.categories): continue - if g in self.private: - self.logger.error("Refusing to add dynamic membership in private group %s for client %s" % (g, imd.hostname)) + if newgroup in self.private: + self.logger.error("Refusing to add dynamic membership " + "in private group %s for client %s" % + (newgroup, imd.hostname)) continue - imd.groups.add(g) + imd.groups.add(newgroup) def merge_additional_data(self, imd, source, data): if not hasattr(imd, source): @@ -709,18 +683,19 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, return True if address in self.addresses: if client in self.addresses[address]: - self.debug_log("Client %s matches address %s" % (client, address)) + self.debug_log("Client %s matches address %s" % + (client, address)) return True else: - self.logger.error("Got request for non-float client %s from %s" \ - % (client, address)) + self.logger.error("Got request for non-float client %s from %s" % + (client, address)) return False resolved = self.resolve_client(addresspair) if resolved.lower() == client.lower(): return True else: - self.logger.error("Got request for %s from incorrect address %s" \ - % (client, address)) + self.logger.error("Got request for %s from incorrect address %s" % + (client, address)) self.logger.error("Resolved to %s" % resolved) return False @@ -738,7 +713,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, try: client = self.resolve_client(address) except MetadataConsistencyError: - self.logger.error("Client %s failed to resolve; metadata problem" % (address[0])) + self.logger.error("Client %s failed to resolve; metadata problem" + % address[0]) return False else: id_method = 'uuid' @@ -770,10 +746,12 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, if client not in self.passwords: if client in self.secure: - self.logger.error("Client %s in secure mode but has no password" % (address[0])) + self.logger.error("Client %s in secure mode but has no password" + % address[0]) return False if password != self.password: - self.logger.error("Client %s used incorrect global password" % (address[0])) + self.logger.error("Client %s used incorrect global password" % + address[0]) return False if client not in self.secure: if client in self.passwords: @@ -781,14 +759,14 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, else: plist = [self.password] if password not in plist: - self.logger.error("Client %s failed to use either allowed password" % \ - (address[0])) + self.logger.error("Client %s failed to use either allowed " + "password" % address[0]) return False else: # client in secure mode and has a client password if password != self.passwords[client]: - self.logger.error("Client %s failed to use client password in secure mode" % \ - (address[0])) + self.logger.error("Client %s failed to use client password in " + "secure mode" % address[0]) return False # populate the session cache if user.decode('utf-8') != 'root': @@ -799,13 +777,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, """Hook into statistics interface to toggle clients in bootstrap mode.""" client = meta.hostname if client in self.auth and self.auth[client] == 'bootstrap': - self.logger.info("Asserting client %s auth mode to cert" % client) - xdict = self.clients_xml.find_xml_for_xpath('.//Client[@name="%s"]' % (client)) - if not xdict: - self.logger.error("Metadata: Unable to update profile for client %s. Use of Xinclude?" % client) - raise MetadataConsistencyError - xdict['xquery'][0].set('auth', 'cert') - self.clients_xml.write_xml(xdict['filename'], xdict['xmltree']) + self.update_client(client, dict(auth='cert')) def viz(self, hosts, bundles, key, only_client, colors): """Admin mode viz support.""" @@ -820,8 +792,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, def include_group(group): return not only_client or group in clientmeta.groups - - groups_tree = lxml.etree.parse(self.data + "/groups.xml", + + groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml"), parser=Bcfg2.Server.XMLParser) try: groups_tree.xinclude() @@ -830,7 +802,7 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, (dest, sys.exc_info()[1])) groups = groups_tree.getroot() categories = {'default': 'grey83'} - viz_str = "" + viz_str = [] egroups = groups.findall("Group") + groups.findall('.//Groups/Group') for group in egroups: if not group.get('category') in categories: @@ -850,10 +822,10 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, instances[profile] = [client] for profile, clist in list(instances.items()): clist.sort() - viz_str += '''\t"%s-instances" [ label="%s", shape="record" ];\n''' \ - % (profile, '|'.join(clist)) - viz_str += '''\t"%s-instances" -> "group-%s";\n''' \ - % (profile, profile) + viz_str.append('"%s-instances" [ label="%s", shape="record" ];' % + (profile, '|'.join(clist))) + viz_str.append('"%s-instances" -> "group-%s";' % + (profile, profile)) if bundles: bundles = [] [bundles.append(bund.get('name')) \ @@ -862,8 +834,8 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, and include_bundle(bund.get('name'))] bundles.sort() for bundle in bundles: - viz_str += '''\t"bundle-%s" [ label="%s", shape="septagon"];\n''' \ - % (bundle, bundle) + viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' % + (bundle, bundle)) gseen = [] for group in egroups: if group.get('profile', 'false') == 'true': @@ -872,24 +844,25 @@ class Metadata(Bcfg2.Server.Plugin.Plugin, style = "filled" gseen.append(group.get('name')) if include_group(group.get('name')): - viz_str += '\t"group-%s" [label="%s", style="%s", fillcolor=%s];\n' % \ - (group.get('name'), group.get('name'), style, group.get('color')) + viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' % + (group.get('name'), group.get('name'), style, + group.get('color'))) if bundles: for bundle in group.findall('Bundle'): - viz_str += '\t"group-%s" -> "bundle-%s";\n' % \ - (group.get('name'), bundle.get('name')) - gfmt = '\t"group-%s" [label="%s", style="filled", fillcolor="grey83"];\n' + viz_str.append('"group-%s" -> "bundle-%s";' % + (group.get('name'), bundle.get('name'))) + gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];' for group in egroups: for parent in group.findall('Group'): if parent.get('name') not in gseen and include_group(parent.get('name')): - viz_str += gfmt % (parent.get('name'), parent.get('name')) + viz_str.append(gfmt % (parent.get('name'), + parent.get('name'))) gseen.append(parent.get("name")) if include_group(group.get('name')): - viz_str += '\t"group-%s" -> "group-%s" ;\n' % \ - (group.get('name'), parent.get('name')) + viz_str.append('"group-%s" -> "group-%s";' % + (group.get('name'), parent.get('name'))) if key: for category in categories: - viz_str += '''\t"''' + category + '''" [label="''' + category + \ - '''", shape="record", style="filled", fillcolor=''' + \ - categories[category] + '''];\n''' - return viz_str + viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' % + (category, category, categories[category])) + return "\n".join("\t" + s for s in viz_str) diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py index 627a83b81..4e8b09f30 100644 --- a/src/lib/Server/Plugins/NagiosGen.py +++ b/src/lib/Bcfg2/Server/Plugins/NagiosGen.py @@ -17,17 +17,30 @@ line_fmt = '\t%-32s %s' class NagiosGenConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, Bcfg2.Server.Plugin.StructFile): def __init__(self, filename, fam): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) + # create config.xml if missing + if not os.path.exists(filename): + LOGGER.warning("NagiosGen: %s missing. " + "Creating empty one for you." % filename) + f = open(filename, "w") + f.write("<NagiosGen></NagiosGen>") + f.close() + + try: + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, + filename, + fam) + except OSError: + LOGGER.error("NagiosGen: Failed to read configuration file: %s" % err) + raise Bcfg2.Server.Plugin.PluginInitError(msg) Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - + class NagiosGen(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Generator): """NagiosGen is a Bcfg2 plugin that dynamically generates Nagios configuration file based on Bcfg2 data. """ name = 'NagiosGen' - __version__ = '0.7' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): @@ -137,14 +150,14 @@ class NagiosGen(Bcfg2.Server.Plugin.Plugin, group_data = [] for host in host_configs: host_data.append(open(host, 'r').read()) - + for group in group_configs: group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group) if "\n".join(host_data).find(group_name) != -1: groupfile = open(group, 'r') group_data.append(groupfile.read()) groupfile.close() - + entry.text = "%s\n\n%s" % ("\n".join(group_data), "\n".join(host_data)) [entry.attrib.__setitem__(key, value) for (key, value) in list(self.server_attrib.items())] diff --git a/src/lib/Server/Plugins/Ohai.py b/src/lib/Bcfg2/Server/Plugins/Ohai.py index 5fff20d98..5fff20d98 100644 --- a/src/lib/Server/Plugins/Ohai.py +++ b/src/lib/Bcfg2/Server/Plugins/Ohai.py diff --git a/src/lib/Server/Plugins/Packages/Apt.py b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py index f76bf7fa1..49e9d417b 100644 --- a/src/lib/Server/Plugins/Packages/Apt.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Apt.py @@ -50,10 +50,9 @@ class AptSource(Source): def read_files(self): bdeps = dict() bprov = dict() + depfnames = ['Depends', 'Pre-Depends'] if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] + depfnames.append('Recommends') for fname in self.files: if not self.rawurl: barch = [x @@ -77,6 +76,8 @@ class AptSource(Source): pkgname = words[1].strip().rstrip() self.pkgnames.add(pkgname) bdeps[barch][pkgname] = [] + elif words[0] == 'Essential' and self.essential: + self.essentialpkgs.add(pkgname) elif words[0] in depfnames: vindex = 0 for dep in words[1].split(','): diff --git a/src/lib/Server/Plugins/Packages/Collection.py b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py index 32eeda1ec..ac78ea0fc 100644 --- a/src/lib/Server/Plugins/Packages/Collection.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Collection.py @@ -1,3 +1,4 @@ +import sys import copy import logging import Bcfg2.Server.Plugin @@ -18,9 +19,16 @@ except ImportError: # startup. (It would also prevent machines that change groups from # working properly; e.g., if you reinstall a machine with a new OS, # then returning a cached Collection object would give the wrong -# sources to that client.) +# sources to that client.) These are keyed by the collection +# cachekey, a unique key identifying the collection by its _config_, +# which could be shared among multiple clients. collections = dict() +# cache mapping of hostname -> collection cachekey. this _is_ used to +# return a Collection object when one is requested, so each entry is +# very short-lived -- it's purged at the end of each client run. +clients = dict() + class Collection(Bcfg2.Server.Plugin.Debuggable): def __init__(self, metadata, sources, basepath, debug=False): """ don't call this directly; use the factory function """ @@ -32,11 +40,11 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): self.virt_pkgs = dict() try: - self.config = sources[0].config + self.setup = sources[0].setup self.cachepath = sources[0].basepath self.ptype = sources[0].ptype except IndexError: - self.config = None + self.setup = None self.cachepath = None self.ptype = "unknown" @@ -44,13 +52,38 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): @property def cachekey(self): - return md5(self.get_config()).hexdigest() + return md5(self.sourcelist().encode(Bcfg2.Server.Plugin.encoding)).hexdigest() def get_config(self): - self.logger.error("Packages: Cannot generate config for host with " - "multiple source types (%s)" % self.metadata.hostname) + self.logger.error("Packages: Cannot generate config for host %s with " + "no sources or multiple source types" % + self.metadata.hostname) return "" + def sourcelist(self): + srcs = [] + for source in self.sources: + # get_urls() loads url_map as a side-effect + source.get_urls() + for url_map in source.url_map: + reponame = source.get_repo_name(url_map) + srcs.append("Name: %s" % reponame) + srcs.append(" Type: %s" % source.ptype) + if url_map['url']: + srcs.append(" URL: %s" % url_map['url']) + elif url_map['rawurl']: + srcs.append(" RAWURL: %s" % url_map['rawurl']) + if source.gpgkeys: + srcs.append(" GPG Key(s): %s" % ", ".join(source.gpgkeys)) + else: + srcs.append(" GPG Key(s): None") + if len(source.blacklist): + srcs.append(" Blacklist: %s" % ", ".join(source.blacklist)) + if len(source.whitelist): + srcs.append(" Whitelist: %s" % ", ".join(source.whitelist)) + srcs.append("") + return "\n".join(srcs) + def get_relevant_groups(self): groups = [] for source in self.sources: @@ -97,6 +130,12 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): return source.get_deps(self.metadata, package) return [] + def get_essential(self): + essential = set() + for source in self.sources: + essential |= source.essentialpkgs + return essential + def get_provides(self, package): for source in self.sources: providers = source.get_provides(self.metadata, package) @@ -284,9 +323,33 @@ class Collection(Bcfg2.Server.Plugin.Debuggable): def sort(self, cmp=None, key=None, reverse=False): self.sources.sort(cmp, key, reverse) +def get_collection_class(source_type): + modname = "Bcfg2.Server.Plugins.Packages.%s" % source_type.title() + + try: + module = sys.modules[modname] + except KeyError: + try: + module = __import__(modname).Server.Plugins.Packages + except ImportError: + msg = "Packages: Unknown source type %s" % source_type + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + try: + cclass = getattr(module, source_type.title() + "Collection") + except AttributeError: + msg = "Packages: No collection class found for %s sources" % source_type + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + return cclass + def clear_cache(): global collections + global clients collections = dict() + clients = dict() def factory(metadata, sources, basepath, debug=False): global collections @@ -295,7 +358,10 @@ def factory(metadata, sources, basepath, debug=False): # if sources.xml has not received a FAM event yet, defer; # instantiate a dummy Collection object return Collection(metadata, [], basepath) - + + if metadata.hostname in clients: + return collections[clients[metadata.hostname]] + sclasses = set() relevant = list() @@ -320,24 +386,16 @@ def factory(metadata, sources, basepath, debug=False): metadata.hostname) cclass = Collection else: - stype = sclasses.pop().__name__.replace("Source", "") - try: - module = \ - getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cclass = getattr(module, "%sCollection" % stype.title()) - except ImportError: - logger.error("Packages: Unknown source type %s" % stype) - except AttributeError: - logger.warning("Packages: No collection class found for %s sources" - % stype) + cclass = get_collection_class(sclasses.pop().__name__.replace("Source", + "")) if debug: logger.error("Packages: Using %s for Collection of sources for %s" % (cclass.__name__, metadata.hostname)) collection = cclass(metadata, relevant, basepath, debug=debug) - collections[metadata.hostname] = collection + ckey = collection.cachekey + clients[metadata.hostname] = ckey + collections[ckey] = collection return collection diff --git a/src/lib/Server/Plugins/Packages/Pac.py b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py index 9db6b0535..99a090739 100644 --- a/src/lib/Server/Plugins/Packages/Pac.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Pac.py @@ -6,7 +6,7 @@ from Bcfg2.Server.Plugins.Packages.Source import Source class PacCollection(Collection): def get_group(self, group): - self.logger.warning("Packages: Package groups are not supported by APT") + self.logger.warning("Packages: Package groups are not supported by Pacman") return [] class PacSource(Source): @@ -51,10 +51,9 @@ class PacSource(Source): bdeps = dict() bprov = dict() + depfnames = ['Depends', 'Pre-Depends'] if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] + depfnames.append('Recommends') for fname in self.files: if not self.rawurl: @@ -63,7 +62,7 @@ class PacSource(Source): # RawURL entries assume that they only have one <Arch></Arch> # element and that it is the architecture of the source. barch = self.arches[0] - + if barch not in bdeps: bdeps[barch] = dict() bprov[barch] = dict() diff --git a/src/lib/Server/Plugins/Packages/PackagesSources.py b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py index d399838ae..e2af287db 100644 --- a/src/lib/Server/Plugins/Packages/PackagesSources.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/PackagesSources.py @@ -8,8 +8,8 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, Bcfg2.Server.Plugin.StructFile, Bcfg2.Server.Plugin.Debuggable): __identifier__ = None - - def __init__(self, filename, cachepath, fam, packages, config): + + def __init__(self, filename, cachepath, fam, packages, setup): Bcfg2.Server.Plugin.Debuggable.__init__(self) try: Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, @@ -24,7 +24,7 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, raise Bcfg2.Server.Plugin.PluginInitError(msg) Bcfg2.Server.Plugin.StructFile.__init__(self, filename) self.cachepath = cachepath - self.config = config + self.setup = setup if not os.path.exists(self.cachepath): # create cache directory if needed try: @@ -43,10 +43,20 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, def HandleEvent(self, event=None): Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event) - if event.filename != self.name: - self.parsed.add(os.path.basename(event.filename)) + if event and event.filename != self.name: + for fname in self.extras: + fpath = None + if fname.startswith("/"): + fpath = os.path.abspath(fname) + else: + fpath = \ + os.path.abspath(os.path.join(os.path.dirname(self.name), + fname)) + if fpath == os.path.abspath(event.filename): + self.parsed.add(fname) + break - if self.config.loaded and self.loaded: + if self.loaded: self.logger.info("Reloading Packages plugin") self.pkg_obj.Reload() @@ -77,11 +87,12 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, stype.title()) cls = getattr(module, "%sSource" % stype.title()) except (ImportError, AttributeError): - self.logger.error("Packages: Unknown source type %s" % stype) + ex = sys.exc_info()[1] + self.logger.error("Packages: Unknown source type %s (%s)" % (stype, ex)) return None try: - source = cls(self.cachepath, xsource, self.config) + source = cls(self.cachepath, xsource, self.setup) except SourceInitError: err = sys.exc_info()[1] self.logger.error("Packages: %s" % err) @@ -96,7 +107,7 @@ class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, return "PackagesSources: %s" % repr(self.entries) def __str__(self): - return "PackagesSources: %s" % str(self.entries) + return "PackagesSources: %s sources" % len(self.entries) def __len__(self): return len(self.entries) diff --git a/src/lib/Server/Plugins/Packages/Source.py b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py index 627ff561d..332d0c488 100644 --- a/src/lib/Server/Plugins/Packages/Source.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Source.py @@ -35,11 +35,12 @@ class Source(Bcfg2.Server.Plugin.Debuggable): genericrepo_re = re.compile('https?://.*?/([^/]+)/?$') basegroups = [] - def __init__(self, basepath, xsource, config): + def __init__(self, basepath, xsource, setup): Bcfg2.Server.Plugin.Debuggable.__init__(self) self.basepath = basepath self.xsource = xsource - self.config = config + self.setup = setup + self.essentialpkgs = set() try: self.version = xsource.find('Version').text @@ -64,8 +65,9 @@ class Source(Bcfg2.Server.Plugin.Debuggable): self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] + self.essential = xsource.get('essential', 'true').lower() == 'true' self.recommended = xsource.get('recommended', 'false').lower() == 'true' - + self.rawurl = xsource.get('rawurl', '') if self.rawurl and not self.rawurl.endswith("/"): self.rawurl += "/" @@ -122,7 +124,7 @@ class Source(Bcfg2.Server.Plugin.Debuggable): if os.path.exists(self.cachefile): try: self.load_state() - should_read = False + should_read = False except: self.logger.error("Packages: Cachefile %s load failed; " "falling back to file read" % self.cachefile) @@ -175,6 +177,9 @@ class Source(Bcfg2.Server.Plugin.Debuggable): else: return self.__class__.__name__ + def __repr__(self): + return str(self) + def get_urls(self): return [] urls = property(get_urls) @@ -282,8 +287,8 @@ class Source(Bcfg2.Server.Plugin.Debuggable): if not found_arch: return False - if self.config.getboolean("global", "magic_groups", - default=True) == False: + if not self.setup.cfp.getboolean("packages", "magic_groups", + default=True): return True else: for group in self.basegroups: diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py index 7fe169fc8..87909dc4c 100644 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/Yum.py @@ -14,7 +14,6 @@ from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, URLError, ConfigParser from Bcfg2.Server.Plugins.Packages.Collection import Collection from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ fetch_url -from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig logger = logging.getLogger(__name__) @@ -49,7 +48,7 @@ PULPSERVER = None PULPCONFIG = None -def _setup_pulp(config): +def _setup_pulp(setup): global PULPSERVER, PULPCONFIG if not has_pulp: msg = "Packages: Cannot create Pulp collection: Pulp libraries not found" @@ -58,8 +57,8 @@ def _setup_pulp(config): if PULPSERVER is None: try: - username = config.get("pulp", "username") - password = config.get("pulp", "password") + username = setup.cfp.get("packages:pulp", "username") + password = setup.cfp.get("packages:pulp", "password") except ConfigParser.NoSectionError: msg = "Packages: No [pulp] section found in Packages/packages.conf" logger.error(msg) @@ -71,7 +70,7 @@ def _setup_pulp(config): PULPCONFIG = ConsumerConfig() serveropts = PULPCONFIG.server - + PULPSERVER = server.PulpServer(serveropts['host'], int(serveropts['port']), serveropts['scheme'], @@ -85,22 +84,17 @@ class YumCollection(Collection): # options that are included in the [yum] section but that should # not be included in the temporary yum.conf we write out option_blacklist = ["use_yum_libraries", "helper"] - + def __init__(self, metadata, sources, basepath, debug=False): Collection.__init__(self, metadata, sources, basepath, debug=debug) self.keypath = os.path.join(self.basepath, "keys") - if len(sources): - self.config = sources[0].config - else: - self.config = PackageConfig('Packages') - if self.use_yum: self.cachefile = os.path.join(self.cachepath, "cache-%s" % self.cachekey) if not os.path.exists(self.cachefile): os.mkdir(self.cachefile) - + self.configdir = os.path.join(self.basepath, "yum") if not os.path.exists(self.configdir): os.mkdir(self.configdir) @@ -108,14 +102,14 @@ class YumCollection(Collection): "%s-yum.conf" % self.cachekey) self.write_config() if has_pulp and self.has_pulp_sources: - _setup_pulp(self.config) + _setup_pulp(self.setup) self._helper = None @property def helper(self): try: - return self.config.get("yum", "helper") + return self.setup.cfp.get("packages:yum", "helper") except: pass @@ -131,8 +125,9 @@ class YumCollection(Collection): @property def use_yum(self): - return has_yum and self.config.getboolean("yum", "use_yum_libraries", - default=False) + return has_yum and self.setup.cfp.getboolean("packages:yum", + "use_yum_libraries", + default=False) @property def has_pulp_sources(self): @@ -146,16 +141,16 @@ class YumCollection(Collection): if not os.path.exists(self.cfgfile): yumconf = self.get_config(raw=True) yumconf.add_section("main") - + mainopts = dict(cachedir=self.cachefile, keepcache="0", sslverify="0", debuglevel="0", reposdir="/dev/null") try: - for opt in self.config.options("yum"): + for opt in self.setup.cfp.options("packages:yum"): if opt not in self.option_blacklist: - mainopts[opt] = self.config.get("yum", opt) + mainopts[opt] = self.setup.cfp.get("packages:yum", opt) except ConfigParser.NoSectionError: pass @@ -186,7 +181,7 @@ class YumCollection(Collection): else: rid = 1 reponame = "%s-%d" % (basereponame, rid) - + config.set(reponame, "name", reponame) config.set(reponame, "baseurl", url_map['url']) config.set(reponame, "enabled", "1") @@ -248,12 +243,12 @@ class YumCollection(Collection): for key in needkeys: # figure out the path of the key on the client - keydir = self.config.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") + keydir = self.setup.cfp.get("global", "gpg_keypath", + default="/etc/pki/rpm-gpg") remotekey = os.path.join(keydir, os.path.basename(key)) localkey = os.path.join(self.keypath, os.path.basename(key)) kdata = open(localkey).read() - + # copy the key to the client keypath = lxml.etree.Element("BoundPath", name=remotekey, encoding='ascii', @@ -319,12 +314,12 @@ class YumCollection(Collection): # include gpg key data if not has_yum: return - + try: kinfo = yum.misc.getgpgkeyinfo(keydata) version = yum.misc.keyIdToRPMVer(kinfo['keyid']) release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) - + lxml.etree.SubElement(keyentry, 'Instance', version=version, release=release, @@ -404,7 +399,7 @@ class YumCollection(Collection): unknown = set([str(p) for p in result['unknown']]) self.filter_unknown(unknown) - + return packages, unknown def call_helper(self, command, input=None): @@ -430,7 +425,7 @@ class YumCollection(Collection): self.logger.error("Packages: Failed to execute %s: %s" % (" ".join(cmd), err)) return None - + if input: idata = json.dumps(input) (stdout, stderr) = helper.communicate(idata) @@ -462,7 +457,7 @@ class YumCollection(Collection): os.unlink(self.cfgfile) self.write_config() - + if force_update: self.call_helper("clean") @@ -471,13 +466,13 @@ class YumSource(Source): basegroups = ['yum', 'redhat', 'centos', 'fedora'] ptype = 'yum' - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) + def __init__(self, basepath, xsource, setup): + Source.__init__(self, basepath, xsource, setup) self.pulp_id = None if has_pulp and xsource.get("pulp_id"): self.pulp_id = xsource.get("pulp_id") - - _setup_pulp(self.config) + + _setup_pulp(self.setup) repoapi = RepositoryAPI() try: self.repo = repoapi.repository(self.pulp_id) @@ -504,7 +499,7 @@ class YumSource(Source): self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'], self.repo['relative_path']) self.arches = [self.repo['arch']] - + if not self.rawurl: self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" else: @@ -519,8 +514,9 @@ class YumSource(Source): @property def use_yum(self): - return has_yum and self.config.getboolean("yum", "use_yum_libraries", - default=False) + return has_yum and self.setup.cfp.getboolean("packages:yum", + "use_yum_libraries", + default=False) def save_state(self): if not self.use_yum: @@ -528,7 +524,7 @@ class YumSource(Source): cPickle.dump((self.packages, self.deps, self.provides, self.filemap, self.url_map), cache, 2) cache.close() - + def load_state(self): if not self.use_yum: @@ -544,7 +540,7 @@ class YumSource(Source): usettings = [{'version':self.version, 'component':comp, 'arch':arch} for comp in self.components] - else: # rawurl given + else: # rawurl given usettings = [{'version':self.version, 'component':None, 'arch':arch}] @@ -562,7 +558,7 @@ class YumSource(Source): def _get_urls_from_repodata(self, url, arch): if self.use_yum: return [url] - + rmdurl = '%srepodata/repomd.xml' % url try: repomd = fetch_url(rmdurl) @@ -684,7 +680,7 @@ class YumSource(Source): def get_vpkgs(self, metadata): if self.use_yum: return dict() - + rv = Source.get_vpkgs(self, metadata) for arch, fmdata in list(self.filemap.items()): if arch not in metadata.groups and arch != 'global': diff --git a/src/lib/Server/Plugins/Packages/__init__.py b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py index 17a3458bd..765d1c89d 100644 --- a/src/lib/Server/Plugins/Packages/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/Packages/__init__.py @@ -10,12 +10,12 @@ import Bcfg2.Server.Plugin from Bcfg2.Bcfg2Py3k import ConfigParser, urlopen from Bcfg2.Server.Plugins.Packages import Collection from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources -from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator, Bcfg2.Server.Plugin.Generator, - Bcfg2.Server.Plugin.Connector): + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.ClientRunHooks): name = 'Packages' conflicts = ['Pkgmgr'] experimental = True @@ -26,7 +26,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.StructureValidator.__init__(self) Bcfg2.Server.Plugin.Generator.__init__(self) Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.Probing.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) self.sentinels = set() self.cachepath = os.path.join(self.data, 'cache') @@ -35,15 +35,14 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # create key directory if needed os.makedirs(self.keypath) - # set up config files - self.config = PackagesConfig(self) self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), self.cachepath, core.fam, self, - self.config) + self.core.setup) def toggle_debug(self): - Bcfg2.Server.Plugin.Plugin.toggle_debug(self) + rv = Bcfg2.Server.Plugin.Plugin.toggle_debug(self) self.sources.toggle_debug() + return rv @property def disableResolver(self): @@ -52,7 +51,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # Things return True try: - return not self.config.getboolean("global", "resolver") + return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: @@ -60,28 +59,29 @@ class Packages(Bcfg2.Server.Plugin.Plugin, # "disabled", which are not handled according to the # Python docs but appear to be handled properly by # ConfigParser in at least some versions - return self.config.get("global", "resolver", - default="enabled").lower() == "disabled" + return self.core.setup.cfp.get("packages", "resolver", + default="enabled").lower() == "disabled" @property def disableMetaData(self): try: - return not self.config.getboolean("global", "resolver") + return not self.core.setup.cfp.getboolean("packages", "resolver") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return False except ValueError: # for historical reasons we also accept "enabled" and # "disabled" - return self.config.get("global", "metadata", - default="enabled").lower() == "disabled" + return self.core.setup.cfp.get("packages", "metadata", + default="enabled").lower() == "disabled" def create_config(self, entry, metadata): """ create yum/apt config for the specified host """ - attrib = {'encoding': 'ascii', - 'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0644'} + attrib = dict(encoding='ascii', + owner='root', + group='root', + type='file', + perms='0644', + important='true') collection = self._get_collection(metadata) entry.text = collection.get_config() @@ -91,21 +91,23 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def HandleEntry(self, entry, metadata): if entry.tag == 'Package': collection = self._get_collection(metadata) - entry.set('version', self.config.get("global", - "version", - default="auto")) + entry.set('version', self.core.setup.cfp.get("packages", + "version", + default="auto")) entry.set('type', collection.ptype) elif entry.tag == 'Path': - if (entry.get("name") == self.config.get("global", "yum_config", - default="") or - entry.get("name") == self.config.get("global", "apt_config", - default="")): + if (entry.get("name") == self.core.setup.cfp.get("packages", + "yum_config", + default="") or + entry.get("name") == self.core.setup.cfp.get("packages", + "apt_config", + default="")): self.create_config(entry, metadata) def HandlesEntry(self, entry, metadata): if entry.tag == 'Package': - if self.config.getboolean("global", "magic_groups", - default=True) == True: + if self.core.setup.cfp.getboolean("packages", "magic_groups", + default=True): collection = self._get_collection(metadata) if collection.magic_groups_match(): return True @@ -113,10 +115,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, return True elif entry.tag == 'Path': # managed entries for yum/apt configs - if (entry.get("name") == self.config.get("global", "yum_config", - default="") or - entry.get("name") == self.config.get("global", "apt_config", - default="")): + if (entry.get("name") == self.core.setup.cfp.get("packages", + "yum_config", + default="") or + entry.get("name") == self.core.setup.cfp.get("packages", + "apt_config", + default="")): return True return False @@ -149,6 +153,8 @@ class Packages(Bcfg2.Server.Plugin.Plugin, initial = set() # base is the set of initial packages with groups expanded base = set() + # essential pkgs are those marked as such by the distribution + essential = collection.get_essential() to_remove = [] for struct in structures: for pkg in struct.xpath('//Package | //BoundPackage'): @@ -170,7 +176,7 @@ class Packages(Bcfg2.Server.Plugin.Plugin, else: self.logger.error("Packages: Malformed Package: %s" % lxml.etree.tostring(pkg)) - base.update(initial) + base.update(initial | essential) for el in to_remove: el.getparent().remove(el) @@ -184,8 +190,9 @@ class Packages(Bcfg2.Server.Plugin.Plugin, newpkgs.sort() for pkg in newpkgs: lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, - version=self.config.get("global", "version", - default="auto"), + version=self.core.setup.cfp.get("packages", + "version", + default="auto"), type=collection.ptype, origin='Packages') def Refresh(self): @@ -268,3 +275,12 @@ class Packages(Bcfg2.Server.Plugin.Plugin, def get_additional_data(self, metadata): collection = self._get_collection(metadata) return dict(sources=collection.get_additional_data()) + + def end_client_run(self, metadata): + """ clear the collection cache for this client, which must + persist only the duration of a client run""" + if metadata.hostname in Collection.clients: + del Collection.clients[metadata.hostname] + + def end_statistics(self, metadata): + self.end_client_run(self, metadata) diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py index 2a8623379..fcf2045a7 100644 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Pkgmgr.py @@ -1,10 +1,12 @@ '''This module implements a package management scheme for all images''' -__revision__ = '$Revision$' -import logging import re +import glob +import logging +import lxml.etree import Bcfg2.Server.Plugin -import lxml +import Bcfg2.Server.Lint + try: set except NameError: @@ -137,7 +139,6 @@ class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles package assignments.""" name = 'Pkgmgr' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' __child__ = PkgSrc __element__ = 'Package' @@ -171,3 +172,40 @@ class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): def HandleEntry(self, entry, metadata): self.BindEntry(entry, metadata) + + +class PkgmgrLint(Bcfg2.Server.Lint.ServerlessPlugin): + """ find duplicate Pkgmgr entries with the same priority """ + def Run(self): + pset = set() + for pfile in glob.glob(os.path.join(self.config['repo'], 'Pkgmgr', + '*.xml')): + if self.HandlesFile(pfile): + xdata = lxml.etree.parse(pfile).getroot() + # get priority, type, group + priority = xdata.get('priority') + ptype = xdata.get('type') + for pkg in xdata.xpath("//Package"): + if pkg.getparent().tag == 'Group': + grp = pkg.getparent().get('name') + if (type(grp) is not str and + grp.getparent().tag == 'Group'): + pgrp = grp.getparent().get('name') + else: + pgrp = 'none' + else: + grp = 'none' + pgrp = 'none' + ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) + # check if package is already listed with same + # priority, type, grp + if ptuple in pset: + self.LintError("duplicate-package", + "Duplicate Package %s, priority:%s, type:%s" % + (pkg.get('name'), priority, ptype)) + else: + pset.add(ptuple) + + @classmethod + def Errors(cls): + return {"duplicate-packages":"error"} diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Bcfg2/Server/Plugins/Probes.py index 4f17add3e..8ef6c8737 100644 --- a/src/lib/Server/Plugins/Probes.py +++ b/src/lib/Bcfg2/Server/Plugins/Probes.py @@ -153,7 +153,6 @@ class Probes(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): """A plugin to gather information from a client machine.""" name = 'Probes' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): @@ -187,9 +186,9 @@ class Probes(Bcfg2.Server.Plugin.Plugin, pretty_print='true') try: datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') + datafile.write(data.decode('utf-8')) except IOError: self.logger.error("Failed to write probed.xml") - datafile.write(data.decode('utf-8')) def load_data(self): try: diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Bcfg2/Server/Plugins/Properties.py index 76945b3a0..0271e89ba 100644 --- a/src/lib/Server/Plugins/Properties.py +++ b/src/lib/Bcfg2/Server/Plugins/Properties.py @@ -5,11 +5,30 @@ import copy import logging import lxml.etree import Bcfg2.Server.Plugin +try: + from Bcfg2.Encryption import ssl_decrypt, EVPError + have_crypto = True +except ImportError: + have_crypto = False + +logger = logging.getLogger(__name__) + +SETUP = None + +def passphrases(): + section = "encryption" + if SETUP.cfp.has_section(section): + return dict([(o, SETUP.cfp.get(section, o)) + for o in SETUP.cfp.options(section)]) + else: + return dict() -logger = logging.getLogger('Bcfg2.Plugins.Properties') class PropertyFile(Bcfg2.Server.Plugin.StructFile): """Class for properties files.""" + def __init__(self, name): + Bcfg2.Server.Plugin.StructFile.__init__(self, name) + def write(self): """ Write the data in this data structure back to the property file """ @@ -47,6 +66,39 @@ class PropertyFile(Bcfg2.Server.Plugin.StructFile): else: return True + def Index(self): + Bcfg2.Server.Plugin.StructFile.Index(self) + if self.xdata.get("encryption", "false").lower() != "false": + if not have_crypto: + msg = "Properties: M2Crypto is not available: %s" % self.name + logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + for el in self.xdata.xpath("*[@encrypted]"): + try: + el.text = self._decrypt(el) + except EVPError: + msg = "Failed to decrypt %s element in %s" % (el.tag, + self.name) + logger.error(msg) + raise Bcfg2.Server.PluginExecutionError(msg) + + def _decrypt(self, element): + passphrases = passphrases() + try: + passphrase = passphrases[element.get("encrypted")] + try: + return ssl_decrypt(crypted, self.passphrase) + except EVPError: + # error is raised below + pass + except KeyError: + for passwd in passphrases.values(): + try: + rv = ssl_decrypt(crypted, passwd) + return rv + except EVPError: + pass + raise EVPError("Failed to decrypt") class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): __child__ = PropertyFile @@ -60,9 +112,9 @@ class Properties(Bcfg2.Server.Plugin.Plugin, files into client metadata instances. """ name = 'Properties' - version = '$Revision$' def __init__(self, core, datastore): + global SETUP Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) Bcfg2.Server.Plugin.Connector.__init__(self) try: @@ -73,5 +125,7 @@ class Properties(Bcfg2.Server.Plugin.Plugin, (e.strerror, e.filename)) raise Bcfg2.Server.Plugin.PluginInitError + SETUP = core.setup + def get_additional_data(self, _): return copy.copy(self.store.entries) diff --git a/src/lib/Bcfg2/Server/Plugins/PuppetENC.py b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py new file mode 100644 index 000000000..c75c0b076 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/PuppetENC.py @@ -0,0 +1,120 @@ +import os +import Bcfg2.Server +import Bcfg2.Server.Plugin +from subprocess import Popen, PIPE + +try: + from syck import load as yaml_load, error as yaml_error +except ImportError: + try: + from yaml import load as yaml_load, YAMLError as yaml_error + except ImportError: + raise ImportError("No yaml library could be found") + +class PuppetENCFile(Bcfg2.Server.Plugin.FileBacked): + def HandleEvent(self, event=None): + return + + def __str__(self): + return "%s: %s" % (self.__class__.__name__, self.name) + + +class PuppetENC(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector, + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.DirectoryBacked): + """ A plugin to run Puppet external node classifiers + (http://docs.puppetlabs.com/guides/external_nodes.html) """ + name = 'PuppetENC' + experimental = True + __child__ = PuppetENCFile + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, + self.core.fam) + self.cache = dict() + + def _run_encs(self, metadata): + cache = dict(groups=[], params=dict()) + for enc in self.entries.keys(): + epath = os.path.join(self.data, enc) + self.debug_log("PuppetENC: Running ENC %s for %s" % + (enc, metadata.hostname)) + proc = Popen([epath, metadata.hostname], stdin=PIPE, stdout=PIPE, + stderr=PIPE) + (out, err) = proc.communicate() + rv = proc.wait() + if rv != 0: + msg = "PuppetENC: Error running ENC %s for %s (%s): %s" % \ + (enc, metadata.hostname, rv) + self.logger.error("%s: %s" % (msg, err)) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + if err: + self.debug_log("ENC Error: %s" % err) + + try: + yaml = yaml_load(out) + self.debug_log("Loaded data from %s for %s: %s" % + (enc, metadata.hostname, yaml)) + except yaml_error: + err = sys.exc_info()[1] + msg = "Error decoding YAML from %s for %s: %s" % \ + (enc, metadata.hostname, err) + self.logger.error(msg) + raise Bcfg2.Server.Plugin.PluginExecutionError(msg) + + groups = [] + if "classes" in yaml: + # stock Puppet ENC output format + groups = yaml['classes'] + elif "groups" in yaml: + # more Bcfg2-ish output format + groups = yaml['groups'] + if groups: + if isinstance(groups, list): + self.debug_log("ENC %s adding groups to %s: %s" % + (enc, metadata.hostname, groups)) + cache['groups'].extend(groups) + else: + self.debug_log("ENC %s adding groups to %s: %s" % + (enc, metadata.hostname, groups.keys())) + for group, params in groups.items(): + cache['groups'].append(group) + if params: + cache['params'].update(params) + if "parameters" in yaml and yaml['parameters']: + cache['params'].update(yaml['parameters']) + if "environment" in yaml: + self.logger.info("Ignoring unsupported environment section of " + "ENC %s for %s" % (enc, metadata.hostname)) + + self.cache[metadata.hostname] = cache + + def get_additional_groups(self, metadata): + if metadata.hostname not in self.cache: + self._run_encs(metadata) + return self.cache[metadata.hostname]['groups'] + + def get_additional_data(self, metadata): + if metadata.hostname not in self.cache: + self._run_encs(metadata) + return self.cache[metadata.hostname]['params'] + + def end_client_run(self, metadata): + """ clear the entire cache at the end of each client run. this + guarantees that each client will run all ENCs at or near the + start of each run; we have to clear the entire cache instead + of just the cache for this client because a client that builds + templates that use metadata for other clients will populate + the cache for those clients, which we don't want. This makes + the caching less than stellar, but it does prevent multiple + runs of ENCs for a single host a) for groups and data + separately; and b) when a single client's metadata is + generated multiple times by separate templates """ + self.cache = dict() + + def end_statistics(self, metadata): + self.end_client_run(self, metadata) diff --git a/src/lib/Server/Plugins/Rules.py b/src/lib/Bcfg2/Server/Plugins/Rules.py index c66276179..e77d08653 100644 --- a/src/lib/Server/Plugins/Rules.py +++ b/src/lib/Bcfg2/Server/Plugins/Rules.py @@ -1,21 +1,15 @@ """This generator provides rule-based entry mappings.""" -__revision__ = '$Revision$' import re import Bcfg2.Server.Plugin -class RulesConfig(Bcfg2.Server.Plugin.SimpleConfig): - _required = False - class Rules(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" name = 'Rules' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) - self.config = RulesConfig(self) self._regex_cache = dict() def HandlesEntry(self, entry, metadata): @@ -52,6 +46,6 @@ class Rules(Bcfg2.Server.Plugin.PrioDir): if self._regex_cache[rule].match(entry.get('name')): return True return False - + def _regex_enabled(self): - return self.config.getboolean("rules", "regex", default=False) + return self.core.setup.cfp.getboolean("rules", "regex", default=False) diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Bcfg2/Server/Plugins/SGenshi.py index ae2623d29..12c125c62 100644 --- a/src/lib/Server/Plugins/SGenshi.py +++ b/src/lib/Bcfg2/Server/Plugins/SGenshi.py @@ -1,5 +1,4 @@ '''This module implements a templating generator based on Genshi''' -__revision__ = '$Revision$' import genshi.input import genshi.template @@ -85,7 +84,6 @@ class SGenshi(SGenshiEntrySet, Bcfg2.Server.Plugin.Structure): """The SGenshi plugin provides templated structures.""" name = 'SGenshi' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' deprecated = True diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Bcfg2/Server/Plugins/SSHbase.py index eb91bea39..a1a29727f 100644 --- a/src/lib/Server/Plugins/SSHbase.py +++ b/src/lib/Bcfg2/Server/Plugins/SSHbase.py @@ -1,5 +1,4 @@ -'''This module manages ssh key files for bcfg2''' -__revision__ = '$Revision$' +"""This module manages ssh key files for bcfg2""" import binascii import re @@ -20,10 +19,15 @@ logger = logging.getLogger(__name__) class KeyData(Bcfg2.Server.Plugin.SpecificData): def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, + Bcfg2.Server.Plugin.SpecificData.__init__(self, + name, + specific, encoding) self.encoding = encoding - + + def __lt__(self, other): + return self.name < other.name + def bind_entry(self, entry, metadata): entry.set('type', 'file') if entry.get('encoding') == 'base64': @@ -99,7 +103,6 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, """ name = 'SSHbase' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' keypatterns = ["ssh_host_dsa_key", @@ -148,7 +151,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, for s in list(self.static.values())] mquery = self.core.metadata.query - + # build hostname cache names = dict() for cmeta in mquery.all(): @@ -206,15 +209,15 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, # have no clients yet. don't warn about # this. continue - + if key not in self.badnames: self.badnames[key] = True self.logger.info("Ignoring key for unknown %s %s" % (ktype, key)) continue - + skn.append("%s %s" % (','.join(hostnames), - entry.data.decode().rstrip())) + entry.data.rstrip())) self.__skn = "\n".join(skn) + "\n" return self.__skn @@ -230,7 +233,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, action = event.code2str() if action == "endExist" or event.filename == self.data: return - + for entry in list(self.entries.values()): if entry.specific.match(event.filename): entry.handle_event(event) @@ -325,8 +328,8 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, pass hostkeys.sort() for hostkey in hostkeys: - entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - hostkey.data.decode()) + entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % \ + (hostkey.data) self.entries[entry.get('name')].bind_info_to_entry(entry, metadata) def build_hk(self, entry, metadata): @@ -371,7 +374,7 @@ class SSHbase(Bcfg2.Server.Plugin.Plugin, else: self.logger.error("Unknown key filename: %s" % filename) return - + fileloc = "%s/%s" % (self.data, hostkey) publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub', "H_%s" % client]) diff --git a/src/lib/Server/Plugins/SSLCA.py b/src/lib/Bcfg2/Server/Plugins/SSLCA.py index dc0aea6d3..d207c45a2 100644 --- a/src/lib/Server/Plugins/SSLCA.py +++ b/src/lib/Bcfg2/Server/Plugins/SSLCA.py @@ -19,7 +19,6 @@ class SSLCA(Bcfg2.Server.Plugin.GroupSpool): management of ssl certificates and their keys. """ name = 'SSLCA' - __version__ = '$Id:$' __author__ = 'g.hagger@gmail.com' __child__ = Bcfg2.Server.Plugin.FileBacked key_specs = {} diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Bcfg2/Server/Plugins/Snapshots.py index aeb3b9f74..232dbb0c3 100644 --- a/src/lib/Server/Plugins/Snapshots.py +++ b/src/lib/Bcfg2/Server/Plugins/Snapshots.py @@ -14,6 +14,7 @@ import threading # Compatibility import from Bcfg2.Bcfg2Py3k import Queue +from Bcfg2.Bcfg2Py3k import u_str logger = logging.getLogger('Snapshots') @@ -28,14 +29,6 @@ datafields = { } -# py3k compatibility -def u_str(string): - if sys.hexversion >= 0x03000000: - return string - else: - return unicode(string) - - def build_snap_ent(entry): basefields = [] if entry.tag in ['Package', 'Service']: diff --git a/src/lib/Server/Plugins/Statistics.py b/src/lib/Bcfg2/Server/Plugins/Statistics.py index 9dbbeec28..9af7549ff 100644 --- a/src/lib/Server/Plugins/Statistics.py +++ b/src/lib/Bcfg2/Server/Plugins/Statistics.py @@ -1,5 +1,4 @@ '''This file manages the statistics collected by the BCFG2 Server''' -__revision__ = '$Revision$' import binascii import copy @@ -8,6 +7,7 @@ import logging from lxml.etree import XML, SubElement, Element, XMLSyntaxError import lxml.etree import os +import sys from time import asctime, localtime, time, strptime, mktime import threading @@ -117,7 +117,6 @@ class Statistics(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.ThreadedStatistics, Bcfg2.Server.Plugin.PullSource): name = 'Statistics' - __version__ = '$Id$' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) diff --git a/src/lib/Server/Plugins/Svcmgr.py b/src/lib/Bcfg2/Server/Plugins/Svcmgr.py index 6d25c1a6d..f4232ad5c 100644 --- a/src/lib/Server/Plugins/Svcmgr.py +++ b/src/lib/Bcfg2/Server/Plugins/Svcmgr.py @@ -1,5 +1,4 @@ """This generator provides service mappings.""" -__revision__ = '$Revision$' import Bcfg2.Server.Plugin @@ -7,6 +6,5 @@ import Bcfg2.Server.Plugin class Svcmgr(Bcfg2.Server.Plugin.PrioDir): """This is a generator that handles service assignments.""" name = 'Svcmgr' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' deprecated = True diff --git a/src/lib/Server/Plugins/Svn.py b/src/lib/Bcfg2/Server/Plugins/Svn.py index 9fd6f1051..ae43388ea 100644 --- a/src/lib/Server/Plugins/Svn.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn.py @@ -12,7 +12,6 @@ class Svn(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" name = 'Svn' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): diff --git a/src/lib/Server/Plugins/Svn2.py b/src/lib/Bcfg2/Server/Plugins/Svn2.py index b8a8e6b7e..e4df9574f 100644 --- a/src/lib/Server/Plugins/Svn2.py +++ b/src/lib/Bcfg2/Server/Plugins/Svn2.py @@ -9,7 +9,6 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Version): """Svn is a version plugin for dealing with Bcfg2 repos.""" name = 'Svn2' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' conflicts = ['Svn'] @@ -72,7 +71,8 @@ class Svn2(Bcfg2.Server.Plugin.Plugin, self.revision = self.client.update(self.datastore, recurse=True)[0] self.logger.info("Svn2: Commited changes. At %s" % self.revision.number) - except Exception, err: + except Exception: + err = sys.exc_info()[1] # try to be smart about the error we got back details = None if "callback_ssl_server_trust_prompt" in str(err): diff --git a/src/lib/Server/Plugins/TCheetah.py b/src/lib/Bcfg2/Server/Plugins/TCheetah.py index 49be88881..8879fdef1 100644 --- a/src/lib/Server/Plugins/TCheetah.py +++ b/src/lib/Bcfg2/Server/Plugins/TCheetah.py @@ -1,5 +1,4 @@ '''This module implements a templating generator based on Cheetah''' -__revision__ = '$Revision$' import binascii import logging @@ -76,7 +75,6 @@ class TemplateFile: class TCheetah(Bcfg2.Server.Plugin.GroupSpool): """The TCheetah generator implements a templating mechanism for configuration files.""" name = 'TCheetah' - __version__ = '$Id$' __author__ = 'bcfg-dev@mcs.anl.gov' filename_pattern = 'template' es_child_cls = TemplateFile diff --git a/src/lib/Server/Plugins/TGenshi.py b/src/lib/Bcfg2/Server/Plugins/TGenshi.py index 072f5cd7b..c4dd40614 100644 --- a/src/lib/Server/Plugins/TGenshi.py +++ b/src/lib/Bcfg2/Server/Plugins/TGenshi.py @@ -1,5 +1,4 @@ """This module implements a templating generator based on Genshi.""" -__revision__ = '$Revision$' import binascii import logging @@ -115,11 +114,13 @@ class TemplateFile: if entry.text == '': entry.set('empty', 'true') except TemplateError: + err = sys.exc_info()[1] logger.exception('Genshi template error') - raise Bcfg2.Server.Plugin.PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template error: %s' % err) except AttributeError: + err = sys.exc_info()[1] logger.exception('Genshi template loading error') - raise Bcfg2.Server.Plugin.PluginExecutionError + raise Bcfg2.Server.Plugin.PluginExecutionError('Genshi template loading error: %s' % err) class TGenshi(Bcfg2.Server.Plugin.GroupSpool): @@ -129,7 +130,6 @@ class TGenshi(Bcfg2.Server.Plugin.GroupSpool): """ name = 'TGenshi' - __version__ = '$Id$' __author__ = 'jeff@ocjtech.us' filename_pattern = 'template\.(txt|newtxt|xml)' es_child_cls = TemplateFile diff --git a/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py new file mode 100644 index 000000000..3712506d6 --- /dev/null +++ b/src/lib/Bcfg2/Server/Plugins/TemplateHelper.py @@ -0,0 +1,149 @@ +import re +import imp +import sys +import glob +import logging +import Bcfg2.Server.Lint +import Bcfg2.Server.Plugin + +logger = logging.getLogger(__name__) + +class HelperModule(Bcfg2.Server.Plugin.SpecificData): + _module_name_re = re.compile(r'([^/]+?)\.py') + + def __init__(self, name, specific, encoding): + Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, + encoding) + match = self._module_name_re.search(self.name) + if match: + self._module_name = match.group(1) + else: + self._module_name = name + self._attrs = [] + + def handle_event(self, event): + Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) + try: + module = imp.load_source(self._module_name, self.name) + except: + err = sys.exc_info()[1] + logger.error("TemplateHelper: Failed to import %s: %s" % + (self.name, err)) + return + + if not hasattr(module, "__export__"): + logger.error("TemplateHelper: %s has no __export__ list" % + self.name) + return + + for sym in module.__export__: + if sym not in self._attrs and hasattr(self, sym): + logger.warning("TemplateHelper: %s: %s is a reserved keyword, " + "skipping export" % (self.name, sym)) + setattr(self, sym, getattr(module, sym)) + # remove old exports + for sym in set(self._attrs) - set(module.__export__): + delattr(self, sym) + + self._attrs = module.__export__ + + +class HelperSet(Bcfg2.Server.Plugin.EntrySet): + ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\.py[co])$") + + def __init__(self, path, fam, encoding, plugin_name): + fpattern = '[0-9A-Za-z_\-]+\.py' + self.plugin_name = plugin_name + Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path, + HelperModule, encoding) + fam.AddMonitor(path, self) + + def HandleEvent(self, event): + if (event.filename != self.path and + not self.ignore.match(event.filename)): + return self.handle_event(event) + + +class TemplateHelper(Bcfg2.Server.Plugin.Plugin, + Bcfg2.Server.Plugin.Connector): + """ A plugin to provide helper classes and functions to templates """ + name = 'TemplateHelper' + __author__ = 'chris.a.st.pierre@gmail.com' + + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Connector.__init__(self) + + try: + self.helpers = HelperSet(self.data, core.fam, core.encoding, + self.name) + except: + raise Bcfg2.Server.Plugin.PluginInitError + + def get_additional_data(self, metadata): + return dict([(h._module_name, h) + for h in list(self.helpers.entries.values())]) + + +class TemplateHelperLint(Bcfg2.Server.Lint.ServerlessPlugin): + """ find duplicate Pkgmgr entries with the same priority """ + def __init__(self, *args, **kwargs): + Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) + hm = HelperModule("foo.py", None, None) + self.reserved_keywords = dir(hm) + + def Run(self): + for helper in glob.glob(os.path.join(self.config['repo'], + "TemplateHelper", + "*.py")): + if not self.HandlesFile(helper): + continue + self.check_helper(helper) + + def check_helper(self, helper): + match = HelperModule._module_name_re.search(helper) + if match: + module_name = match.group(1) + else: + module_name = helper + + try: + module = imp.load_source(module_name, helper) + except: + err = sys.exc_info()[1] + self.LintError("templatehelper-import-error", + "Failed to import %s: %s" % + (helper, err)) + continue + + if not hasattr(module, "__export__"): + self.LintError("templatehelper-no-export", + "%s has no __export__ list" % helper) + continue + elif not isinstance(module.__export__, list): + self.LintError("templatehelper-nonlist-export", + "__export__ is not a list in %s" % helper) + continue + + for sym in module.__export__: + if not hasattr(module, sym): + self.LintError("templatehelper-nonexistent-export", + "%s: exported symbol %s does not exist" % + (helper, sym)) + elif sym in self.reserved_keywords: + self.LintError("templatehelper-reserved-export", + "%s: exported symbol %s is reserved" % + (helper, sym)) + elif sym.startswith("_"): + self.LintError("templatehelper-underscore-export", + "%s: exported symbol %s starts with underscore" % + (helper, sym)) + + @classmethod + def Errors(cls): + return {"templatehelper-import-error":"error", + "templatehelper-no-export":"error", + "templatehelper-nonlist-export":"error", + "templatehelper-nonexistent-export":"error", + "templatehelper-reserved-export":"error", + "templatehelper-underscore-export":"warning"} diff --git a/src/lib/Server/Plugins/Trigger.py b/src/lib/Bcfg2/Server/Plugins/Trigger.py index e16b06914..313a1bf03 100644 --- a/src/lib/Server/Plugins/Trigger.py +++ b/src/lib/Bcfg2/Server/Plugins/Trigger.py @@ -3,22 +3,26 @@ import pipes import Bcfg2.Server.Plugin from subprocess import Popen, PIPE +class TriggerFile(Bcfg2.Server.Plugin.FileBacked): + def HandleEvent(self, event=None): + return + + def __str__(self): + return "%s: %s" % (self.__class__.__name__, self.name) + + class Trigger(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Statistics): + Bcfg2.Server.Plugin.ClientRunHooks, + Bcfg2.Server.Plugin.DirectoryBacked): """Trigger is a plugin that calls external scripts (on the server).""" name = 'Trigger' - __version__ = '$Id' __author__ = 'bcfg-dev@mcs.anl.gov' def __init__(self, core, datastore): Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Statistics.__init__(self) - try: - os.stat(self.data) - except: - self.logger.error("Trigger: spool directory %s does not exist; " - "unloading" % self.data) - raise Bcfg2.Server.Plugin.PluginInitError + Bcfg2.Server.Plugin.ClientRunHooks.__init__(self) + Bcfg2.Server.Plugin.DirectoryBacked.__init__(self, self.data, + self.core.fam) def async_run(self, args): pid = os.fork() @@ -39,14 +43,10 @@ class Trigger(Bcfg2.Server.Plugin.Plugin, self.debug_log("Trigger: Error: %s" % err) os._exit(0) - def process_statistics(self, metadata, _): + + def end_client_run(self, metadata): args = [metadata.hostname, '-p', metadata.profile, '-g', ':'.join([g for g in metadata.groups])] - for notifier in os.listdir(self.data): - if ((notifier[-1] == '~') or - (notifier[:2] == '.#') or - (notifier[-4:] == '.swp') or - (notifier in ['SCCS', '.svn', '4913'])): - continue + for notifier in self.entries.keys(): npath = os.path.join(self.data, notifier) self.async_run([npath] + args) diff --git a/src/lib/Server/Plugins/__init__.py b/src/lib/Bcfg2/Server/Plugins/__init__.py index c69c37452..f9f1b4e52 100644 --- a/src/lib/Server/Plugins/__init__.py +++ b/src/lib/Bcfg2/Server/Plugins/__init__.py @@ -1,5 +1,4 @@ """Imports for Bcfg2.Server.Plugins.""" -__revision__ = '$Revision$' __all__ = [ 'Account', @@ -8,7 +7,7 @@ __all__ = [ 'Bzr', 'Cfg', 'Cvs', - 'Darcs', + 'Darcs', 'Decisions', 'Fossil', 'Git', diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py new file mode 100644 index 000000000..54ba07554 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_0_x.py @@ -0,0 +1,11 @@ +""" +1_0_x.py + +This file should contain updates relevant to the 1.0.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import UnsupportedUpdate + +def updates(): + return UnsupportedUpdate("1.0", 10) + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py new file mode 100644 index 000000000..26194cb67 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_1_x.py @@ -0,0 +1,59 @@ +""" +1_1_x.py + +This file should contain updates relevant to the 1.1.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import Updater +from Bcfg2.Server.Reports.Updater.Routines import updatercallable + +from django.db import connection +import sys +import Bcfg2.Server.Reports.settings +from Bcfg2.Server.Reports.reports.models import \ + TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA + +@updatercallable +def _interactions_constraint_or_idx(): + """sqlite doesn't support alter tables.. or constraints""" + cursor = connection.cursor() + try: + cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') + except: + cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') + + +@updatercallable +def _populate_interaction_entry_counts(): + '''Populate up the type totals for the interaction table''' + cursor = connection.cursor() + count_field = {TYPE_BAD: 'bad_entries', + TYPE_MODIFIED: 'modified_entries', + TYPE_EXTRA: 'extra_entries'} + + for type in list(count_field.keys()): + cursor.execute("select count(type), interaction_id " + + "from reports_entries_interactions where type = %s group by interaction_id" % type) + updates = [] + for row in cursor.fetchall(): + updates.append(row) + try: + cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) + except Exception: + e = sys.exc_info()[1] + print(e) + cursor.close() + + +def updates(): + fixes = Updater("1.1") + fixes.override_base_version(12) # Do not do this in new code + + fixes.add('alter table reports_interaction add column bad_entries integer not null default -1;') + fixes.add('alter table reports_interaction add column modified_entries integer not null default -1;') + fixes.add('alter table reports_interaction add column extra_entries integer not null default -1;') + fixes.add(_populate_interaction_entry_counts()) + fixes.add(_interactions_constraint_or_idx()) + fixes.add('alter table reports_reason add is_binary bool NOT NULL default False;') + return fixes + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py new file mode 100644 index 000000000..22bd937c2 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_2_x.py @@ -0,0 +1,15 @@ +""" +1_2_x.py + +This file should contain updates relevant to the 1.2.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import Updater +from Bcfg2.Server.Reports.Updater.Routines import updatercallable + +def updates(): + fixes = Updater("1.2") + fixes.override_base_version(18) # Do not do this in new code + fixes.add('alter table reports_reason add is_sensitive bool NOT NULL default False;') + return fixes + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py new file mode 100644 index 000000000..1a2fff1ea --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/1_3_0.py @@ -0,0 +1,27 @@ +""" +1_3_0.py + +This file should contain updates relevant to the 1.3.x branches ONLY. +The updates() method must be defined and it should return an Updater object +""" +from Bcfg2.Server.Reports.Updater import Updater, UpdaterError +from Bcfg2.Server.Reports.Updater.Routines import AddColumns, \ + RemoveColumns, RebuildTable, DropTable + +from Bcfg2.Server.Reports.reports.models import Reason, Interaction + + +def updates(): + fixes = Updater("1.3") + fixes.add(RemoveColumns(Interaction, 'client_version')) + fixes.add(AddColumns(Reason)) + fixes.add(RebuildTable(Reason, [ + 'owner', 'current_owner', + 'group', 'current_group', + 'perms', 'current_perms', + 'status', 'current_status', + 'to', 'current_to'])) + fixes.add(DropTable('reports_ping')) + + return fixes + diff --git a/src/lib/Server/Reports/reports/templatetags/__init__.py b/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/lib/Server/Reports/reports/templatetags/__init__.py +++ b/src/lib/Bcfg2/Server/Reports/Updater/Changes/__init__.py diff --git a/src/lib/Bcfg2/Server/Reports/Updater/Routines.py b/src/lib/Bcfg2/Server/Reports/Updater/Routines.py new file mode 100644 index 000000000..edb21c321 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/Routines.py @@ -0,0 +1,279 @@ +import logging +import traceback +from django.db.models.fields import NOT_PROVIDED +from django.db import connection, DatabaseError, backend, models +from django.core.management.color import no_style +from django.core.management.sql import sql_create +import django.core.management + +import Bcfg2.Server.Reports.settings + +logger = logging.getLogger(__name__) + +def _quote(value): + """ + Quote a string to use as a table name or column + """ + return backend.DatabaseOperations().quote_name(value) + + +def _rebuild_sqlite_table(model): + """Sqlite doesn't support most alter table statments. This streamlines the + rebuild process""" + try: + cursor = connection.cursor() + table_name = model._meta.db_table + + # Build create staement from django + model._meta.db_table = "%s_temp" % table_name + sql, references = connection.creation.sql_create_model(model, no_style()) + columns = ",".join([_quote(f.column) \ + for f in model._meta.fields]) + + # Create a temp table + [cursor.execute(s) for s in sql] + + # Fill the table + tbl_name = _quote(table_name) + tmp_tbl_name = _quote(model._meta.db_table) + # Reset this + model._meta.db_table = table_name + cursor.execute("insert into %s(%s) select %s from %s;" % ( + tmp_tbl_name, + columns, + columns, + tbl_name)) + cursor.execute("drop table %s" % tbl_name) + + # Call syncdb to create the table again + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + # syncdb closes our cursor + cursor = connection.cursor() + # Repopulate + cursor.execute('insert into %s(%s) select %s from %s;' % (tbl_name, + columns, + columns, + tmp_tbl_name)) + cursor.execute('DROP TABLE %s;' % tmp_tbl_name) + except DatabaseError: + logger.error("Failed to rebuild sqlite table %s" % table_name, exc_info=1) + raise UpdaterRoutineException + + +class UpdaterRoutineException(Exception): + pass + + +class UpdaterRoutine(object): + """Base for routines.""" + def __init__(self): + pass + + def __str__(self): + return __name__ + + def run(self): + """Called to execute the action""" + raise UpdaterRoutineException + + + +class AddColumns(UpdaterRoutine): + """ + Routine to add new columns to an existing model + """ + def __init__(self, model): + self.model = model + self.model_name = model.__name__ + + def __str__(self): + return "Add new columns for model %s" % self.model_name + + def run(self): + try: + cursor = connection.cursor() + except DatabaseError: + logger.error("Failed to connect to the db") + raise UpdaterRoutineException + + try: + desc = {} + for d in connection.introspection.get_table_description(cursor, + self.model._meta.db_table): + desc[d[0]] = d + except DatabaseError: + logger.error("Failed to get table description", exc_info=1) + raise UpdaterRoutineException + + for field in self.model._meta.fields: + if field.column in desc: + continue + logger.debug("Column %s does not exist yet" % field.column) + if field.default == NOT_PROVIDED: + logger.error("Cannot add a column with out a default value") + raise UpdaterRoutineException + + sql = "ALTER TABLE %s ADD %s %s NOT NULL DEFAULT " % ( + _quote(self.model._meta.db_table), + _quote(field.column), field.db_type(), ) + db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] + if db_engine == 'django.db.backends.sqlite3': + sql += _quote(field.default) + sql_values = () + else: + sql += '%s' + sql_values = (field.default, ) + try: + cursor.execute(sql, sql_values) + logger.debug("Added column %s to %s" % + (field.column, self.model._meta.db_table)) + except DatabaseError: + logger.error("Unable to add column %s" % field.column) + raise UpdaterRoutineException + + +class RebuildTable(UpdaterRoutine): + """ + Rebuild the table for an existing model. Use this if field types have changed. + """ + def __init__(self, model, columns): + self.model = model + self.model_name = model.__name__ + + if type(columns) == str: + self.columns = [columns] + elif type(columns) in (tuple, list): + self.columns = columns + else: + logger.error("Columns must be a str, tuple, or list") + raise UpdaterRoutineException + + + def __str__(self): + return "Rebuild columns for model %s" % self.model_name + + def run(self): + try: + cursor = connection.cursor() + except DatabaseError: + logger.error("Failed to connect to the db") + raise UpdaterRoutineException + + db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] + if db_engine == 'django.db.backends.sqlite3': + """ Sqlite is a special case. Altering columns is not supported. """ + _rebuild_sqlite_table(self.model) + return + + if db_engine == 'django.db.backends.mysql': + modify_cmd = 'MODIFY ' + else: + modify_cmd = 'ALTER COLUMN ' + + col_strings = [] + for column in self.columns: + col_strings.append("%s %s %s" % ( \ + modify_cmd, + _quote(column), + self.model._meta.get_field(column).db_type() + )) + + try: + cursor.execute('ALTER TABLE %s %s' % + (_quote(self.model._meta.db_table), ", ".join(col_strings))) + except DatabaseError: + logger.debug("Failed modify table %s" % self.model._meta.db_table) + raise UpdaterRoutineException + + + +class RemoveColumns(RebuildTable): + """ + Routine to remove columns from an existing model + """ + def __init__(self, model, columns): + super(RemoveColumns, self).__init__(model, columns) + + + def __str__(self): + return "Remove columns from model %s" % self.model_name + + def run(self): + try: + cursor = connection.cursor() + except DatabaseError: + logger.error("Failed to connect to the db") + raise UpdaterRoutineException + + try: + columns = [d[0] for d in connection.introspection.get_table_description(cursor, + self.model._meta.db_table)] + except DatabaseError: + logger.error("Failed to get table description", exc_info=1) + raise UpdaterRoutineException + + for column in self.columns: + if column not in columns: + logger.warning("Cannot drop column %s: does not exist" % column) + continue + + logger.debug("Dropping column %s" % column) + + db_engine = Bcfg2.Server.Reports.settings.DATABASES['default']['ENGINE'] + if db_engine == 'django.db.backends.sqlite3': + _rebuild_sqlite_table(self.model) + else: + sql = "alter table %s drop column %s" % \ + (_quote(self.model._meta.db_table), _quote(column), ) + try: + cursor.execute(sql) + except DatabaseError: + logger.debug("Failed to drop column %s from %s" % + (column, self.model._meta.db_table)) + raise UpdaterRoutineException + + +class DropTable(UpdaterRoutine): + """ + Drop a table + """ + def __init__(self, table_name): + self.table_name = table_name + + def __str__(self): + return "Drop table %s" % self.table_name + + def run(self): + try: + cursor = connection.cursor() + cursor.execute('DROP TABLE %s' % _quote(self.table_name)) + except DatabaseError: + logger.error("Failed to drop table: %s" % + traceback.format_exc().splitlines()[-1]) + raise UpdaterRoutineException + + +class UpdaterCallable(UpdaterRoutine): + """Helper for routines. Basically delays execution""" + def __init__(self, fn): + self.fn = fn + self.args = [] + self.kwargs = {} + + def __call__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + return self + + def __str__(self): + return self.fn.__name__ + + def run(self): + self.fn(*self.args, **self.kwargs) + +def updatercallable(fn): + """Decorator for UpdaterCallable. Use for any function passed + into the fixes list""" + return UpdaterCallable(fn) + + diff --git a/src/lib/Bcfg2/Server/Reports/Updater/__init__.py b/src/lib/Bcfg2/Server/Reports/Updater/__init__.py new file mode 100644 index 000000000..3038e9691 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/Updater/__init__.py @@ -0,0 +1,239 @@ +from django.db import connection, DatabaseError +import django.core.management +import logging +import pkgutil +import re +import sys +import traceback + +from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion +from Bcfg2.Server.Reports.Updater.Routines import UpdaterRoutineException, \ + UpdaterRoutine +from Bcfg2.Server.Reports.Updater import Changes + +logger = logging.getLogger(__name__) + +class UpdaterError(Exception): + pass + + +class SchemaTooOldError(UpdaterError): + pass + + +def _walk_packages(path): + """Python 2.4 lacks this routine""" + import glob + submodules = [] + for path in __path__: + for submodule in glob.glob("%s/*.py" % path): + mod = '.'.join(submodule.split("/")[-1].split('.')[:-1]) + if mod != '__init__': + submodules.append((None, mod, False)) + return submodules + + +def _release_to_version(release): + """ + Build a release base for a version + + Expects a string of the form 00.00 + + returns an integer of the form MMmm00 + """ + regex = re.compile("^(\d+)\.(\d+)$") + m = regex.match(release) + if not m: + logger.error("Invalid release string: %s" % release) + raise TypeError + return int("%02d%02d00" % (int(m.group(1)), int(m.group(2)))) + + +class Updater(object): + """Database updater to standardize updates""" + + def __init__(self, release): + self._cursor = None + self._release = release + try: + self._base_version = _release_to_version(release) + except: + raise UpdaterError + + self._fixes = [] + self._version = -1 + + + def __cmp__(self, other): + return self._base_version - other._base_version + + @property + def release(self): + return self._release + + @property + def version(self): + if self._version < 0: + try: + iv = InternalDatabaseVersion.objects.latest() + self._version = iv.version + except InternalDatabaseVersion.DoesNotExist: + raise UpdaterError + return self._version + + @property + def cursor(self): + if not self._cursor: + self._cursor = connection.cursor() + return self._cursor + + @property + def target_version(self): + if(len(self._fixes) == 0): + return self._base_version + else: + return self._base_version + len(self._fixes) - 1 + + + def add(self, update): + if type(update) == str or isinstance(update, UpdaterRoutine): + self._fixes.append(update) + else: + raise TypeError + + + def override_base_version(self, version): + """Override our starting point for old releases. New code should + not use this method""" + self._base_version = int(version) + + + @staticmethod + def get_current_version(): + """Queries the db for the latest version. Returns 0 for a fresh install""" + + if "call_command" in dir(django.core.management): + django.core.management.call_command("syncdb", interactive=False, verbosity=0) + else: + logger.warning("Unable to call syndb routine") + raise UpdaterError + + try: + iv = InternalDatabaseVersion.objects.latest() + version = iv.version + except InternalDatabaseVersion.DoesNotExist: + version = 0 + + return version + + + def syncdb(self): + """Function to do the syncronisation for the models""" + + self._version = Updater.get_current_version() + self._cursor = None + + + def increment(self): + """Increment schema version in the database""" + if self._version < self._base_version: + self._version = self._base_version + else: + self._version += 1 + InternalDatabaseVersion.objects.create(version=self._version) + + def apply(self): + """Apply pending schema changes""" + + if self.version >= self.target_version: + logger.debug("No updates for release %s" % self._release) + return + + logger.debug("Applying updates for release %s" % self._release) + + if self.version < self._base_version: + start = 0 + else: + start = self.version - self._base_version + 1 + + try: + for fix in self._fixes[start:]: + if type(fix) == str: + self.cursor.execute(fix) + elif isinstance(fix, UpdaterRoutine): + fix.run() + else: + logger.error("Invalid schema change at %s" % \ + self._version + 1) + self.increment() + logger.debug("Applied schema change number %s: %s" % \ + (self.version, fix)) + logger.info("Applied schema changes for release %s" % self._release) + except: + logger.error("Failed to perform db update %s (%s): %s" % \ + (self._version + 1, fix, traceback.format_exc().splitlines()[-1])) + raise UpdaterError + + +class UnsupportedUpdate(Updater): + """Handle an unsupported update""" + + def __init__(self, release, version): + super(UnsupportedUpdate, self).__init__(release) + self._base_version = version + + def apply(self): + """Raise an exception if we're too old""" + + if self.version < self.target_version: + logger.error("Upgrade from release %s unsupported" % self._release) + raise SchemaTooOldError + + +def update_database(): + """method to search where we are in the revision + of the database models and update them""" + try: + logger.debug("Verifying database schema") + + updaters = [] + if hasattr(pkgutil, 'walk_packages'): + submodules = pkgutil.walk_packages(path=Changes.__path__) + else: + #python 2.4 + submodules = _walk_packages(Changes.__path__) + for loader, submodule, ispkg in submodules: + if ispkg: + continue + try: + updates = getattr( + __import__("%s.%s" % (Changes.__name__, submodule), + globals(), locals(), ['*']), + "updates") + updaters.append(updates()) + except ImportError: + logger.error("Failed to import %s" % submodule) + except AttributeError: + logger.warning("Module %s does not have an updates function" % submodule) + except: + logger.error("Failed to build updater for %s" % submodule, exc_info=1) + raise UpdaterError + + current_version = Updater.get_current_version() + logger.debug("Database version at %s" % current_version) + + if current_version > 0: + [u.apply() for u in sorted(updaters)] + logger.debug("Database version at %s" % Updater.get_current_version()) + else: + target = updaters[-1].target_version + InternalDatabaseVersion.objects.create(version=target) + logger.info("A new database was created") + + except UpdaterError: + raise + except: + logger.error("Error while updating the database") + for x in traceback.format_exc().splitlines(): + logger.error(x) + raise UpdaterError diff --git a/src/lib/Server/Reports/__init__.py b/src/lib/Bcfg2/Server/Reports/__init__.py index bdf908f4a..bdf908f4a 100644 --- a/src/lib/Server/Reports/__init__.py +++ b/src/lib/Bcfg2/Server/Reports/__init__.py diff --git a/src/lib/Server/Reports/backends.py b/src/lib/Bcfg2/Server/Reports/backends.py index 85241932f..85241932f 100644 --- a/src/lib/Server/Reports/backends.py +++ b/src/lib/Bcfg2/Server/Reports/backends.py diff --git a/src/lib/Bcfg2/Server/Reports/importscript.py b/src/lib/Bcfg2/Server/Reports/importscript.py new file mode 100755 index 000000000..14f2bb1f9 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/importscript.py @@ -0,0 +1,334 @@ +#! /usr/bin/env python +""" +Imports statistics.xml and clients.xml files in to database backend for +new statistics engine +""" + +import binascii +import os +import sys +import traceback +try: + import Bcfg2.Server.Reports.settings +except Exception: + e = sys.exc_info()[1] + sys.stderr.write("Failed to load configuration settings. %s\n" % e) + sys.exit(1) + +project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) +project_name = os.path.basename(project_directory) +sys.path.append(os.path.join(project_directory, '..')) +project_module = __import__(project_name, '', '', ['']) +sys.path.pop() +# Set DJANGO_SETTINGS_MODULE appropriately. +os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name + +from Bcfg2.Server.Reports.reports.models import * +from lxml.etree import XML, XMLSyntaxError +from getopt import getopt, GetoptError +from datetime import datetime +from time import strptime +from django.db import connection, transaction +from Bcfg2.Server.Plugins.Metadata import ClientMetadata +from Bcfg2.Server.Reports.Updater import update_database, UpdaterError +import logging +import Bcfg2.Logger +import platform + +# Compatibility import +from Bcfg2.Bcfg2Py3k import ConfigParser + + +def build_reason_kwargs(r_ent, encoding, logger): + binary_file = False + sensitive_file = False + unpruned_entries = '' + if r_ent.get('sensitive') in ['true', 'True']: + sensitive_file = True + rc_diff = '' + elif r_ent.get('current_bfile', False): + binary_file = True + rc_diff = r_ent.get('current_bfile') + if len(rc_diff) > 1024 * 1024: + rc_diff = '' + elif len(rc_diff) == 0: + # No point in flagging binary if we have no data + binary_file = False + elif r_ent.get('current_bdiff', False): + rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) + elif r_ent.get('current_diff', False): + rc_diff = r_ent.get('current_diff') + else: + rc_diff = '' + # detect unmanaged entries in pruned directories + if r_ent.get('prune', 'false') == 'true' and r_ent.get('qtest'): + unpruned_elist = [e.get('path') for e in r_ent.findall('Prune')] + unpruned_entries = "\n".join(unpruned_elist) + if not binary_file: + try: + rc_diff = rc_diff.decode(encoding) + except: + logger.error("Reason isn't %s encoded, cannot decode it" % encoding) + rc_diff = '' + return dict(owner=r_ent.get('owner', default=""), + current_owner=r_ent.get('current_owner', default=""), + group=r_ent.get('group', default=""), + current_group=r_ent.get('current_group', default=""), + perms=r_ent.get('perms', default=""), + current_perms=r_ent.get('current_perms', default=""), + status=r_ent.get('status', default=""), + current_status=r_ent.get('current_status', default=""), + to=r_ent.get('to', default=""), + current_to=r_ent.get('current_to', default=""), + version=r_ent.get('version', default=""), + current_version=r_ent.get('current_version', default=""), + current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", + current_diff=rc_diff, + is_binary=binary_file, + is_sensitive=sensitive_file, + unpruned=unpruned_entries) + +def _fetch_reason(elem, kargs, logger): + try: + rr = None + try: + rr = Reason.objects.filter(**kargs)[0] + except IndexError: + rr = Reason(**kargs) + rr.save() + logger.info("Created reason: %s" % rr.id) + except Exception: + ex = sys.exc_info()[1] + logger.error("Failed to create reason for %s: %s" % (elem.get('name'), ex)) + rr = Reason(current_exists=elem.get('current_exists', + default="True").capitalize() == "True") + rr.save() + return rr + + +def load_stats(sdata, encoding, vlevel, logger, quick=False, location=''): + for node in sdata.findall('Node'): + name = node.get('name') + for statistics in node.findall('Statistics'): + try: + load_stat(name, statistics, encoding, vlevel, logger, quick, location) + except: + logger.error("Failed to create interaction for %s: %s" % + (name, traceback.format_exc().splitlines()[-1])) + +@transaction.commit_on_success +def load_stat(cobj, statistics, encoding, vlevel, logger, quick, location): + if isinstance(cobj, ClientMetadata): + client_name = cobj.hostname + else: + client_name = cobj + client, created = Client.objects.get_or_create(name=client_name) + if created and vlevel > 0: + logger.info("Client %s added to db" % client_name) + + timestamp = datetime(*strptime(statistics.get('time'))[0:6]) + ilist = Interaction.objects.filter(client=client, + timestamp=timestamp) + if ilist: + current_interaction = ilist[0] + if vlevel > 0: + logger.info("Interaction for %s at %s with id %s already exists" % \ + (client.id, timestamp, current_interaction.id)) + return + else: + newint = Interaction(client=client, + timestamp=timestamp, + state=statistics.get('state', + default="unknown"), + repo_rev_code=statistics.get('revision', + default="unknown"), + goodcount=statistics.get('good', + default="0"), + totalcount=statistics.get('total', + default="0"), + server=location) + newint.save() + current_interaction = newint + if vlevel > 0: + logger.info("Interaction for %s at %s with id %s INSERTED in to db" % (client.id, + timestamp, current_interaction.id)) + + if isinstance(cobj, ClientMetadata): + try: + imeta = InteractionMetadata(interaction=current_interaction) + profile, created = Group.objects.get_or_create(name=cobj.profile) + imeta.profile = profile + imeta.save() # save here for m2m + + #FIXME - this should be more efficient + group_set = [] + for group_name in cobj.groups: + group, created = Group.objects.get_or_create(name=group_name) + if created: + logger.debug("Added group %s" % group) + imeta.groups.add(group) + for bundle_name in cobj.bundles: + bundle, created = Bundle.objects.get_or_create(name=bundle_name) + if created: + logger.debug("Added bundle %s" % bundle) + imeta.bundles.add(bundle) + imeta.save() + except: + logger.error("Failed to save interaction metadata for %s: %s" % + (client_name, traceback.format_exc().splitlines()[-1])) + + + entries_cache = {} + [entries_cache.__setitem__((e.kind, e.name), e) \ + for e in Entries.objects.all()] + counter_fields = {TYPE_BAD: 0, + TYPE_MODIFIED: 0, + TYPE_EXTRA: 0} + pattern = [('Bad/*', TYPE_BAD), + ('Extra/*', TYPE_EXTRA), + ('Modified/*', TYPE_MODIFIED)] + for (xpath, type) in pattern: + for x in statistics.findall(xpath): + counter_fields[type] = counter_fields[type] + 1 + rr = _fetch_reason(x, build_reason_kwargs(x, encoding, logger), logger) + + try: + entry = entries_cache[(x.tag, x.get('name'))] + except KeyError: + entry, created = Entries.objects.get_or_create(\ + name=x.get('name'), kind=x.tag) + + Entries_interactions(entry=entry, reason=rr, + interaction=current_interaction, + type=type).save() + if vlevel > 0: + logger.info("%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id)) + + # add good entries + good_reason = None + for x in statistics.findall('Good/*'): + if good_reason == None: + # Do this once. Really need to fix Reasons... + good_reason = _fetch_reason(x, build_reason_kwargs(x, encoding, logger), logger) + try: + entry = entries_cache[(x.tag, x.get('name'))] + except KeyError: + entry, created = Entries.objects.get_or_create(\ + name=x.get('name'), kind=x.tag) + Entries_interactions(entry=entry, reason=good_reason, + interaction=current_interaction, + type=TYPE_GOOD).save() + if vlevel > 0: + logger.info("%s interaction created with reason id %s and entry %s" % (xpath, good_reason.id, entry.id)) + + # Update interaction counters + current_interaction.bad_entries = counter_fields[TYPE_BAD] + current_interaction.modified_entries = counter_fields[TYPE_MODIFIED] + current_interaction.extra_entries = counter_fields[TYPE_EXTRA] + current_interaction.save() + + mperfs = [] + for times in statistics.findall('OpStamps'): + for metric, value in list(times.items()): + mmatch = [] + if not quick: + mmatch = Performance.objects.filter(metric=metric, value=value) + + if mmatch: + mperf = mmatch[0] + else: + mperf = Performance(metric=metric, value=value) + mperf.save() + mperfs.append(mperf) + current_interaction.performance_items.add(*mperfs) + + +if __name__ == '__main__': + from sys import argv + verb = 0 + cpath = "/etc/bcfg2.conf" + clientpath = False + statpath = False + syslog = False + + try: + opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", + "verbose", + "updates", + "debug", + "clients=", + "stats=", + "config=", + "syslog"]) + except GetoptError: + mesg = sys.exc_info()[1] + # print help information and exit: + print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-s statistics-file]" % (mesg)) + raise SystemExit(2) + + for o, a in opts: + if o in ("-h", "--help"): + print("Usage:\nimportscript.py [-h] [-v] -s <statistics-file> \n") + print("h : help; this message") + print("v : verbose; print messages on record insertion/skip") + print("u : updates; print status messages as items inserted semi-verbose") + print("d : debug; print most SQL used to manipulate database") + print("C : path to bcfg2.conf config file.") + print("s : statistics.xml file") + print("S : syslog; output to syslog") + raise SystemExit + if o in ["-C", "--config"]: + cpath = a + + if o in ("-v", "--verbose"): + verb = 1 + if o in ("-u", "--updates"): + verb = 2 + if o in ("-d", "--debug"): + verb = 3 + if o in ("-c", "--clients"): + print "DeprecationWarning: %s is no longer used" % o + + if o in ("-s", "--stats"): + statpath = a + if o in ("-S", "--syslog"): + syslog = True + + logger = logging.getLogger('importscript.py') + logging.getLogger().setLevel(logging.INFO) + Bcfg2.Logger.setup_logging('importscript.py', + True, + syslog, level=logging.INFO) + + cf = ConfigParser.ConfigParser() + cf.read([cpath]) + + if not statpath: + try: + statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + print("Could not read bcfg2.conf; exiting") + raise SystemExit(1) + try: + statsdata = XML(open(statpath).read()) + except (IOError, XMLSyntaxError): + print("StatReports: Failed to parse %s" % (statpath)) + raise SystemExit(1) + + try: + encoding = cf.get('components', 'encoding') + except: + encoding = 'UTF-8' + + q = '-O3' in sys.argv + # Be sure the database is ready for new schema + try: + update_database() + except UpdaterError: + raise SystemExit(1) + load_stats(statsdata, + encoding, + verb, + logger, + quick=q, + location=platform.node()) diff --git a/src/lib/Server/Reports/manage.py b/src/lib/Bcfg2/Server/Reports/manage.py index 858bddeca..858bddeca 100755 --- a/src/lib/Server/Reports/manage.py +++ b/src/lib/Bcfg2/Server/Reports/manage.py diff --git a/src/lib/Server/Reports/nisauth.py b/src/lib/Bcfg2/Server/Reports/nisauth.py index 6fc346f1e..b3e37113b 100644 --- a/src/lib/Server/Reports/nisauth.py +++ b/src/lib/Bcfg2/Server/Reports/nisauth.py @@ -4,8 +4,6 @@ from Bcfg2.Server.Reports.settings import AUTHORIZED_GROUP """Checks with NIS to see if the current user is in the support group""" -__revision__ = "$Revision: $" - class NISAUTHError(Exception): """NISAUTHError is raised when somehting goes boom.""" diff --git a/src/lib/Server/Reports/reports/__init__.py b/src/lib/Bcfg2/Server/Reports/reports/__init__.py index ccdce8943..ccdce8943 100644 --- a/src/lib/Server/Reports/reports/__init__.py +++ b/src/lib/Bcfg2/Server/Reports/reports/__init__.py diff --git a/src/lib/Server/Reports/reports/models.py b/src/lib/Bcfg2/Server/Reports/reports/models.py index 870239641..4b078eb2c 100644 --- a/src/lib/Server/Reports/reports/models.py +++ b/src/lib/Bcfg2/Server/Reports/reports/models.py @@ -1,5 +1,14 @@ """Django models for Bcfg2 reports.""" -from django.db import models +import sys + +from django.core.exceptions import ImproperlyConfigured +try: + from django.db import models +except ImproperlyConfigured: + e = sys.exc_info()[1] + print("Reports: unable to import django models: %s" % e) + sys.exit(1) + from django.db import connection, transaction from django.db.models import Q from datetime import datetime, timedelta @@ -14,16 +23,13 @@ KIND_CHOICES = ( ('Path', 'symlink'), ('Service', 'Service'), ) -PING_CHOICES = ( - #These are possible ping states - ('Up (Y)', 'Y'), - ('Down (N)', 'N') -) +TYPE_GOOD = 0 TYPE_BAD = 1 TYPE_MODIFIED = 2 TYPE_EXTRA = 3 TYPE_CHOICES = ( + (TYPE_GOOD, 'Good'), (TYPE_BAD, 'Bad'), (TYPE_MODIFIED, 'Modified'), (TYPE_EXTRA, 'Extra'), @@ -78,30 +84,9 @@ class Client(models.Model): pass -class Ping(models.Model): - """Represents a ping of a client (sparsely).""" - client = models.ForeignKey(Client, related_name="pings") - starttime = models.DateTimeField() - endtime = models.DateTimeField() - status = models.CharField(max_length=4, choices=PING_CHOICES) # up/down - - class Meta: - get_latest_by = 'endtime' - - class InteractiveManager(models.Manager): """Manages interactions objects.""" - def recent_interactions_dict(self, maxdate=None, active_only=True): - """ - Return the most recent interactions for clients as of a date. - - This method uses aggregated queries to return a ValuesQueryDict object. - Faster then raw sql since this is executed as a single query. - """ - - return list(self.values('client').annotate(max_timestamp=Max('timestamp')).values()) - def interaction_per_client(self, maxdate=None, active_only=True): """ Returns the most recent interactions for clients as of a date @@ -145,18 +130,17 @@ class InteractiveManager(models.Manager): cursor.execute(sql) return [item[0] for item in cursor.fetchall()] except: - '''FIXME - really need some error hadling''' + '''FIXME - really need some error handling''' pass return [] class Interaction(models.Model): """Models each reconfiguration operation interaction between client and server.""" - client = models.ForeignKey(Client, related_name="interactions",) - timestamp = models.DateTimeField() # Timestamp for this record + client = models.ForeignKey(Client, related_name="interactions") + timestamp = models.DateTimeField(db_index=True) # Timestamp for this record state = models.CharField(max_length=32) # good/bad/modified/etc repo_rev_code = models.CharField(max_length=64) # repo revision at time of interaction - client_version = models.CharField(max_length=32) # Client Version goodcount = models.IntegerField() # of good config-items totalcount = models.IntegerField() # of total config-items server = models.CharField(max_length=256) # Name of the server used for the interaction @@ -262,26 +246,47 @@ class Interaction(models.Model): class Reason(models.Model): """reason why modified or bad entry did not verify, or changed.""" - owner = models.TextField(max_length=128, blank=True) - current_owner = models.TextField(max_length=128, blank=True) - group = models.TextField(max_length=128, blank=True) - current_group = models.TextField(max_length=128, blank=True) - perms = models.TextField(max_length=4, blank=True) # txt fixes typing issue - current_perms = models.TextField(max_length=4, blank=True) - status = models.TextField(max_length=3, blank=True) # on/off/(None) - current_status = models.TextField(max_length=1, blank=True) # on/off/(None) - to = models.TextField(max_length=256, blank=True) - current_to = models.TextField(max_length=256, blank=True) - version = models.TextField(max_length=128, blank=True) - current_version = models.TextField(max_length=128, blank=True) + owner = models.CharField(max_length=255, blank=True) + current_owner = models.CharField(max_length=255, blank=True) + group = models.CharField(max_length=255, blank=True) + current_group = models.CharField(max_length=255, blank=True) + perms = models.CharField(max_length=4, blank=True) + current_perms = models.CharField(max_length=4, blank=True) + status = models.CharField(max_length=128, blank=True) + current_status = models.CharField(max_length=128, blank=True) + to = models.CharField(max_length=1024, blank=True) + current_to = models.CharField(max_length=1024, blank=True) + version = models.CharField(max_length=1024, blank=True) + current_version = models.CharField(max_length=1024, blank=True) current_exists = models.BooleanField() # False means its missing. Default True - current_diff = models.TextField(max_length=1280, blank=True) + current_diff = models.TextField(max_length=1024*1024, blank=True) is_binary = models.BooleanField(default=False) is_sensitive = models.BooleanField(default=False) + unpruned = models.TextField(max_length=4096, blank=True, default='') def _str_(self): return "Reason" + def short_list(self): + rv = [] + if self.current_owner or self.current_group or self.current_perms: + rv.append("File permissions") + if self.current_status: + rv.append("Incorrect status") + if self.current_to: + rv.append("Incorrect target") + if self.current_version or self.version == 'auto': + rv.append("Wrong version") + if not self.current_exists: + rv.append("Missing") + if self.current_diff or self.is_sensitive: + rv.append("Incorrect data") + if self.unpruned: + rv.append("Directory has extra files") + if len(rv) == 0: + rv.append("Exists") + return rv + @staticmethod @transaction.commit_on_success def prune_orphans(): @@ -307,6 +312,9 @@ class Entries(models.Model): cursor.execute('delete from reports_entries where not exists (select rei.id from reports_entries_interactions rei where rei.entry_id = reports_entries.id)') transaction.set_dirty() + class Meta: + unique_together = ("name", "kind") + class Entries_interactions(models.Model): """Define the relation between the reason, the interaction and the entry.""" @@ -341,3 +349,57 @@ class InternalDatabaseVersion(models.Model): def __str__(self): return "version %d updated the %s" % (self.version, self.updated.isoformat()) + + class Meta: + get_latest_by = "version" + + +class Group(models.Model): + """ + Groups extracted from interactions + + name - The group name + + TODO - Most of this is for future use + TODO - set a default group + """ + + name = models.CharField(max_length=255, unique=True) + profile = models.BooleanField(default=False) + public = models.BooleanField(default=False) + category = models.CharField(max_length=1024, blank=True) + comment = models.TextField(blank=True) + + groups = models.ManyToManyField("self", symmetrical=False) + bundles = models.ManyToManyField("Bundle") + + def __unicode__(self): + return self.name + + +class Bundle(models.Model): + """ + Bundles extracted from interactions + + name - The bundle name + """ + + name = models.CharField(max_length=255, unique=True) + + def __unicode__(self): + return self.name + + +class InteractionMetadata(models.Model): + """ + InteractionMetadata + + Hold extra data associated with the client and interaction + """ + + interaction = models.OneToOneField(Interaction, primary_key=True, related_name='metadata') + profile = models.ForeignKey(Group, related_name="+") + groups = models.ManyToManyField(Group) + bundles = models.ManyToManyField(Bundle) + + diff --git a/src/lib/Server/Reports/reports/templates/404.html b/src/lib/Bcfg2/Server/Reports/reports/templates/404.html index 168bd9fec..168bd9fec 100644 --- a/src/lib/Server/Reports/reports/templates/404.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/404.html diff --git a/src/lib/Server/Reports/reports/templates/base-timeview.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html index 842de36f0..9a5ef651c 100644 --- a/src/lib/Server/Reports/reports/templates/base-timeview.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/base-timeview.html @@ -20,6 +20,9 @@ document.write(getCalendarStyles()); {% if not timestamp %}Rendered at {% now "Y-m-d H:i" %} | {% else %}View as of {{ timestamp|date:"Y-m-d H:i" }} | {% endif %}{% spaceless %} <a id='cal_link' name='cal_link' href='#' onclick='showCalendar(); return false;' >[change]</a> - <form method='post' action='{{ path }}' id='cal_form' name='cal_form'><input id='cal_date' name='cal_date' type='hidden' value=''/></form> + <form method='post' action='{{ path }}' id='cal_form' name='cal_form'> + <input id='cal_date' name='cal_date' type='hidden' value=''/> + <input name='op' type='hidden' value='timeview'/> + </form> {% endspaceless %} {% endblock %} diff --git a/src/lib/Server/Reports/reports/templates/base.html b/src/lib/Bcfg2/Server/Reports/reports/templates/base.html index f541c0d2b..625177390 100644 --- a/src/lib/Server/Reports/reports/templates/base.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/base.html @@ -62,6 +62,7 @@ <li>Entries Configured</li> </ul> <ul class='menu-level2'> + <li><a href="{% url reports_common_problems %}">Common problems</a></li> <li><a href="{% url reports_item_list "bad" %}">Bad</a></li> <li><a href="{% url reports_item_list "modified" %}">Modified</a></li> <li><a href="{% url reports_item_list "extra" %}">Extra</a></li> diff --git a/src/lib/Server/Reports/reports/templates/clients/detail.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html index dd4295f21..9b86b609f 100644 --- a/src/lib/Server/Reports/reports/templates/clients/detail.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detail.html @@ -50,6 +50,9 @@ span.history_links a { {% if interaction.server %} <tr><td>Served by</td><td>{{interaction.server}}</td></tr> {% endif %} + {% if interaction.metadata %} + <tr><td>Profile</td><td>{{interaction.metadata.profile}}</td></tr> + {% endif %} {% if interaction.repo_rev_code %} <tr><td>Revision</td><td>{{interaction.repo_rev_code}}</td></tr> {% endif %} @@ -60,58 +63,57 @@ span.history_links a { {% endif %} </table> - {% if interaction.bad_entry_count %} + {% if interaction.metadata.groups.count %} <div class='entry_list'> - <div class='entry_list_head dirty-lineitem' onclick='javascript:toggleMe("bad_table");'> - <h3>Bad Entries — {{ interaction.bad_entry_count }}</h3> - <div class='entry_expand_tab' id='plusminus_bad_table'>[+]</div> + <div class='entry_list_head' onclick='javascript:toggleMe("groups_table");'> + <h3>Group membership</h3> + <div class='entry_expand_tab' id='plusminus_groups_table'>[+]</div> </div> - <table id='bad_table' class='entry_list'> - {% for e in interaction.bad|sortwell %} + <table id='groups_table' class='entry_list' style='display: none'> + {% for group in interaction.metadata.groups.all %} <tr class='{% cycle listview,listview_alt %}'> - <td class='entry_list_type'>{{e.entry.kind}}:</td> - <td><a href="{% url reports_item "bad",e.id %}"> - {{e.entry.name}}</a></td> + <td class='entry_list_type'>{{group}}</td> </tr> {% endfor %} </table> </div> {% endif %} - {% if interaction.modified_entry_count %} + {% if interaction.metadata.bundles.count %} <div class='entry_list'> - <div class='entry_list_head modified-lineitem' onclick='javascript:toggleMe("modified_table");'> - <h3>Modified Entries — {{ interaction.modified_entry_count }}</h3> - <div class='entry_expand_tab' id='plusminus_modified_table'>[+]</div> + <div class='entry_list_head' onclick='javascript:toggleMe("bundles_table");'> + <h3>Bundle membership</h3> + <div class='entry_expand_tab' id='plusminus_bundless_table'>[+]</div> </div> - <table id='modified_table' class='entry_list'> - {% for e in interaction.modified|sortwell %} + <table id='bundles_table' class='entry_list' style='display: none'> + {% for bundle in interaction.metadata.bundles.all %} <tr class='{% cycle listview,listview_alt %}'> - <td class='entry_list_type'>{{e.entry.kind}}:</td> - <td><a href="{% url reports_item "modified",e.id %}"> - {{e.entry.name}}</a></td> + <td class='entry_list_type'>{{bundle}}</td> </tr> {% endfor %} </table> </div> {% endif %} - {% if interaction.extra_entry_count %} + {% for type, ei_list in ei_lists %} + {% if ei_list %} <div class='entry_list'> - <div class='entry_list_head extra-lineitem' onclick='javascript:toggleMe("extra_table");'> - <h3>Extra Entries — {{ interaction.extra_entry_count }}</h3> - <div class='entry_expand_tab' id='plusminus_extra_table'>[+]</div> + <div class='entry_list_head {{type}}-lineitem' onclick='javascript:toggleMe("{{type}}_table");'> + <h3>{{ type|capfirst }} Entries — {{ ei_list|length }}</h3> + <div class='entry_expand_tab' id='plusminus_{{type}}_table'>[+]</div> </div> - <table id='extra_table' class='entry_list'> - {% for e in interaction.extra|sortwell %} + <table id='{{type}}_table' class='entry_list'> + {% for ei in ei_list %} <tr class='{% cycle listview,listview_alt %}'> - <td class='entry_list_type'>{{e.entry.kind}}:</td> - <td><a href="{% url reports_item "extra",e.id %}">{{e.entry.name}}</a></td> + <td class='entry_list_type'>{{ei.entry.kind}}</td> + <td><a href="{% url reports_item type ei.id %}"> + {{ei.entry.name}}</a></td> </tr> - {% endfor %} + {% endfor %} </table> </div> {% endif %} + {% endfor %} {% if entry_list %} <div class="entry_list recent_history_wrapper"> diff --git a/src/lib/Server/Reports/reports/templates/clients/detailed-list.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html index 0c1fae8d5..9be59e7d2 100644 --- a/src/lib/Server/Reports/reports/templates/clients/detailed-list.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/detailed-list.html @@ -6,24 +6,24 @@ {% block content %} <div class='client_list_box'> -{% if entry_list %} {% filter_navigator %} +{% if entry_list %} <table cellpadding="3"> <tr id='table_list_header' class='listview'> - <td class='left_column'>Node</td> - <td class='right_column' style='width:75px'>State</td> - <td class='right_column_narrow'>Good</td> - <td class='right_column_narrow'>Bad</td> - <td class='right_column_narrow'>Modified</td> - <td class='right_column_narrow'>Extra</td> - <td class='right_column'>Last Run</td> - <td class='right_column_wide'>Server</td> + <td class='left_column'>{% sort_link 'client' 'Node' %}</td> + <td class='right_column' style='width:75px'>{% sort_link 'state' 'State' %}</td> + <td class='right_column_narrow'>{% sort_link '-good' 'Good' %}</td> + <td class='right_column_narrow'>{% sort_link '-bad' 'Bad' %}</td> + <td class='right_column_narrow'>{% sort_link '-modified' 'Modified' %}</td> + <td class='right_column_narrow'>{% sort_link '-extra' 'Extra' %}</td> + <td class='right_column'>{% sort_link 'timestamp' 'Last Run' %}</td> + <td class='right_column_wide'>{% sort_link 'server' 'Server' %}</td> </tr> {% for entry in entry_list %} <tr class='{% cycle listview,listview_alt %}'> <td class='left_column'><a href='{% url Bcfg2.Server.Reports.reports.views.client_detail hostname=entry.client.name, pk=entry.id %}'>{{ entry.client.name }}</a></td> <td class='right_column' style='width:75px'><a href='{% add_url_filter state=entry.state %}' - {% ifequal entry.state 'dirty' %}class='dirty-lineitem'{% endifequal %}>{{ entry.state }}</a></td> + class='{{entry|determine_client_state}}'>{{ entry.state }}</a></td> <td class='right_column_narrow'>{{ entry.goodcount }}</td> <td class='right_column_narrow'>{{ entry.bad_entry_count }}</td> <td class='right_column_narrow'>{{ entry.modified_entry_count }}</td> diff --git a/src/lib/Server/Reports/reports/templates/clients/history.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html index 01d4ec2f4..01d4ec2f4 100644 --- a/src/lib/Server/Reports/reports/templates/clients/history.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/history.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html new file mode 100644 index 000000000..45ba20b86 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/index.html @@ -0,0 +1,35 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block extra_header_info %} +{% endblock%} + +{% block title %}Bcfg2 - Client Grid View{% endblock %} + +{% block pagebanner %}Clients - Grid View{% endblock %} + +{% block content %} +{% filter_navigator %} +{% if inter_list %} + <table class='grid-view' align='center'> + {% for inter in inter_list %} + {% if forloop.first %}<tr>{% endif %} + <td class='{{ inter|determine_client_state }}'> + <a href="{% spaceless %} + {% if not timestamp %} + {% url reports_client_detail inter.client.name %} + {% else %} + {% url reports_client_detail_pk inter.client.name,inter.id %} + {% endif %} + {% endspaceless %}">{{ inter.client.name }}</a> + </td> + {% if forloop.last %} + </tr> + {% else %} + {% if forloop.counter|divisibleby:"4" %}</tr><tr>{% endif %} + {% endif %} + {% endfor %} + </table> +{% else %}<p>No client records are available.</p> +{% endif %} +{% endblock %} diff --git a/src/lib/Server/Reports/reports/templates/clients/manage.html b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html index 5725ae577..5725ae577 100644 --- a/src/lib/Server/Reports/reports/templates/clients/manage.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/clients/manage.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html new file mode 100644 index 000000000..d6ad303fc --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/common.html @@ -0,0 +1,42 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Common Problems{% endblock %} + +{% block extra_header_info %} +{% endblock%} + +{% block pagebanner %}Common configuration problems{% endblock %} + +{% block content %} + <div id='threshold_box'> + <form method='post' action='{{ request.path }}'> + <span>Showing items with more then {{ threshold }} entries</span> + <input type='text' name='threshold' value='{{ threshold }}' maxlength='5' size='5' /> + <input type='submit' value='Change' /> + </form> + </div> + {% for type_name, type_list in lists %} + <div class='entry_list'> + <div class='entry_list_head element_list_head' onclick='javascript:toggleMe("table_{{ type_name }}");'> + <h3>{{ type_name|capfirst }} entries</h3> + <div class='entry_expand_tab' id='plusminus_table_{{ type_name }}'>[–]</div> + </div> + {% if type_list %} + <table id='table_{{ type_name }}' class='entry_list'> + <tr style='text-align: left'><th>Type</th><th>Name</th><th>Count</th><th>Reason</th></tr> + {% for entry, reason, interaction in type_list %} + <tr class='{% cycle listview,listview_alt %}'> + <td>{{ entry.kind }}</td> + <td><a href="{% url reports_entry eid=entry.pk %}">{{ entry.name }}</a></td> + <td>{{ interaction|length }}</td> + <td><a href="{% url reports_item type=type_name pk=interaction.0 %}">{{ reason.short_list|join:"," }}</a></td> + </tr> + {% endfor %} + </table> + {% else %} + <p>There are currently no inconsistent {{ type_name }} configuration entries.</p> + {% endif %} + </div> + {% endfor %} +{% endblock %} diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html new file mode 100644 index 000000000..5f7579eb9 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/entry_status.html @@ -0,0 +1,30 @@ +{% extends "base-timeview.html" %} +{% load bcfg2_tags %} + +{% block title %}Bcfg2 - Entry Status{% endblock %} + +{% block extra_header_info %} +{% endblock%} + +{% block pagebanner %}{{ entry.kind }} entry {{ entry.name }} status{% endblock %} + +{% block content %} +{% filter_navigator %} +{% if item_data %} + <div class='entry_list'> + <table class='entry_list'> + <tr style='text-align: left' ><th>Name</th><th>Timestamp</th><th>State</th><th>Reason</th></tr> + {% for ei, inter, reason in item_data %} + <tr class='{% cycle listview,listview_alt %}'> + <td><a href='{% url Bcfg2.Server.Reports.reports.views.client_detail hostname=inter.client.name, pk=inter.id %}'>{{ inter.client.name }}</a></td> + <td style='white-space: nowrap'><a href='{% url Bcfg2.Server.Reports.reports.views.client_detail hostname=inter.client.name, pk=inter.id %}'>{{ inter.timestamp|date:"Y-m-d\&\n\b\s\p\;H:i"|safe }}</a></td> + <td>{{ ei.get_type_display }}</td> + <td style='white-space: nowrap'><a href="{% url reports_item type=ei.get_type_display pk=ei.pk %}">{{ reason.short_list|join:"," }}</a></td> + </tr> + {% endfor %} + </table> + </div> +{% else %} + <p>There are currently no hosts with this configuration entry.</p> +{% endif %} +{% endblock %} diff --git a/src/lib/Server/Reports/reports/templates/config_items/item.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html index cc99ef503..42c3e8349 100644 --- a/src/lib/Server/Reports/reports/templates/config_items/item.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/item.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% load split %} {% load syntax_coloring %} @@ -91,6 +92,20 @@ div.entry_list h3 { </div> {% endif %} + <!-- display extra directory entries --> + {% if item.reason.unpruned != '' %} + <div class='entry_list'> + <div class='entry_list_head'> + <h3>Extra entries found</h3> + </div> + <table class='entry_list' cellpadding='3'> + {% for unpruned_item in item.reason.unpruned|split %} + <tr><td>{{ unpruned_item }}</td></tr> + {% endfor %} + </table> + </div> + {% endif %} + <div class='entry_list'> <div class='entry_list_head'> diff --git a/src/lib/Server/Reports/reports/templates/config_items/listing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html index 9b1026a08..0a92e7fc0 100644 --- a/src/lib/Server/Reports/reports/templates/config_items/listing.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/config_items/listing.html @@ -9,19 +9,21 @@ {% block pagebanner %}{{mod_or_bad|capfirst}} Element Listing{% endblock %} {% block content %} -{% if item_list_dict %} - {% for kind, entries in item_list_dict.items %} - +{% filter_navigator %} +{% if item_list %} + {% for type_name, type_data in item_list %} <div class='entry_list'> - <div class='entry_list_head element_list_head' onclick='javascript:toggleMe("table_{{ kind }}");'> - <h3>{{ kind }} — {{ entries|length }}</h3> - <div class='entry_expand_tab' id='plusminus_table_{{ kind }}'>[–]</div> + <div class='entry_list_head element_list_head' onclick='javascript:toggleMe("table_{{ type_name }}");'> + <h3>{{ type_name }} — {{ type_data|length }}</h3> + <div class='entry_expand_tab' id='plusminus_table_{{ type_name }}'>[–]</div> </div> - - <table id='table_{{ kind }}' class='entry_list'> - {% for e in entries %} + <table id='table_{{ type_name }}' class='entry_list'> + <tr style='text-align: left' ><th>Name</th><th>Count</th><th>Reason</th></tr> + {% for entry, reason, eis in type_data %} <tr class='{% cycle listview,listview_alt %}'> - <td><a href="{% url reports_item type=mod_or_bad,pk=e.id %}">{{e.entry.name}}</a></td> + <td><a href="{% url reports_entry eid=entry.pk %}">{{entry.name}}</a></td> + <td>{{ eis|length }}</td> + <td><a href="{% url reports_item type=mod_or_bad,pk=eis.0 %}">{{ reason.short_list|join:"," }}</a></td> </tr> {% endfor %} </table> diff --git a/src/lib/Server/Reports/reports/templates/displays/summary.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html index b9847cf96..b9847cf96 100644 --- a/src/lib/Server/Reports/reports/templates/displays/summary.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/summary.html diff --git a/src/lib/Server/Reports/reports/templates/displays/timing.html b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html index 47accb2cb..ff775ded5 100644 --- a/src/lib/Server/Reports/reports/templates/displays/timing.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/displays/timing.html @@ -20,7 +20,7 @@ <td>Install</td> <td>Config</td> <td>Total</td> - </tr> + </tr> {% for metric in metrics|dictsort:"name" %} <tr class='{% cycle listview,listview_alt %}'> <td><a style='font-size: 100%' diff --git a/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html new file mode 100644 index 000000000..759415507 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/filter_bar.html @@ -0,0 +1,25 @@ +{% spaceless %} +<div class="filter_bar"> +<form name='filter_form'> +{% if filters %} +{% for filter, filter_url in filters %} + {% if forloop.first %} + Active filters (click to remove): + {% endif %} + <a href='{{ filter_url }}'>{{ filter|capfirst }}</a>{% if not forloop.last %}, {% endif %} + {% if forloop.last %} + {% if groups %}|{% endif %} + {% endif %} +{% endfor %} +{% endif %} +{% if groups %} +<label for="id_group">Group filter:</label> +<select id="id_group" name="group" onchange="javascript:url=document.forms['filter_form'].group.value; if(url) { location.href=url }"> + {% for group, group_url, selected in groups %} + <option label="{{group}}" value="{{group_url}}" {% if selected %}selected {% endif %}/> + {% endfor %} +</select> +{% endif %} +</form> +</div> +{% endspaceless %} diff --git a/src/lib/Server/Reports/reports/templates/widgets/interaction_list.inc b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc index 8f2dec1dc..6fe7e6547 100644 --- a/src/lib/Server/Reports/reports/templates/widgets/interaction_list.inc +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/interaction_list.inc @@ -20,7 +20,7 @@ <td class='right_column_wide'><a href='{% add_url_filter hostname=entry.client.name %}'>{{ entry.client.name }}</a></td> {% endif %} <td class='right_column' style='width:75px'><a href='{% add_url_filter state=entry.state %}' - {% ifequal entry.state 'dirty' %}class='dirty-lineitem'{% endifequal %}>{{ entry.state }}</a></td> + class='{{entry|determine_client_state}}'>{{ entry.state }}</a></td> <td class='right_column_narrow'>{{ entry.goodcount }}</td> <td class='right_column_narrow'>{{ entry.bad_entry_count }}</td> <td class='right_column_narrow'>{{ entry.modified_entry_count }}</td> diff --git a/src/lib/Server/Reports/reports/templates/widgets/page_bar.html b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html index aa0def83e..aa0def83e 100644 --- a/src/lib/Server/Reports/reports/templates/widgets/page_bar.html +++ b/src/lib/Bcfg2/Server/Reports/reports/templates/widgets/page_bar.html diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/__init__.py diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py new file mode 100644 index 000000000..894353bba --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/bcfg2_tags.py @@ -0,0 +1,415 @@ +import sys +from copy import copy + +from django import template +from django.conf import settings +from django.core.urlresolvers import resolve, reverse, \ + Resolver404, NoReverseMatch +from django.template.loader import get_template, \ + get_template_from_string,TemplateDoesNotExist +from django.utils.encoding import smart_unicode, smart_str +from django.utils.safestring import mark_safe +from datetime import datetime, timedelta +from Bcfg2.Server.Reports.utils import filter_list +from Bcfg2.Server.Reports.reports.models import Group + +register = template.Library() + +__PAGE_NAV_LIMITS__ = (10, 25, 50, 100) + + +@register.inclusion_tag('widgets/page_bar.html', takes_context=True) +def page_navigator(context): + """ + Creates paginated links. + + Expects the context to be a RequestContext and + views.prepare_paginated_list() to have populated page information. + """ + fragment = dict() + try: + path = context['request'].META['PATH_INFO'] + total_pages = int(context['total_pages']) + records_per_page = int(context['records_per_page']) + except KeyError: + return fragment + except ValueError: + return fragment + + if total_pages < 2: + return {} + + try: + view, args, kwargs = resolve(path) + current_page = int(kwargs.get('page_number', 1)) + fragment['current_page'] = current_page + fragment['page_number'] = current_page + fragment['total_pages'] = total_pages + fragment['records_per_page'] = records_per_page + if current_page > 1: + kwargs['page_number'] = current_page - 1 + fragment['prev_page'] = reverse(view, args=args, kwargs=kwargs) + if current_page < total_pages: + kwargs['page_number'] = current_page + 1 + fragment['next_page'] = reverse(view, args=args, kwargs=kwargs) + + view_range = 5 + if total_pages > view_range: + pager_start = current_page - 2 + pager_end = current_page + 2 + if pager_start < 1: + pager_end += (1 - pager_start) + pager_start = 1 + if pager_end > total_pages: + pager_start -= (pager_end - total_pages) + pager_end = total_pages + else: + pager_start = 1 + pager_end = total_pages + + if pager_start > 1: + kwargs['page_number'] = 1 + fragment['first_page'] = reverse(view, args=args, kwargs=kwargs) + if pager_end < total_pages: + kwargs['page_number'] = total_pages + fragment['last_page'] = reverse(view, args=args, kwargs=kwargs) + + pager = [] + for page in range(pager_start, int(pager_end) + 1): + kwargs['page_number'] = page + pager.append((page, reverse(view, args=args, kwargs=kwargs))) + + kwargs['page_number'] = 1 + page_limits = [] + for limit in __PAGE_NAV_LIMITS__: + kwargs['page_limit'] = limit + page_limits.append((limit, + reverse(view, args=args, kwargs=kwargs))) + # resolver doesn't like this + del kwargs['page_number'] + del kwargs['page_limit'] + page_limits.append(('all', + reverse(view, args=args, kwargs=kwargs) + "|all")) + + fragment['pager'] = pager + fragment['page_limits'] = page_limits + + except Resolver404: + path = "404" + except NoReverseMatch: + nr = sys.exc_info()[1] + path = "NoReverseMatch: %s" % nr + except ValueError: + path = "ValueError" + #FIXME - Handle these + + fragment['path'] = path + return fragment + + +@register.inclusion_tag('widgets/filter_bar.html', takes_context=True) +def filter_navigator(context): + try: + path = context['request'].META['PATH_INFO'] + view, args, kwargs = resolve(path) + + # Strip any page limits and numbers + if 'page_number' in kwargs: + del kwargs['page_number'] + if 'page_limit' in kwargs: + del kwargs['page_limit'] + + filters = [] + for filter in filter_list: + if filter == 'group': + continue + if filter in kwargs: + myargs = kwargs.copy() + del myargs[filter] + filters.append((filter, + reverse(view, args=args, kwargs=myargs))) + filters.sort(lambda x, y: cmp(x[0], y[0])) + + myargs = kwargs.copy() + selected=True + if 'group' in myargs: + del myargs['group'] + selected=False + groups = [('---', reverse(view, args=args, kwargs=myargs), selected)] + for group in Group.objects.values('name'): + myargs['group'] = group['name'] + groups.append((group['name'], reverse(view, args=args, kwargs=myargs), + group['name'] == kwargs.get('group', ''))) + + return {'filters': filters, 'groups': groups} + except (Resolver404, NoReverseMatch, ValueError, KeyError): + pass + return dict() + + +def _subtract_or_na(mdict, x, y): + """ + Shortcut for build_metric_list + """ + try: + return round(mdict[x] - mdict[y], 4) + except: + return "n/a" + + +@register.filter +def build_metric_list(mdict): + """ + Create a list of metric table entries + + Moving this here to simplify the view. + Should really handle the case where these are missing... + """ + td_list = [] + # parse + td_list.append(_subtract_or_na(mdict, 'config_parse', 'config_download')) + #probe + td_list.append(_subtract_or_na(mdict, 'probe_upload', 'start')) + #inventory + td_list.append(_subtract_or_na(mdict, 'inventory', 'initialization')) + #install + td_list.append(_subtract_or_na(mdict, 'install', 'inventory')) + #cfg download & parse + td_list.append(_subtract_or_na(mdict, 'config_parse', 'probe_upload')) + #total + td_list.append(_subtract_or_na(mdict, 'finished', 'start')) + return td_list + + +@register.filter +def isstale(timestamp, entry_max=None): + """ + Check for a stale timestamp + + Compares two timestamps and returns True if the + difference is greater then 24 hours. + """ + if not entry_max: + entry_max = datetime.now() + return entry_max - timestamp > timedelta(hours=24) + + +@register.filter +def sort_interactions_by_name(value): + """ + Sort an interaction list by client name + """ + inters = list(value) + inters.sort(lambda a, b: cmp(a.client.name, b.client.name)) + return inters + + +class AddUrlFilter(template.Node): + def __init__(self, filter_name, filter_value): + self.filter_name = filter_name + self.filter_value = filter_value + self.fallback_view = 'Bcfg2.Server.Reports.reports.views.render_history_view' + + def render(self, context): + link = '#' + try: + path = context['request'].META['PATH_INFO'] + view, args, kwargs = resolve(path) + filter_value = self.filter_value.resolve(context, True) + if filter_value: + filter_name = smart_str(self.filter_name) + filter_value = smart_unicode(filter_value) + kwargs[filter_name] = filter_value + # These two don't make sense + if filter_name == 'server' and 'hostname' in kwargs: + del kwargs['hostname'] + elif filter_name == 'hostname' and 'server' in kwargs: + del kwargs['server'] + try: + link = reverse(view, args=args, kwargs=kwargs) + except NoReverseMatch: + link = reverse(self.fallback_view, args=None, + kwargs={filter_name: filter_value}) + except NoReverseMatch: + rm = sys.exc_info()[1] + raise rm + except (Resolver404, ValueError): + pass + return link + + +@register.tag +def add_url_filter(parser, token): + """ + Return a url with the filter added to the current view. + + Takes a new filter and resolves the current view with the new filter + applied. Resolves to Bcfg2.Server.Reports.reports.views.client_history + by default. + + {% add_url_filter server=interaction.server %} + """ + try: + tag_name, filter_pair = token.split_contents() + filter_name, filter_value = filter_pair.split('=', 1) + filter_name = filter_name.strip() + filter_value = parser.compile_filter(filter_value) + except ValueError: + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) + if not filter_name or not filter_value: + raise template.TemplateSyntaxError("argument should be a filter=value pair") + + return AddUrlFilter(filter_name, filter_value) + + +class MediaTag(template.Node): + def __init__(self, filter_value): + self.filter_value = filter_value + + def render(self, context): + base = context['MEDIA_URL'] + try: + request = context['request'] + try: + base = request.environ['bcfg2.media_url'] + except: + if request.path != request.META['PATH_INFO']: + offset = request.path.find(request.META['PATH_INFO']) + if offset > 0: + base = "%s/%s" % (request.path[:offset], \ + context['MEDIA_URL'].strip('/')) + except: + pass + return "%s/%s" % (base, self.filter_value) + + +@register.tag +def to_media_url(parser, token): + """ + Return a url relative to the media_url. + + {% to_media_url /bcfg2.css %} + """ + try: + filter_value = token.split_contents()[1] + filter_value = parser.compile_filter(filter_value) + except ValueError: + raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) + + return MediaTag(filter_value) + +@register.filter +def determine_client_state(entry): + """ + Determine client state. + + This is used to determine whether a client is reporting clean or + dirty. If the client is reporting dirty, this will figure out just + _how_ dirty and adjust the color accordingly. + """ + if entry.state == 'clean': + return "clean-lineitem" + + bad_percentage = 100 * (float(entry.badcount()) / entry.totalcount) + if bad_percentage < 33: + thisdirty = "slightly-dirty-lineitem" + elif bad_percentage < 66: + thisdirty = "dirty-lineitem" + else: + thisdirty = "very-dirty-lineitem" + return thisdirty + + +@register.tag(name='qs') +def do_qs(parser, token): + """ + qs tag + + accepts a name value pair and inserts or replaces it in the query string + """ + try: + tag, name, value = token.split_contents() + except ValueError: + raise TemplateSyntaxError, "%r tag requires exactly two arguments" \ + % token.contents.split()[0] + return QsNode(name, value) + +class QsNode(template.Node): + def __init__(self, name, value): + self.name = template.Variable(name) + self.value = template.Variable(value) + + def render(self, context): + try: + name = self.name.resolve(context) + value = self.value.resolve(context) + request = context['request'] + qs = copy(request.GET) + qs[name] = value + return "?%s" % qs.urlencode() + except template.VariableDoesNotExist: + return '' + except KeyError: + if settings.TEMPLATE_DEBUG: + raise Exception, "'qs' tag requires context['request']" + return '' + except: + return '' + + +@register.tag +def sort_link(parser, token): + ''' + Create a sort anchor tag. Reverse it if active. + + {% sort_link sort_key text %} + ''' + try: + tag, sort_key, text = token.split_contents() + except ValueError: + raise TemplateSyntaxError("%r tag requires at least four arguments" \ + % token.split_contents()[0]) + + return SortLinkNode(sort_key, text) + +class SortLinkNode(template.Node): + __TMPL__ = "{% load bcfg2_tags %}<a href='{% qs 'sort' key %}'>{{ text }}</a>" + + def __init__(self, sort_key, text): + self.sort_key = template.Variable(sort_key) + self.text = template.Variable(text) + + def render(self, context): + try: + try: + sort = context['request'].GET['sort'] + except KeyError: + #fall back on this + sort = context.get('sort', '') + sort_key = self.sort_key.resolve(context) + text = self.text.resolve(context) + + # add arrows + try: + sort_base = sort_key.lstrip('-') + if sort[0] == '-' and sort[1:] == sort_base: + text = text + '▼' + sort_key = sort_base + elif sort_base == sort: + text = text + '▲' + sort_key = '-' + sort_base + except IndexError: + pass + + context.push() + context['key'] = sort_key + context['text'] = mark_safe(text) + output = get_template_from_string(self.__TMPL__).render(context) + context.pop() + return output + except: + if settings.DEBUG: + raise + raise + return '' + diff --git a/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py new file mode 100644 index 000000000..a9b4f0371 --- /dev/null +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/split.py @@ -0,0 +1,8 @@ +from django import template +register = template.Library() + + +@register.filter +def split(s): + """split by newlines""" + return s.split('\n') diff --git a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py index 2e30125f9..0d4c6501d 100644 --- a/src/lib/Server/Reports/reports/templatetags/syntax_coloring.py +++ b/src/lib/Bcfg2/Server/Reports/reports/templatetags/syntax_coloring.py @@ -1,9 +1,11 @@ import sys from django import template -from django.utils.encoding import smart_unicode, smart_str +from django.utils.encoding import smart_unicode from django.utils.html import conditional_escape from django.utils.safestring import mark_safe +from Bcfg2.Bcfg2Py3k import u_str + register = template.Library() try: @@ -15,17 +17,12 @@ try: except: colorize = False -# py3k compatibility -def u_str(string): - if sys.hexversion >= 0x03000000: - return string - else: - return unicode(string) @register.filter def syntaxhilight(value, arg="diff", autoescape=None): """ - Returns a syntax-hilighted version of Code; requires code/language arguments + Returns a syntax-hilighted version of Code; + requires code/language arguments """ if autoescape: @@ -44,6 +41,6 @@ def syntaxhilight(value, arg="diff", autoescape=None): except: return value else: - return mark_safe(u_str('<div class="note-box">Tip: Install pygments for highlighting</div><pre>%s</pre>') % value) + return mark_safe(u_str('<div class="note-box">Tip: Install pygments ' + 'for highlighting</div><pre>%s</pre>') % value) syntaxhilight.needs_autoescape = True - diff --git a/src/lib/Server/Reports/reports/urls.py b/src/lib/Bcfg2/Server/Reports/reports/urls.py index 434ce07b7..1cfe725c2 100644 --- a/src/lib/Server/Reports/reports/urls.py +++ b/src/lib/Bcfg2/Server/Reports/reports/urls.py @@ -17,20 +17,23 @@ urlpatterns = patterns('Bcfg2.Server.Reports.reports', url(r'^client/(?P<hostname>[^/]+)/(?P<pk>\d+)/?$', 'views.client_detail', name='reports_client_detail_pk'), url(r'^client/(?P<hostname>[^/]+)/?$', 'views.client_detail', name='reports_client_detail'), url(r'^elements/(?P<type>\w+)/(?P<pk>\d+)/?$', 'views.config_item', name='reports_item'), + url(r'^entry/(?P<eid>\w+)/?$', 'views.entry_status', name='reports_entry'), ) urlpatterns += patterns('Bcfg2.Server.Reports.reports', *timeviewUrls( - (r'^grid/?$', 'views.client_index', None, 'reports_grid_view'), (r'^summary/?$', 'views.display_summary', None, 'reports_summary'), (r'^timing/?$', 'views.display_timing', None, 'reports_timing'), - (r'^elements/(?P<type>\w+)/?$', 'views.config_item_list', None, 'reports_item_list'), + (r'^common/(?P<threshold>\d+)/?$', 'views.common_problems', None, 'reports_common_problems'), + (r'^common/?$', 'views.common_problems', None, 'reports_common_problems'), )) urlpatterns += patterns('Bcfg2.Server.Reports.reports', *filteredUrls(*timeviewUrls( + (r'^grid/?$', 'views.client_index', None, 'reports_grid_view'), (r'^detailed/?$', - 'views.client_detailed_list', None, 'reports_detailed_list') + 'views.client_detailed_list', None, 'reports_detailed_list'), + (r'^elements/(?P<type>\w+)/?$', 'views.config_item_list', None, 'reports_item_list'), ))) urlpatterns += patterns('Bcfg2.Server.Reports.reports', diff --git a/src/lib/Server/Reports/reports/views.py b/src/lib/Bcfg2/Server/Reports/reports/views.py index ccd71a60e..e4c38363f 100644 --- a/src/lib/Server/Reports/reports/views.py +++ b/src/lib/Bcfg2/Server/Reports/reports/views.py @@ -13,16 +13,41 @@ from django.http import \ from django.shortcuts import render_to_response, get_object_or_404 from django.core.urlresolvers import \ resolve, reverse, Resolver404, NoReverseMatch -from django.db import connection +from django.db import connection, DatabaseError +from django.db.models import Q from Bcfg2.Server.Reports.reports.models import * +__SORT_FIELDS__ = ( 'client', 'state', 'good', 'bad', 'modified', 'extra', \ + 'timestamp', 'server' ) + class PaginationError(Exception): """This error is raised when pagination cannot be completed.""" pass +def _in_bulk(model, ids): + """ + Short cut to fetch in bulk and trap database errors. sqlite will raise + a "too many SQL variables" exception if this list is too long. Try using + django and fetch manually if an error occurs + + returns a dict of this form { id: <model instance> } + """ + + try: + return model.objects.in_bulk(ids) + except DatabaseError: + pass + + # if objects.in_bulk fails so will obejcts.filter(pk__in=ids) + bulk_dict = {} + [bulk_dict.__setitem__(i.id, i) \ + for i in model.objects.all() if i.id in ids] + return bulk_dict + + def server_error(request): """ 500 error handler. @@ -44,7 +69,7 @@ def timeview(fn): """ def _handle_timeview(request, **kwargs): """Send any posts back.""" - if request.method == 'POST': + if request.method == 'POST' and request.POST.get('op', '') == 'timeview': cal_date = request.POST['cal_date'] try: fmt = "%Y/%m/%d" @@ -84,6 +109,30 @@ def timeview(fn): return _handle_timeview +def _handle_filters(query, **kwargs): + """ + Applies standard filters to a query object + + Returns an updated query object + + query - query object to filter + + server -- Filter interactions by server + state -- Filter interactions by state + group -- Filter interactions by group + + """ + if 'state' in kwargs and kwargs['state']: + query = query.filter(state__exact=kwargs['state']) + if 'server' in kwargs and kwargs['server']: + query = query.filter(server__exact=kwargs['server']) + + if 'group' in kwargs and kwargs['group']: + group = get_object_or_404(Group, name=kwargs['group']) + query = query.filter(metadata__groups__id=group.pk) + return query + + def config_item(request, pk, type="bad"): """ Display a single entry. @@ -121,47 +170,138 @@ def config_item(request, pk, type="bad"): @timeview -def config_item_list(request, type, timestamp=None): +def config_item_list(request, type, timestamp=None, **kwargs): """Render a listing of affected elements""" mod_or_bad = type.lower() type = convert_entry_type_to_id(type) if type < 0: raise Http404 - current_clients = Interaction.objects.get_interaction_per_client_ids(timestamp) - item_list_dict = {} - seen = dict() - for x in Entries_interactions.objects.filter(interaction__in=current_clients, - type=type).select_related(): - if (x.entry, x.reason) in seen: - continue - seen[(x.entry, x.reason)] = 1 - if item_list_dict.get(x.entry.kind, None): - item_list_dict[x.entry.kind].append(x) - else: - item_list_dict[x.entry.kind] = [x] + current_clients = Interaction.objects.interaction_per_client(timestamp) + current_clients = [q['id'] for q in _handle_filters(current_clients, **kwargs).values('id')] + + ldata = list(Entries_interactions.objects.filter( + interaction__in=current_clients, type=type).values()) + entry_ids = set([x['entry_id'] for x in ldata]) + reason_ids = set([x['reason_id'] for x in ldata]) - for kind in item_list_dict: - item_list_dict[kind].sort(lambda a, b: cmp(a.entry.name, b.entry.name)) + entries = _in_bulk(Entries, entry_ids) + reasons = _in_bulk(Reason, reason_ids) + + kind_list = {} + [kind_list.__setitem__(kind, {}) for kind in set([e.kind for e in entries.values()])] + for x in ldata: + kind = entries[x['entry_id']].kind + data_key = (x['entry_id'], x['reason_id']) + try: + kind_list[kind][data_key].append(x['id']) + except KeyError: + kind_list[kind][data_key] = [x['id']] + + lists = [] + for kind in kind_list.keys(): + lists.append((kind, [(entries[e[0][0]], reasons[e[0][1]], e[1]) + for e in sorted(kind_list[kind].iteritems(), key=lambda x: entries[x[0][0]].name)])) return render_to_response('config_items/listing.html', - {'item_list_dict': item_list_dict, + {'item_list': lists, 'mod_or_bad': mod_or_bad, 'timestamp': timestamp}, context_instance=RequestContext(request)) @timeview -def client_index(request, timestamp=None): +def entry_status(request, eid, timestamp=None, **kwargs): + """Render a listing of affected elements""" + entry = get_object_or_404(Entries, pk=eid) + + current_clients = Interaction.objects.interaction_per_client(timestamp) + inters = {} + [inters.__setitem__(i.id, i) \ + for i in _handle_filters(current_clients, **kwargs).select_related('client')] + + eis = Entries_interactions.objects.filter( + interaction__in=inters.keys(), entry=entry) + + reasons = _in_bulk(Reason, set([x.reason_id for x in eis])) + + item_data = [] + for ei in eis: + item_data.append((ei, inters[ei.interaction_id], reasons[ei.reason_id])) + + return render_to_response('config_items/entry_status.html', + {'entry': entry, + 'item_data': item_data, + 'timestamp': timestamp}, + context_instance=RequestContext(request)) + + +@timeview +def common_problems(request, timestamp=None, threshold=None): + """Mine config entries""" + + if request.method == 'POST': + try: + threshold = int(request.POST['threshold']) + view, args, kw = resolve(request.META['PATH_INFO']) + kw['threshold'] = threshold + return HttpResponseRedirect(reverse(view, + args=args, + kwargs=kw)) + except: + pass + + try: + threshold = int(threshold) + except: + threshold = 10 + + c_intr = Interaction.objects.get_interaction_per_client_ids(timestamp) + data_list = {} + [data_list.__setitem__(t_id, {}) \ + for t_id, t_label in TYPE_CHOICES if t_id != TYPE_GOOD] + ldata = list(Entries_interactions.objects.filter( + interaction__in=c_intr).exclude(type=TYPE_GOOD).values()) + + entry_ids = set([x['entry_id'] for x in ldata]) + reason_ids = set([x['reason_id'] for x in ldata]) + for x in ldata: + type = x['type'] + data_key = (x['entry_id'], x['reason_id']) + try: + data_list[type][data_key].append(x['id']) + except KeyError: + data_list[type][data_key] = [x['id']] + + entries = _in_bulk(Entries, entry_ids) + reasons = _in_bulk(Reason, reason_ids) + + lists = [] + for type, type_name in TYPE_CHOICES: + if type == TYPE_GOOD: + continue + lists.append([type_name.lower(), [(entries[e[0][0]], reasons[e[0][1]], e[1]) + for e in sorted(data_list[type].items(), key=lambda x: len(x[1]), reverse=True) + if len(e[1]) > threshold]]) + + return render_to_response('config_items/common.html', + {'lists': lists, + 'timestamp': timestamp, + 'threshold': threshold}, + context_instance=RequestContext(request)) + + +@timeview +def client_index(request, timestamp=None, **kwargs): """ Render a grid view of active clients. Keyword parameters: - timestamp -- datetime objectto render from + timestamp -- datetime object to render from """ - list = Interaction.objects.interaction_per_client(timestamp).select_related()\ - .order_by("client__name").all() + list = _handle_filters(Interaction.objects.interaction_per_client(timestamp), **kwargs).\ + select_related().order_by("client__name").all() return render_to_response('clients/index.html', {'inter_list': list, @@ -177,8 +317,29 @@ def client_detailed_list(request, timestamp=None, **kwargs): """ + try: + sort = request.GET['sort'] + if sort[0] == '-': + sort_key = sort[1:] + else: + sort_key = sort + if not sort_key in __SORT_FIELDS__: + raise ValueError + + if sort_key == "client": + kwargs['orderby'] = "%s__name" % sort + elif sort_key == "good": + kwargs['orderby'] = "%scount" % sort + elif sort_key in ["bad", "modified", "extra"]: + kwargs['orderby'] = "%s_entries" % sort + else: + kwargs['orderby'] = sort + kwargs['sort'] = sort + except (ValueError, KeyError): + kwargs['orderby'] = "client__name" + kwargs['sort'] = "client" + kwargs['interaction_base'] = Interaction.objects.interaction_per_client(timestamp).select_related() - kwargs['orderby'] = "client__name" kwargs['page_limit'] = 0 return render_history_view(request, 'clients/detailed-list.html', **kwargs) @@ -187,13 +348,25 @@ def client_detail(request, hostname=None, pk=None): context = dict() client = get_object_or_404(Client, name=hostname) if(pk == None): - context['interaction'] = client.current_interaction - return render_history_view(request, 'clients/detail.html', page_limit=5, - client=client, context=context) + inter = client.current_interaction + maxdate = None else: - context['interaction'] = client.interactions.get(pk=pk) - return render_history_view(request, 'clients/detail.html', page_limit=5, - client=client, maxdate=context['interaction'].timestamp, context=context) + inter = client.interactions.get(pk=pk) + maxdate = inter.timestamp + + ei = Entries_interactions.objects.filter(interaction=inter).select_related('entry').order_by('entry__kind', 'entry__name') + #ei = Entries_interactions.objects.filter(interaction=inter).select_related('entry') + #ei = sorted(Entries_interactions.objects.filter(interaction=inter).select_related('entry'), + # key=lambda x: (x.entry.kind, x.entry.name)) + context['ei_lists'] = ( + ('bad', [x for x in ei if x.type == TYPE_BAD]), + ('modified', [x for x in ei if x.type == TYPE_MODIFIED]), + ('extra', [x for x in ei if x.type == TYPE_EXTRA]) + ) + + context['interaction']=inter + return render_history_view(request, 'clients/detail.html', page_limit=5, + client=client, maxdate=maxdate, context=context) def client_manage(request): @@ -230,9 +403,9 @@ def display_summary(request, timestamp=None): """ Display a summary of the bcfg2 world """ - query = Interaction.objects.interaction_per_client(timestamp).select_related() - node_count = query.count() - recent_data = query.all() + recent_data = Interaction.objects.interaction_per_client(timestamp) \ + .select_related().all() + node_count = len(recent_data) if not timestamp: timestamp = datetime.now() @@ -240,18 +413,11 @@ def display_summary(request, timestamp=None): bad=[], modified=[], extra=[], - stale=[], - pings=[]) + stale=[]) for node in recent_data: if timestamp - node.timestamp > timedelta(hours=24): collected_data['stale'].append(node) # If stale check for uptime - try: - if node.client.pings.latest().status == 'N': - collected_data['pings'].append(node) - except Ping.DoesNotExist: - collected_data['pings'].append(node) - continue if node.bad_entry_count() > 0: collected_data['bad'].append(node) else: @@ -281,9 +447,6 @@ def display_summary(request, timestamp=None): if len(collected_data['stale']) > 0: summary_data.append(get_dict('stale', 'nodes did not run within the last 24 hours.')) - if len(collected_data['pings']) > 0: - summary_data.append(get_dict('pings', - 'are down.')) return render_to_response('displays/summary.html', {'summary_data': summary_data, 'node_count': node_count, @@ -299,7 +462,11 @@ def display_timing(request, timestamp=None): for inter in inters] for metric in Performance.objects.filter(interaction__in=list(mdict.keys())).all(): for i in metric.interaction.all(): - mdict[i][metric.metric] = metric.value + try: + mdict[i][metric.metric] = metric.value + except KeyError: + #In the unlikely event two interactions share a metric, ignore it + pass return render_to_response('displays/timing.html', {'metrics': list(mdict.values()), 'timestamp': timestamp}, @@ -324,6 +491,7 @@ def render_history_view(request, template='clients/history.html', **kwargs): not found server -- Filter interactions by server state -- Filter interactions by state + group -- Filter interactions by group entry_max -- Most recent interaction to display orderby -- Sort results using this field @@ -345,15 +513,15 @@ def render_history_view(request, template='clients/history.html', **kwargs): # Either filter by client or limit by clients iquery = kwargs.get('interaction_base', Interaction.objects) if client: - iquery = iquery.filter(client__exact=client).select_related() + iquery = iquery.filter(client__exact=client) + iquery = iquery.select_related() if 'orderby' in kwargs and kwargs['orderby']: iquery = iquery.order_by(kwargs['orderby']) + if 'sort' in kwargs: + context['sort'] = kwargs['sort'] - if 'state' in kwargs and kwargs['state']: - iquery = iquery.filter(state__exact=kwargs['state']) - if 'server' in kwargs and kwargs['server']: - iquery = iquery.filter(server__exact=kwargs['server']) + iquery = _handle_filters(iquery, **kwargs) if entry_max: iquery = iquery.filter(timestamp__lte=entry_max) diff --git a/src/lib/Server/Reports/settings.py b/src/lib/Bcfg2/Server/Reports/settings.py index 957c4237a..b27348aee 100644 --- a/src/lib/Server/Reports/settings.py +++ b/src/lib/Bcfg2/Server/Reports/settings.py @@ -1,11 +1,25 @@ -import django +import os import sys +import getopt +import Bcfg2.Options + +try: + import django +except ImportError: + raise ImportError('Import of Django module failed. Is Django installed?') + +cfile_opt=Bcfg2.Options.CFILE +cfiles=[cfile_opt.default, '/etc/bcfg2-web.conf'] +for i in range(1, len(sys.argv)): + if sys.argv[i] == cfile_opt.cmd: + cfiles = sys.argv[i+1] + break # Compatibility import from Bcfg2.Bcfg2Py3k import ConfigParser # Django settings for bcfg2 reports project. c = ConfigParser.ConfigParser() -if len(c.read(['/etc/bcfg2.conf', '/etc/bcfg2-web.conf'])) == 0: +if len(c.read(cfiles)) == 0: raise ImportError("Please check that bcfg2.conf or bcfg2-web.conf exists " "and is readable by your web server.") @@ -67,10 +81,10 @@ if django.VERSION[0] == 1 and django.VERSION[1] < 2: # Local time zone for this installation. All choices can be found here: # http://docs.djangoproject.com/en/dev/ref/settings/#time-zone -try: - TIME_ZONE = c.get('statistics', 'time_zone') -except: - if django.VERSION[0] == 1 and django.VERSION[1] > 2: +if django.VERSION[0] == 1 and django.VERSION[1] > 2: + try: + TIME_ZONE = c.get('statistics', 'time_zone') + except: TIME_ZONE = None # Language code for this installation. All choices can be found here: @@ -120,21 +134,16 @@ AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend', # The NIS group authorized to login to BCFG2's reportinvg system AUTHORIZED_GROUP = '' #create login url area: -try: - import django.contrib.auth -except ImportError: - raise ImportError('Import of Django module failed. Is Django installed?') -django.contrib.auth.LOGIN_URL = '/login' +LOGIN_URL = '/login' SESSION_EXPIRE_AT_BROWSER_CLOSE = True - - + + TEMPLATE_DIRS = ( # Put strings here, like "/home/html/django_templates". # Always use forward slashes, even on Windows. '/usr/share/python-support/python-django/django/contrib/admin/templates/', - 'Bcfg2.Server.Reports.reports' ) if django.VERSION[0] == 1 and django.VERSION[1] < 2: diff --git a/src/lib/Server/Reports/urls.py b/src/lib/Bcfg2/Server/Reports/urls.py index d7ff1eee5..d7ff1eee5 100644 --- a/src/lib/Server/Reports/urls.py +++ b/src/lib/Bcfg2/Server/Reports/urls.py diff --git a/src/lib/Server/Reports/utils.py b/src/lib/Bcfg2/Server/Reports/utils.py index e0b6ead59..c47763e39 100755 --- a/src/lib/Server/Reports/utils.py +++ b/src/lib/Bcfg2/Server/Reports/utils.py @@ -3,7 +3,7 @@ from django.conf.urls.defaults import * import re """List of filters provided by filteredUrls""" -filter_list = ('server', 'state') +filter_list = ('server', 'state', 'group') class BatchFetch(object): @@ -97,6 +97,8 @@ def filteredUrls(pattern, view, kwargs=None, name=None): tail = mtail.group(1) pattern = pattern[:len(pattern) - len(tail)] for filter in ('/state/(?P<state>\w+)', + '/group/(?P<group>[\w\-\.]+)', + '/group/(?P<group>[\w\-\.]+)/(?P<state>[A-Za-z]+)', '/server/(?P<server>[\w\-\.]+)', '/server/(?P<server>[\w\-\.]+)/(?P<state>[A-Za-z]+)'): results += [(pattern + filter + tail, view, kwargs)] diff --git a/src/lib/Server/Snapshots/__init__.py b/src/lib/Bcfg2/Server/Snapshots/__init__.py index 7c901adb2..7c901adb2 100644 --- a/src/lib/Server/Snapshots/__init__.py +++ b/src/lib/Bcfg2/Server/Snapshots/__init__.py diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Bcfg2/Server/Snapshots/model.py index 5d7973c16..0bbd206da 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Bcfg2/Server/Snapshots/model.py @@ -6,13 +6,7 @@ import sqlalchemy.exceptions from sqlalchemy.orm import relation, backref from sqlalchemy.ext.declarative import declarative_base - -# py3k compatibility -def u_str(string): - if sys.hexversion >= 0x03000000: - return string - else: - return unicode(string) +from Bcfg2.Bcfg2Py3k import u_str class Uniquer(object): diff --git a/src/lib/Server/__init__.py b/src/lib/Bcfg2/Server/__init__.py index bca73ded7..320371284 100644 --- a/src/lib/Server/__init__.py +++ b/src/lib/Bcfg2/Server/__init__.py @@ -1,9 +1,7 @@ -# $Id$ """This is the set of modules for Bcfg2.Server.""" import lxml.etree -__revision__ = '$Revision$' __all__ = ["Admin", "Core", "FileMonitor", "Plugin", "Plugins", "Hostbase", "Reports", "Snapshots", "XMLParser"] diff --git a/src/lib/Statistics.py b/src/lib/Bcfg2/Statistics.py index a0cb8f39b..a0cb8f39b 100644 --- a/src/lib/Statistics.py +++ b/src/lib/Bcfg2/Statistics.py diff --git a/src/lib/__init__.py b/src/lib/Bcfg2/__init__.py index d36c0a00a..357f66f6d 100644 --- a/src/lib/__init__.py +++ b/src/lib/Bcfg2/__init__.py @@ -1,4 +1,3 @@ """Base modules definition.""" -__revision__ = '$Revision$' __all__ = ['Server', 'Client', 'Component', 'Logger', 'Options', 'Proxy', 'Statistics'] diff --git a/src/lib/Bcfg2Py3Incompat.py b/src/lib/Bcfg2Py3Incompat.py deleted file mode 100644 index 6b66e72b0..000000000 --- a/src/lib/Bcfg2Py3Incompat.py +++ /dev/null @@ -1,2 +0,0 @@ -def fprint(s, f): - print(s, file=f) diff --git a/src/lib/Client/Tools/Portage.py b/src/lib/Client/Tools/Portage.py deleted file mode 100644 index 17163afa9..000000000 --- a/src/lib/Client/Tools/Portage.py +++ /dev/null @@ -1,72 +0,0 @@ -"""This is the Bcfg2 tool for the Gentoo Portage system.""" -__revision__ = '$Revision$' - -import re -import Bcfg2.Client.Tools - - -class Portage(Bcfg2.Client.Tools.PkgTool): - """The Gentoo toolset implements package and service operations and - inherits the rest from Toolset.Toolset.""" - name = 'Portage' - __execs__ = ['/usr/bin/emerge', '/usr/bin/equery'] - __handles__ = [('Package', 'ebuild')] - __req__ = {'Package': ['name', 'version']} - pkgtype = 'ebuild' - # requires a working PORTAGE_BINHOST in make.conf - pkgtool = ('emerge --getbinpkgonly %s', ('=%s-%s', ['name', 'version'])) - - def __init__(self, logger, cfg, setup): - Bcfg2.Client.Tools.PkgTool.__init__(self, logger, cfg, setup) - self.__important__ = self.__important__ + ['/etc/make.conf'] - self.cfg = cfg - self.installed = {} - self.RefreshPackages() - - def RefreshPackages(self): - """Refresh memory hashes of packages.""" - ret, cache = self.cmd.run("equery -q list '*'") - if ret == 2: - cache = self.cmd.run("equery -q list '*'")[1] - pattern = re.compile('(.*)-(\d.*)') - self.installed = {} - for pkg in cache: - if pattern.match(pkg): - name = pattern.match(pkg).group(1) - version = pattern.match(pkg).group(2) - self.installed[name] = version - else: - self.logger.info("Failed to parse pkg name %s" % pkg) - - def VerifyPackage(self, entry, modlist): - """Verify package for entry.""" - if not 'version' in entry.attrib: - self.logger.info("Cannot verify unversioned package %s" % - (entry.attrib['name'])) - return False - if entry.attrib['name'] in self.installed: - if self.installed[entry.attrib['name']] == entry.attrib['version']: - if not self.setup['quick'] and \ - entry.get('verify', 'true') == 'true': - output = self.cmd.run("/usr/bin/equery -N check '=%s-%s' 2>&1 " - "| grep '!!!' | awk '{print $2}'" \ - % (entry.get('name'), entry.get('version')))[1] - if [filename for filename in output \ - if filename not in modlist]: - return False - return True - else: - entry.set('current_version', self.installed[entry.get('name')]) - return False - entry.set('current_exists', 'false') - return False - - def RemovePackages(self, packages): - """Deal with extra configuration detected.""" - pkgnames = " ".join([pkg.get('name') for pkg in packages]) - if len(packages) > 0: - self.logger.info('Removing packages:') - self.logger.info(pkgnames) - self.cmd.run("emerge --unmerge --quiet %s" % " ".join(pkgnames.split(' '))) - self.RefreshPackages() - self.extra = self.FindExtraPackages() diff --git a/src/lib/Options.py b/src/lib/Options.py deleted file mode 100644 index b55a8a55a..000000000 --- a/src/lib/Options.py +++ /dev/null @@ -1,392 +0,0 @@ -"""Option parsing library for utilities.""" -__revision__ = '$Revision$' - -import getopt -import os -import sys -import shlex -import Bcfg2.Client.Tools -# Compatibility imports -from Bcfg2.Bcfg2Py3k import ConfigParser - -def bool_cook(x): - if x: - return True - else: - return False - -class OptionFailure(Exception): - pass - -DEFAULT_CONFIG_LOCATION = '/etc/bcfg2.conf' #/etc/bcfg2.conf -DEFAULT_INSTALL_PREFIX = '/usr' #/usr - -class Option(object): - cfpath = DEFAULT_CONFIG_LOCATION - __cfp = False - - def getCFP(self): - if not self.__cfp: - self.__cfp = ConfigParser.ConfigParser() - self.__cfp.readfp(open(self.cfpath)) - return self.__cfp - cfp = property(getCFP) - - def get_cooked_value(self, value): - if self.boolean: - return True - if self.cook: - return self.cook(value) - else: - return value - - def __init__(self, desc, default, cmd=False, odesc=False, - env=False, cf=False, cook=False, long_arg=False): - self.desc = desc - self.default = default - self.cmd = cmd - self.long = long_arg - if not self.long: - if cmd and (cmd[0] != '-' or len(cmd) != 2): - raise OptionFailure("Poorly formed command %s" % cmd) - else: - if cmd and (not cmd.startswith('--')): - raise OptionFailure("Poorly formed command %s" % cmd) - self.odesc = odesc - self.env = env - self.cf = cf - self.boolean = False - if not odesc and not cook: - self.boolean = True - self.cook = cook - - def buildHelpMessage(self): - msg = '' - if self.cmd: - if not self.long: - msg = self.cmd.ljust(3) - else: - msg = self.cmd - if self.odesc: - if self.long: - msg = "%-28s" % ("%s=%s" % (self.cmd, self.odesc)) - else: - msg += '%-25s' % (self.odesc) - else: - msg += '%-25s' % ('') - msg += "%s\n" % self.desc - return msg - - def buildGetopt(self): - gstr = '' - if self.long: - return gstr - if self.cmd: - gstr = self.cmd[1] - if self.odesc: - gstr += ':' - return gstr - - def buildLongGetopt(self): - if self.odesc: - return self.cmd[2:]+'=' - else: - return self.cmd[2:] - - def parse(self, opts, rawopts): - if self.cmd and opts: - # Processing getopted data - optinfo = [opt[1] for opt in opts if opt[0] == self.cmd] - if optinfo: - if optinfo[0]: - self.value = self.get_cooked_value(optinfo[0]) - else: - self.value = True - return - if self.cmd and self.cmd in rawopts: - data = rawopts[rawopts.index(self.cmd) + 1] - self.value = self.get_cooked_value(data) - return - # No command line option found - if self.env and self.env in os.environ: - self.value = self.get_cooked_value(os.environ[self.env]) - return - if self.cf: - # FIXME: This is potentially masking a lot of errors - try: - self.value = self.get_cooked_value(self.cfp.get(*self.cf)) - return - except: - pass - # Default value not cooked - self.value = self.default - -class OptionSet(dict): - def __init__(self, *args): - dict.__init__(self, *args) - self.hm = self.buildHelpMessage() - - def buildGetopt(self): - return ''.join([opt.buildGetopt() for opt in list(self.values())]) - - def buildLongGetopt(self): - return [opt.buildLongGetopt() for opt in list(self.values()) if opt.long] - - def buildHelpMessage(self): - if hasattr(self, 'hm'): - return self.hm - return ' '.join([opt.buildHelpMessage() for opt in list(self.values())]) - - def helpExit(self, msg='', code=1): - if msg: - print(msg) - print("Usage:\n %s" % self.buildHelpMessage()) - raise SystemExit(code) - - def parse(self, argv, do_getopt=True): - '''Parse options from command line.''' - if do_getopt: - try: - opts, args = getopt.getopt(argv, self.buildGetopt(), - self.buildLongGetopt()) - except getopt.GetoptError: - err = sys.exc_info()[1] - self.helpExit(err) - if '-h' in argv: - self.helpExit('', 0) - self['args'] = args - for key in list(self.keys()): - if key == 'args': - continue - option = self[key] - if do_getopt: - option.parse(opts, []) - else: - option.parse([], argv) - if hasattr(option, 'value'): - val = option.value - self[key] = val - -list_split = lambda x:x.replace(' ','').split(',') -flist_split = lambda x:list_split(x.replace(':', '').lower()) - -def colon_split(c_string): - if c_string: - return c_string.split(':') - return [] - -def get_bool(s): - # these values copied from ConfigParser.RawConfigParser.getboolean - # with the addition of True and False - truelist = ["1", "yes", "True", "true", "on"] - falselist = ["0", "no", "False", "false", "off"] - if s in truelist: - return True - elif s in falselist: - return False - else: - raise ValueError - -# General options -CFILE = Option('Specify configuration file', DEFAULT_CONFIG_LOCATION, cmd='-C', - odesc='<conffile>') -LOCKFILE = Option('Specify lockfile', - "/var/lock/bcfg2.run", - cf=('components', 'lockfile'), - odesc='<Path to lockfile>') -HELP = Option('Print this usage message', False, cmd='-h') -DEBUG = Option("Enable debugging output", False, cmd='-d') -VERBOSE = Option("Enable verbose output", False, cmd='-v') -DAEMON = Option("Daemonize process, storing pid", False, - cmd='-D', odesc="<pidfile>") -INSTALL_PREFIX = Option('Installation location', cf=('server', 'prefix'), - default=DEFAULT_INSTALL_PREFIX, odesc='</path>') -SENDMAIL_PATH = Option('Path to sendmail', cf=('reports', 'sendmailpath'), - default='/usr/lib/sendmail') -INTERACTIVE = Option('Run interactively, prompting the user for each change', - default=False, - cmd='-I', ) -ENCODING = Option('Encoding of cfg files', - default='UTF-8', - cmd='-E', - odesc='<encoding>', - cf=('components', 'encoding')) -PARANOID_PATH = Option('Specify path for paranoid file backups', - default='/var/cache/bcfg2', cf=('paranoid', 'path'), - odesc='<paranoid backup path>') -PARANOID_MAX_COPIES = Option('Specify the number of paranoid copies you want', - default=1, cf=('paranoid', 'max_copies'), - odesc='<max paranoid copies>') -OMIT_LOCK_CHECK = Option('Omit lock check', default=False, cmd='-O') -CORE_PROFILE = Option('profile', - default=False, cmd='-p', ) -FILES_ON_STDIN = Option('Operate on a list of files supplied on stdin', - cmd='--stdin', default=False, long_arg=True) -SCHEMA_PATH = Option('Path to XML Schema files', cmd='--schema', - odesc='<schema path>', - default="%s/share/bcfg2/schemas" % DEFAULT_INSTALL_PREFIX, - long_arg=True) -REQUIRE_SCHEMA = Option("Require property files to have matching schema files", - cmd="--require-schema", default=False, long_arg=True) - -# Metadata options -MDATA_OWNER = Option('Default Path owner', - default='root', cf=('mdata', 'owner'), - odesc='owner permissions') -MDATA_GROUP = Option('Default Path group', - default='root', cf=('mdata', 'group'), - odesc='group permissions') -MDATA_IMPORTANT = Option('Default Path priority (importance)', - default='False', cf=('mdata', 'important'), - odesc='Important entries are installed first') -MDATA_PERMS = Option('Default Path permissions', - '644', cf=('mdata', 'perms'), - odesc='octal permissions') -MDATA_PARANOID = Option('Default Path paranoid setting', - 'false', cf=('mdata', 'paranoid'), - odesc='Path paranoid setting') -MDATA_SENSITIVE = Option('Default Path sensitive setting', - 'false', cf=('mdata', 'sensitive'), - odesc='Path sensitive setting') - -# Server options -SERVER_REPOSITORY = Option('Server repository path', '/var/lib/bcfg2', - cf=('server', 'repository'), cmd='-Q', - odesc='<repository path>') -SERVER_PLUGINS = Option('Server plugin list', cf=('server', 'plugins'), - # default server plugins - default=[ - 'Bundler', - 'Cfg', - 'Metadata', - 'Pkgmgr', - 'Rules', - 'SSHbase', - ], - cook=list_split) -SERVER_MCONNECT = Option('Server Metadata Connector list', cook=list_split, - cf=('server', 'connectors'), default=['Probes'], ) -SERVER_FILEMONITOR = Option('Server file monitor', cf=('server', 'filemonitor'), - default='default', odesc='File monitoring driver') -SERVER_LISTEN_ALL = Option('Listen on all interfaces', - cf=('server', 'listen_all'), - cmd='--listen-all', - default=False, - long_arg=True, - cook=get_bool, - odesc='True|False') -SERVER_LOCATION = Option('Server Location', cf=('components', 'bcfg2'), - default='https://localhost:6789', cmd='-S', - odesc='https://server:port') -SERVER_STATIC = Option('Server runs on static port', cf=('components', 'bcfg2'), - default=False, cook=bool_cook) -SERVER_KEY = Option('Path to SSL key', cf=('communication', 'key'), - default=False, cmd='--ssl-key', odesc='<ssl key>', - long_arg=True) -SERVER_CERT = Option('Path to SSL certificate', default='/etc/bcfg2.key', - cf=('communication', 'certificate'), odesc='<ssl cert>') -SERVER_CA = Option('Path to SSL CA Cert', default=None, - cf=('communication', 'ca'), odesc='<ca cert>') -SERVER_PASSWORD = Option('Communication Password', cmd='-x', odesc='<password>', - cf=('communication', 'password'), default=False) -SERVER_PROTOCOL = Option('Server Protocol', cf=('communication', 'procotol'), - default='xmlrpc/ssl') -# Client options -CLIENT_KEY = Option('Path to SSL key', cf=('communication', 'key'), - default=None, cmd="--ssl-key", odesc='<ssl key>', - long_arg=True) -CLIENT_CERT = Option('Path to SSL certificate', default=None, cmd="--ssl-cert", - cf=('communication', 'certificate'), odesc='<ssl cert>', - long_arg=True) -CLIENT_CA = Option('Path to SSL CA Cert', default=None, cmd="--ca-cert", - cf=('communication', 'ca'), odesc='<ca cert>', - long_arg=True) -CLIENT_SCNS = Option('List of server commonNames', default=None, cmd="--ssl-cns", - cf=('communication', 'serverCommonNames'), - odesc='<commonName1:commonName2>', cook=list_split, - long_arg=True) -CLIENT_PROFILE = Option('Assert the given profile for the host', - default=False, cmd='-p', odesc="<profile>") -CLIENT_RETRIES = Option('The number of times to retry network communication', - default='3', cmd='-R', cf=('communication', 'retries'), - odesc="<retry count>") -CLIENT_DRYRUN = Option('Do not actually change the system', - default=False, cmd='-n', ) -CLIENT_EXTRA_DISPLAY = Option('enable extra entry output', - default=False, cmd='-e', ) -CLIENT_PARANOID = Option('Make automatic backups of config files', - default=False, - cmd='-P', - cook=get_bool, - cf=('client', 'paranoid')) -CLIENT_DRIVERS = Option('Specify tool driver set', cmd='-D', - cf=('client', 'drivers'), - odesc="<driver1,driver2>", cook=list_split, - default=Bcfg2.Client.Tools.default) -CLIENT_CACHE = Option('Store the configuration in a file', - default=False, cmd='-c', odesc="<cache path>") -CLIENT_REMOVE = Option('Force removal of additional configuration items', - default=False, cmd='-r', odesc="<entry type|all>") -CLIENT_BUNDLE = Option('Only configure the given bundle(s)', default=[], - cmd='-b', odesc='<bundle:bundle>', cook=colon_split) -CLIENT_BUNDLEQUICK = Option('only verify/configure the given bundle(s)', default=False, - cmd='-Q') -CLIENT_INDEP = Option('Only configure independent entries, ignore bundles', default=False, - cmd='-z') -CLIENT_KEVLAR = Option('Run in kevlar (bulletproof) mode', default=False, - cmd='-k', ) -CLIENT_DLIST = Option('Run client in server decision list mode', default='none', - cf=('client', 'decision'), - cmd='-l', odesc='<whitelist|blacklist|none>') -CLIENT_FILE = Option('Configure from a file rather than querying the server', - default=False, cmd='-f', odesc='<specification path>') -CLIENT_QUICK = Option('Disable some checksum verification', default=False, - cmd='-q', ) -CLIENT_USER = Option('The user to provide for authentication', default='root', - cmd='-u', cf=('communication', 'user'), odesc='<user>') -CLIENT_SERVICE_MODE = Option('Set client service mode', default='default', - cmd='-s', odesc='<default|disabled|build>') -CLIENT_TIMEOUT = Option('Set the client XML-RPC timeout', default=90, - cmd='-t', cf=('communication', 'timeout'), - odesc='<timeout>') - -# bcfg2-test options -TEST_NOSEOPTS = Option('Options to pass to nosetests', default=[], - cmd='--nose-options', cf=('bcfg2_test', 'nose_options'), - odesc='<opts>', long_arg=True, cook=shlex.split) -TEST_IGNORE = Option('Ignore these entries if they fail to build.', default=[], - cmd='--ignore', - cf=('bcfg2_test', 'ignore_entries'), long_arg=True, - odesc='<Type>:<name>,<Type>:<name>', cook=list_split) - -# APT client tool options -CLIENT_APT_TOOLS_INSTALL_PATH = Option('Apt tools install path', - cf=('APT', 'install_path'), - default='/usr') -CLIENT_APT_TOOLS_VAR_PATH = Option('Apt tools var path', - cf=('APT', 'var_path'), default='/var') -CLIENT_SYSTEM_ETC_PATH = Option('System etc path', cf=('APT', 'etc_path'), - default='/etc') - -# Logging options -LOGGING_FILE_PATH = Option('Set path of file log', default=None, - cmd='-o', odesc='<path>', cf=('logging', 'path')) - -class OptionParser(OptionSet): - """ - OptionParser bootstraps option parsing, - getting the value of the config file - """ - def __init__(self, args): - self.Bootstrap = OptionSet([('configfile', CFILE)]) - self.Bootstrap.parse(sys.argv[1:], do_getopt=False) - if self.Bootstrap['configfile'] != Option.cfpath: - Option.cfpath = self.Bootstrap['configfile'] - Option.__cfp = False - OptionSet.__init__(self, args) - try: - f = open(Option.cfpath, 'r') - f.close() - except IOError: - e = sys.exc_info()[1] - print("Warning! Unable to read specified configuration file: %s" % e) diff --git a/src/lib/Server/FileMonitor.py b/src/lib/Server/FileMonitor.py deleted file mode 100644 index d6b313e6b..000000000 --- a/src/lib/Server/FileMonitor.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Bcfg2.Server.FileMonitor provides the support for monitorung files.""" - -import logging -import os -import stat -from time import sleep, time - -logger = logging.getLogger('Bcfg2.Server.FileMonitor') - - -def ShouldIgnore(event): - """Test if the event should be suppresed.""" - # FIXME should move event suppression out of the core - if event.filename.split('/')[-1] == '.svn': - return True - if event.filename.endswith('~') or \ - event.filename.startswith('#') or event.filename.startswith('.#'): - #logger.error("Suppressing event for file %s" % (event.filename)) - return True - return False - - -class Event(object): - def __init__(self, request_id, filename, code): - self.requestID = request_id - self.filename = filename - self.action = code - - def code2str(self): - """return static code for event""" - return self.action - -available = {} - - -class FileMonitor(object): - """File Monitor baseclass.""" - def __init__(self, debug=False): - object.__init__(self) - self.debug = debug - self.handles = dict() - - def get_event(self): - return None - - def pending(self): - return False - - def fileno(self): - return 0 - - def handle_one_event(self, event): - if ShouldIgnore(event): - return - if event.requestID not in self.handles: - logger.info("Got event for unexpected id %s, file %s" % - (event.requestID, event.filename)) - return - if self.debug: - logger.info("Dispatching event %s %s to obj %s" \ - % (event.code2str(), event.filename, - self.handles[event.requestID])) - try: - self.handles[event.requestID].HandleEvent(event) - except: - logger.error("error in handling of gamin event for %s" % \ - (event.filename), exc_info=1) - - def handle_event_set(self, lock=None): - count = 1 - event = self.get_event() - start = time() - if lock: - lock.acquire() - try: - self.handle_one_event(event) - while self.pending(): - self.handle_one_event(self.get_event()) - count += 1 - except: - pass - if lock: - lock.release() - end = time() - logger.info("Handled %d events in %.03fs" % (count, (end - start))) - - def handle_events_in_interval(self, interval): - end = time() + interval - while time() < end: - if self.pending(): - self.handle_event_set() - end = time() + interval - else: - sleep(0.5) - - -class FamFam(object): - """The fam object is a set of callbacks for - file alteration events (FAM support). - """ - - def __init__(self): - object.__init__(self) - self.fm = _fam.open() - self.users = {} - self.handles = {} - self.debug = False - - def fileno(self): - """Return fam file handle number.""" - return self.fm.fileno() - - def handle_event_set(self, _): - self.Service() - - def handle_events_in_interval(self, interval): - now = time() - while (time() - now) < interval: - if self.Service(): - now = time() - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - mode = os.stat(path)[stat.ST_MODE] - if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) - else: - handle = self.fm.monitorFile(path, None) - self.handles[handle.requestID()] = handle - if obj != None: - self.users[handle.requestID()] = obj - return handle.requestID() - - def Service(self, interval=0.50): - """Handle all fam work.""" - count = 0 - collapsed = 0 - rawevents = [] - start = time() - now = time() - while (time() - now) < interval: - if self.fm.pending(): - while self.fm.pending(): - count += 1 - rawevents.append(self.fm.nextEvent()) - now = time() - unique = [] - bookkeeping = [] - for event in rawevents: - if ShouldIgnore(event): - continue - if event.code2str() != 'changed': - # process all non-change events - unique.append(event) - else: - if (event.filename, event.requestID) not in bookkeeping: - bookkeeping.append((event.filename, event.requestID)) - unique.append(event) - else: - collapsed += 1 - for event in unique: - if event.requestID in self.users: - try: - self.users[event.requestID].HandleEvent(event) - except: - logger.error("handling event for file %s" % (event.filename), exc_info=1) - end = time() - logger.info("Processed %s fam events in %03.03f seconds. %s coalesced" % - (count, (end - start), collapsed)) - return count - - -class Fam(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (FAM support). - """ - - def __init__(self, debug=False): - FileMonitor.__init__(self, debug) - self.fm = _fam.open() - - def fileno(self): - return self.fm.fileno() - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - mode = os.stat(path)[stat.ST_MODE] - if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) - else: - handle = self.fm.monitorFile(path, None) - if obj != None: - self.handles[handle.requestID()] = obj - return handle.requestID() - - def pending(self): - return self.fm.pending() - - def get_event(self): - return self.fm.nextEvent() - - -class Pseudo(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (static monitor support). - """ - - def __init__(self, debug=False): - FileMonitor.__init__(self, debug=False) - self.pending_events = [] - - def pending(self): - return len(self.pending_events) != 0 - - def get_event(self): - return self.pending_events.pop() - - def AddMonitor(self, path, obj): - """add a monitor to path, installing a callback to obj.HandleEvent""" - handleID = len(list(self.handles.keys())) - mode = os.stat(path)[stat.ST_MODE] - handle = Event(handleID, path, 'exists') - if stat.S_ISDIR(mode): - dirList = os.listdir(path) - self.pending_events.append(handle) - for includedFile in dirList: - self.pending_events.append(Event(handleID, - includedFile, - 'exists')) - self.pending_events.append(Event(handleID, path, 'endExist')) - else: - self.pending_events.append(Event(handleID, path, 'exists')) - if obj != None: - self.handles[handleID] = obj - return handleID - - -try: - from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ - GAMChanged, GAMDeleted, GAMMoved - - class GaminEvent(Event): - """ - This class provides an event analogous to - python-fam events based on gamin sources. - """ - def __init__(self, request_id, filename, code): - Event.__init__(self, request_id, filename, code) - action_map = {GAMCreated: 'created', GAMExists: 'exists', - GAMChanged: 'changed', GAMDeleted: 'deleted', - GAMEndExist: 'endExist', GAMMoved: 'moved'} - if code in action_map: - self.action = action_map[code] - - class Gamin(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (Gamin support) - """ - def __init__(self, debug=False): - FileMonitor.__init__(self, debug) - self.mon = WatchMonitor() - self.counter = 0 - self.events = [] - - def fileno(self): - return self.mon.get_fd() - - def queue(self, path, action, request_id): - """queue up the event for later handling""" - self.events.append(GaminEvent(request_id, path, action)) - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - handle = self.counter - self.counter += 1 - mode = os.stat(path)[stat.ST_MODE] - - # Flush queued gamin events - while self.mon.event_pending(): - self.mon.handle_one_event() - - if stat.S_ISDIR(mode): - self.mon.watch_directory(path, self.queue, handle) - else: - self.mon.watch_file(path, self.queue, handle) - self.handles[handle] = obj - return handle - - def pending(self): - return len(self.events) > 0 or self.mon.event_pending() - - def get_event(self): - if self.mon.event_pending(): - self.mon.handle_one_event() - return self.events.pop(0) - - available['gamin'] = Gamin -except ImportError: - # fall back to _fam - pass - -try: - import _fam - available['fam'] = FamFam -except ImportError: - pass -available['pseudo'] = Pseudo - -for fdrv in ['gamin', 'fam', 'pseudo']: - if fdrv in available: - available['default'] = available[fdrv] - break diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py deleted file mode 100644 index 472915cfd..000000000 --- a/src/lib/Server/Lint/Bundles.py +++ /dev/null @@ -1,61 +0,0 @@ -import lxml.etree -import Bcfg2.Server.Lint - -class Bundles(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ - - def Run(self): - """ run plugin """ - if 'Bundler' in self.core.plugins: - self.missing_bundles() - for bundle in self.core.plugins['Bundler'].entries.values(): - if self.HandlesFile(bundle.name): - if (not Bcfg2.Server.Plugins.Bundler.have_genshi or - type(bundle) is not - Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): - self.bundle_names(bundle) - - def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ - if self.files is None: - # when given a list of files on stdin, this check is - # useless, so skip it - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) - - allbundles = self.core.plugins['Bundler'].entries.keys() - for bundle in ref_bundles: - xmlbundle = "%s.xml" % bundle - genshibundle = "%s.genshi" % bundle - if (xmlbundle not in allbundles and - genshibundle not in allbundles): - self.LintError("bundle-not-found", - "Bundle %s referenced, but does not exist" % - bundle) - - def bundle_names(self, bundle): - """ verify bundle name attribute matches filename """ - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - - fname = bundle.name.split('Bundler/')[1].split('.')[0] - bname = xdata.get('name') - if fname != bname: - self.LintError("inconsistent-bundle-name", - "Inconsistent bundle name: filename is %s, bundle name is %s" % - (fname, bname)) - - def sgenshi_groups(self, bundle): - """ ensure that Genshi Bundles do not include <Group> tags, - which are not supported """ - xdata = lxml.etree.parse(bundle.name) - groups = [self.RenderXML(g) - for g in xdata.getroottree().findall("//Group")] - if groups: - self.LintError("group-tag-not-allowed", - "<Group> tag is not allowed in SGenshi Bundle:\n%s" % - "\n".join(groups)) diff --git a/src/lib/Server/Lint/Deltas.py b/src/lib/Server/Lint/Deltas.py deleted file mode 100644 index cf91d1d13..000000000 --- a/src/lib/Server/Lint/Deltas.py +++ /dev/null @@ -1,20 +0,0 @@ -import Bcfg2.Server.Lint - -class Deltas(Bcfg2.Server.Lint.ServerPlugin): - """ Warn about usage of .cat and .diff files """ - - def Run(self): - """ run plugin """ - if 'Cfg' in self.core.plugins: - cfg = self.core.plugins['Cfg'] - for basename, entry in list(cfg.entries.items()): - self.check_entry(basename, entry) - - def check_entry(self, basename, entry): - for fname in list(entry.entries.keys()): - if self.HandlesFile(fname): - match = entry.specific.delta_reg.match(fname) - if match: - self.LintError("%s-file-used" % match.group('delta'), - "%s file used on %s: %s" % - (match.group('delta'), basename, fname)) diff --git a/src/lib/Server/Lint/GroupPatterns.py b/src/lib/Server/Lint/GroupPatterns.py deleted file mode 100644 index b69d7a5d8..000000000 --- a/src/lib/Server/Lint/GroupPatterns.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import Bcfg2.Server.Lint -from Bcfg2.Server.Plugins.GroupPatterns import PatternMap - -class GroupPatterns(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ - - def Run(self): - """ run plugin """ - if 'GroupPatterns' in self.core.plugins: - cfg = self.core.plugins['GroupPatterns'].config - for entry in cfg.xdata.xpath('//GroupPattern'): - groups = [g.text for g in entry.findall('Group')] - self.check(entry, groups, ptype='NamePattern') - self.check(entry, groups, ptype='NameRange') - - def check(self, entry, groups, ptype="NamePattern"): - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - - for el in entry.findall(ptype): - pat = el.text - try: - pmap(pat) - except: - err = sys.exc_info()[1] - self.LintError("pattern-fails-to-initialize", - "Failed to initialize %s %s for %s: %s" % - (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Server/Lint/Pkgmgr.py deleted file mode 100644 index 8f099163a..000000000 --- a/src/lib/Server/Lint/Pkgmgr.py +++ /dev/null @@ -1,35 +0,0 @@ -import glob -import lxml.etree -import Bcfg2.Server.Lint - -class Pkgmgr(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ - - def Run(self): - pset = set() - for pfile in glob.glob("%s/Pkgmgr/*.xml" % self.config['repo']): - if self.HandlesFile(pfile): - xdata = lxml.etree.parse(pfile).getroot() - # get priority, type, group - priority = xdata.get('priority') - ptype = xdata.get('type') - for pkg in xdata.xpath("//Package"): - if pkg.getparent().tag == 'Group': - grp = pkg.getparent().get('name') - if (type(grp) is not str and - grp.getparent().tag == 'Group'): - pgrp = grp.getparent().get('name') - else: - pgrp = 'none' - else: - grp = 'none' - pgrp = 'none' - ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) - # check if package is already listed with same - # priority, type, grp - if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) - else: - pset.add(ptuple) diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py deleted file mode 100644 index 9ec39e108..000000000 --- a/src/lib/Server/Plugins/Cfg.py +++ /dev/null @@ -1,295 +0,0 @@ -"""This module implements a config file repository.""" -__revision__ = '$Revision$' - -import binascii -import logging -import lxml -import operator -import os -import os.path -import re -import stat -import sys -import tempfile -from subprocess import Popen, PIPE -from Bcfg2.Bcfg2Py3k import u_str - -import Bcfg2.Server.Plugin - -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, NewTextTemplate - have_genshi = True -except: - have_genshi = False - -try: - import Cheetah.Template - import Cheetah.Parser - have_cheetah = True -except: - have_cheetah = False - -# setup logging -logger = logging.getLogger('Bcfg2.Plugins.Cfg') - - -# snipped from TGenshi -def removecomment(stream): - """A genshi filter that removes comments from the stream.""" - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos - - -def process_delta(data, delta): - if not delta.specific.delta: - return data - if delta.specific.delta == 'cat': - datalines = data.strip().split('\n') - for line in delta.data.split('\n'): - if not line: - continue - if line[0] == '+': - datalines.append(line[1:]) - elif line[0] == '-': - if line[1:] in datalines: - datalines.remove(line[1:]) - return "\n".join(datalines) + "\n" - elif delta.specific.delta == 'diff': - basehandle, basename = tempfile.mkstemp() - basefile = open(basename, 'w') - basefile.write(data) - basefile.close() - os.close(basehandle) - - cmd = ["patch", "-u", "-f", basefile.name] - patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stderr = patch.communicate(input=delta.data)[1] - ret = patch.wait() - output = open(basefile.name, 'r').read() - os.unlink(basefile.name) - if ret >> 8 != 0: - logger.error("Error applying diff %s: %s" % (delta.name, stderr)) - raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) - return output - - -class CfgMatcher: - - def __init__(self, fname): - name = re.escape(fname) - self.basefile_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+?)|.G(?P<prio>\d+)_(?P<group>\S+?))((?P<genshi>\\.genshi)|(?P<cheetah>\\.cheetah))?$' % name) - self.delta_reg = re.compile('^(?P<basename>%s)(|\\.H_(?P<hostname>\S+)|\\.G(?P<prio>\d+)_(?P<group>\S+))\\.(?P<delta>(cat|diff))$' % name) - self.cat_count = fname.count(".cat") - self.diff_count = fname.count(".diff") - - def match(self, fname): - if fname.count(".cat") > self.cat_count \ - or fname.count('.diff') > self.diff_count: - return self.delta_reg.match(fname) - return self.basefile_reg.match(fname) - - -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): - - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - self.specific = CfgMatcher(path.split('/')[-1]) - path = path - - def debug_log(self, message, flag=None): - if (flag is None and self.debug_flag) or flag: - logger.error(message) - - def sort_by_specific(self, one, other): - return cmp(one.specific, other.specific) - - def get_pertinent_entries(self, entry, metadata): - """return a list of all entries pertinent - to a client => [base, delta1, delta2] - """ - matching = [ent for ent in list(self.entries.values()) if \ - ent.specific.matches(metadata)] - matching.sort(key=operator.attrgetter('specific')) - # base entries which apply to a client - # (e.g. foo, foo.G##_groupname, foo.H_hostname) - base_files = [matching.index(m) for m in matching - if not m.specific.delta] - if not base_files: - msg = "No base file found for %s" % entry.get('name') - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - base = min(base_files) - used = matching[:base + 1] - used.reverse() - return used - - def bind_entry(self, entry, metadata): - self.bind_info_to_entry(entry, metadata) - used = self.get_pertinent_entries(entry, metadata) - basefile = used.pop(0) - if entry.get('perms').lower() == 'inherit': - # use on-disk permissions - fname = os.path.join(self.path, entry.get('name')) - entry.set('perms', - str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) - if entry.tag == 'Path': - entry.set('type', 'file') - if basefile.name.endswith(".genshi"): - if not have_genshi: - msg = "Cfg: Genshi is not available: %s" % entry.get("name") - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - try: - template_cls = NewTextTemplate - loader = TemplateLoader() - template = loader.load(basefile.name, cls=template_cls, - encoding=self.encoding) - fname = entry.get('realname', entry.get('name')) - stream = template.generate(name=fname, - metadata=metadata, - path=basefile.name).filter(removecomment) - try: - data = stream.render('text', encoding=self.encoding, - strip_whitespace=False) - except TypeError: - data = stream.render('text', encoding=self.encoding) - if data == '': - entry.set('empty', 'true') - except Exception: - msg = "Cfg: genshi exception (%s): %s" % (entry.get("name"), - sys.exc_info()[1]) - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - elif basefile.name.endswith(".cheetah"): - if not have_cheetah: - msg = "Cfg: Cheetah is not available: %s" % entry.get("name") - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - try: - fname = entry.get('realname', entry.get('name')) - s = {'useStackFrames': False} - template = Cheetah.Template.Template(open(basefile.name).read(), - compilerSettings=s) - template.metadata = metadata - template.path = fname - template.source_path = basefile.name - data = template.respond() - if data == '': - entry.set('empty', 'true') - except Exception: - msg = "Cfg: cheetah exception (%s): %s" % (entry.get("name"), - sys.exc_info()[1]) - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - else: - data = basefile.data - for delta in used: - data = process_delta(data, delta) - if entry.get('encoding') == 'base64': - entry.text = binascii.b2a_base64(data) - else: - try: - entry.text = u_str(data, self.encoding) - except UnicodeDecodeError: - msg = "Failed to decode %s: %s" % (entry.get('name'), - sys.exc_info()[1]) - logger.error(msg) - logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - except ValueError: - msg = "Error in specification for %s: %s" % (entry.get('name'), - sys.exc_info()[1]) - logger.error(msg) - logger.error("You need to specify base64 encoding for %s." % - entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - if entry.text in ['', None]: - entry.set('empty', 'true') - - def list_accept_choices(self, entry, metadata): - '''return a list of candidate pull locations''' - used = self.get_pertinent_entries(entry, metadata) - ret = [] - if used: - ret.append(used[0].specific) - if not ret[0].hostname: - ret.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) - return ret - - def build_filename(self, specific): - bfname = self.path + '/' + self.path.split('/')[-1] - if specific.all: - return bfname - elif specific.group: - return "%s.G%02d_%s" % (bfname, specific.prio, specific.group) - elif specific.hostname: - return "%s.H_%s" % (bfname, specific.hostname) - - def write_update(self, specific, new_entry, log): - if 'text' in new_entry: - name = self.build_filename(specific) - if os.path.exists("%s.genshi" % name): - msg = "Cfg: Unable to pull data for genshi types" - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - elif os.path.exists("%s.cheetah" % name): - msg = "Cfg: Unable to pull data for cheetah types" - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - try: - etext = new_entry['text'].encode(self.encoding) - except: - msg = "Cfg: Cannot encode content of %s as %s" % (name, - self.encoding) - logger.error(msg) - raise Bcfg2.Server.Plugin.PluginExecutionError(msg) - open(name, 'w').write(etext) - self.debug_log("Wrote file %s" % name, flag=log) - badattr = [attr for attr in ['owner', 'group', 'perms'] - if attr in new_entry] - if badattr: - # check for info files and inform user of their removal - if os.path.exists(self.path + "/:info"): - logger.info("Removing :info file and replacing with " - "info.xml") - os.remove(self.path + "/:info") - if os.path.exists(self.path + "/info"): - logger.info("Removing info file and replacing with " - "info.xml") - os.remove(self.path + "/info") - metadata_updates = {} - metadata_updates.update(self.metadata) - for attr in badattr: - metadata_updates[attr] = new_entry.get(attr) - infoxml = lxml.etree.Element('FileInfo') - infotag = lxml.etree.SubElement(infoxml, 'Info') - [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ - for attr in metadata_updates] - ofile = open(self.path + "/info.xml", "w") - ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) - ofile.close() - self.debug_log("Wrote file %s" % (self.path + "/info.xml"), - flag=log) - - -class Cfg(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.PullTarget): - """This generator in the configuration file repository for Bcfg2.""" - name = 'Cfg' - __version__ = '$Id$' - __author__ = 'bcfg-dev@mcs.anl.gov' - es_cls = CfgEntrySet - es_child_cls = Bcfg2.Server.Plugin.SpecificData - - def AcceptChoices(self, entry, metadata): - return self.entries[entry.get('name')].list_accept_choices(entry, metadata) - - def AcceptPullData(self, specific, new_entry, log): - return self.entries[new_entry.get('name')].write_update(specific, - new_entry, - log) diff --git a/src/lib/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Server/Plugins/Packages/PackagesConfig.py deleted file mode 100644 index 7950f15e6..000000000 --- a/src/lib/Server/Plugins/Packages/PackagesConfig.py +++ /dev/null @@ -1,15 +0,0 @@ -import Bcfg2.Server.Plugin - -class PackagesConfig(Bcfg2.Server.Plugin.SimpleConfig): - _required = False - - def Index(self): - """ Build local data structures """ - Bcfg2.Server.Plugin.SimpleConfig.Index(self) - - if hasattr(self.plugin, "sources") and self.plugin.sources.loaded: - # only reload Packages plugin if sources have been loaded. - # otherwise, this is getting called on server startup, and - # we have to wait until all sources have been indexed - # before we can call Packages.Reload() - self.plugin.Reload() diff --git a/src/lib/Server/Reports/importscript.py b/src/lib/Server/Reports/importscript.py deleted file mode 100755 index 7dfac6fae..000000000 --- a/src/lib/Server/Reports/importscript.py +++ /dev/null @@ -1,311 +0,0 @@ -#! /usr/bin/env python -""" -Imports statistics.xml and clients.xml files in to database backend for -new statistics engine -""" -__revision__ = '$Revision$' - -import binascii -import os -import sys -try: - import Bcfg2.Server.Reports.settings -except Exception: - e = sys.exc_info()[1] - sys.stderr.write("Failed to load configuration settings. %s\n" % e) - sys.exit(1) - -project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name - -from Bcfg2.Server.Reports.reports.models import * -from lxml.etree import XML, XMLSyntaxError -from getopt import getopt, GetoptError -from datetime import datetime -from time import strptime -from django.db import connection -from Bcfg2.Server.Reports.updatefix import update_database -import logging -import Bcfg2.Logger -import platform - -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - - -def build_reason_kwargs(r_ent, encoding, logger): - binary_file = False - sensitive_file = False - if r_ent.get('sensitive') in ['true', 'True']: - sensitive_file = True - rc_diff = '' - elif r_ent.get('current_bfile', False): - binary_file = True - rc_diff = r_ent.get('current_bfile') - if len(rc_diff) > 1024 * 1024: - rc_diff = '' - elif len(rc_diff) == 0: - # No point in flagging binary if we have no data - binary_file = False - elif r_ent.get('current_bdiff', False): - rc_diff = binascii.a2b_base64(r_ent.get('current_bdiff')) - elif r_ent.get('current_diff', False): - rc_diff = r_ent.get('current_diff') - else: - rc_diff = '' - if not binary_file: - try: - rc_diff = rc_diff.decode(encoding) - except: - logger.error("Reason isn't %s encoded, cannot decode it" % encoding) - rc_diff = '' - return dict(owner=r_ent.get('owner', default=""), - current_owner=r_ent.get('current_owner', default=""), - group=r_ent.get('group', default=""), - current_group=r_ent.get('current_group', default=""), - perms=r_ent.get('perms', default=""), - current_perms=r_ent.get('current_perms', default=""), - status=r_ent.get('status', default=""), - current_status=r_ent.get('current_status', default=""), - to=r_ent.get('to', default=""), - current_to=r_ent.get('current_to', default=""), - version=r_ent.get('version', default=""), - current_version=r_ent.get('current_version', default=""), - current_exists=r_ent.get('current_exists', default="True").capitalize() == "True", - current_diff=rc_diff, - is_binary=binary_file, - is_sensitive=sensitive_file) - - -def load_stats(cdata, sdata, encoding, vlevel, logger, quick=False, location=''): - clients = {} - [clients.__setitem__(c.name, c) \ - for c in Client.objects.all()] - - pingability = {} - [pingability.__setitem__(n.get('name'), n.get('pingable', default='N')) \ - for n in cdata.findall('Client')] - - for node in sdata.findall('Node'): - name = node.get('name') - c_inst, created = Client.objects.get_or_create(name=name) - if vlevel > 0: - logger.info("Client %s added to db" % name) - clients[name] = c_inst - try: - pingability[name] - except KeyError: - pingability[name] = 'N' - for statistics in node.findall('Statistics'): - timestamp = datetime(*strptime(statistics.get('time'))[0:6]) - ilist = Interaction.objects.filter(client=c_inst, - timestamp=timestamp) - if ilist: - current_interaction = ilist[0] - if vlevel > 0: - logger.info("Interaction for %s at %s with id %s already exists" % \ - (c_inst.id, timestamp, current_interaction.id)) - continue - else: - newint = Interaction(client=c_inst, - timestamp=timestamp, - state=statistics.get('state', - default="unknown"), - repo_rev_code=statistics.get('revision', - default="unknown"), - client_version=statistics.get('client_version', - default="unknown"), - goodcount=statistics.get('good', - default="0"), - totalcount=statistics.get('total', - default="0"), - server=location) - newint.save() - current_interaction = newint - if vlevel > 0: - logger.info("Interaction for %s at %s with id %s INSERTED in to db" % (c_inst.id, - timestamp, current_interaction.id)) - - counter_fields = {TYPE_CHOICES[0]: 0, - TYPE_CHOICES[1]: 0, - TYPE_CHOICES[2]: 0} - pattern = [('Bad/*', TYPE_CHOICES[0]), - ('Extra/*', TYPE_CHOICES[2]), - ('Modified/*', TYPE_CHOICES[1])] - for (xpath, type) in pattern: - for x in statistics.findall(xpath): - counter_fields[type] = counter_fields[type] + 1 - kargs = build_reason_kwargs(x, encoding, logger) - - try: - rr = None - try: - rr = Reason.objects.filter(**kargs)[0] - except IndexError: - rr = Reason(**kargs) - rr.save() - if vlevel > 0: - logger.info("Created reason: %s" % rr.id) - except Exception: - ex = sys.exc_info()[1] - logger.error("Failed to create reason for %s: %s" % (x.get('name'), ex)) - rr = Reason(current_exists=x.get('current_exists', - default="True").capitalize() == "True") - rr.save() - - entry, created = Entries.objects.get_or_create(\ - name=x.get('name'), kind=x.tag) - - Entries_interactions(entry=entry, reason=rr, - interaction=current_interaction, - type=type[0]).save() - if vlevel > 0: - logger.info("%s interaction created with reason id %s and entry %s" % (xpath, rr.id, entry.id)) - - # Update interaction counters - current_interaction.bad_entries = counter_fields[TYPE_CHOICES[0]] - current_interaction.modified_entries = counter_fields[TYPE_CHOICES[1]] - current_interaction.extra_entries = counter_fields[TYPE_CHOICES[2]] - current_interaction.save() - - mperfs = [] - for times in statistics.findall('OpStamps'): - for metric, value in list(times.items()): - mmatch = [] - if not quick: - mmatch = Performance.objects.filter(metric=metric, value=value) - - if mmatch: - mperf = mmatch[0] - else: - mperf = Performance(metric=metric, value=value) - mperf.save() - mperfs.append(mperf) - current_interaction.performance_items.add(*mperfs) - - for key in list(pingability.keys()): - if key not in clients: - continue - try: - pmatch = Ping.objects.filter(client=clients[key]).order_by('-endtime')[0] - if pmatch.status == pingability[key]: - pmatch.endtime = datetime.now() - pmatch.save() - continue - except IndexError: - pass - Ping(client=clients[key], status=pingability[key], - starttime=datetime.now(), - endtime=datetime.now()).save() - - if vlevel > 1: - logger.info("---------------PINGDATA SYNCED---------------------") - - #Clients are consistent - -if __name__ == '__main__': - from sys import argv - verb = 0 - cpath = "/etc/bcfg2.conf" - clientpath = False - statpath = False - syslog = False - - try: - opts, args = getopt(argv[1:], "hvudc:s:CS", ["help", - "verbose", - "updates", - "debug", - "clients=", - "stats=", - "config=", - "syslog"]) - except GetoptError: - mesg = sys.exc_info()[1] - # print help information and exit: - print("%s\nUsage:\nimportscript.py [-h] [-v] [-u] [-d] [-S] [-C bcfg2 config file] [-c clients-file] [-s statistics-file]" % (mesg)) - raise SystemExit(2) - - for o, a in opts: - if o in ("-h", "--help"): - print("Usage:\nimportscript.py [-h] [-v] -c <clients-file> -s <statistics-file> \n") - print("h : help; this message") - print("v : verbose; print messages on record insertion/skip") - print("u : updates; print status messages as items inserted semi-verbose") - print("d : debug; print most SQL used to manipulate database") - print("C : path to bcfg2.conf config file.") - print("c : clients.xml file") - print("s : statistics.xml file") - print("S : syslog; output to syslog") - raise SystemExit - if o in ["-C", "--config"]: - cpath = a - - if o in ("-v", "--verbose"): - verb = 1 - if o in ("-u", "--updates"): - verb = 2 - if o in ("-d", "--debug"): - verb = 3 - if o in ("-c", "--clients"): - clientspath = a - - if o in ("-s", "--stats"): - statpath = a - if o in ("-S", "--syslog"): - syslog = True - - logger = logging.getLogger('importscript.py') - logging.getLogger().setLevel(logging.INFO) - Bcfg2.Logger.setup_logging('importscript.py', - True, - syslog) - - cf = ConfigParser.ConfigParser() - cf.read([cpath]) - - if not statpath: - try: - statpath = "%s/etc/statistics.xml" % cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print("Could not read bcfg2.conf; exiting") - raise SystemExit(1) - try: - statsdata = XML(open(statpath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s" % (statpath)) - raise SystemExit(1) - - try: - encoding = cf.get('components', 'encoding') - except: - encoding = 'UTF-8' - - if not clientpath: - try: - clientspath = "%s/Metadata/clients.xml" % \ - cf.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - print("Could not read bcfg2.conf; exiting") - raise SystemExit(1) - try: - clientsdata = XML(open(clientspath).read()) - except (IOError, XMLSyntaxError): - print("StatReports: Failed to parse %s" % (clientspath)) - raise SystemExit(1) - - q = '-O3' in sys.argv - # Be sure the database is ready for new schema - update_database() - load_stats(clientsdata, - statsdata, - encoding, - verb, - logger, - quick=q, - location=platform.node()) diff --git a/src/lib/Server/Reports/reports/fixtures/initial_version.xml b/src/lib/Server/Reports/reports/fixtures/initial_version.xml deleted file mode 100644 index 919265d48..000000000 --- a/src/lib/Server/Reports/reports/fixtures/initial_version.xml +++ /dev/null @@ -1,39 +0,0 @@ -<?xml version='1.0' encoding='utf-8' ?> -<django-objects version="1.0"> - <object pk="1" model="reports.internaldatabaseversion"> - <field type="IntegerField" name="version">0</field> - <field type="DateTimeField" name="updated">2008-08-05 11:03:50</field> - </object> - <object pk="2" model="reports.internaldatabaseversion"> - <field type="IntegerField" name="version">1</field> - <field type="DateTimeField" name="updated">2008-08-05 11:04:10</field> - </object> - <object pk="3" model="reports.internaldatabaseversion"> - <field type="IntegerField" name="version">2</field> - <field type="DateTimeField" name="updated">2008-08-05 13:37:19</field> - </object> - <object pk="4" model="reports.internaldatabaseversion"> - <field type='IntegerField' name='version'>3</field> - <field type='DateTimeField' name='updated'>2008-08-11 08:44:36</field> - </object> - <object pk="5" model="reports.internaldatabaseversion"> - <field type='IntegerField' name='version'>10</field> - <field type='DateTimeField' name='updated'>2008-08-22 11:28:50</field> - </object> - <object pk="5" model="reports.internaldatabaseversion"> - <field type='IntegerField' name='version'>11</field> - <field type='DateTimeField' name='updated'>2009-01-13 12:26:10</field> - </object> - <object pk="6" model="reports.internaldatabaseversion"> - <field type='IntegerField' name='version'>16</field> - <field type='DateTimeField' name='updated'>2010-06-01 12:26:10</field> - </object> - <object pk="7" model="reports.internaldatabaseversion"> - <field type='IntegerField' name='version'>17</field> - <field type='DateTimeField' name='updated'>2010-07-02 00:00:00</field> - </object> - <object pk="8" model="reports.internaldatabaseversion"> - <field type='IntegerField' name='version'>18</field> - <field type='DateTimeField' name='updated'>2011-06-30 00:00:00</field> - </object> -</django-objects> diff --git a/src/lib/Server/Reports/reports/sql/client.sql b/src/lib/Server/Reports/reports/sql/client.sql deleted file mode 100644 index 8c63754c9..000000000 --- a/src/lib/Server/Reports/reports/sql/client.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE VIEW reports_current_interactions AS SELECT x.client_id AS client_id, reports_interaction.id AS interaction_id FROM (select client_id, MAX(timestamp) as timer FROM reports_interaction GROUP BY client_id) x, reports_interaction WHERE reports_interaction.client_id = x.client_id AND reports_interaction.timestamp = x.timer; - -create index reports_interaction_client_id on reports_interaction (client_id); -create index reports_extra_interactions_client_id on reports_extra_interactions(interaction_id); -create index reports_modified_interactions_client_id on reports_modified_interactions(interaction_id); -create index reports_client_current_interaction_id on reports_client (current_interaction_id); -create index reports_performance_interaction_performance_id on reports_performance_interaction (performance_id); -create index reports_interaction_timestamp on reports_interaction (timestamp); -create index reports_performance_interation_interaction_id on reports_performance_interaction (interaction_id);
\ No newline at end of file diff --git a/src/lib/Server/Reports/reports/templates/clients/index.html b/src/lib/Server/Reports/reports/templates/clients/index.html deleted file mode 100644 index e0c0d2d7a..000000000 --- a/src/lib/Server/Reports/reports/templates/clients/index.html +++ /dev/null @@ -1,34 +0,0 @@ -{% extends "base-timeview.html" %} - -{% block extra_header_info %} -{% endblock%} - -{% block title %}Bcfg2 - Client Grid View{% endblock %} - -{% block pagebanner %}Clients - Grid View{% endblock %} - -{% block content %} - -{% if inter_list %} - <table class='grid-view' align='center'> - {% for inter in inter_list %} - {% if forloop.first %}<tr>{% endif %} - <td class="{{inter.state}}-lineitem"> - <a href="{% spaceless %}{% if not timestamp %} - {% url reports_client_detail inter.client.name %} - {% else %} - {% url reports_client_detail_pk inter.client.name,inter.id %} - {% endif %} - {% endspaceless %}">{{ inter.client.name }}</a> - </td> - {% if forloop.last %} - </tr> - {% else %} - {% if forloop.counter|divisibleby:"4" %}</tr><tr>{% endif %} - {% endif %} - {% endfor %} - </table> -{% else %} - <p>No client records are available.</p> -{% endif %} -{% endblock %} diff --git a/src/lib/Server/Reports/reports/templates/widgets/filter_bar.html b/src/lib/Server/Reports/reports/templates/widgets/filter_bar.html deleted file mode 100644 index 6b57baf6a..000000000 --- a/src/lib/Server/Reports/reports/templates/widgets/filter_bar.html +++ /dev/null @@ -1,13 +0,0 @@ -{% spaceless %} -{% if filters %} -{% for filter, filter_url in filters %} - {% if forloop.first %} - <div class="filter_bar">Active filters (click to remove): - {% endif %} - <a href='{{ filter_url }}'>{{ filter|capfirst }}</a>{% if not forloop.last %}, {% endif %} - {% if forloop.last %} - </div> - {% endif %} -{% endfor %} -{% endif %} -{% endspaceless %} diff --git a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py b/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py deleted file mode 100644 index 629984f26..000000000 --- a/src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py +++ /dev/null @@ -1,276 +0,0 @@ -from django import template -from django.core.urlresolvers import resolve, reverse, Resolver404, NoReverseMatch -from django.utils.encoding import smart_unicode, smart_str -from datetime import datetime, timedelta -from Bcfg2.Server.Reports.utils import filter_list - -register = template.Library() - -__PAGE_NAV_LIMITS__ = (10, 25, 50, 100) - -@register.inclusion_tag('widgets/page_bar.html', takes_context=True) -def page_navigator(context): - """ - Creates paginated links. - - Expects the context to be a RequestContext and views.prepare_paginated_list() - to have populated page information. - """ - fragment = dict() - try: - path = context['request'].META['PATH_INFO'] - total_pages = int(context['total_pages']) - records_per_page = int(context['records_per_page']) - except KeyError: - return fragment - except ValueError: - return fragment - - if total_pages < 2: - return {} - - try: - view, args, kwargs = resolve(path) - current_page = int(kwargs.get('page_number',1)) - fragment['current_page'] = current_page - fragment['page_number'] = current_page - fragment['total_pages'] = total_pages - fragment['records_per_page'] = records_per_page - if current_page > 1: - kwargs['page_number'] = current_page - 1 - fragment['prev_page'] = reverse(view, args=args, kwargs=kwargs) - if current_page < total_pages: - kwargs['page_number'] = current_page + 1 - fragment['next_page'] = reverse(view, args=args, kwargs=kwargs) - - view_range = 5 - if total_pages > view_range: - pager_start = current_page - 2 - pager_end = current_page + 2 - if pager_start < 1: - pager_end += (1 - pager_start) - pager_start = 1 - if pager_end > total_pages: - pager_start -= (pager_end - total_pages) - pager_end = total_pages - else: - pager_start = 1 - pager_end = total_pages - - if pager_start > 1: - kwargs['page_number'] = 1 - fragment['first_page'] = reverse(view, args=args, kwargs=kwargs) - if pager_end < total_pages: - kwargs['page_number'] = total_pages - fragment['last_page'] = reverse(view, args=args, kwargs=kwargs) - - pager = [] - for page in range(pager_start, int(pager_end) + 1): - kwargs['page_number'] = page - pager.append( (page, reverse(view, args=args, kwargs=kwargs)) ) - - kwargs['page_number'] = 1 - page_limits = [] - for limit in __PAGE_NAV_LIMITS__: - kwargs['page_limit'] = limit - page_limits.append( (limit, reverse(view, args=args, kwargs=kwargs)) ) - # resolver doesn't like this - del kwargs['page_number'] - del kwargs['page_limit'] - page_limits.append( ('all', reverse(view, args=args, kwargs=kwargs) + "|all") ) - - fragment['pager'] = pager - fragment['page_limits'] = page_limits - - except Resolver404: - path = "404" - except NoReverseMatch: - nr = sys.exc_info()[1] - path = "NoReverseMatch: %s" % nr - except ValueError: - path = "ValueError" - #FIXME - Handle these - - fragment['path'] = path - return fragment - -@register.inclusion_tag('widgets/filter_bar.html', takes_context=True) -def filter_navigator(context): - try: - path = context['request'].META['PATH_INFO'] - view, args, kwargs = resolve(path) - - # Strip any page limits and numbers - if 'page_number' in kwargs: - del kwargs['page_number'] - if 'page_limit' in kwargs: - del kwargs['page_limit'] - - filters = [] - for filter in filter_list: - if filter in kwargs: - myargs = kwargs.copy() - del myargs[filter] - filters.append( (filter, reverse(view, args=args, kwargs=myargs) ) ) - filters.sort(lambda x,y: cmp(x[0], y[0])) - return { 'filters': filters } - except (Resolver404, NoReverseMatch, ValueError, KeyError): - pass - return dict() - -def _subtract_or_na(mdict, x, y): - """ - Shortcut for build_metric_list - """ - try: - return round(mdict[x] - mdict[y], 4) - except: - return "n/a" - -@register.filter -def build_metric_list(mdict): - """ - Create a list of metric table entries - - Moving this here it simplify the view. Should really handle the case where these - are missing... - """ - td_list = [] - # parse - td_list.append( _subtract_or_na(mdict, 'config_parse', 'config_download')) - #probe - td_list.append( _subtract_or_na(mdict, 'probe_upload', 'start')) - #inventory - td_list.append( _subtract_or_na(mdict, 'inventory', 'initialization')) - #install - td_list.append( _subtract_or_na(mdict, 'install', 'inventory')) - #cfg download & parse - td_list.append( _subtract_or_na(mdict, 'config_parse', 'probe_upload')) - #total - td_list.append( _subtract_or_na(mdict, 'finished', 'start')) - return td_list - -@register.filter -def isstale(timestamp, entry_max=None): - """ - Check for a stale timestamp - - Compares two timestamps and returns True if the - difference is greater then 24 hours. - """ - if not entry_max: - entry_max = datetime.now() - return entry_max - timestamp > timedelta(hours=24) - -@register.filter -def sort_interactions_by_name(value): - """ - Sort an interaction list by client name - """ - inters = list(value) - inters.sort(lambda a,b: cmp(a.client.name, b.client.name)) - return inters - -class AddUrlFilter(template.Node): - def __init__(self, filter_name, filter_value): - self.filter_name = filter_name - self.filter_value = filter_value - self.fallback_view = 'Bcfg2.Server.Reports.reports.views.render_history_view' - - def render(self, context): - link = '#' - try: - path = context['request'].META['PATH_INFO'] - view, args, kwargs = resolve(path) - filter_value = self.filter_value.resolve(context, True) - if filter_value: - filter_name = smart_str(self.filter_name) - filter_value = smart_unicode(filter_value) - kwargs[filter_name] = filter_value - # These two don't make sense - if filter_name == 'server' and 'hostname' in kwargs: - del kwargs['hostname'] - elif filter_name == 'hostname' and 'server' in kwargs: - del kwargs['server'] - try: - link = reverse(view, args=args, kwargs=kwargs) - except NoReverseMatch: - link = reverse(self.fallback_view, args=None, - kwargs={ filter_name: filter_value }) - except NoReverseMatch: - rm = sys.exc_info()[1] - raise rm - except (Resolver404, ValueError): - pass - return link - -@register.tag -def add_url_filter(parser, token): - """ - Return a url with the filter added to the current view. - - Takes a new filter and resolves the current view with the new filter - applied. Resolves to Bcfg2.Server.Reports.reports.views.client_history - by default. - - {% add_url_filter server=interaction.server %} - """ - try: - tag_name, filter_pair = token.split_contents() - filter_name, filter_value = filter_pair.split('=', 1) - filter_name = filter_name.strip() - filter_value = parser.compile_filter(filter_value) - except ValueError: - raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) - if not filter_name or not filter_value: - raise template.TemplateSyntaxError("argument should be a filter=value pair") - - return AddUrlFilter(filter_name, filter_value) - -@register.filter -def sortwell(value): - """ - Sorts a list(or evaluates queryset to list) of bad, extra, or modified items in the best - way for presentation - """ - - configItems = list(value) - configItems.sort(lambda x,y: cmp(x.entry.name, y.entry.name)) - configItems.sort(lambda x,y: cmp(x.entry.kind, y.entry.kind)) - return configItems - -class MediaTag(template.Node): - def __init__(self, filter_value): - self.filter_value = filter_value - - def render(self, context): - base = context['MEDIA_URL'] - try: - request = context['request'] - try: - base = request.environ['bcfg2.media_url'] - except: - if request.path != request.META['PATH_INFO']: - offset = request.path.find(request.META['PATH_INFO']) - if offset > 0: - base = "%s/%s" % (request.path[:offset], \ - context['MEDIA_URL'].strip('/')) - except: - pass - return "%s/%s" % (base, self.filter_value) - -@register.tag -def to_media_url(parser, token): - """ - Return a url relative to the media_url. - - {% to_media_url /bcfg2.css %} - """ - try: - tag_name, filter_value = token.split_contents() - filter_value = parser.compile_filter(filter_value) - except ValueError: - raise template.TemplateSyntaxError("%r tag requires exactly one argument" % token.contents.split()[0]) - - return MediaTag(filter_value) - diff --git a/src/lib/Server/Reports/updatefix.py b/src/lib/Server/Reports/updatefix.py deleted file mode 100644 index 7cebaaca9..000000000 --- a/src/lib/Server/Reports/updatefix.py +++ /dev/null @@ -1,190 +0,0 @@ -import Bcfg2.Server.Reports.settings - -from django.db import connection -import django.core.management -import logging -import traceback -from Bcfg2.Server.Reports.reports.models import InternalDatabaseVersion, \ - TYPE_BAD, TYPE_MODIFIED, TYPE_EXTRA -logger = logging.getLogger('Bcfg2.Server.Reports.UpdateFix') - - -# all update function should go here -def _merge_database_table_entries(): - cursor = connection.cursor() - insert_cursor = connection.cursor() - find_cursor = connection.cursor() - cursor.execute(""" - Select name, kind from reports_bad - union - select name, kind from reports_modified - union - select name, kind from reports_extra - """) - # this fetch could be better done - entries_map = {} - for row in cursor.fetchall(): - insert_cursor.execute("insert into reports_entries (name, kind) \ - values (%s, %s)", (row[0], row[1])) - entries_map[(row[0], row[1])] = insert_cursor.lastrowid - - cursor.execute(""" - Select name, kind, reason_id, interaction_id, 1 from reports_bad - inner join reports_bad_interactions on reports_bad.id=reports_bad_interactions.bad_id - union - Select name, kind, reason_id, interaction_id, 2 from reports_modified - inner join reports_modified_interactions on reports_modified.id=reports_modified_interactions.modified_id - union - Select name, kind, reason_id, interaction_id, 3 from reports_extra - inner join reports_extra_interactions on reports_extra.id=reports_extra_interactions.extra_id - """) - for row in cursor.fetchall(): - key = (row[0], row[1]) - if entries_map.get(key, None): - entry_id = entries_map[key] - else: - find_cursor.execute("Select id from reports_entries where name=%s and kind=%s", key) - rowe = find_cursor.fetchone() - entry_id = rowe[0] - insert_cursor.execute("insert into reports_entries_interactions \ - (entry_id, interaction_id, reason_id, type) values (%s, %s, %s, %s)", (entry_id, row[3], row[2], row[4])) - - -def _interactions_constraint_or_idx(): - '''sqlite doesn't support alter tables.. or constraints''' - cursor = connection.cursor() - try: - cursor.execute('alter table reports_interaction add constraint reports_interaction_20100601 unique (client_id,timestamp)') - except: - cursor.execute('create unique index reports_interaction_20100601 on reports_interaction (client_id,timestamp)') - - -def _populate_interaction_entry_counts(): - '''Populate up the type totals for the interaction table''' - cursor = connection.cursor() - count_field = {TYPE_BAD: 'bad_entries', - TYPE_MODIFIED: 'modified_entries', - TYPE_EXTRA: 'extra_entries'} - - for type in list(count_field.keys()): - cursor.execute("select count(type), interaction_id " + - "from reports_entries_interactions where type = %s group by interaction_id" % type) - updates = [] - for row in cursor.fetchall(): - updates.append(row) - try: - cursor.executemany("update reports_interaction set " + count_field[type] + "=%s where id = %s", updates) - except Exception: - e = sys.exc_info()[1] - print(e) - cursor.close() - - -# be sure to test your upgrade query before reflecting the change in the models -# the list of function and sql command to do should go here -_fixes = [_merge_database_table_entries, - # this will remove unused tables - "drop table reports_bad;", - "drop table reports_bad_interactions;", - "drop table reports_extra;", - "drop table reports_extra_interactions;", - "drop table reports_modified;", - "drop table reports_modified_interactions;", - "drop table reports_repository;", - "drop table reports_metadata;", - "alter table reports_interaction add server varchar(256) not null default 'N/A';", - # fix revision data type to support $VCS hashes - "alter table reports_interaction add repo_rev_code varchar(64) default '';", - # Performance enhancements for large sites - 'alter table reports_interaction add column bad_entries integer not null default -1;', - 'alter table reports_interaction add column modified_entries integer not null default -1;', - 'alter table reports_interaction add column extra_entries integer not null default -1;', - _populate_interaction_entry_counts, - _interactions_constraint_or_idx, - 'alter table reports_reason add is_binary bool NOT NULL default False;', - 'alter table reports_reason add is_sensitive bool NOT NULL default False;', -] - -# this will calculate the last possible version of the database -lastversion = len(_fixes) - - -def rollupdate(current_version): - """ function responsible to coordinates all the updates - need current_version as integer - """ - ret = None - if current_version < lastversion: - for i in range(current_version, lastversion): - try: - if type(_fixes[i]) == str: - connection.cursor().execute(_fixes[i]) - else: - _fixes[i]() - except: - logger.error("Failed to perform db update %s" % (_fixes[i]), exc_info=1) - # since array start at 0 but version start at 1 we add 1 to the normal count - ret = InternalDatabaseVersion.objects.create(version=i + 1) - return ret - else: - return None - - -def dosync(): - """Function to do the syncronisation for the models""" - # try to detect if it's a fresh new database - try: - cursor = connection.cursor() - # If this table goes missing then don't forget to change it to the new one - cursor.execute("Select * from reports_client") - # if we get here with no error then the database has existing tables - fresh = False - except: - logger.debug("there was an error while detecting the freshness of the database") - #we should get here if the database is new - fresh = True - - # ensure database connection are close, so that the management can do it's job right - try: - cursor.close() - connection.close() - except: - # ignore any errors from missing/invalid dbs - pass - # Do the syncdb according to the django version - if "call_command" in dir(django.core.management): - # this is available since django 1.0 alpha. - # not yet tested for full functionnality - django.core.management.call_command("syncdb", interactive=False, verbosity=0) - if fresh: - django.core.management.call_command("loaddata", 'initial_version.xml', verbosity=0) - elif "syncdb" in dir(django.core.management): - # this exist only for django 0.96.* - django.core.management.syncdb(interactive=False, verbosity=0) - if fresh: - logger.debug("loading the initial_version fixtures") - django.core.management.load_data(fixture_labels=['initial_version'], verbosity=0) - else: - logger.warning("Don't forget to run syncdb") - - -def update_database(): - ''' methode to search where we are in the revision of the database models and update them ''' - try: - logger.debug("Running upgrade of models to the new one") - dosync() - know_version = InternalDatabaseVersion.objects.order_by('-version') - if not know_version: - logger.debug("No version, creating initial version") - know_version = InternalDatabaseVersion.objects.create(version=0) - else: - know_version = know_version[0] - logger.debug("Presently at %s" % know_version) - if know_version.version < lastversion: - new_version = rollupdate(know_version.version) - if new_version: - logger.debug("upgraded to %s" % new_version) - except: - logger.error("Error while updating the database") - for x in traceback.format_exc().splitlines(): - logger.error(x) diff --git a/src/sbin/bcfg2 b/src/sbin/bcfg2 index 1d1cc8424..2a7e5f585 100755 --- a/src/sbin/bcfg2 +++ b/src/sbin/bcfg2 @@ -1,7 +1,6 @@ #!/usr/bin/env python """Bcfg2 Client""" -__revision__ = '$Revision$' import fcntl import logging @@ -28,10 +27,6 @@ def cb_sigint_handler(signum, frame): """Exit upon CTRL-C.""" os._exit(1) -DECISION_LIST = Bcfg2.Options.Option('Decision List', default=False, - cmd="--decision-list", odesc='<file>', - long_arg=True) - class Client: """The main bcfg2 client class""" @@ -39,46 +34,40 @@ class Client: def __init__(self): self.toolset = None self.config = None - - optinfo = { - # 'optname': (('-a', argdesc, optdesc), - # env, cfpath, default, boolean)), - 'verbose': Bcfg2.Options.VERBOSE, - 'extra': Bcfg2.Options.CLIENT_EXTRA_DISPLAY, - 'quick': Bcfg2.Options.CLIENT_QUICK, - 'debug': Bcfg2.Options.DEBUG, - 'lockfile': Bcfg2.Options.LOCKFILE, - 'drivers': Bcfg2.Options.CLIENT_DRIVERS, - 'dryrun': Bcfg2.Options.CLIENT_DRYRUN, - 'paranoid': Bcfg2.Options.CLIENT_PARANOID, - 'bundle': Bcfg2.Options.CLIENT_BUNDLE, - 'bundle-quick': Bcfg2.Options.CLIENT_BUNDLEQUICK, - 'indep': Bcfg2.Options.CLIENT_INDEP, - 'file': Bcfg2.Options.CLIENT_FILE, - 'interactive': Bcfg2.Options.INTERACTIVE, - 'cache': Bcfg2.Options.CLIENT_CACHE, - 'profile': Bcfg2.Options.CLIENT_PROFILE, - 'remove': Bcfg2.Options.CLIENT_REMOVE, - 'help': Bcfg2.Options.HELP, - 'setup': Bcfg2.Options.CFILE, - 'server': Bcfg2.Options.SERVER_LOCATION, - 'user': Bcfg2.Options.CLIENT_USER, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'retries': Bcfg2.Options.CLIENT_RETRIES, - 'kevlar': Bcfg2.Options.CLIENT_KEVLAR, - 'decision-list': DECISION_LIST, - 'encoding': Bcfg2.Options.ENCODING, - 'omit-lock-check': Bcfg2.Options.OMIT_LOCK_CHECK, - 'filelog': Bcfg2.Options.LOGGING_FILE_PATH, - 'decision': Bcfg2.Options.CLIENT_DLIST, - 'servicemode': Bcfg2.Options.CLIENT_SERVICE_MODE, - 'key': Bcfg2.Options.CLIENT_KEY, - 'certificate': Bcfg2.Options.CLIENT_CERT, - 'ca': Bcfg2.Options.CLIENT_CA, - 'serverCN': Bcfg2.Options.CLIENT_SCNS, - 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, - } - + + optinfo = \ + dict(extra=Bcfg2.Options.CLIENT_EXTRA_DISPLAY, + quick=Bcfg2.Options.CLIENT_QUICK, + lockfile=Bcfg2.Options.LOCKFILE, + drivers=Bcfg2.Options.CLIENT_DRIVERS, + dryrun=Bcfg2.Options.CLIENT_DRYRUN, + paranoid=Bcfg2.Options.CLIENT_PARANOID, + bundle=Bcfg2.Options.CLIENT_BUNDLE, + skipbundle=Bcfg2.Options.CLIENT_SKIPBUNDLE, + bundle_quick=Bcfg2.Options.CLIENT_BUNDLEQUICK, + indep=Bcfg2.Options.CLIENT_INDEP, + skipindep=Bcfg2.Options.CLIENT_SKIPINDEP, + file=Bcfg2.Options.CLIENT_FILE, + interactive=Bcfg2.Options.INTERACTIVE, + cache=Bcfg2.Options.CLIENT_CACHE, + profile=Bcfg2.Options.CLIENT_PROFILE, + remove=Bcfg2.Options.CLIENT_REMOVE, + server=Bcfg2.Options.SERVER_LOCATION, + user=Bcfg2.Options.CLIENT_USER, + password=Bcfg2.Options.SERVER_PASSWORD, + retries=Bcfg2.Options.CLIENT_RETRIES, + kevlar=Bcfg2.Options.CLIENT_KEVLAR, + omit_lock_check=Bcfg2.Options.OMIT_LOCK_CHECK, + decision=Bcfg2.Options.CLIENT_DLIST, + servicemode=Bcfg2.Options.CLIENT_SERVICE_MODE, + key=Bcfg2.Options.CLIENT_KEY, + certificate=Bcfg2.Options.CLIENT_CERT, + ca=Bcfg2.Options.CLIENT_CA, + serverCN=Bcfg2.Options.CLIENT_SCNS, + timeout=Bcfg2.Options.CLIENT_TIMEOUT, + decision_list=Bcfg2.Options.CLIENT_DECISION_LIST) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.DRIVER_OPTIONS) self.setup = Bcfg2.Options.OptionParser(optinfo) self.setup.parse(sys.argv[1:]) @@ -94,30 +83,29 @@ class Client: Bcfg2.Logger.setup_logging('bcfg2', to_syslog=False, level=level, - to_file=self.setup['filelog']) + to_file=self.setup['logging']) self.logger = logging.getLogger('bcfg2') self.logger.debug(self.setup) - if self.setup['bundle-quick']: + if self.setup['bundle_quick']: if self.setup['bundle'] == []: self.logger.error("-Q option requires -b") raise SystemExit(1) - elif self.setup['remove'] != False: + elif self.setup['remove']: self.logger.error("-Q option incompatible with -r") raise SystemExit(1) if 'drivers' in self.setup and self.setup['drivers'] == 'help': self.logger.info("The following drivers are available:") self.logger.info(Bcfg2.Client.Tools.drivers) raise SystemExit(0) - if self.setup['remove'] and 'services' in self.setup['remove']: - self.logger.error("Service removal is nonsensical; removed services will only be disabled") - if self.setup['remove'] not in [False, - 'all', - 'Services', - 'Packages', - 'services', - 'packages']: - self.logger.error("Got unknown argument %s for -r" % (self.setup['remove'])) - if (self.setup["file"] != False) and (self.setup["cache"] != False): + if self.setup['remove'] and 'services' in self.setup['remove'].lower(): + self.logger.error("Service removal is nonsensical; " + "removed services will only be disabled") + if (self.setup['remove'] and + self.setup['remove'].lower() not in ['all', 'services', + 'packages']): + self.logger.error("Got unknown argument %s for -r" % + self.setup['remove']) + if self.setup["file"] and self.setup["cache"]: print("cannot use -f and -c together") raise SystemExit(1) if not self.setup['server'].startswith('https://'): @@ -283,10 +271,12 @@ class Client: self.fatal_error("Server error: %s" % (self.config.text)) return(1) - if self.setup['bundle-quick']: + if self.setup['bundle_quick']: newconfig = Bcfg2.Client.XML.XML('<Configuration/>') - [newconfig.append(bundle) for bundle in self.config.getchildren() if \ - bundle.tag == 'Bundle' and bundle.get('name') in self.setup['bundle']] + [newconfig.append(bundle) + for bundle in self.config.getchildren() + if (bundle.tag == 'Bundle' and + bundle.get('name') in self.setup['bundle'])] self.config = newconfig self.tools = Bcfg2.Client.Frame.Frame(self.config, @@ -294,7 +284,7 @@ class Client: times, self.setup['drivers'], self.setup['dryrun']) - if not self.setup['omit-lock-check']: + if not self.setup['omit_lock_check']: #check lock here try: lockfile = open(self.setup['lockfile'], 'w') @@ -310,7 +300,7 @@ class Client: # execute the said configuration self.tools.Execute() - if not self.setup['omit-lock-check']: + if not self.setup['omit_lock_check']: #unlock here if lockfile: try: @@ -319,7 +309,7 @@ class Client: except OSError: self.logger.error("Failed to unlock lockfile %s" % lockfile.name) - if not self.setup['file'] and not self.setup['bundle-quick']: + if not self.setup['file'] and not self.setup['bundle_quick']: # upload statistics feedback = self.tools.GenerateStats() diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 5cb69d747..9b28d9bd5 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -3,7 +3,6 @@ import sys import logging -import Bcfg2.Server.Core import Bcfg2.Logger import Bcfg2.Options import Bcfg2.Server.Admin @@ -31,32 +30,27 @@ def create_description(): for mode in modes: try: description.write((" %-15s %s\n" % - (mode, mode_import(mode).__shorthelp__))) + (mode, mode_import(mode).__shorthelp__))) except (ImportError, SystemExit): pass return description.getvalue() def main(): - optinfo = { - 'configfile': Bcfg2.Options.CFILE, - 'help': Bcfg2.Options.HELP, - 'verbose': Bcfg2.Options.VERBOSE, - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'event debug': Bcfg2.Options.DEBUG, - 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'encoding': Bcfg2.Options.ENCODING, - } + optinfo = dict() + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) # override default help message to include description of all modes - setup.hm = "Usage:\n %s\n%s" % (setup.buildHelpMessage(), - create_description()) + setup.hm = "%s\n%s" % (setup.buildHelpMessage(), create_description()) setup.parse(sys.argv[1:]) - log_args = dict(to_syslog=False, to_console=logging.WARNING) - if setup['verbose']: - log_args['to_console'] = logging.DEBUG + if setup['debug']: + level = logging.DEBUG + elif setup['verbose']: + level = logging.INFO + else: + level = logging.WARNING + Bcfg2.Logger.setup_logging('bcfg2-admin', to_syslog=False, level=level) # Provide help if requested or no args were specified if (not setup['args'] or len(setup['args']) < 1 or @@ -86,7 +80,6 @@ def main(): mode.bcore.shutdown() else: log.error("Unknown mode %s" % setup['args'][0]) - print("Usage:\n %s" % setup.buildHelpMessage()) print(create_description()) raise SystemExit(1) diff --git a/src/sbin/bcfg2-build-reports b/src/sbin/bcfg2-build-reports index 7122fb300..7fa08110a 100755 --- a/src/sbin/bcfg2-build-reports +++ b/src/sbin/bcfg2-build-reports @@ -4,8 +4,6 @@ bcfg2-build-reports generates & distributes reports of statistic information for Bcfg2.""" -__revision__ = '$Revision$' - import copy import getopt import re diff --git a/src/sbin/bcfg2-crypt b/src/sbin/bcfg2-crypt new file mode 100755 index 000000000..89dfe3e2a --- /dev/null +++ b/src/sbin/bcfg2-crypt @@ -0,0 +1,356 @@ +#!/usr/bin/env python +""" helper for encrypting/decrypting Cfg and Properties files """ + +import os +import sys +import logging +import lxml.etree +import Bcfg2.Logger +import Bcfg2.Options +import Bcfg2.Encryption + +LOGGER = None + +def get_logger(verbose=0): + """ set up logging according to the verbose level given on the + command line """ + global LOGGER + if LOGGER is None: + LOGGER = logging.getLogger(sys.argv[0]) + stderr = logging.StreamHandler() + if verbose: + level = logging.DEBUG + else: + level = logging.WARNING + LOGGER.setLevel(level) + LOGGER.addHandler(stderr) + syslog = logging.handlers.SysLogHandler("/dev/log") + syslog.setFormatter(logging.Formatter("%(name)s: %(message)s")) + LOGGER.addHandler(syslog) + return LOGGER + + +class Encryptor(object): + def __init__(self, setup): + self.setup = setup + self.logger = get_logger() + self.passphrase = None + self.pname = None + + def get_encrypted_filename(self, plaintext_filename): + return plaintext_filename + + def get_plaintext_filename(self, encrypted_filename): + return encrypted_filename + + def chunk(self, data): + yield data + + def unchunk(self, data, original): + return data[0] + + def set_passphrase(self): + if (not self.setup.cfp.has_section("encryption") or + self.setup.cfp.options("encryption") == 0): + self.logger.error("No passphrases available in %s" % + self.setup['configfile']) + return False + + if self.passphrase: + self.logger.debug("Using previously determined passphrase %s" % + self.pname) + return True + + if self.setup['passphrase']: + self.pname = self.setup['passphrase'] + + if self.pname: + if self.setup.cfp.has_option("encryption", self.pname): + self.passphrase = self.setup.cfp.get("encryption", + self.pname) + self.logger.debug("Using passphrase %s specified on command " + "line" % self.pname) + return True + else: + self.logger.error("Could not find passphrase %s in %s" % + (self.pname, self.setup['configfile'])) + return False + else: + pnames = self.setup.cfp.options("encryption") + if len(pnames) == 1: + self.passphrase = self.setup.cfp.get(pnames[0]) + self.pname = pnames[0] + self.logger.info("Using passphrase %s" % pnames[0]) + return True + self.logger.info("No passphrase could be determined") + return False + + def encrypt(self, fname): + try: + plaintext = open(fname).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % (fname, err)) + return False + + self.set_passphrase() + + crypted = [] + for chunk in self.chunk(plaintext): + try: + passphrase, pname = self.get_passphrase(chunk) + except TypeError: + return False + + crypted.append(self._encrypt(chunk, passphrase, name=pname)) + + new_fname = self.get_encrypted_filename(fname) + try: + open(new_fname, "wb").write(self.unchunk(crypted, plaintext)) + self.logger.info("Wrote encrypted data to %s" % new_fname) + return True + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error writing encrypted data from %s to %s: %s" % + (fname, new_fname, err)) + return False + + def _encrypt(self, plaintext, passphrase, name=None): + return Bcfg2.Encryption.ssl_encrypt(plaintext, passphrase) + + def decrypt(self, fname): + try: + crypted = open(fname).read() + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error reading %s, skipping: %s" % (fname, err)) + return False + + self.set_passphrase() + + plaintext = [] + for chunk in self.chunk(crypted): + try: + passphrase, pname = self.get_passphrase(chunk) + try: + plaintext.append(self._decrypt(chunk, passphrase)) + except Bcfg2.Encryption.EVPError: + self.logger.info("Could not decrypt %s with the specified " + "passphrase" % fname) + return False + except: + err = sys.exc_info()[1] + self.logger.error("Error decrypting %s: %s" % (fname, err)) + return False + except TypeError: + pchunk = None + for pname in self.setup.cfp.options('encryption'): + self.logger.debug("Trying passphrase %s" % pname) + passphrase = self.setup.cfp.get('encryption', pname) + try: + pchunk = self._decrypt(chunk, passphrase) + break + except Bcfg2.Encryption.EVPError: + pass + except: + err = sys.exc_info()[1] + self.logger.error("Error decrypting %s: %s" % + (fname, err)) + if pchunk is not None: + plaintext.append(pchunk) + else: + self.logger.error("Could not decrypt %s with any " + "passphrase in %s" % + (fname, self.setup['configfile'])) + return False + + new_fname = self.get_plaintext_filename(fname) + try: + open(new_fname, "wb").write(self.unchunk(plaintext, crypted)) + self.logger.info("Wrote decrypted data to %s" % new_fname) + return True + except IOError: + err = sys.exc_info()[1] + self.logger.error("Error writing encrypted data from %s to %s: %s" % + (fname, new_fname, err)) + return False + + def get_passphrase(self, chunk): + pname = self._get_passphrase(chunk) + if not self.pname: + if not pname: + self.logger.info("No passphrase given on command line or " + "found in file") + return False + elif self.setup.cfp.has_option("encryption", pname): + passphrase = self.setup.cfp.get("encryption", pname) + else: + self.logger.error("Could not find passphrase %s in %s" % + (pname, self.setup['configfile'])) + return False + else: + pname = self.pname + passphrase = self.passphrase + if self.pname != pname: + self.logger.warning("Passphrase given on command line (%s) " + "differs from passphrase embedded in " + "file (%s), using command-line option" % + (self.pname, pname)) + return (passphrase, pname) + + def _get_passphrase(self, chunk): + return None + + def _decrypt(self, crypted, passphrase): + return Bcfg2.Encryption.ssl_decrypt(crypted, passphrase) + + +class CfgEncryptor(Encryptor): + def get_encrypted_filename(self, plaintext_filename): + return plaintext_filename + ".crypt" + + def get_plaintext_filename(self, encrypted_filename): + if encrypted_filename.endswith(".crypt"): + return encrypted_filename[:-6] + else: + return Encryptor.get_plaintext_filename(self, encrypted_filename) + + +class PropertiesEncryptor(Encryptor): + def _encrypt(self, plaintext, passphrase, name=None): + # plaintext is an lxml.etree._Element + if name is None: + name = "true" + if plaintext.text and plaintext.text.strip(): + plaintext.text = Bcfg2.Encryption.ssl_encrypt(plaintext.text, + passphrase) + plaintext.set("encrypted", name) + return plaintext + + def chunk(self, data): + xdata = lxml.etree.XML(data) + if self.setup['xpath']: + elements = xdata.xpath(self.setup['xpath']) + else: + elements = xdata.xpath('//*[@encrypted]') + if not elements: + elements = list(xdata.getiterator()) + # this is not a good use of a generator, but we need to + # generate the full list of elements in order to ensure that + # some exist before we know what to return + for elt in elements: + yield elt + + def unchunk(self, data, original): + # Properties elements are modified in-place, so we don't + # actually need to unchunk anything + xdata = data[0] + # find root element + while xdata.getparent() != None: + xdata = xdata.getparent() + xdata.set("encryption", "true") + return lxml.etree.tostring(xdata) + + def _get_passphrase(self, chunk): + pname = chunk.get("encrypted") or chunk.get("encryption") + if pname and pname.lower() != "true": + return pname + return None + + def _decrypt(self, crypted, passphrase): + # crypted is in lxml.etree._Element + if not crypted.text or not crypted.text.strip(): + self.logger.warning("Skipping empty element %s" % crypted.tag) + return crypted + rv = Bcfg2.Encryption.ssl_decrypt(crypted.text, passphrase) + crypted.text = rv + return crypted + + +def main(): + optinfo = dict() + optinfo.update(Bcfg2.Options.CRYPT_OPTIONS) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + setup = Bcfg2.Options.OptionParser(optinfo) + setup.hm = " bcfg2-crypt [options] <filename>\nOptions:\n%s" % \ + setup.buildHelpMessage() + setup.parse(sys.argv[1:]) + + if not setup['args']: + print(setup.hm) + raise SystemExit(1) + elif setup['encrypt'] and setup['decrypt']: + print("You cannot specify both --encrypt) and --decrypt") + raise SystemExit(1) + elif setup['cfg'] and setup['properties']: + print("You cannot specify both --cfg and --properties") + raise SystemExit(1) + elif setup['cfg'] and setup['properties']: + print("Specifying --xpath with --cfg is nonsensical, ignoring --xpath") + setup['xpath'] = Bcfg2.Options.CRYPT_XPATH.default + elif setup['decrypt'] and setup['remove']: + print("--remove cannot be used with --decrypt, ignoring") + setup['remove'] = Bcfg2.Options.CRYPT_REMOVE.default + + logger = get_logger(setup['verbose']) + + props_crypt = PropertiesEncryptor(setup) + cfg_crypt = CfgEncryptor(setup) + + for fname in setup['args']: + if not os.path.exists(fname): + logger.error("%s does not exist, skipping" % fname) + continue + + # figure out if we need to encrypt this as a Properties file + # or as a Cfg file + props = False + if setup['properties']: + props = True + elif setup['cfg']: + props = False + elif fname.endswith(".xml"): + try: + xroot = lxml.etree.parse(fname).getroot() + if xroot.tag == "Properties": + props = True + else: + props = False + except IOError: + err = sys.exc_info()[1] + logger.error("Error reading %s, skipping: %s" % (fname, err)) + continue + except lxml.etree.XMLSyntaxError: + props = False + else: + props = False + + if props: + encryptor = props_crypt + else: + encryptor = cfg_crypt + + if setup['encrypt']: + if not encryptor.encrypt(fname): + print("Failed to encrypt %s, skipping" % fname) + elif setup['decrypt']: + if not encryptor.decrypt(fname): + print("Failed to decrypt %s, skipping" % fname) + else: + logger.info("Neither --encrypt nor --decrypt specified, " + "determining mode") + if not encryptor.decrypt(fname): + logger.info("Failed to decrypt %s, trying encryption" % fname) + if not encryptor.encrypt(fname): + print("Failed to encrypt %s, skipping" % fname) + + if setup['remove'] and encryptor.get_encrypted_filename(fname) != fname: + try: + os.unlink(fname) + except IOError: + err = sys.exc_info()[1] + logger.error("Error removing %s: %s" % (fname, err)) + continue + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 656532155..617584d3d 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -1,17 +1,17 @@ #!/usr/bin/env python - """This tool loads the Bcfg2 core into an interactive debugger.""" -__revision__ = '$Revision$' -from code import InteractiveConsole +import os +import sys import cmd import errno import getopt +import fnmatch import logging -import lxml.etree -import os -import sys import tempfile +import lxml.etree +import traceback +from code import InteractiveConsole try: try: @@ -27,14 +27,20 @@ import Bcfg2.Logger import Bcfg2.Options import Bcfg2.Server.Core import Bcfg2.Server.Plugins.Metadata -import Bcfg2.Server.Plugins.SGenshi import Bcfg2.Server.Plugin +try: + import Bcfg2.Server.Plugins.SGenshi + has_genshi = True +except ImportError: + has_genshi = False + logger = logging.getLogger('bcfg2-info') USAGE = """Commands: build <hostname> <filename> - Build config for hostname, writing to filename builddir <hostname> <dirname> - Build config for hostname, writing separate files to dirname -buildall <directory> - Build configs for all clients in directory +buildall <directory> [<hostnames*>] - Build configs for all clients in directory +buildallfile <directory> <filename> [<hostnames*>] - Build config file for all clients in directory buildfile <filename> <hostname> - Build config file for hostname (not written to disk) buildbundle <bundle> <hostname> - Render a templated bundle for hostname (not written to disk) bundles - Print out group/bundle information @@ -51,8 +57,7 @@ profile <command> <args> - Profile a single bcfg2-info command quit - Exit the bcfg2-info command line showentries <hostname> <type> - Show abstract configuration entries for a given host showclient <client1> <client2> - Show metadata for given hosts -update - Process pending file events -version - Print version of this tool""" +update - Process pending file events""" BUILDDIR_USAGE = """Usage: builddir [-f] <hostname> <output dir> @@ -78,10 +83,12 @@ class mockLog(object): def debug(self, *args, **kwargs): pass + class dummyError(Exception): """This is just a dummy.""" pass + class FileNotBuilt(Exception): """Thrown when File entry contains no content.""" def __init__(self, value): @@ -90,6 +97,30 @@ class FileNotBuilt(Exception): def __str__(self): return repr(self.value) + +def getClientList(hostglobs): + """ given a host glob, get a list of clients that match it """ + # special cases to speed things up: + if '*' in hostglobs: + return list(self.metadata.clients.keys()) + has_wildcards = False + for glob in hostglobs: + # check if any wildcard characters are in the string + if set('*?[]') & set(glob): + has_wildcards = True + break + if not has_wildcards: + return hostglobs + + rv = set() + clist = set(self.metadata.clients.keys()) + for glob in hostglobs: + for client in clist: + if fnmatch.fnmatch(client, glob): + rv.update(client) + clist.difference_update(rv) + return list(rv) + def printTabular(rows): """Print data in tabular format.""" cmax = tuple([max([len(str(row[index])) for row in rows]) + 1 \ @@ -109,12 +140,12 @@ def displayTrace(trace, num=80, sort=('time', 'calls')): class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): """Main class for bcfg2-info.""" def __init__(self, repo, plgs, passwd, encoding, event_debug, - cfile='/etc/bcfg2.conf', filemonitor='default'): + filemonitor='default', setup=None): cmd.Cmd.__init__(self) try: Bcfg2.Server.Core.Core.__init__(self, repo, plgs, passwd, - encoding, cfile=cfile, - filemonitor=filemonitor) + encoding, filemonitor=filemonitor, + setup=setup) if event_debug: self.fam.debug = True except Bcfg2.Server.Core.CoreInitError: @@ -177,7 +208,11 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): else: raise ImportError except ImportError: - sh.interact() + try: + import bpython.cli + bpython.cli.main(locals_=locals()) + except ImportError: + sh.interact() def do_quit(self, _): """ @@ -199,10 +234,6 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): """Process pending filesystem events.""" self.fam.handle_events_in_interval(0.1) - def do_version(self, _): - """Print out code version.""" - print(__revision__) - def do_build(self, args): """Build client configuration.""" alist = args.split() @@ -260,50 +291,101 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): def do_buildall(self, args): alist = args.split() - flags = [] - for arg in alist: - if arg == '-f': - alist.remove('-f') - flags.append(arg) - if len(alist) != 1: - print("Usage: buildall [-f] <directory>") + if len(alist) < 1: + print("Usage: buildall <directory> [<hostnames*>]") return - if not os.path.exists(alist[0]): - try: - os.mkdir(alist[0]) - except OSError: - err = sys.exc_info()[1] - logger.error("Could not create %s: %s" % (alist[0], err)) - for client in self.metadata.clients: - self.do_build("%s %s %s/%s.xml" % (" ".join(flags), - client, args, client)) + + destdir = alist[0] + try: + os.mkdir(destdir) + except OSError: + err = sys.exc_info()[1] + if err.errno != 17: + print("Could not create %s: %s" % (destdir, err)) + if len(alist) > 1: + clients = getClientList(alist[1:]) + else: + clients = list(self.metadata.clients.keys()) + for client in clients: + self.do_build("%s %s" % (client, os.path.join(destdir, + client + ".xml"))) + + def do_buildallfile(self, args): + """Build a config file for all clients.""" + usage = 'Usage: buildallfile [--altsrc=<altsrc>] <directory> <filename> [<hostnames*>]' + try: + opts, args = getopt.gnu_getopt(args.split(), '', ['altsrc=']) + except: + print(usage) + return + altsrc = None + for opt in opts: + if opt[0] == '--altsrc': + altsrc = opt[1] + if len(args) < 2: + print(usage) + return + + destdir = args[0] + filename = args[1] + try: + os.mkdir(destdir) + except OSError: + err = sys.exc_info()[1] + if err.errno != 17: + print("Could not create %s: %s" % (destdir, err)) + if len(args) > 2: + clients = getClientList(args[1:]) + else: + clients = list(self.metadata.clients.keys()) + if altsrc: + args = "--altsrc %s -f %%s %%s %%s" % altsrc + else: + args = "-f %s %s %s" + for client in clients: + self.do_buildfile(args % (os.path.join(destdir, client), + filename, client)) def do_buildfile(self, args): """Build a config file for client.""" - usage = 'Usage: buildfile [--altsrc=<altsrc>] filename hostname' + usage = 'Usage: buildfile [-f <outfile>] [--altsrc=<altsrc>] filename hostname' try: - opts, alist = getopt.gnu_getopt(args.split(), '', ['altsrc=']) + opts, alist = getopt.gnu_getopt(args.split(), 'f:', ['altsrc=']) except: print(usage) return altsrc = None + outfile = None for opt in opts: if opt[0] == '--altsrc': altsrc = opt[1] - if len(alist) == 2: - fname, client = alist - entry = lxml.etree.Element('Path', type='file', name=fname) - if altsrc: - entry.set("altsrc", altsrc) - try: - metadata = self.build_metadata(client) - self.Bind(entry, metadata) - print(lxml.etree.tostring(entry, encoding="UTF-8", - xml_declaration=True)) - except: - print("Failed to build entry %s for host %s" % (fname, client)) - else: + elif opt[0] == '-f': + outfile = opt[1] + if len(alist) != 2: print(usage) + return + + fname, client = alist + entry = lxml.etree.Element('Path', type='file', name=fname) + if altsrc: + entry.set("altsrc", altsrc) + try: + metadata = self.build_metadata(client) + self.Bind(entry, metadata) + data = lxml.etree.tostring(entry, encoding="UTF-8", + xml_declaration=True) + if outfile: + open(outfile, 'w').write(data) + else: + print(data) + except IOError: + err = sys.exc_info()[1] + print("Could not write to %s: %s" % (outfile, err)) + print(data) + except Exception: + print("Failed to build entry %s for host %s: %s" % + (fname, client, traceback.format_exc().splitlines()[-1])) + raise def do_buildbundle(self, args): """Render a bundle for client.""" @@ -313,8 +395,9 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): metadata = self.build_metadata(client) if bname in self.plugins['Bundler'].entries: bundle = self.plugins['Bundler'].entries[bname] - if isinstance(bundle, - Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): + if (has_genshi and + isinstance(bundle, + Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile)): stream = bundle.template.generate(metadata=metadata) print(stream.render("xml")) else: @@ -523,31 +606,8 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): print("Unable to build metadata for host %s" % args) return collection = self.plugins['Packages']._get_collection(metadata) - for source in collection.sources: - # get_urls() loads url_map as a side-effect - source.get_urls() - for url_map in source.url_map: - for arch in url_map['arches']: - # make sure client is in all the proper arch groups - if arch not in metadata.groups: - continue - reponame = source.get_repo_name(url_map) - print("Name: %s" % reponame) - print(" Type: %s" % source.ptype) - if url_map['url'] != '': - print(" URL: %s" % url_map['url']) - elif url_map['rawurl'] != '': - print(" RAWURL: %s" % url_map['rawurl']) - if source.gpgkeys: - print(" GPG Key(s): %s" % ", ".join(source.gpgkeys)) - else: - print(" GPG Key(s): None") - if len(source.blacklist): - print(" Blacklist: %s" % ", ".join(source.blacklist)) - if len(source.whitelist): - print(" Whitelist: %s" % ", ".join(source.whitelist)) - print("") - + print collection.sourcelist() + def do_profile(self, arg): """.""" if not have_profile: @@ -568,32 +628,16 @@ class infoCore(cmd.Cmd, Bcfg2.Server.Core.Core): if __name__ == '__main__': Bcfg2.Logger.setup_logging('bcfg2-info', to_syslog=False) - optinfo = { - 'configfile': Bcfg2.Options.CFILE, - 'help': Bcfg2.Options.HELP, - 'event debug': Bcfg2.Options.DEBUG, - 'profile': Bcfg2.Options.CORE_PROFILE, - 'encoding': Bcfg2.Options.ENCODING, - # Server options - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'mconnect': Bcfg2.Options.SERVER_MCONNECT, - 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, - 'location': Bcfg2.Options.SERVER_LOCATION, - 'static': Bcfg2.Options.SERVER_STATIC, - 'key': Bcfg2.Options.SERVER_KEY, - 'cert': Bcfg2.Options.SERVER_CERT, - 'ca': Bcfg2.Options.SERVER_CA, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'protocol': Bcfg2.Options.SERVER_PROTOCOL, - # More options - 'logging': Bcfg2.Options.LOGGING_FILE_PATH, - 'interactive': Bcfg2.Options.INTERACTIVE, - } + optinfo = dict(profile=Bcfg2.Options.CORE_PROFILE, + mconnect=Bcfg2.Options.SERVER_MCONNECT, + interactive=Bcfg2.Options.INTERACTIVE) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) - setup.hm = "Usage:\n %s\n%s" % (setup.buildHelpMessage(), - USAGE) + setup.hm = "\n".join([" bcfg2-info [options] [command <command args>]", + "Options:", + setup.buildHelpMessage(), + USAGE]) setup.parse(sys.argv[1:]) if setup['args'] and setup['args'][0] == 'help': @@ -603,15 +647,14 @@ if __name__ == '__main__': prof = profile.Profile() loop = prof.runcall(infoCore, setup['repo'], setup['plugins'], setup['password'], setup['encoding'], - setup['event debug'], cfile=setup['configfile'], - filemonitor=setup['filemonitor']) + setup['debug'], setup['filemonitor'], + setup) displayTrace(prof) else: if setup['profile']: print("Profiling functionality not available.") loop = infoCore(setup['repo'], setup['plugins'], setup['password'], - setup['encoding'], setup['event debug'], - cfile=setup['configfile'], - filemonitor=setup['filemonitor']) + setup['encoding'], setup['debug'], + setup['filemonitor'], setup) loop.Run(setup['args']) diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 2d371f4aa..423c63ba3 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -1,7 +1,6 @@ #!/usr/bin/env python """This tool examines your Bcfg2 specifications for errors.""" -__revision__ = '$Revision$' import sys import inspect @@ -63,46 +62,36 @@ def get_errorhandler(config): def load_server(setup): """ load server """ core = Bcfg2.Server.Core.Core(setup['repo'], setup['plugins'], - setup['password'], setup['encoding']) - if setup['event debug']: - core.fam.debug = True + setup['password'], setup['encoding'], + filemonitor=setup['filemonitor'], + setup=setup) core.fam.handle_events_in_interval(4) return core +def load_plugin(module, obj_name=None): + parts = module.split(".") + if obj_name is None: + obj_name = parts[-1] + + try: + mod = __import__(module) + except ImportError: + err = sys.exc_info()[1] + logger.error("Failed to load plugin %s: %s" % (obj_name, err)) + raise + + for p in parts[1:]: + mod = getattr(mod, p) + return getattr(mod, obj_name) + if __name__ == '__main__': - optinfo = { - 'configfile': Bcfg2.Options.CFILE, - 'help': Bcfg2.Options.HELP, - 'verbose': Bcfg2.Options.VERBOSE, - } - optinfo.update({ - 'event debug': Bcfg2.Options.DEBUG, - 'encoding': Bcfg2.Options.ENCODING, - # Server options - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'mconnect': Bcfg2.Options.SERVER_MCONNECT, - 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, - 'location': Bcfg2.Options.SERVER_LOCATION, - 'static': Bcfg2.Options.SERVER_STATIC, - 'key': Bcfg2.Options.SERVER_KEY, - 'cert': Bcfg2.Options.SERVER_CERT, - 'ca': Bcfg2.Options.SERVER_CA, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'protocol': Bcfg2.Options.SERVER_PROTOCOL, - # More options - 'logging': Bcfg2.Options.LOGGING_FILE_PATH, - 'stdin': Bcfg2.Options.FILES_ON_STDIN, - 'schema': Bcfg2.Options.SCHEMA_PATH, - 'config': Bcfg2.Options.Option('Specify bcfg2-lint configuration file', - '/etc/bcfg2-lint.conf', - cmd='--lint-config', - odesc='<conffile>', - long_arg=True), - 'showerrors': Bcfg2.Options.Option('Show error handling', False, - cmd='--list-errors', - long_arg=True), - }) + optinfo = dict(config=Bcfg2.Options.LINT_CONFIG, + showerrors=Bcfg2.Options.LINT_SHOW_ERRORS, + stdin=Bcfg2.Options.LINT_FILES_ON_STDIN, + schema=Bcfg2.Options.SCHEMA_PATH, + plugins=Bcfg2.Options.SERVER_PLUGINS) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) @@ -115,53 +104,38 @@ if __name__ == '__main__': config.read(setup['configfile']) config.read(setup['config']) - if setup['showerrors']: - if config.has_section("errors"): - econf = dict(config.items("errors")) - else: - econf = dict() - - print("%-35s %-35s" % ("Error name", "Handler (Default)")) - for err, default in Bcfg2.Server.Lint.ErrorHandler._errors.items(): - if err in econf and econf[err] != default: - handler = "%s (%s)" % (econf[err], default) - else: - handler = default - print("%-35s %-35s" % (err, handler)) - raise SystemExit(0) - # get list of plugins to run if setup['args']: - allplugins = setup['args'] + plugin_list = setup['args'] elif "bcfg2-repo-validate" in sys.argv[0]: - allplugins = 'Duplicates,RequiredAttrs,Validate'.split(',') + plugin_list = 'Duplicates,RequiredAttrs,Validate'.split(',') else: try: - allplugins = config.get('lint', 'plugins').split(',') + plugin_list = config.get('lint', 'plugins').split(',') except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - allplugins = Bcfg2.Server.Lint.__all__ + plugin_list = Bcfg2.Server.Lint.__all__ if setup['stdin']: files = [s.strip() for s in sys.stdin.readlines()] else: files = None - # load plugins - serverplugins = {} - serverlessplugins = {} - for plugin_name in allplugins: + # load plugins specified in the config first + allplugins = dict() + for plugin in plugin_list: + allplugins[plugin] = load_plugin("Bcfg2.Server.Lint." + plugin) + + # load lint plugins bundled with bcfg2-server plugins + for plugin in setup['plugins']: try: - mod = getattr(__import__("Bcfg2.Server.Lint.%s" % - (plugin_name)).Server.Lint, plugin_name) - except ImportError: - try: - mod = __import__(plugin_name) - except Exception: - err = sys.exc_info()[1] - logger.error("Failed to load plugin %s: %s" % (plugin_name, - err)) - raise SystemExit(1) - plugin = getattr(mod, plugin_name) + allplugins[plugin] = load_plugin("Bcfg2.Server.Plugins." + plugin, + obj_name=plugin + "Lint") + except AttributeError: + pass + + serverplugins = dict() + serverlessplugins = dict() + for plugin_name, plugin in allplugins.items(): if [c for c in inspect.getmro(plugin) if c == Bcfg2.Server.Lint.ServerPlugin]: serverplugins[plugin_name] = plugin @@ -170,6 +144,15 @@ if __name__ == '__main__': errorhandler = get_errorhandler(config) + if setup['showerrors']: + for plugin in serverplugins.values() + serverlessplugins.values(): + errorhandler.RegisterErrors(getattr(plugin, 'Errors')()) + + print("%-35s %-35s" % ("Error name", "Handler")) + for err, handler in errorhandler._handlers.items(): + print("%-35s %-35s" % (err, handler.__name__)) + raise SystemExit(0) + run_serverless_plugins(serverlessplugins, errorhandler=errorhandler, config=config, setup=setup) diff --git a/src/sbin/bcfg2-ping-sweep b/src/sbin/bcfg2-ping-sweep index 70f718690..be8994be3 100755 --- a/src/sbin/bcfg2-ping-sweep +++ b/src/sbin/bcfg2-ping-sweep @@ -3,8 +3,6 @@ """Generates hostinfo.xml at a regular interval.""" -__revision__ = '$Revision$' - from os import dup2, execl, fork, uname, wait import sys import time diff --git a/src/sbin/bcfg2-reports b/src/sbin/bcfg2-reports index 3920f519a..cb553c0ba 100755 --- a/src/sbin/bcfg2-reports +++ b/src/sbin/bcfg2-reports @@ -1,9 +1,10 @@ #!/usr/bin/env python """Query reporting system for client status.""" -__revision__ = '$Revision$' import os import sys +import datetime +from optparse import OptionParser, OptionGroup, make_option from Bcfg2.Bcfg2Py3k import ConfigParser try: @@ -22,376 +23,277 @@ sys.path.pop() # Set DJANGO_SETTINGS_MODULE appropriately. os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name -from Bcfg2.Server.Reports.reports.models import Client -import getopt -import datetime -import fileinput - -usage = """Usage: bcfg2-reports [option] ... - -Options and arguments (and corresponding environment variables): --a : shows all hosts, including expired hosts --b NAME : single-host mode - shows bad entries from the - current interaction of NAME --c : shows only clean hosts --d : shows only dirty hosts --e NAME : single-host mode - shows extra entries from the - current interaction of NAME --h : shows help and usage info about bcfg2-reports --m NAME : single-host mode - shows modified entries from the - current interaction of NAME --s NAME : single-host mode - shows bad, modified, and extra - entries from the current interaction of NAME --t NAME : single-host mode - shows total number of managed and - good entries from the current interaction of NAME --x NAME : toggles expired/unexpired state of NAME ---badentry=KIND,NAME : shows only hosts whose current interaction has bad - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 ---modifiedentry=KIND,NAME : shows only hosts whose current interaction has - modified entries in of KIND kind and NAME name; if a - single argument ARG1 is given, then KIND,NAME pairs - will be read from a file of name ARG1 ---extraentry=KIND,NAME : shows only hosts whose current interaction has extra - entries in of KIND kind and NAME name; if a single - argument ARG1 is given, then KIND,NAME pairs will be - read from a file of name ARG1 ---fields=ARG1,ARG2,... : only displays the fields ARG1,ARG2,... - (name,time,state) ---sort=ARG1,ARG2,... : sorts output on ARG1,ARG2,... (name,time,state) ---stale : shows hosts which haven't run in the last 24 hours -""" - -def timecompare(client1, client2): - """Compares two clients by their timestamps.""" - return cmp(client1.current_interaction.timestamp, \ - client2.current_interaction.timestamp) - -def namecompare(client1, client2): - """Compares two clients by their names.""" - return cmp(client1.name, client2.name) - -def statecompare(client1, client2): - """Compares two clients by their states.""" - clean1 = client1.current_interaction.isclean() - clean2 = client2.current_interaction.isclean() - - if clean1 and not clean2: - return -1 - elif clean2 and not clean1: - return 1 - else: - return 0 - -def totalcompare(client1, client2): - """Compares two clients by their total entry counts.""" - return cmp(client2.current_interaction.totalcount, \ - client1.current_interaction.totalcount) - -def goodcompare(client1, client2): - """Compares two clients by their good entry counts.""" - return cmp(client2.current_interaction.goodcount, \ - client1.current_interaction.goodcount) +from Bcfg2.Server.Reports.reports.models import (Client, Entries_interactions, + Entries, TYPE_CHOICES) -def badcompare(client1, client2): - """Compares two clients by their bad entry counts.""" - return cmp(client2.current_interaction.totalcount - \ - client2.current_interaction.goodcount, \ - client1.current_interaction.totalcount - \ - client1.current_interaction.goodcount) +def hosts_by_entry_type(clients, etype, entryspec): + result = [] + for entry in entryspec: + for client in clients: + items = getattr(client.current_interaction, etype)() + for item in items: + if (item.entry.kind == entry[0] and + item.entry.name == entry[1]): + result.append(client) + return result -def crit_compare(criterion, client1, client2): - """Compares two clients by the criteria provided in criterion.""" - for crit in criterion: - comp = 0 - if crit == 'name': - comp = namecompare(client1, client2) - elif crit == 'state': - comp = statecompare(client1, client2) - elif crit == 'time': - comp = timecompare(client1, client2) - elif crit == 'total': - comp = totalcompare(client1, client2) - elif crit == 'good': - comp = goodcompare(client1, client2) - elif crit == 'bad': - comp = badcompare(client1, client2) - - if comp != 0: - return comp - - return 0 - -def print_fields(fields, cli, max_name, entrydict): +def print_fields(fields, client, fmt, extra=None): """ - Prints the fields specified in fields of cli, max_name + Prints the fields specified in fields of client, max_name specifies the column width of the name column. """ - fmt = '' - for field in fields: - if field == 'name': - fmt += ("%%-%ds " % (max_name)) - else: - fmt += "%s " fdata = [] + if extra is None: + extra = dict() for field in fields: if field == 'time': - fdata.append(str(cli.current_interaction.timestamp)) + fdata.append(str(client.current_interaction.timestamp)) elif field == 'state': - if cli.current_interaction.isclean(): + if client.current_interaction.isclean(): fdata.append("clean") else: fdata.append("dirty") elif field == 'total': - fdata.append("%5d" % cli.current_interaction.totalcount) + fdata.append(client.current_interaction.totalcount) elif field == 'good': - fdata.append("%5d" % cli.current_interaction.goodcount) + fdata.append(client.current_interaction.goodcount) + elif field == 'modified': + fdata.append(client.current_interaction.modified_entry_count()) + elif field == 'extra': + fdata.append(client.current_interaction.extra_entry_count()) elif field == 'bad': - fdata.append("%5d" % cli.current_interaction.totalcount \ - - cli.current_interaction.goodcount) + fdata.append((client.current_interaction.badcount())) else: try: - fdata.append(getattr(cli, field)) + fdata.append(getattr(client, field)) except: - fdata.append("N/A") + fdata.append(extra.get(field, "N/A")) - display = fmt % tuple(fdata) - if len(entrydict) > 0: - display += " " - display += str(entrydict[cli]) - print(display) + print(fmt % tuple(fdata)) -def print_entry(item, max_name): - fmt = ("%%-%ds " % (max_name)) - fdata = item.entry.kind + ":" + item.entry.name - display = fmt % (fdata) - print(display) - -fields = "" -sort = "" -badentry = "" -modifiedentry = "" -extraentry = "" -expire = "" -singlehost = "" +def print_entries(interaction, etype): + items = getattr(interaction, etype)() + for item in items: + print("%-70s %s" % (item.entry.kind + ":" + item.entry.name, etype)) -c_list = Client.objects.all() +def main(): + parser = OptionParser(usage="%prog [options] <mode> [arg]") -result = list() -entrydict = dict() + # single host modes + multimodes = [] + singlemodes = [] + multimodes.append(make_option("-b", "--bad", action="store_true", + default=False, + help="Show bad entries from HOST")) + multimodes.append(make_option("-e", "--extra", action="store_true", + default=False, + help="Show extra entries from HOST")) + multimodes.append(make_option("-m", "--modified", action="store_true", + default=False, + help="Show modified entries from HOST")) + multimodes.append(make_option("-s", "--show", action="store_true", + default=False, + help="Equivalent to --bad --extra --modified")) + singlemodes.append(make_option("-t", "--total", action="store_true", + default=False, + help="Show total number of managed and good " + "entries from HOST")) + singlemodes.append(make_option("-x", "--expire", action="store_true", + default=False, + help="Toggle expired/unexpired state of " + "HOST")) + hostmodes = \ + OptionGroup(parser, "Single-Host Modes", + "The following mode flags require a single HOST argument") + hostmodes.add_options(multimodes) + hostmodes.add_options(singlemodes) + parser.add_option_group(hostmodes) -args = sys.argv[1:] -try: - opts, pargs = getopt.getopt(args, 'ab:cde:hm:s:t:x:', - ['stale', - 'sort=', - 'fields=', - 'badentry=', - 'modifiedentry=', - 'extraentry=']) -except getopt.GetoptError: - msg = sys.exc_info()[1] - print(msg) - print(usage) - sys.exit(2) + # all host modes + allhostmodes = OptionGroup(parser, "Host Selection Modes", + "The following mode flags require no arguments") + allhostmodes.add_option("-a", "--all", action="store_true", default=False, + help="Show all hosts, including expired hosts") + allhostmodes.add_option("-c", "--clean", action="store_true", default=False, + help="Show only clean hosts") + allhostmodes.add_option("-d", "--dirty", action="store_true", default=False, + help="Show only dirty hosts") + allhostmodes.add_option("--stale", action="store_true", default=False, + help="Show hosts that haven't run in the last 24 " + "hours") + parser.add_option_group(allhostmodes) + + # entry modes + entrymodes = \ + OptionGroup(parser, "Entry Modes", + "The following mode flags require either any number of " + "TYPE:NAME arguments describing entries, or the --file " + "option") + entrymodes.add_option("--badentry", action="store_true", default=False, + help="Show hosts that have bad entries that match " + "the argument") + entrymodes.add_option("--modifiedentry", action="store_true", default=False, + help="Show hosts that have modified entries that " + "match the argument") + entrymodes.add_option("--extraentry", action="store_true", default=False, + help="Show hosts that have extra entries that match " + "the argument") + entrymodes.add_option("--entrystatus", action="store_true", default=False, + help="Show the status of the named entry on all " + "hosts. Only supports a single entry.") + parser.add_option_group(entrymodes) + + # entry options + entryopts = OptionGroup(parser, "Entry Options", + "Options that can be used with entry modes") + entryopts.add_option("--fields", metavar="FIELD,FIELD,...", + help="Only display the listed fields", + default='name,time,state') + entryopts.add_option("--file", metavar="FILE", + help="Read TYPE:NAME pairs from the specified file " + "instead of the command line") + parser.add_option_group(entryopts) -for option in opts: - if len(option) > 0: - if option[0] == '--fields': - fields = option[1] - if option[0] == '--sort': - sort = option[1] - if option[0] == '--badentry': - badentry = option[1] - if option[0] == '--modifiedentry': - modifiedentry = option[1] - if option[0] == '--extraentry': - extraentry = option[1] - if option[0] == '-x': - expire = option[1] - if option[0] == '-s' or \ - option[0] == '-t' or \ - option[0] == '-b' or \ - option[0] == '-m' or \ - option[0] == '-e': - singlehost = option[1] + options, args = parser.parse_args() -if expire != "": - for c_inst in c_list: - if expire == c_inst.name: - if c_inst.expiration == None: - c_inst.expiration = datetime.datetime.now() + # make sure we've specified exactly one mode + mode_family = None + mode = None + for opt in allhostmodes.option_list + entrymodes.option_list + \ + singlemodes: + if getattr(options, opt.dest): + if mode is not None: + parser.error("Only one mode can be specified; found %s and %s" % + (mode.get_opt_string(), opt.get_opt_string())) + mode = opt + mode_family = parser.get_option_group(opt.get_opt_string()) + + # you can specify more than one of --bad, --extra, --modified, --show, so + # consider single-host options separately + if not mode_family: + for opt in multimodes: + if getattr(options, opt.dest): + mode_family = parser.get_option_group(opt.get_opt_string()) + break + + if not mode_family: + parser.error("You must specify a mode") + + if mode_family == hostmodes: + try: + cname = args.pop() + client = Client.objects.select_related().get(name=cname) + except IndexError: + parser.error("%s require a single HOST argument" % hostmodes.title) + except Client.DoesNotExist: + print("No such host: %s" % cname) + return 2 + + if options.expire: + if client.expiration == None: + client.expiration = datetime.datetime.now() print("Host expired.") else: - c_inst.expiration = None + client.expiration = None print("Host un-expired.") - c_inst.save() + client.save() + elif options.total: + managed = client.current_interaction.totalcount + good = client.current_interaction.goodcount + print("Total managed entries: %d (good: %d)" % (managed, good)) + elif mode_family == hostmodes: + if options.bad or options.show: + print_entries(client.current_interaction, "bad") -elif '-h' in args: - print(usage) -elif singlehost != "": - for c_inst in c_list: - if singlehost == c_inst.name: - if '-t' in args: - managed = c_inst.current_interaction.totalcount - good = c_inst.current_interaction.goodcount - print("Total managed entries: %d (good: %d)" % (managed, good)) - baditems = c_inst.current_interaction.bad() - if len(baditems) > 0 and ('-b' in args or '-s' in args): - print("Bad Entries:") - max_name = -1 - for item in baditems: - if len(item.entry.name) > max_name: - max_name = len(item.entry.name) - for item in baditems: - print_entry(item, max_name) - modifieditems = c_inst.current_interaction.modified() - if len(modifieditems) > 0 and ('-m' in args or '-s' in args): - print "Modified Entries:" - max_name = -1 - for item in modifieditems: - if len(item.entry.name) > max_name: - max_name = len(item.entry.name) - for item in modifieditems: - print_entry(item, max_name) - extraitems = c_inst.current_interaction.extra() - if len(extraitems) > 0 and ('-e' in args or '-s' in args): - print("Extra Entries:") - max_name = -1 - for item in extraitems: - if len(item.entry.name) > max_name: - max_name = len(item.entry.name) - for item in extraitems: - print_entry(item, max_name) - + if options.modified or options.show: + print_entries(client.current_interaction, "modified") -else: - if fields == "": - fields = ['name', 'time', 'state'] + if options.extra or options.show: + print_entries(client.current_interaction, "extra") else: - fields = fields.split(',') - - if sort != "": - sort = sort.split(',') + clients = Client.objects.exclude(current_interaction__isnull=True) + result = list() + edata = dict() + fields = options.fields.split(',') - if badentry != "": - badentry = badentry.split(',') + if mode_family == allhostmodes: + if args: + print("%s do not take any arguments, ignoring" % + allhostmodes.title) - if modifiedentry != "": - modifiedentry = modifiedentry.split(',') + for client in clients: + interaction = client.current_interaction + if (options.all or + (options.stale and interaction.isstale()) or + (options.clean and interaction.isclean()) or + (options.dirty and not interaction.isclean())): + result.append(client) + else: + # entry query modes + if options.file: + try: + entries = [l.strip().split(":") + for l in open(options.file)] + except IOError, err: + print("Cannot read entries from %s: %s" % (options.file, + err)) + return 2 + elif args: + entries = [a.split(":") for a in args] + else: + parser.error("%s require either a list of entries on the " + "command line or the --file options" % + mode_family.title) + + if options.badentry: + result = hosts_by_entry_type(clients, "bad", entries) + elif options.modifiedentry: + result = hosts_by_entry_type(clients, "modified", entries) + elif options.extraentry: + result = hosts_by_entry_type(clients, "extra", entries) + elif options.entrystatus: + if 'state' in fields: + fields.remove('state') + fields.append("entry state") + try: + entry_obj = Entries.objects.get( + kind=entries[0][0], + name=entries[0][1]) + except Entries.DoesNotExist: + print("No entry %s found" % ":".join(entries[0])) + return 2 - if extraentry != "": - extraentry = extraentry.split(',') - - # stale hosts - if '--stale' in args: - for c_inst in c_list: - if c_inst.current_interaction.isstale(): - result.append(c_inst) - # clean hosts - elif '-c' in args: - for c_inst in c_list: - if c_inst.current_interaction.isclean(): - result.append(c_inst) - # dirty hosts - elif '-d' in args: - for c_inst in c_list: - if not c_inst.current_interaction.isclean(): - result.append(c_inst) + for client in clients: + try: + entry = \ + Entries_interactions.objects.select_related().get( + interaction=client.current_interaction, + entry=entry_obj) + edata[client] = \ + {"entry state":dict(TYPE_CHOICES)[entry.type], + "reason":entry.reason} + result.append(client) + except Entries_interactions.DoesNotExist: + pass - elif badentry != "": - if len(badentry) == 1: - fileread = fileinput.input(badentry[0]) - try: - for line in fileread: - badentry = line.strip().split(',') - for c_inst in c_list: - baditems = c_inst.current_interaction.bad() - for item in baditems: - if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: - result.append(c_inst) - if c_inst in entrydict: - entrydict.get(c_inst).append(badentry[1]) - else: - entrydict[c_inst] = [badentry[1]] - break - except IOError: - e = sys.exc_info()[1] - print("Cannot read %s: %s" % (e.filename, e.strerror)) - else: - for c_inst in c_list: - baditems = c_inst.current_interaction.bad() - for item in baditems: - if item.entry.name == badentry[1] and item.entry.kind == badentry[0]: - result.append(c_inst) - break - elif modifiedentry != "": - if len(modifiedentry) == 1: - fileread = fileinput.input(modifiedentry[0]) - try: - for line in fileread: - modifiedentry = line.strip().split(',') - for c_inst in c_list: - modifieditems = c_inst.current_interaction.modified() - for item in modifieditems: - if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]: - result.append(c_inst) - if c_inst in entrydict: - entrydict.get(c_inst).append(modifiedentry[1]) - else: - entrydict[c_inst] = [modifiedentry[1]] - break - except IOError: - e = sys.exc_info()[1] - print("Cannot read %s: %s" % (e.filename, e.strerror)) - else: - for c_inst in c_list: - modifieditems = c_inst.current_interaction.modified() - for item in modifieditems: - if item.entry.name == modifiedentry[1] and item.entry.kind == modifiedentry[0]: - result.append(c_inst) - break - elif extraentry != "": - if len(extraentry) == 1: - fileread = fileinput.input(extraentry[0]) - try: - for line in fileread: - extraentry = line.strip().split(',') - for c_inst in c_list: - extraitems = c_inst.current_interaction.extra() - for item in extraitems: - if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]: - result.append(c_inst) - if c_inst in entrydict: - entrydict.get(c_inst).append(extraentry[1]) - else: - entrydict[c_inst] = [extraentry[1]] - break - except IOError: - e = sys.exc_info()[1] - print("Cannot read %s: %s" % (e.filename, e.strerror)) - else: - for c_inst in c_list: - extraitems = c_inst.current_interaction.extra() - for item in extraitems: - if item.entry.name == extraentry[1] and item.entry.kind == extraentry[0]: - result.append(c_inst) - break - else: - for c_inst in c_list: - result.append(c_inst) - max_name = -1 - if 'name' in fields: - for c_inst in result: - if len(c_inst.name) > max_name: - max_name = len(c_inst.name) + if 'name' not in fields: + fields.insert(0, "name") + max_name = max(len(c.name) for c in result) + ffmt = [] + for field in fields: + if field == "name": + ffmt.append("%%-%ds" % max_name) + elif field == "time": + ffmt.append("%-19s") + else: + ffmt.append("%%-%ds" % len(field)) + fmt = " ".join(ffmt) + print(fmt % tuple(f.title() for f in fields)) + for client in result: + if not client.expiration: + print_fields(fields, client, fmt, + extra=edata.get(client, None)) - if sort != "": - result.sort(lambda x, y: crit_compare(sort, x, y)) - - if fields != "": - for c_inst in result: - if '-a' in args or c_inst.expiration == None: - print_fields(fields, c_inst, max_name, entrydict) +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index 546d5a249..d03edc93e 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -1,7 +1,6 @@ #!/usr/bin/env python """The XML-RPC Bcfg2 server.""" -__revision__ = '$Revision$' import logging import os.path @@ -16,35 +15,11 @@ from Bcfg2.Server.Core import CoreInitError logger = logging.getLogger('bcfg2-server') if __name__ == '__main__': - - OPTINFO = { - 'configfile': Bcfg2.Options.CFILE, - 'daemon' : Bcfg2.Options.DAEMON, - 'debug' : Bcfg2.Options.DEBUG, - 'help' : Bcfg2.Options.HELP, - 'verbose' : Bcfg2.Options.VERBOSE, - 'to_file' : Bcfg2.Options.LOGGING_FILE_PATH, - } - - OPTINFO.update({'repo' : Bcfg2.Options.SERVER_REPOSITORY, - 'plugins' : Bcfg2.Options.SERVER_PLUGINS, - 'password' : Bcfg2.Options.SERVER_PASSWORD, - 'fm' : Bcfg2.Options.SERVER_FILEMONITOR, - }) - - OPTINFO.update({'key' : Bcfg2.Options.SERVER_KEY, - 'cert' : Bcfg2.Options.SERVER_CERT, - 'ca' : Bcfg2.Options.SERVER_CA, - 'listen_all' : Bcfg2.Options.SERVER_LISTEN_ALL, - 'location' : Bcfg2.Options.SERVER_LOCATION, - 'passwd' : Bcfg2.Options.SERVER_PASSWORD, - 'static' : Bcfg2.Options.SERVER_STATIC, - 'encoding' : Bcfg2.Options.ENCODING, - 'filelog' : Bcfg2.Options.LOGGING_FILE_PATH, - 'protocol' : Bcfg2.Options.SERVER_PROTOCOL, - }) - - setup = Bcfg2.Options.OptionParser(OPTINFO) + optinfo = dict() + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.DAEMON_COMMON_OPTIONS) + setup = Bcfg2.Options.OptionParser(optinfo) setup.parse(sys.argv[1:]) try: # check whether the specified bcfg2.conf exists @@ -54,10 +29,10 @@ if __name__ == '__main__': Bcfg2.Component.run_component(Bcfg2.Server.Core.Core, listen_all=setup['listen_all'], location=setup['location'], - daemon = setup['daemon'], - pidfile_name = setup['daemon'], - protocol = setup['protocol'], - to_file=setup['to_file'], + daemon=setup['daemon'], + pidfile_name=setup['daemon'], + protocol=setup['protocol'], + to_file=setup['logging'], cfile=setup['configfile'], register=False, cls_kwargs={'repo':setup['repo'], @@ -65,11 +40,12 @@ if __name__ == '__main__': 'password':setup['password'], 'encoding':setup['encoding'], 'ca':setup['ca'], - 'filemonitor':setup['fm'], - 'start_fam_thread':True}, + 'filemonitor':setup['filemonitor'], + 'start_fam_thread':True, + 'setup':setup}, keyfile=setup['key'], certfile=setup['cert'], - ca=setup['ca'], + ca=setup['ca'] ) except CoreInitError: msg = sys.exc_info()[1] diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index 01a2a4893..653c24124 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -61,17 +61,11 @@ class ClientTest(TestCase): id = __str__ def main(): - optinfo = { - 'configfile': Bcfg2.Options.CFILE, - 'help': Bcfg2.Options.HELP, - 'encoding': Bcfg2.Options.ENCODING, - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'verbose': Bcfg2.Options.VERBOSE, - 'noseopts': Bcfg2.Options.TEST_NOSEOPTS, - 'ignore': Bcfg2.Options.TEST_IGNORE, - } + optinfo = dict(noseopts=Bcfg2.Options.TEST_NOSEOPTS, + test_ignore=Bcfg2.Options.TEST_IGNORE, + validate=Bcfg2.Options.CFG_VALIDATION) + optinfo.update(Bcfg2.Options.CLI_COMMON_OPTIONS) + optinfo.update(Bcfg2.Options.SERVER_COMMON_OPTIONS) setup = Bcfg2.Options.OptionParser(optinfo) setup.hm = \ "bcfg2-test [options] [client] [client] [...]\nOptions:\n %s" % \ @@ -86,11 +80,12 @@ def main(): setup['plugins'], setup['password'], setup['encoding'], - filemonitor='pseudo' + filemonitor=setup['filemonitor'], + setup=setup ) ignore = dict() - for entry in setup['ignore']: + for entry in setup['test_ignore']: tag, name = entry.split(":") try: ignore[tag].append(name) diff --git a/src/sbin/bcfg2-yum-helper b/src/sbin/bcfg2-yum-helper index dc46bb81a..2da7c6336 100755 --- a/src/sbin/bcfg2-yum-helper +++ b/src/sbin/bcfg2-yum-helper @@ -5,8 +5,6 @@ the right way to get around that in long-running processes it to have a short-lived helper. No, seriously -- check out the yum-updatesd code. It's pure madness. """ -__revision__ = '$Revision$' - import os import sys import yum |