From dab1d03d81c538966d03fb9318a4588a9e803b44 Mon Sep 17 00:00:00 2001 From: Sol Jerome Date: Sat, 24 Mar 2012 11:20:07 -0500 Subject: Allow to run directly from a git checkout (#1037) Signed-off-by: Sol Jerome --- src/lib/Server/Admin/Backup.py | 25 - src/lib/Server/Admin/Bundle.py | 96 -- src/lib/Server/Admin/Client.py | 64 - src/lib/Server/Admin/Compare.py | 151 --- src/lib/Server/Admin/Group.py | 63 - src/lib/Server/Admin/Init.py | 357 ------ src/lib/Server/Admin/Minestruct.py | 65 -- src/lib/Server/Admin/Perf.py | 37 - src/lib/Server/Admin/Pull.py | 154 --- src/lib/Server/Admin/Query.py | 72 -- src/lib/Server/Admin/Reports.py | 376 ------ src/lib/Server/Admin/Snapshots.py | 165 --- src/lib/Server/Admin/Tidy.py | 69 -- src/lib/Server/Admin/Viz.py | 119 -- src/lib/Server/Admin/Xcmd.py | 60 - src/lib/Server/Admin/__init__.py | 136 --- src/lib/Server/Core.py | 479 -------- src/lib/Server/FileMonitor.py | 315 ----- src/lib/Server/Hostbase/.gitignore | 3 - src/lib/Server/Hostbase/__init__.py | 0 src/lib/Server/Hostbase/backends.py | 68 -- src/lib/Server/Hostbase/hostbase/__init__.py | 0 src/lib/Server/Hostbase/hostbase/admin.py | 15 - src/lib/Server/Hostbase/hostbase/models.py | 210 ---- src/lib/Server/Hostbase/hostbase/sql/zone.sql | 2 - src/lib/Server/Hostbase/hostbase/urls.py | 68 -- src/lib/Server/Hostbase/hostbase/views.py | 970 --------------- .../Hostbase/hostbase/webtemplates/base.html | 34 - .../Hostbase/hostbase/webtemplates/confirm.html | 117 -- .../Hostbase/hostbase/webtemplates/copy.html | 122 -- .../Server/Hostbase/hostbase/webtemplates/dns.html | 40 - .../Hostbase/hostbase/webtemplates/dnsedit.html | 98 -- .../Hostbase/hostbase/webtemplates/edit.html | 191 --- .../Hostbase/hostbase/webtemplates/errors.html | 31 - .../Hostbase/hostbase/webtemplates/host.html | 80 -- .../webtemplates/hostbase/host_confirm_delete.html | 89 -- .../hostbase/webtemplates/hostbase/log_detail.html | 23 - .../Hostbase/hostbase/webtemplates/index.html | 16 - .../Hostbase/hostbase/webtemplates/login.html | 37 - .../Hostbase/hostbase/webtemplates/logout.html | 13 - .../Hostbase/hostbase/webtemplates/logout.tmpl | 6 - .../Hostbase/hostbase/webtemplates/logviewer.html | 27 - .../Hostbase/hostbase/webtemplates/navbar.tmpl | 5 - .../Server/Hostbase/hostbase/webtemplates/new.html | 102 -- .../Hostbase/hostbase/webtemplates/remove.html | 89 -- .../Hostbase/hostbase/webtemplates/results.html | 45 - .../Hostbase/hostbase/webtemplates/search.html | 57 - .../Hostbase/hostbase/webtemplates/zoneedit.html | 81 -- .../Hostbase/hostbase/webtemplates/zonenew.html | 43 - .../Hostbase/hostbase/webtemplates/zones.html | 37 - .../Hostbase/hostbase/webtemplates/zoneview.html | 71 -- src/lib/Server/Hostbase/ldapauth.py | 180 --- src/lib/Server/Hostbase/manage.py | 11 - src/lib/Server/Hostbase/media/base.css | 5 - src/lib/Server/Hostbase/media/boxypastel.css | 179 --- src/lib/Server/Hostbase/media/global.css | 8 - src/lib/Server/Hostbase/media/layout.css | 62 - src/lib/Server/Hostbase/nisauth.py | 40 - src/lib/Server/Hostbase/regex.py | 6 - src/lib/Server/Hostbase/settings.py | 143 --- src/lib/Server/Hostbase/templates/batchadd.tmpl | 29 - src/lib/Server/Hostbase/templates/dhcpd.conf.head | 5 - src/lib/Server/Hostbase/templates/dhcpd.tmpl | 17 - src/lib/Server/Hostbase/templates/hosts.tmpl | 26 - src/lib/Server/Hostbase/templates/hostsappend.tmpl | 5 - src/lib/Server/Hostbase/templates/named.tmpl | 69 -- src/lib/Server/Hostbase/templates/namedviews.tmpl | 92 -- .../Server/Hostbase/templates/reverseappend.tmpl | 4 - src/lib/Server/Hostbase/templates/reversesoa.tmpl | 13 - src/lib/Server/Hostbase/templates/zone.tmpl | 18 - src/lib/Server/Hostbase/test/harness.py | 11 - .../Server/Hostbase/test/test_environ_settings.py | 35 - src/lib/Server/Hostbase/test/test_ldapauth.py | 10 - src/lib/Server/Hostbase/test/test_settings.py | 12 - src/lib/Server/Hostbase/urls.py | 27 - src/lib/Server/Lint/Bundles.py | 61 - src/lib/Server/Lint/Comments.py | 187 --- src/lib/Server/Lint/Duplicates.py | 81 -- src/lib/Server/Lint/Genshi.py | 27 - src/lib/Server/Lint/GroupPatterns.py | 31 - src/lib/Server/Lint/InfoXML.py | 42 - src/lib/Server/Lint/MergeFiles.py | 69 -- src/lib/Server/Lint/Pkgmgr.py | 35 - src/lib/Server/Lint/RequiredAttrs.py | 138 --- src/lib/Server/Lint/Validate.py | 200 ---- src/lib/Server/Lint/__init__.py | 215 ---- src/lib/Server/Plugin.py | 1232 -------------------- src/lib/Server/Plugins/Account.py | 101 -- src/lib/Server/Plugins/BB.py | 83 -- src/lib/Server/Plugins/Base.py | 43 - src/lib/Server/Plugins/Bundler.py | 99 -- src/lib/Server/Plugins/Bzr.py | 35 - src/lib/Server/Plugins/Cfg.py | 283 ----- src/lib/Server/Plugins/Cvs.py | 46 - src/lib/Server/Plugins/DBStats.py | 113 -- src/lib/Server/Plugins/Darcs.py | 48 - src/lib/Server/Plugins/Decisions.py | 66 -- src/lib/Server/Plugins/Defaults.py | 53 - src/lib/Server/Plugins/Deps.py | 134 --- src/lib/Server/Plugins/Editor.py | 80 -- src/lib/Server/Plugins/FileProbes.py | 230 ---- src/lib/Server/Plugins/Fossil.py | 51 - src/lib/Server/Plugins/Git.py | 44 - src/lib/Server/Plugins/GroupPatterns.py | 124 -- src/lib/Server/Plugins/Guppy.py | 62 - src/lib/Server/Plugins/Hg.py | 45 - src/lib/Server/Plugins/Hostbase.py | 593 ---------- src/lib/Server/Plugins/Ldap.py | 245 ---- src/lib/Server/Plugins/Metadata.py | 866 -------------- src/lib/Server/Plugins/NagiosGen.py | 151 --- src/lib/Server/Plugins/Ohai.py | 92 -- src/lib/Server/Plugins/Packages/Apt.py | 139 --- src/lib/Server/Plugins/Packages/Collection.py | 343 ------ src/lib/Server/Plugins/Packages/Pac.py | 120 -- src/lib/Server/Plugins/Packages/PackagesConfig.py | 15 - src/lib/Server/Plugins/Packages/PackagesSources.py | 97 -- src/lib/Server/Plugins/Packages/Source.py | 278 ----- src/lib/Server/Plugins/Packages/Yum.py | 688 ----------- src/lib/Server/Plugins/Packages/__init__.py | 263 ----- src/lib/Server/Plugins/Pkgmgr.py | 169 --- src/lib/Server/Plugins/Probes.py | 285 ----- src/lib/Server/Plugins/Properties.py | 76 -- src/lib/Server/Plugins/Rules.py | 55 - src/lib/Server/Plugins/SGenshi.py | 97 -- src/lib/Server/Plugins/SSHbase.py | 413 ------- src/lib/Server/Plugins/SSLCA.py | 274 ----- src/lib/Server/Plugins/Snapshots.py | 142 --- src/lib/Server/Plugins/Statistics.py | 161 --- src/lib/Server/Plugins/Svcmgr.py | 10 - src/lib/Server/Plugins/Svn.py | 46 - src/lib/Server/Plugins/Svn2.py | 125 -- src/lib/Server/Plugins/TCheetah.py | 80 -- src/lib/Server/Plugins/TGenshi.py | 135 --- src/lib/Server/Plugins/TemplateHelper.py | 83 -- src/lib/Server/Plugins/Trigger.py | 43 - src/lib/Server/Plugins/__init__.py | 34 - src/lib/Server/Reports/__init__.py | 1 - src/lib/Server/Reports/backends.py | 34 - src/lib/Server/Reports/importscript.py | 310 ----- src/lib/Server/Reports/manage.py | 11 - src/lib/Server/Reports/nisauth.py | 44 - src/lib/Server/Reports/reports/__init__.py | 1 - .../Reports/reports/fixtures/initial_version.xml | 39 - src/lib/Server/Reports/reports/models.py | 343 ------ src/lib/Server/Reports/reports/sql/client.sql | 9 - src/lib/Server/Reports/reports/templates/404.html | 8 - .../Reports/reports/templates/base-timeview.html | 25 - src/lib/Server/Reports/reports/templates/base.html | 95 -- .../Reports/reports/templates/clients/detail.html | 127 -- .../reports/templates/clients/detailed-list.html | 46 - .../Reports/reports/templates/clients/history.html | 20 - .../Reports/reports/templates/clients/index.html | 34 - .../Reports/reports/templates/clients/manage.html | 45 - .../reports/templates/config_items/item.html | 115 -- .../reports/templates/config_items/listing.html | 33 - .../reports/templates/displays/summary.html | 42 - .../Reports/reports/templates/displays/timing.html | 38 - .../reports/templates/widgets/filter_bar.html | 13 - .../reports/templates/widgets/interaction_list.inc | 38 - .../reports/templates/widgets/page_bar.html | 23 - .../Reports/reports/templatetags/__init__.py | 0 .../Reports/reports/templatetags/bcfg2_tags.py | 276 ----- .../reports/templatetags/syntax_coloring.py | 49 - src/lib/Server/Reports/reports/urls.py | 55 - src/lib/Server/Reports/reports/views.py | 415 ------- src/lib/Server/Reports/settings.py | 161 --- src/lib/Server/Reports/updatefix.py | 190 --- src/lib/Server/Reports/urls.py | 14 - src/lib/Server/Reports/utils.py | 124 -- src/lib/Server/Snapshots/__init__.py | 31 - src/lib/Server/Snapshots/model.py | 329 ------ src/lib/Server/__init__.py | 4 - 172 files changed, 19918 deletions(-) delete mode 100644 src/lib/Server/Admin/Backup.py delete mode 100644 src/lib/Server/Admin/Bundle.py delete mode 100644 src/lib/Server/Admin/Client.py delete mode 100644 src/lib/Server/Admin/Compare.py delete mode 100644 src/lib/Server/Admin/Group.py delete mode 100644 src/lib/Server/Admin/Init.py delete mode 100644 src/lib/Server/Admin/Minestruct.py delete mode 100644 src/lib/Server/Admin/Perf.py delete mode 100644 src/lib/Server/Admin/Pull.py delete mode 100644 src/lib/Server/Admin/Query.py delete mode 100644 src/lib/Server/Admin/Reports.py delete mode 100644 src/lib/Server/Admin/Snapshots.py delete mode 100644 src/lib/Server/Admin/Tidy.py delete mode 100644 src/lib/Server/Admin/Viz.py delete mode 100644 src/lib/Server/Admin/Xcmd.py delete mode 100644 src/lib/Server/Admin/__init__.py delete mode 100644 src/lib/Server/Core.py delete mode 100644 src/lib/Server/FileMonitor.py delete mode 100644 src/lib/Server/Hostbase/.gitignore delete mode 100644 src/lib/Server/Hostbase/__init__.py delete mode 100644 src/lib/Server/Hostbase/backends.py delete mode 100644 src/lib/Server/Hostbase/hostbase/__init__.py delete mode 100644 src/lib/Server/Hostbase/hostbase/admin.py delete mode 100644 src/lib/Server/Hostbase/hostbase/models.py delete mode 100644 src/lib/Server/Hostbase/hostbase/sql/zone.sql delete mode 100644 src/lib/Server/Hostbase/hostbase/urls.py delete mode 100644 src/lib/Server/Hostbase/hostbase/views.py delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/base.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/copy.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/dns.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/edit.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/errors.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/host.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/index.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/login.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/logout.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/new.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/remove.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/results.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/search.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/zones.html delete mode 100644 src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html delete mode 100644 src/lib/Server/Hostbase/ldapauth.py delete mode 100755 src/lib/Server/Hostbase/manage.py delete mode 100644 src/lib/Server/Hostbase/media/base.css delete mode 100644 src/lib/Server/Hostbase/media/boxypastel.css delete mode 100644 src/lib/Server/Hostbase/media/global.css delete mode 100644 src/lib/Server/Hostbase/media/layout.css delete mode 100644 src/lib/Server/Hostbase/nisauth.py delete mode 100644 src/lib/Server/Hostbase/regex.py delete mode 100644 src/lib/Server/Hostbase/settings.py delete mode 100644 src/lib/Server/Hostbase/templates/batchadd.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/dhcpd.conf.head delete mode 100644 src/lib/Server/Hostbase/templates/dhcpd.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/hosts.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/hostsappend.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/named.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/namedviews.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/reverseappend.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/reversesoa.tmpl delete mode 100644 src/lib/Server/Hostbase/templates/zone.tmpl delete mode 100644 src/lib/Server/Hostbase/test/harness.py delete mode 100644 src/lib/Server/Hostbase/test/test_environ_settings.py delete mode 100644 src/lib/Server/Hostbase/test/test_ldapauth.py delete mode 100644 src/lib/Server/Hostbase/test/test_settings.py delete mode 100644 src/lib/Server/Hostbase/urls.py delete mode 100644 src/lib/Server/Lint/Bundles.py delete mode 100644 src/lib/Server/Lint/Comments.py delete mode 100644 src/lib/Server/Lint/Duplicates.py delete mode 100755 src/lib/Server/Lint/Genshi.py delete mode 100644 src/lib/Server/Lint/GroupPatterns.py delete mode 100644 src/lib/Server/Lint/InfoXML.py delete mode 100644 src/lib/Server/Lint/MergeFiles.py delete mode 100644 src/lib/Server/Lint/Pkgmgr.py delete mode 100644 src/lib/Server/Lint/RequiredAttrs.py delete mode 100644 src/lib/Server/Lint/Validate.py delete mode 100644 src/lib/Server/Lint/__init__.py delete mode 100644 src/lib/Server/Plugin.py delete mode 100644 src/lib/Server/Plugins/Account.py delete mode 100644 src/lib/Server/Plugins/BB.py delete mode 100644 src/lib/Server/Plugins/Base.py delete mode 100644 src/lib/Server/Plugins/Bundler.py delete mode 100644 src/lib/Server/Plugins/Bzr.py delete mode 100644 src/lib/Server/Plugins/Cfg.py delete mode 100644 src/lib/Server/Plugins/Cvs.py delete mode 100644 src/lib/Server/Plugins/DBStats.py delete mode 100644 src/lib/Server/Plugins/Darcs.py delete mode 100644 src/lib/Server/Plugins/Decisions.py delete mode 100644 src/lib/Server/Plugins/Defaults.py delete mode 100644 src/lib/Server/Plugins/Deps.py delete mode 100644 src/lib/Server/Plugins/Editor.py delete mode 100644 src/lib/Server/Plugins/FileProbes.py delete mode 100644 src/lib/Server/Plugins/Fossil.py delete mode 100644 src/lib/Server/Plugins/Git.py delete mode 100644 src/lib/Server/Plugins/GroupPatterns.py delete mode 100644 src/lib/Server/Plugins/Guppy.py delete mode 100644 src/lib/Server/Plugins/Hg.py delete mode 100644 src/lib/Server/Plugins/Hostbase.py delete mode 100644 src/lib/Server/Plugins/Ldap.py delete mode 100644 src/lib/Server/Plugins/Metadata.py delete mode 100644 src/lib/Server/Plugins/NagiosGen.py delete mode 100644 src/lib/Server/Plugins/Ohai.py delete mode 100644 src/lib/Server/Plugins/Packages/Apt.py delete mode 100644 src/lib/Server/Plugins/Packages/Collection.py delete mode 100644 src/lib/Server/Plugins/Packages/Pac.py delete mode 100644 src/lib/Server/Plugins/Packages/PackagesConfig.py delete mode 100644 src/lib/Server/Plugins/Packages/PackagesSources.py delete mode 100644 src/lib/Server/Plugins/Packages/Source.py delete mode 100644 src/lib/Server/Plugins/Packages/Yum.py delete mode 100644 src/lib/Server/Plugins/Packages/__init__.py delete mode 100644 src/lib/Server/Plugins/Pkgmgr.py delete mode 100644 src/lib/Server/Plugins/Probes.py delete mode 100644 src/lib/Server/Plugins/Properties.py delete mode 100644 src/lib/Server/Plugins/Rules.py delete mode 100644 src/lib/Server/Plugins/SGenshi.py delete mode 100644 src/lib/Server/Plugins/SSHbase.py delete mode 100644 src/lib/Server/Plugins/SSLCA.py delete mode 100644 src/lib/Server/Plugins/Snapshots.py delete mode 100644 src/lib/Server/Plugins/Statistics.py delete mode 100644 src/lib/Server/Plugins/Svcmgr.py delete mode 100644 src/lib/Server/Plugins/Svn.py delete mode 100644 src/lib/Server/Plugins/Svn2.py delete mode 100644 src/lib/Server/Plugins/TCheetah.py delete mode 100644 src/lib/Server/Plugins/TGenshi.py delete mode 100644 src/lib/Server/Plugins/TemplateHelper.py delete mode 100644 src/lib/Server/Plugins/Trigger.py delete mode 100644 src/lib/Server/Plugins/__init__.py delete mode 100644 src/lib/Server/Reports/__init__.py delete mode 100644 src/lib/Server/Reports/backends.py delete mode 100755 src/lib/Server/Reports/importscript.py delete mode 100755 src/lib/Server/Reports/manage.py delete mode 100644 src/lib/Server/Reports/nisauth.py delete mode 100644 src/lib/Server/Reports/reports/__init__.py delete mode 100644 src/lib/Server/Reports/reports/fixtures/initial_version.xml delete mode 100644 src/lib/Server/Reports/reports/models.py delete mode 100644 src/lib/Server/Reports/reports/sql/client.sql delete mode 100644 src/lib/Server/Reports/reports/templates/404.html delete mode 100644 src/lib/Server/Reports/reports/templates/base-timeview.html delete mode 100644 src/lib/Server/Reports/reports/templates/base.html delete mode 100644 src/lib/Server/Reports/reports/templates/clients/detail.html delete mode 100644 src/lib/Server/Reports/reports/templates/clients/detailed-list.html delete mode 100644 src/lib/Server/Reports/reports/templates/clients/history.html delete mode 100644 src/lib/Server/Reports/reports/templates/clients/index.html delete mode 100644 src/lib/Server/Reports/reports/templates/clients/manage.html delete mode 100644 src/lib/Server/Reports/reports/templates/config_items/item.html delete mode 100644 src/lib/Server/Reports/reports/templates/config_items/listing.html delete mode 100644 src/lib/Server/Reports/reports/templates/displays/summary.html delete mode 100644 src/lib/Server/Reports/reports/templates/displays/timing.html delete mode 100644 src/lib/Server/Reports/reports/templates/widgets/filter_bar.html delete mode 100644 src/lib/Server/Reports/reports/templates/widgets/interaction_list.inc delete mode 100644 src/lib/Server/Reports/reports/templates/widgets/page_bar.html delete mode 100644 src/lib/Server/Reports/reports/templatetags/__init__.py delete mode 100644 src/lib/Server/Reports/reports/templatetags/bcfg2_tags.py delete mode 100644 src/lib/Server/Reports/reports/templatetags/syntax_coloring.py delete mode 100644 src/lib/Server/Reports/reports/urls.py delete mode 100644 src/lib/Server/Reports/reports/views.py delete mode 100644 src/lib/Server/Reports/settings.py delete mode 100644 src/lib/Server/Reports/updatefix.py delete mode 100644 src/lib/Server/Reports/urls.py delete mode 100755 src/lib/Server/Reports/utils.py delete mode 100644 src/lib/Server/Snapshots/__init__.py delete mode 100644 src/lib/Server/Snapshots/model.py delete mode 100644 src/lib/Server/__init__.py (limited to 'src/lib/Server') diff --git a/src/lib/Server/Admin/Backup.py b/src/lib/Server/Admin/Backup.py deleted file mode 100644 index 3744abca3..000000000 --- a/src/lib/Server/Admin/Backup.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import sys -import time -import tarfile -import Bcfg2.Server.Admin -import Bcfg2.Options - - -class Backup(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Make a backup of the Bcfg2 repository" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin backup\n") - #"\n\nbcfg2-admin backup restore") - __usage__ = ("bcfg2-admin backup") - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - self.datastore = self.setup['repo'] - timestamp = time.strftime('%Y%m%d%H%M%S') - format = 'gz' - mode = 'w:' + format - filename = timestamp + '.tar' + '.' + format - out = tarfile.open(self.datastore + '/' + filename, mode=mode) - out.add(self.datastore, os.path.basename(self.datastore)) - out.close() - print("Archive %s was stored under %s" % (filename, self.datastore)) diff --git a/src/lib/Server/Admin/Bundle.py b/src/lib/Server/Admin/Bundle.py deleted file mode 100644 index 89c099602..000000000 --- a/src/lib/Server/Admin/Bundle.py +++ /dev/null @@ -1,96 +0,0 @@ -import lxml.etree -import glob -import sys -import re -import Bcfg2.Server.Admin -import Bcfg2.Options -from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError - - -class Bundle(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Create or delete bundle entries" - # TODO: add/del functions - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin bundle list-xml" - "\nbcfg2-admin bundle list-genshi" - "\nbcfg2-admin bundle show\n") - __usage__ = ("bcfg2-admin bundle [options] [add|del] [group]") - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - rg = re.compile(r'([^.]+\.(?:[a-z][a-z\-]+))(?![\w\.])', - re.IGNORECASE | re.DOTALL) - - # Get all bundles out of the Bundle/ directory - repo = self.setup['repo'] - xml_list = glob.glob("%s/Bundler/*.xml" % repo) - genshi_list = glob.glob("%s/Bundler/*.genshi" % repo) - - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin bundle help for usage.") -# if args[0] == 'add': -# try: -# self.metadata.add_bundle(args[1]) -# except MetadataConsistencyError: -# print("Error in adding bundle.") -# raise SystemExit(1) -# elif args[0] in ['delete', 'remove', 'del', 'rm']: -# try: -# self.metadata.remove_bundle(args[1]) -# except MetadataConsistencyError: -# print("Error in deleting bundle.") -# raise SystemExit(1) - # Lists all available xml bundles - elif args[0] in ['list-xml', 'ls-xml']: - bundle_name = [] - for bundle_path in xml_list: - bundle_name.append(rg.search(bundle_path).group(1)) - for bundle in bundle_name: - print(bundle.split('.')[0]) - # Lists all available genshi bundles - elif args[0] in ['list-genshi', 'ls-gen']: - bundle_name = [] - for bundle_path in genshi_list: - bundle_name.append(rg.search(bundle_path).group(1)) - for bundle in bundle_name: - print(bundle.split('.')[0]) - # Shows a list of all available bundles and prints bundle - # details after the user choose one bundle. - # FIXME: Add support for detailed output of genshi bundles - # FIXME: This functionality is almost identical with - # bcfg2-info bundles - elif args[0] in ['show']: - bundle_name = [] - bundle_list = xml_list + genshi_list - for bundle_path in bundle_list: - print "matching %s" % bundle_path - bundle_name.append(rg.search(bundle_path).group(1)) - text = "Available bundles (Number of bundles: %s)" % \ - (len(bundle_list)) - print(text) - print("%s" % (len(text) * "-")) - for i in range(len(bundle_list)): - print("[%i]\t%s" % (i, bundle_name[i])) - try: - lineno = raw_input("Enter the line number of a bundle for details: ") - except NameError: - lineno = input("Enter the line number of a bundle for details: ") - if int(lineno) >= int(len(bundle_list)): - print("No line with this number.") - else: - if '%s/Bundler/%s' % \ - (repo, bundle_name[int(lineno)]) in genshi_list: - print("Detailed output for *.genshi bundles is not supported.") - else: - print('Details for the "%s" bundle:' % \ - (bundle_name[int(lineno)].split('.')[0])) - tree = lxml.etree.parse(bundle_list[int(lineno)]) - #Prints bundle content - #print(lxml.etree.tostring(tree)) - names = ['Action', 'Package', 'Path', 'Service'] - for name in names: - for node in tree.findall("//" + name): - print("%s:\t%s" % (name, node.attrib["name"])) - else: - print("No command specified") - raise SystemExit(1) diff --git a/src/lib/Server/Admin/Client.py b/src/lib/Server/Admin/Client.py deleted file mode 100644 index 4d580c54c..000000000 --- a/src/lib/Server/Admin/Client.py +++ /dev/null @@ -1,64 +0,0 @@ -import lxml.etree -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError - - -class Client(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Create, delete, or modify client entries" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin client add " - "attr1=val1 attr2=val2" - "\nbcfg2-admin client update " - "attr1=val1 attr2=val2" - "\nbcfg2-admin client list" - "\nbcfg2-admin client del \n") - __usage__ = ("bcfg2-admin client [options] [add|del|update|list] [attr=val]") - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin client help for usage.") - if args[0] == 'add': - attr_d = {} - for i in args[2:]: - attr, val = i.split('=', 1) - if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address', - 'auth']: - print("Attribute %s unknown" % attr) - raise SystemExit(1) - attr_d[attr] = val - try: - self.metadata.add_client(args[1], attr_d) - except MetadataConsistencyError: - print("Error in adding client") - raise SystemExit(1) - elif args[0] in ['update', 'up']: - attr_d = {} - for i in args[2:]: - attr, val = i.split('=', 1) - if attr not in ['profile', 'uuid', 'password', - 'location', 'secure', 'address', - 'auth']: - print("Attribute %s unknown" % attr) - raise SystemExit(1) - attr_d[attr] = val - try: - self.metadata.update_client(args[1], attr_d) - except MetadataConsistencyError: - print("Error in updating client") - raise SystemExit(1) - elif args[0] in ['delete', 'remove', 'del', 'rm']: - try: - self.metadata.remove_client(args[1]) - except MetadataConsistencyError: - print("Error in deleting client") - raise SystemExit(1) - elif args[0] in ['list', 'ls']: - tree = lxml.etree.parse(self.metadata.data + "/clients.xml") - tree.xinclude() - for node in tree.findall("//Client"): - print(node.attrib["name"]) - else: - print("No command specified") - raise SystemExit(1) diff --git a/src/lib/Server/Admin/Compare.py b/src/lib/Server/Admin/Compare.py deleted file mode 100644 index 050dd69f8..000000000 --- a/src/lib/Server/Admin/Compare.py +++ /dev/null @@ -1,151 +0,0 @@ -import lxml.etree -import os - -import Bcfg2.Server.Admin - - -class Compare(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("Determine differences between files or " - "directories of client specification instances") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin compare " - "\nbcfg2-admin compare -r ") - __usage__ = ("bcfg2-admin compare \n\n" - " -r\trecursive") - - def __init__(self, setup): - Bcfg2.Server.Admin.Mode.__init__(self, setup) - self.important = {'Path': ['name', 'type', 'owner', 'group', 'perms', - 'important', 'paranoid', 'sensitive', - 'dev_type', 'major', 'minor', 'prune', - 'encoding', 'empty', 'to', 'recursive', - 'vcstype', 'sourceurl', 'revision'], - 'Package': ['name', 'type', 'version', 'simplefile', - 'verify'], - 'Service': ['name', 'type', 'status', 'mode', - 'target', 'sequence', 'parameters'], - 'Action': ['name', 'timing', 'when', 'status', - 'command'], - 'PostInstall': ['name'] - } - - def compareStructures(self, new, old): - if new.tag == 'Independent': - bundle = 'Base' - else: - bundle = new.get('name') - - identical = True - - for child in new.getchildren(): - if child.tag not in self.important: - print(" %s in (new) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - continue - equiv = old.xpath('%s[@name="%s"]' % - (child.tag, child.get('name'))) - if len(equiv) == 0: - print(" %s %s in bundle %s:\n only in new configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - continue - diff = [] - if child.tag == 'Path' and child.get('type') == 'file' and \ - child.text != equiv[0].text: - diff.append('contents') - attrdiff = [field for field in self.important[child.tag] if \ - child.get(field) != equiv[0].get(field)] - if attrdiff: - diff.append('attributes (%s)' % ', '.join(attrdiff)) - if diff: - print(" %s %s in bundle %s:\n %s differ" % (child.tag, \ - child.get('name'), bundle, ' and '.join(diff))) - identical = False - - for child in old.getchildren(): - if child.tag not in self.important: - print(" %s in (old) bundle %s:\n tag type not handled!" % - (child.tag, bundle)) - elif len(new.xpath('%s[@name="%s"]' % - (child.tag, child.get('name')))) == 0: - print(" %s %s in bundle %s:\n only in old configuration" % - (child.tag, child.get('name'), bundle)) - identical = False - - return identical - - def compareSpecifications(self, path1, path2): - try: - new = lxml.etree.parse(path1).getroot() - except IOError: - print("Failed to read %s" % (path1)) - raise SystemExit(1) - - try: - old = lxml.etree.parse(path2).getroot() - except IOError: - print("Failed to read %s" % (path2)) - raise SystemExit(1) - - for src in [new, old]: - for bundle in src.findall('./Bundle'): - if bundle.get('name')[-4:] == '.xml': - bundle.set('name', bundle.get('name')[:-4]) - - identical = True - - for bundle in old.findall('./Bundle'): - if len(new.xpath('Bundle[@name="%s"]' % (bundle.get('name')))) == 0: - print(" Bundle %s only in old configuration" % - bundle.get('name')) - identical = False - for bundle in new.findall('./Bundle'): - equiv = old.xpath('Bundle[@name="%s"]' % (bundle.get('name'))) - if len(equiv) == 0: - print(" Bundle %s only in new configuration" % - bundle.get('name')) - identical = False - elif not self.compareStructures(bundle, equiv[0]): - identical = False - - i1 = lxml.etree.Element('Independent') - i2 = lxml.etree.Element('Independent') - i1.extend(new.findall('./Independent/*')) - i2.extend(old.findall('./Independent/*')) - if not self.compareStructures(i1, i2): - identical = False - - return identical - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin compare help for usage.") - if '-r' in args: - args = list(args) - args.remove('-r') - (oldd, newd) = args - (old, new) = [os.listdir(spot) for spot in args] - old_extra = [] - for item in old: - if item not in new: - old_extra.append(item) - continue - print("File: %s" % item) - state = self.__call__([oldd + '/' + item, newd + '/' + item]) - new.remove(item) - if state: - print("File %s is good" % item) - else: - print("File %s is bad" % item) - if new: - print("%s has extra files: %s" % (newd, ', '.join(new))) - if old_extra: - print("%s has extra files: %s" % (oldd, ', '.join(old_extra))) - return - try: - (old, new) = args - return self.compareSpecifications(new, old) - except IndexError: - print(self.__call__.__doc__) - raise SystemExit(1) diff --git a/src/lib/Server/Admin/Group.py b/src/lib/Server/Admin/Group.py deleted file mode 100644 index 16a773d6f..000000000 --- a/src/lib/Server/Admin/Group.py +++ /dev/null @@ -1,63 +0,0 @@ -import lxml.etree -import Bcfg2.Server.Admin -from Bcfg2.Server.Plugins.Metadata import MetadataConsistencyError - - -class Group(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Create, delete, or modify group entries" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin group add " - "attr1=val1 attr2=val2" - "\nbcfg2-admin group update " - "attr1=val1 attr2=val2" - "\nbcfg2-admin group list" - "\nbcfg2-admin group del \n") - __usage__ = ("bcfg2-admin group [options] [add|del|update|list] [attr=val]") - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin group help for usage.") - if args[0] == 'add': - attr_d = {} - for i in args[2:]: - attr, val = i.split('=', 1) - if attr not in ['profile', 'public', 'default', - 'name', 'auth', 'toolset', 'category', - 'comment']: - print("Attribute %s unknown" % attr) - raise SystemExit(1) - attr_d[attr] = val - try: - self.metadata.add_group(args[1], attr_d) - except MetadataConsistencyError: - print("Error in adding group") - raise SystemExit(1) - elif args[0] in ['update', 'up']: - attr_d = {} - for i in args[2:]: - attr, val = i.split('=', 1) - if attr not in ['profile', 'public', 'default', - 'name', 'auth', 'toolset', 'category', - 'comment']: - print("Attribute %s unknown" % attr) - raise SystemExit(1) - attr_d[attr] = val - try: - self.metadata.update_group(args[1], attr_d) - except MetadataConsistencyError: - print("Error in updating group") - raise SystemExit(1) - elif args[0] in ['delete', 'remove', 'del', 'rm']: - try: - self.metadata.remove_group(args[1]) - except MetadataConsistencyError: - print("Error in deleting group") - raise SystemExit(1) - elif args[0] in ['list', 'ls']: - tree = lxml.etree.parse(self.metadata.data + "/groups.xml") - for node in tree.findall("//Group"): - print(node.attrib["name"]) - else: - print("No command specified") - raise SystemExit(1) diff --git a/src/lib/Server/Admin/Init.py b/src/lib/Server/Admin/Init.py deleted file mode 100644 index 832190b7d..000000000 --- a/src/lib/Server/Admin/Init.py +++ /dev/null @@ -1,357 +0,0 @@ -import getpass -import os -import random -import socket -import stat -import string -import sys -import subprocess -import Bcfg2.Server.Admin -import Bcfg2.Server.Plugin -import Bcfg2.Options - -# default config file -config = ''' -[server] -repository = %s -plugins = %s - -[statistics] -sendmailpath = %s -database_engine = sqlite3 -# 'postgresql', 'mysql', 'mysql_old', 'sqlite3' or 'ado_mssql'. -database_name = -# Or path to database file if using sqlite3. -#/etc/brpt.sqlite is default path if left empty -database_user = -# Not used with sqlite3. -database_password = -# Not used with sqlite3. -database_host = -# Not used with sqlite3. -database_port = - -[communication] -protocol = %s -password = %s -certificate = %s -key = %s -ca = %s - -[components] -bcfg2 = %s -''' - -# Default groups -groups = ''' - - - - - - - - - - - - -''' - -# Default contents of clients.xml -clients = ''' - - -''' - -# Mapping of operating system names to groups -os_list = [('Red Hat/Fedora/RHEL/RHAS/Centos', 'redhat'), - ('SUSE/SLES', 'suse'), - ('Mandrake', 'mandrake'), - ('Debian', 'debian'), - ('Ubuntu', 'ubuntu'), - ('Gentoo', 'gentoo'), - ('FreeBSD', 'freebsd')] - -# Complete list of plugins -plugin_list = ['Account', - 'Base', - 'Bundler', - 'Bzr', - 'Cfg', - 'Decisions', - 'Deps', - 'Git', - 'Guppy', - 'Hg', - 'Metadata', - 'NagiosGen', - 'Ohai', - 'Packages', - 'Pkgmgr', - 'Probes', - 'Properties', - 'Rules', - 'Snapshots', - 'SSHbase', - 'SSLCA', - 'Statistics', - 'Svcmgr', - 'TCheetah', - 'TGenshi'] - -# Default list of plugins to use -default_plugins = Bcfg2.Options.SERVER_PLUGINS.default - - -def get_input(prompt): - """py3k compatible function to get input""" - try: - return raw_input(prompt) - except NameError: - return input(prompt) - - -def gen_password(length): - """Generates a random alphanumeric password with length characters.""" - chars = string.letters + string.digits - newpasswd = '' - for i in range(length): - newpasswd = newpasswd + random.choice(chars) - return newpasswd - - -def create_key(hostname, keypath, certpath, country, state, location): - """Creates a bcfg2.key at the directory specifed by keypath.""" - kcstr = ("openssl req -batch -x509 -nodes -subj '/C=%s/ST=%s/L=%s/CN=%s' " - "-days 1000 -newkey rsa:2048 -keyout %s -noout" % (country, - state, - location, - hostname, - keypath)) - subprocess.call((kcstr), shell=True) - ccstr = ("openssl req -batch -new -subj '/C=%s/ST=%s/L=%s/CN=%s' -key %s " - "| openssl x509 -req -days 1000 -signkey %s -out %s" % (country, - state, - location, - hostname, - keypath, - keypath, - certpath)) - subprocess.call((ccstr), shell=True) - os.chmod(keypath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - - -def create_conf(confpath, confdata, keypath): - # Don't overwrite existing bcfg2.conf file - if os.path.exists(confpath): - result = get_input("\nWarning: %s already exists. " - "Overwrite? [y/N]: " % confpath) - if result not in ['Y', 'y']: - print("Leaving %s unchanged" % confpath) - return - try: - open(confpath, "w").write(confdata) - os.chmod(confpath, stat.S_IRUSR | stat.S_IWUSR) # 0600 - except Exception: - e = sys.exc_info()[1] - print("Error %s occured while trying to write configuration " - "file to '%s'.\n" % - (e, confpath)) - raise SystemExit(1) - - -class Init(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("Interactively initialize a new repository.") - __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin init" - __usage__ = "bcfg2-admin init" - options = {'configfile': Bcfg2.Options.CFILE, - 'plugins': Bcfg2.Options.SERVER_PLUGINS, - 'proto': Bcfg2.Options.SERVER_PROTOCOL, - 'repo': Bcfg2.Options.SERVER_REPOSITORY, - 'sendmail': Bcfg2.Options.SENDMAIL_PATH} - repopath = "" - response = "" - - def _set_defaults(self): - """Set default parameters.""" - self.configfile = self.opts['configfile'] - self.repopath = self.opts['repo'] - self.password = gen_password(8) - self.server_uri = "https://%s:6789" % socket.getfqdn() - self.plugins = default_plugins - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - - # Parse options - self.opts = Bcfg2.Options.OptionParser(self.options) - self.opts.parse(args) - self._set_defaults() - - # Prompt the user for input - self._prompt_config() - self._prompt_repopath() - self._prompt_password() - self._prompt_hostname() - self._prompt_server() - self._prompt_groups() - # self._prompt_plugins() - self._prompt_certificate() - - # Initialize the repository - self.init_repo() - - def _prompt_hostname(self): - """Ask for the server hostname.""" - data = get_input("What is the server's hostname [%s]: " % - socket.getfqdn()) - if data != '': - self.shostname = data - else: - self.shostname = socket.getfqdn() - - def _prompt_config(self): - """Ask for the configuration file path.""" - newconfig = get_input("Store Bcfg2 configuration in [%s]: " % - self.configfile) - if newconfig != '': - self.configfile = os.path.abspath(newconfig) - - def _prompt_repopath(self): - """Ask for the repository path.""" - while True: - newrepo = get_input("Location of Bcfg2 repository [%s]: " % - self.repopath) - if newrepo != '': - self.repopath = os.path.abspath(newrepo) - if os.path.isdir(self.repopath): - response = get_input("Directory %s exists. Overwrite? [y/N]:" \ - % self.repopath) - if response.lower().strip() == 'y': - break - else: - break - - def _prompt_password(self): - """Ask for a password or generate one if none is provided.""" - newpassword = getpass.getpass( - "Input password used for communication verification " - "(without echoing; leave blank for a random): ").strip() - if len(newpassword) != 0: - self.password = newpassword - - def _prompt_server(self): - """Ask for the server name.""" - newserver = get_input("Input the server location [%s]: " % - self.server_uri) - if newserver != '': - self.server_uri = newserver - - def _prompt_groups(self): - """Create the groups.xml file.""" - prompt = '''Input base Operating System for clients:\n''' - for entry in os_list: - prompt += "%d: %s\n" % (os_list.index(entry) + 1, entry[0]) - prompt += ': ' - while True: - try: - osidx = int(get_input(prompt)) - self.os_sel = os_list[osidx - 1][1] - break - except ValueError: - continue - - def _prompt_plugins(self): - default = get_input("Use default plugins? (%s) [Y/n]: " % - ''.join(default_plugins)).lower() - if default != 'y' or default != '': - while True: - plugins_are_valid = True - plug_str = get_input("Specify plugins: ") - plugins = plug_str.split(',') - for plugin in plugins: - plugin = plugin.strip() - if not plugin in plugin_list: - plugins_are_valid = False - print("ERROR: Plugin %s not recognized" % plugin) - if plugins_are_valid: - break - - def _prompt_certificate(self): - """Ask for the key details (country, state, and location).""" - print("The following questions affect SSL certificate generation.") - print("If no data is provided, the default values are used.") - newcountry = get_input("Country name (2 letter code) for certificate: ") - if newcountry != '': - if len(newcountry) == 2: - self.country = newcountry - else: - while len(newcountry) != 2: - newcountry = get_input("2 letter country code (eg. US): ") - if len(newcountry) == 2: - self.country = newcountry - break - else: - self.country = 'US' - - newstate = get_input("State or Province Name (full name) for certificate: ") - if newstate != '': - self.state = newstate - else: - self.state = 'Illinois' - - newlocation = get_input("Locality Name (eg, city) for certificate: ") - if newlocation != '': - self.location = newlocation - else: - self.location = 'Argonne' - - def _init_plugins(self): - """Initialize each plugin-specific portion of the repository.""" - for plugin in self.plugins: - if plugin == 'Metadata': - Bcfg2.Server.Plugins.Metadata.Metadata.init_repo(self.repopath, - groups, - self.os_sel, - clients) - else: - try: - module = __import__("Bcfg2.Server.Plugins.%s" % plugin, '', - '', ["Bcfg2.Server.Plugins"]) - cls = getattr(module, plugin) - cls.init_repo(self.repopath) - except Exception: - e = sys.exc_info()[1] - print("Plugin setup for %s failed: %s\n" - "Check that dependencies are installed?" % (plugin, e)) - - def init_repo(self): - """Setup a new repo and create the content of the configuration file.""" - keypath = os.path.dirname(self.configfile) - kpath = os.path.join(keypath, 'bcfg2.key') - cpath = os.path.join(keypath, 'bcfg2.crt') - - confdata = config % (self.repopath, - ','.join(self.plugins), - self.opts['sendmail'], - self.opts['proto'], - self.password, - cpath, - kpath, - cpath, - self.server_uri) - - # Create the configuration file and SSL key - create_conf(self.configfile, confdata, keypath) - create_key(self.shostname, kpath, cpath, self.country, - self.state, self.location) - - # Create the repository - path = os.path.join(self.repopath, 'etc') - try: - os.makedirs(path) - self._init_plugins() - print("Repository created successfuly in %s" % (self.repopath)) - except OSError: - print("Failed to create %s." % path) diff --git a/src/lib/Server/Admin/Minestruct.py b/src/lib/Server/Admin/Minestruct.py deleted file mode 100644 index b929a9a8c..000000000 --- a/src/lib/Server/Admin/Minestruct.py +++ /dev/null @@ -1,65 +0,0 @@ -import getopt -import lxml.etree -import sys - -import Bcfg2.Server.Admin - -class Minestruct(Bcfg2.Server.Admin.StructureMode): - """Pull extra entries out of statistics.""" - __shorthelp__ = "Extract extra entry lists from statistics" - __longhelp__ = (__shorthelp__ + - "\n\nbcfg2-admin minestruct [-f filename] " - "[-g groups] client\n") - __usage__ = ("bcfg2-admin minestruct [options] \n\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-f ", - "build a particular file", - "-g ", - "only build config for groups")) - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - if len(args) == 0: - self.errExit("No argument specified.\n" - "Please see bcfg2-admin minestruct help for usage.") - try: - (opts, args) = getopt.getopt(args, 'f:g:h') - except: - self.log.error(self.__shorthelp__) - raise SystemExit(1) - - client = args[0] - output = sys.stdout - groups = [] - - for (opt, optarg) in opts: - if opt == '-f': - try: - output = open(optarg, 'w') - except IOError: - self.log.error("Failed to open file: %s" % (optarg)) - raise SystemExit(1) - elif opt == '-g': - groups = optarg.split(':') - - try: - extra = set() - for source in self.bcore.pull_sources: - for item in source.GetExtra(client): - extra.add(item) - except: - self.log.error("Failed to find extra entry info for client %s" % - client) - raise SystemExit(1) - root = lxml.etree.Element("Base") - self.log.info("Found %d extra entries" % (len(extra))) - add_point = root - for g in groups: - add_point = lxml.etree.SubElement(add_point, "Group", name=g) - for tag, name in extra: - self.log.info("%s: %s" % (tag, name)) - lxml.etree.SubElement(add_point, tag, name=name) - - tree = lxml.etree.ElementTree(root) - tree.write(output, pretty_print=True) diff --git a/src/lib/Server/Admin/Perf.py b/src/lib/Server/Admin/Perf.py deleted file mode 100644 index 411442698..000000000 --- a/src/lib/Server/Admin/Perf.py +++ /dev/null @@ -1,37 +0,0 @@ -import sys - -import Bcfg2.Options -import Bcfg2.Proxy -import Bcfg2.Server.Admin - - -class Perf(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("Query server for performance data") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin perf\n") - __usage__ = ("bcfg2-admin perf") - - def __call__(self, args): - output = [('Name', 'Min', 'Max', 'Mean', 'Count')] - optinfo = { - 'ca': Bcfg2.Options.CLIENT_CA, - 'certificate': Bcfg2.Options.CLIENT_CERT, - 'key': Bcfg2.Options.SERVER_KEY, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'server': Bcfg2.Options.SERVER_LOCATION, - 'user': Bcfg2.Options.CLIENT_USER, - 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, - } - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(sys.argv[1:]) - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - data = proxy.get_statistics() - for key, value in list(data.items()): - data = tuple(["%.06f" % (item) for item in value[:-1]] + [value[-1]]) - output.append((key, ) + data) - self.print_table(output) diff --git a/src/lib/Server/Admin/Pull.py b/src/lib/Server/Admin/Pull.py deleted file mode 100644 index daf353107..000000000 --- a/src/lib/Server/Admin/Pull.py +++ /dev/null @@ -1,154 +0,0 @@ -import getopt -import sys - -import Bcfg2.Server.Admin - - -class Pull(Bcfg2.Server.Admin.MetadataCore): - """Pull mode retrieves entries from clients and - integrates the information into the repository. - """ - __shorthelp__ = ("Integrate configuration information " - "from clients into the server repository") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin pull [-v] [-f][-I] [-s] " - " \n") - __usage__ = ("bcfg2-admin pull [options] " - "\n\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-v", - "be verbose", - "-f", - "force", - "-I", - "interactive", - "-s", - "stdin")) - allowed = ['Metadata', 'BB', "DBStats", "Statistics", "Cfg", "SSHbase"] - - def __init__(self, setup): - Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) - self.log = False - self.mode = 'interactive' - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - use_stdin = False - try: - opts, gargs = getopt.getopt(args, 'vfIs') - except: - print(self.__shorthelp__) - raise SystemExit(1) - for opt in opts: - if opt[0] == '-v': - self.log = True - elif opt[0] == '-f': - self.mode = 'force' - elif opt[0] == '-I': - self.mode == 'interactive' - elif opt[0] == '-s': - use_stdin = True - - if use_stdin: - for line in sys.stdin: - try: - self.PullEntry(*line.split(None, 3)) - except SystemExit: - print(" for %s" % line) - except: - print("Bad entry: %s" % line.strip()) - elif len(gargs) < 3: - print(self.__longhelp__) - raise SystemExit(1) - else: - self.PullEntry(gargs[0], gargs[1], gargs[2]) - - def BuildNewEntry(self, client, etype, ename): - """Construct a new full entry for - given client/entry from statistics. - """ - new_entry = {'type': etype, 'name': ename} - for plugin in self.bcore.pull_sources: - try: - (owner, group, perms, contents) = \ - plugin.GetCurrentEntry(client, etype, ename) - break - except Bcfg2.Server.Plugin.PluginExecutionError: - if plugin == self.bcore.pull_sources[-1]: - print("Pull Source failure; could not fetch current state") - raise SystemExit(1) - - try: - data = {'owner': owner, - 'group': group, - 'perms': perms, - 'text': contents} - except UnboundLocalError: - print("Unable to build entry. " - "Do you have a statistics plugin enabled?") - raise SystemExit(1) - for k, v in list(data.items()): - if v: - new_entry[k] = v - #print new_entry - return new_entry - - def Choose(self, choices): - """Determine where to put pull data.""" - if self.mode == 'interactive': - for choice in choices: - print("Plugin returned choice:") - if id(choice) == id(choices[0]): - print("(current entry) ") - if choice.all: - print(" => global entry") - elif choice.group: - print(" => group entry: %s (prio %d)" % - (choice.group, choice.prio)) - else: - print(" => host entry: %s" % (choice.hostname)) - # py3k compatibility - try: - ans = raw_input("Use this entry? [yN]: ") in ['y', 'Y'] - except NameError: - ans = input("Use this entry? [yN]: ") in ['y', 'Y'] - if ans: - return choice - return False - else: - # mode == 'force' - if not choices: - return False - return choices[0] - - def PullEntry(self, client, etype, ename): - """Make currently recorded client state correct for entry.""" - new_entry = self.BuildNewEntry(client, etype, ename) - - meta = self.bcore.build_metadata(client) - # Find appropriate plugin in bcore - glist = [gen for gen in self.bcore.generators if - ename in gen.Entries.get(etype, {})] - if len(glist) != 1: - self.errExit("Got wrong numbers of matching generators for entry:" \ - + "%s" % ([g.name for g in glist])) - plugin = glist[0] - if not isinstance(plugin, Bcfg2.Server.Plugin.PullTarget): - self.errExit("Configuration upload not supported by plugin %s" \ - % (plugin.name)) - try: - choices = plugin.AcceptChoices(new_entry, meta) - specific = self.Choose(choices) - if specific: - plugin.AcceptPullData(specific, new_entry, self.log) - except Bcfg2.Server.Plugin.PluginExecutionError: - self.errExit("Configuration upload not supported by plugin %s" \ - % (plugin.name)) - # Commit if running under a VCS - for vcsplugin in list(self.bcore.plugins.values()): - if isinstance(vcsplugin, Bcfg2.Server.Plugin.Version): - files = "%s/%s" % (plugin.data, ename) - comment = 'file "%s" pulled from host %s' % (files, client) - vcsplugin.commit_data([files], comment) diff --git a/src/lib/Server/Admin/Query.py b/src/lib/Server/Admin/Query.py deleted file mode 100644 index 3dd326645..000000000 --- a/src/lib/Server/Admin/Query.py +++ /dev/null @@ -1,72 +0,0 @@ -import sys -import logging -import Bcfg2.Logger -import Bcfg2.Server.Admin - - -class Query(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Query clients" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin query [-n] [-c] " - "[-f filename] g=group p=profile") - __usage__ = ("bcfg2-admin query [options] \n\n" - " %-25s%s\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-n", - "query results delimited with newlines", - "-c", - "query results delimited with commas", - "-f filename", - "write query to file")) - - def __init__(self, setup): - Bcfg2.Server.Admin.MetadataCore.__init__(self, setup) - logging.root.setLevel(100) - Bcfg2.Logger.setup_logging(100, to_console=False, to_syslog=False) - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - clients = list(self.metadata.clients.keys()) - filename_arg = False - filename = None - for arg in args: - if filename_arg == True: - filename = arg - filename_arg = False - continue - if arg in ['-n', '-c']: - continue - if arg in ['-f']: - filename_arg = True - continue - try: - k, v = arg.split('=') - except: - print("Unknown argument %s" % arg) - continue - if k == 'p': - nc = self.metadata.get_client_names_by_profiles(v.split(',')) - elif k == 'g': - nc = self.metadata.get_client_names_by_groups(v.split(',')) - # add probed groups (if present) - for conn in self.bcore.connectors: - if isinstance(conn, Bcfg2.Server.Plugins.Probes.Probes): - for c, glist in list(conn.cgroups.items()): - for g in glist: - if g in v.split(','): - nc.append(c) - else: - print("One of g= or p= must be specified") - raise SystemExit(1) - clients = [c for c in clients if c in nc] - if '-n' in args: - for client in clients: - print(client) - else: - print(','.join(clients)) - if '-f' in args: - f = open(filename, "w") - for client in clients: - f.write(client + "\n") - f.close() - print("Wrote results to %s" % (filename)) diff --git a/src/lib/Server/Admin/Reports.py b/src/lib/Server/Admin/Reports.py deleted file mode 100644 index 974cdff9d..000000000 --- a/src/lib/Server/Admin/Reports.py +++ /dev/null @@ -1,376 +0,0 @@ -'''Admin interface for dynamic reports''' -import Bcfg2.Logger -import Bcfg2.Server.Admin -import datetime -import os -import logging -import pickle -import platform -import sys -import traceback -from lxml.etree import XML, XMLSyntaxError - -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - -# FIXME: Remove when server python dep is 2.5 or greater -if sys.version_info >= (2, 5): - from hashlib import md5 -else: - from md5 import md5 - -# Prereq issues can be signaled with ImportError, so no try needed -# FIXME - settings file uses a hardcoded path for /etc/bcfg2.conf -import Bcfg2.Server.Reports.settings - -# Load django and reports stuff _after_ we know we can load settings -import django.core.management -from Bcfg2.Server.Reports.importscript import load_stats -from Bcfg2.Server.Reports.updatefix import update_database -from Bcfg2.Server.Reports.utils import * - -project_directory = os.path.dirname(Bcfg2.Server.Reports.settings.__file__) -project_name = os.path.basename(project_directory) -sys.path.append(os.path.join(project_directory, '..')) -project_module = __import__(project_name, '', '', ['']) -sys.path.pop() - -# Set DJANGO_SETTINGS_MODULE appropriately. -os.environ['DJANGO_SETTINGS_MODULE'] = '%s.settings' % project_name -from django.db import connection, transaction - -from Bcfg2.Server.Reports.reports.models import Client, Interaction, Entries, \ - Entries_interactions, Performance, \ - Reason, Ping - - -def printStats(fn): - """ - Print db stats. - - Decorator for purging. Prints database statistics after a run. - """ - def print_stats(self, *data): - start_client = Client.objects.count() - start_i = Interaction.objects.count() - start_ei = Entries_interactions.objects.count() - start_perf = Performance.objects.count() - start_ping = Ping.objects.count() - - fn(self, *data) - - self.log.info("Clients removed: %s" % - (start_client - Client.objects.count())) - self.log.info("Interactions removed: %s" % - (start_i - Interaction.objects.count())) - self.log.info("Interactions->Entries removed: %s" % - (start_ei - Entries_interactions.objects.count())) - self.log.info("Metrics removed: %s" % - (start_perf - Performance.objects.count())) - self.log.info("Ping metrics removed: %s" % - (start_ping - Ping.objects.count())) - - return print_stats - - -class Reports(Bcfg2.Server.Admin.Mode): - '''Admin interface for dynamic reports''' - __shorthelp__ = "Manage dynamic reports" - __longhelp__ = (__shorthelp__) - django_commands = ['syncdb', 'sqlall', 'validate'] - __usage__ = ("bcfg2-admin reports [command] [options]\n" - " -v|--verbose Be verbose\n" - " -q|--quiet Print only errors\n" - "\n" - " Commands:\n" - " init Initialize the database\n" - " load_stats Load statistics data\n" - " -s|--stats Path to statistics.xml file\n" - " -c|--clients-file Path to clients.xml file\n" - " -O3 Fast mode. Duplicates data!\n" - " purge Purge records\n" - " --client [n] Client to operate on\n" - " --days [n] Records older then n days\n" - " --expired Expired clients only\n" - " scrub Scrub the database for duplicate reasons and orphaned entries\n" - " update Apply any updates to the reporting database\n" - "\n" - " Django commands:\n " - "\n ".join(django_commands)) - - def __init__(self, setup): - Bcfg2.Server.Admin.Mode.__init__(self, setup) - self.log.setLevel(logging.INFO) - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - if len(args) == 0 or args[0] == '-h': - print(self.__usage__) - raise SystemExit(0) - - verb = 0 - - if '-v' in args or '--verbose' in args: - self.log.setLevel(logging.DEBUG) - verb = 1 - if '-q' in args or '--quiet' in args: - self.log.setLevel(logging.WARNING) - - # FIXME - dry run - - if args[0] in self.django_commands: - self.django_command_proxy(args[0]) - elif args[0] == 'scrub': - self.scrub() - elif args[0] == 'init': - update_database() - elif args[0] == 'update': - update_database() - elif args[0] == 'load_stats': - quick = '-O3' in args - stats_file = None - clients_file = None - i = 1 - while i < len(args): - if args[i] == '-s' or args[i] == '--stats': - stats_file = args[i + 1] - if stats_file[0] == '-': - self.errExit("Invalid statistics file: %s" % stats_file) - elif args[i] == '-c' or args[i] == '--clients-file': - clients_file = args[i + 1] - if clients_file[0] == '-': - self.errExit("Invalid clients file: %s" % clients_file) - i = i + 1 - self.load_stats(stats_file, clients_file, verb, quick) - elif args[0] == 'purge': - expired = False - client = None - maxdate = None - state = None - i = 1 - while i < len(args): - if args[i] == '-c' or args[i] == '--client': - if client: - self.errExit("Only one client per run") - client = args[i + 1] - print(client) - i = i + 1 - elif args[i] == '--days': - if maxdate: - self.errExit("Max date specified multiple times") - try: - maxdate = datetime.datetime.now() - datetime.timedelta(days=int(args[i + 1])) - except: - self.log.error("Invalid number of days: %s" % args[i + 1]) - raise SystemExit(-1) - i = i + 1 - elif args[i] == '--expired': - expired = True - i = i + 1 - if expired: - if state: - self.log.error("--state is not valid with --expired") - raise SystemExit(-1) - self.purge_expired(maxdate) - else: - self.purge(client, maxdate, state) - else: - print("Unknown command: %s" % args[0]) - - @transaction.commit_on_success - def scrub(self): - ''' Perform a thorough scrub and cleanup of the database ''' - - # Currently only reasons are a problem - try: - start_count = Reason.objects.count() - except Exception: - e = sys.exc_info()[1] - self.log.error("Failed to load reason objects: %s" % e) - return - dup_reasons = [] - - cmp_reasons = dict() - batch_update = [] - for reason in BatchFetch(Reason.objects): - ''' Loop through each reason and create a key out of the data. \ - This lets us take advantage of a fast hash lookup for \ - comparisons ''' - id = reason.id - reason.id = None - key = md5(pickle.dumps(reason)).hexdigest() - reason.id = id - - if key in cmp_reasons: - self.log.debug("Update interactions from %d to %d" \ - % (reason.id, cmp_reasons[key])) - dup_reasons.append([reason.id]) - batch_update.append([cmp_reasons[key], reason.id]) - else: - cmp_reasons[key] = reason.id - self.log.debug("key %d" % reason.id) - - self.log.debug("Done with updates, deleting dupes") - try: - cursor = connection.cursor() - cursor.executemany('update reports_entries_interactions set reason_id=%s where reason_id=%s', batch_update) - cursor.executemany('delete from reports_reason where id = %s', dup_reasons) - transaction.set_dirty() - except Exception: - ex = sys.exc_info()[1] - self.log.error("Failed to delete reasons: %s" % ex) - raise - - self.log.info("Found %d dupes out of %d" % (len(dup_reasons), start_count)) - - # Cleanup orphans - start_count = Reason.objects.count() - Reason.prune_orphans() - self.log.info("Pruned %d Reason records" % (start_count - Reason.objects.count())) - - start_count = Entries.objects.count() - Entries.prune_orphans() - self.log.info("Pruned %d Entries records" % (start_count - Entries.objects.count())) - - def django_command_proxy(self, command): - '''Call a django command''' - if command == 'sqlall': - django.core.management.call_command(command, 'reports') - else: - django.core.management.call_command(command) - - def load_stats(self, stats_file=None, clientspath=None, verb=0, quick=False): - '''Load statistics data into the database''' - location = '' - - if not stats_file: - try: - stats_file = "%s/etc/statistics.xml" % self.cfp.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.errExit("Could not read bcfg2.conf; exiting") - try: - statsdata = XML(open(stats_file).read()) - except (IOError, XMLSyntaxError): - self.errExit("StatReports: Failed to parse %s" % (stats_file)) - - try: - encoding = self.cfp.get('components', 'encoding') - except: - encoding = 'UTF-8' - - if not clientspath: - try: - clientspath = "%s/Metadata/clients.xml" % \ - self.cfp.get('server', 'repository') - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - self.errExit("Could not read bcfg2.conf; exiting") - try: - clientsdata = XML(open(clientspath).read()) - except (IOError, XMLSyntaxError): - self.errExit("StatReports: Failed to parse %s" % (clientspath)) - - try: - load_stats(clientsdata, - statsdata, - encoding, - verb, - self.log, - quick=quick, - location=platform.node()) - except: - pass - - @printStats - def purge(self, client=None, maxdate=None, state=None): - '''Purge historical data from the database''' - - filtered = False # indicates whether or not a client should be deleted - - if not client and not maxdate and not state: - self.errExit("Reports.prune: Refusing to prune all data") - - ipurge = Interaction.objects - if client: - try: - cobj = Client.objects.get(name=client) - ipurge = ipurge.filter(client=cobj) - except Client.DoesNotExist: - self.log.error("Client %s not in database" % client) - raise SystemExit(-1) - self.log.debug("Filtering by client: %s" % client) - - if maxdate: - filtered = True - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - ipurge = ipurge.filter(timestamp__lt=maxdate) - - # Handle ping data as well - ping = Ping.objects.filter(endtime__lt=maxdate) - if client: - ping = ping.filter(client=cobj) - ping.delete() - - if state: - filtered = True - if state not in ('dirty', 'clean', 'modified'): - raise TypeError("state is not one of the following values " + \ - "('dirty','clean','modified')") - self.log.debug("Filtering by state: %s" % state) - ipurge = ipurge.filter(state=state) - - count = ipurge.count() - rnum = 0 - try: - while rnum < count: - grp = list(ipurge[:1000].values("id")) - # just in case... - if not grp: - break - Interaction.objects.filter(id__in=[x['id'] for x in grp]).delete() - rnum += len(grp) - self.log.debug("Deleted %s of %s" % (rnum, count)) - except: - self.log.error("Failed to remove interactions") - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - # bulk operations bypass the Interaction.delete method - self.log.debug("Pruning orphan Performance objects") - Performance.prune_orphans() - self.log.debug("Pruning orphan Reason objects") - Reason.prune_orphans() - - if client and not filtered: - '''Delete the client, ping data is automatic''' - try: - self.log.debug("Purging client %s" % client) - cobj.delete() - except: - self.log.error("Failed to delete client %s" % client) - (a, b, c) = sys.exc_info() - msg = traceback.format_exception(a, b, c, limit=2)[-1][:-1] - del a, b, c - self.log.error(msg) - - @printStats - def purge_expired(self, maxdate=None): - '''Purge expired clients from the database''' - - if maxdate: - if not isinstance(maxdate, datetime.datetime): - raise TypeError("maxdate is not a DateTime object") - self.log.debug("Filtering by maxdate: %s" % maxdate) - clients = Client.objects.filter(expiration__lt=maxdate) - else: - clients = Client.objects.filter(expiration__isnull=False) - - for client in clients: - self.log.debug("Purging client %s" % client) - Interaction.objects.filter(client=client).delete() - client.delete() - self.log.debug("Pruning orphan Performance objects") - Performance.prune_orphans() diff --git a/src/lib/Server/Admin/Snapshots.py b/src/lib/Server/Admin/Snapshots.py deleted file mode 100644 index 8bc56f1f1..000000000 --- a/src/lib/Server/Admin/Snapshots.py +++ /dev/null @@ -1,165 +0,0 @@ -from datetime import date -import sys - -# Prereq issues can be signaled with ImportError, so no try needed -import sqlalchemy, sqlalchemy.orm -import Bcfg2.Server.Admin -import Bcfg2.Server.Snapshots -import Bcfg2.Server.Snapshots.model -from Bcfg2.Server.Snapshots.model import Snapshot, Client, Metadata, Base, \ - File, Group, Package, Service -# Compatibility import -from Bcfg2.Bcfg2Py3k import u_str - -class Snapshots(Bcfg2.Server.Admin.Mode): - __shorthelp__ = "Interact with the Snapshots system" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin snapshots init" - "\nbcfg2-admin query qtype\n") - __usage__ = ("bcfg2-admin snapshots [init|query qtype]") - - q_dispatch = {'client': Client, - 'group': Group, - 'metadata': Metadata, - 'package': Package, - 'snapshot': Snapshot} - - def __init__(self, setup): - Bcfg2.Server.Admin.Mode.__init__(self, setup) - self.session = Bcfg2.Server.Snapshots.setup_session(self.configfile) - self.cfile = self.configfile - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - if len(args) == 0 or args[0] == '-h': - print(self.__usage__) - raise SystemExit(0) - - if args[0] == 'query': - if args[1] in self.q_dispatch: - q_obj = self.q_dispatch[args[1]] - if q_obj == Client: - rows = [] - labels = ('Client', 'Active') - for host in \ - self.session.query(q_obj).filter(q_obj.active == False): - rows.append([host.name, 'No']) - for host in \ - self.session.query(q_obj).filter(q_obj.active == True): - rows.append([host.name, 'Yes']) - self.print_table([labels]+rows, - justify='left', - hdr=True, - vdelim=" ", - padding=1) - elif q_obj == Group: - print("Groups:") - for group in self.session.query(q_obj).all(): - print(" %s" % group.name) - else: - results = self.session.query(q_obj).all() - else: - print('error') - raise SystemExit(1) - elif args[0] == 'init': - # Initialize the Snapshots database - dbpath = Bcfg2.Server.Snapshots.db_from_config(self.cfile) - engine = sqlalchemy.create_engine(dbpath, echo=True) - metadata = Base.metadata - metadata.create_all(engine) - Session = sqlalchemy.orm.sessionmaker() - Session.configure(bind=engine) - session = Session() - session.commit() - elif args[0] == 'dump': - client = args[1] - snap = Snapshot.get_current(self.session, u_str(client)) - if not snap: - print("Current snapshot for %s not found" % client) - sys.exit(1) - print("Client %s last run at %s" % (client, snap.timestamp)) - for pkg in snap.packages: - print("C:", pkg.correct, 'M:', pkg.modified) - print("start", pkg.start.name, pkg.start.version) - print("end", pkg.end.name, pkg.end.version) - elif args[0] == 'reports': - # bcfg2-admin reporting interface for Snapshots - if '-a' in args[1:]: - # Query all hosts for Name, Status, Revision, Timestamp - q = self.session.query(Client.name, - Snapshot.correct, - Snapshot.revision, - Snapshot.timestamp)\ - .filter(Client.id==Snapshot.client_id)\ - .group_by(Client.id) - rows = [] - labels = ('Client', 'Correct', 'Revision', 'Time') - for item in q.all(): - cli, cor, time, rev = item - rows.append([cli, cor, time, rev]) - self.print_table([labels]+rows, - justify='left', - hdr=True, vdelim=" ", - padding=1) - elif '-b' in args[1:]: - # Query a single host for bad entries - if len(args) < 3: - print("Usage: bcfg2-admin snapshots -b ") - return - client = args[2] - snap = Snapshot.get_current(self.session, u_str(client)) - if not snap: - print("Current snapshot for %s not found" % client) - sys.exit(1) - print("Bad entries:") - bad_pkgs = [self.session.query(Package) - .filter(Package.id==p.start_id).one().name \ - for p in snap.packages if p.correct == False] - for p in bad_pkgs: - print(" Package:%s" % p) - bad_files = [self.session.query(File) - .filter(File.id==f.start_id).one().name \ - for f in snap.files if f.correct == False] - for filename in bad_files: - print(" File:%s" % filename) - bad_svcs = [self.session.query(Service) - .filter(Service.id==s.start_id).one().name \ - for s in snap.services if s.correct == False] - for svc in bad_svcs: - print(" Service:%s" % svc) - elif '-e' in args[1:]: - # Query a single host for extra entries - client = args[2] - snap = Snapshot.get_current(self.session, u_str(client)) - if not snap: - print("Current snapshot for %s not found" % client) - sys.exit(1) - print("Extra entries:") - for pkg in snap.extra_packages: - print(" Package:%s" % pkg.name) - # FIXME: Do we know about extra files yet? - for f in snap.extra_files: - print(" File:%s" % f.name) - for svc in snap.extra_services: - print(" Service:%s" % svc.name) - elif '--date' in args[1:]: - year, month, day = args[2:] - timestamp = date(int(year), int(month), int(day)) - snaps = [] - for client in self.session.query(Client).filter(Client.active == True): - snaps.append(Snapshot.get_by_date(self.session, - client.name, - timestamp)) - rows = [] - labels = ('Client', 'Correct', 'Revision', 'Time') - for snap in snaps: - rows.append([snap.client.name, - snap.correct, - snap.revision, - snap.timestamp]) - self.print_table([labels]+rows, - justify='left', - hdr=True, - vdelim=" ", - padding=1) - else: - print("Unknown options: ", args[1:]) diff --git a/src/lib/Server/Admin/Tidy.py b/src/lib/Server/Admin/Tidy.py deleted file mode 100644 index 82319b93e..000000000 --- a/src/lib/Server/Admin/Tidy.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import re -import socket - -import Bcfg2.Server.Admin - - -class Tidy(Bcfg2.Server.Admin.Mode): - __shorthelp__ = "Clean up useless files in the repo" - __longhelp__ = __shorthelp__ + "\n\nbcfg2-admin tidy [-f] [-I]\n" - __usage__ = ("bcfg2-admin tidy [options]\n\n" - " %-25s%s\n" - " %-25s%s\n" % - ("-f", - "force", - "-I", - "interactive")) - - def __call__(self, args): - Bcfg2.Server.Admin.Mode.__call__(self, args) - badfiles = self.buildTidyList() - if '-f' in args or '-I' in args: - if '-I' in args: - for name in badfiles[:]: - # py3k compatibility - try: - answer = raw_input("Unlink file %s? [yN] " % name) - except NameError: - answer = input("Unlink file %s? [yN] " % name) - if answer not in ['y', 'Y']: - badfiles.remove(name) - for name in badfiles: - try: - os.unlink(name) - except IOError: - print("Failed to unlink %s" % name) - else: - for name in badfiles: - print(name) - - def buildTidyList(self): - """Clean up unused or unusable files from the repository.""" - hostmatcher = re.compile('.*\.H_(\S+)$') - to_remove = [] - good = [] - bad = [] - - # clean up unresolvable hosts in SSHbase - for name in os.listdir("%s/SSHbase" % self.setup['repo']): - if hostmatcher.match(name): - hostname = hostmatcher.match(name).group(1) - if hostname in good + bad: - continue - try: - socket.gethostbyname(hostname) - good.append(hostname) - except: - bad.append(hostname) - for name in os.listdir("%s/SSHbase" % self.setup['repo']): - if not hostmatcher.match(name): - to_remove.append("%s/SSHbase/%s" % (self.setup['repo'], - name)) - else: - if hostmatcher.match(name).group(1) in bad: - to_remove.append("%s/SSHbase/%s" % - (self.setup['repo'], name)) - # clean up file~ - # clean up files without parsable names in Cfg - return to_remove diff --git a/src/lib/Server/Admin/Viz.py b/src/lib/Server/Admin/Viz.py deleted file mode 100644 index 2faa423c1..000000000 --- a/src/lib/Server/Admin/Viz.py +++ /dev/null @@ -1,119 +0,0 @@ -import getopt -from subprocess import Popen, PIPE -import sys -import pipes -import Bcfg2.Server.Admin - - -class Viz(Bcfg2.Server.Admin.MetadataCore): - __shorthelp__ = "Produce graphviz diagrams of metadata structures" - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin viz [--includehosts] " - "[--includebundles] [--includekey] " - "[--only-client clientname] " - "[-o output.]\n") - __usage__ = ("bcfg2-admin viz [options]\n\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" - " %-32s%s\n" % - ("-H, --includehosts", - "include hosts in the viz output", - "-b, --includebundles", - "include bundles in the viz output", - "-k, --includekey", - "show a key for different digraph shapes", - "-c, --only-client ", - "show only the groups, bundles for the named client", - "-o, --outfile ", - "write viz output to an output file")) - - colors = ['steelblue1', 'chartreuse', 'gold', 'magenta', - 'indianred1', 'limegreen', 'orange1', 'lightblue2', - 'green1', 'blue1', 'yellow1', 'darkturquoise', 'gray66'] - - __plugin_blacklist__ = ['DBStats', 'Snapshots', 'Cfg', 'Pkgmgr', 'Packages', - 'Rules', 'Account', 'Decisions', 'Deps', 'Git', - 'Svn', 'Fossil', 'Bzr', 'Bundler', 'TGenshi', - 'SGenshi', 'Base'] - - def __call__(self, args): - Bcfg2.Server.Admin.MetadataCore.__call__(self, args) - # First get options to the 'viz' subcommand - try: - opts, args = getopt.getopt(args, 'Hbkc:o:', - ['includehosts', 'includebundles', - 'includekey', 'only-client=', 'outfile=']) - except getopt.GetoptError: - msg = sys.exc_info()[1] - print(msg) - print(self.__longhelp__) - raise SystemExit(1) - - hset = False - bset = False - kset = False - only_client = None - outputfile = False - for opt, arg in opts: - if opt in ("-H", "--includehosts"): - hset = True - elif opt in ("-b", "--includebundles"): - bset = True - elif opt in ("-k", "--includekey"): - kset = True - elif opt in ("-c", "--only-client"): - only_client = arg - elif opt in ("-o", "--outfile"): - outputfile = arg - - data = self.Visualize(self.setup['repo'], hset, bset, - kset, only_client, outputfile) - if data: - print(data) - raise SystemExit(0) - - def Visualize(self, repopath, hosts=False, - bundles=False, key=False, only_client=None, output=False): - """Build visualization of groups file.""" - if output: - format = output.split('.')[-1] - else: - format = 'png' - - cmd = ["dot", "-T", format] - if output: - cmd.extend(["-o", output]) - try: - dotpipe = Popen(cmd, stdin=PIPE, stdout=PIPE, close_fds=True) - except OSError: - # on some systems (RHEL 6), you cannot run dot with - # shell=True. on others (Gentoo with Python 2.7), you - # must. In yet others (RHEL 5), either way works. I have - # no idea what the difference is, but it's kind of a PITA. - cmd = ["dot", "-T", pipes.quote(format)] - if output: - cmd.extend(["-o", pipes.quote(output)]) - dotpipe = Popen(cmd, shell=True, - stdin=PIPE, stdout=PIPE, close_fds=True) - try: - dotpipe.stdin.write("digraph groups {\n") - except: - print("write to dot process failed. Is graphviz installed?") - raise SystemExit(1) - dotpipe.stdin.write('\trankdir="LR";\n') - dotpipe.stdin.write(self.metadata.viz(hosts, bundles, - key, only_client, self.colors)) - if key: - dotpipe.stdin.write("\tsubgraph cluster_key {\n") - dotpipe.stdin.write('''\tstyle="filled";\n''') - dotpipe.stdin.write('''\tcolor="lightblue";\n''') - dotpipe.stdin.write('''\tBundle [ shape="septagon" ];\n''') - dotpipe.stdin.write('''\tGroup [shape="ellipse"];\n''') - dotpipe.stdin.write('''\tProfile [style="bold", shape="ellipse"];\n''') - dotpipe.stdin.write('''\tHblock [label="Host1|Host2|Host3", shape="record"];\n''') - dotpipe.stdin.write('''\tlabel="Key";\n''') - dotpipe.stdin.write("\t}\n") - dotpipe.stdin.write("}\n") - dotpipe.stdin.close() - return dotpipe.stdout.read() diff --git a/src/lib/Server/Admin/Xcmd.py b/src/lib/Server/Admin/Xcmd.py deleted file mode 100644 index 140465468..000000000 --- a/src/lib/Server/Admin/Xcmd.py +++ /dev/null @@ -1,60 +0,0 @@ -import sys - -import Bcfg2.Options -import Bcfg2.Proxy -import Bcfg2.Server.Admin - -# Compatibility import -from Bcfg2.Bcfg2Py3k import xmlrpclib - - -class Xcmd(Bcfg2.Server.Admin.Mode): - __shorthelp__ = ("XML-RPC Command Interface") - __longhelp__ = (__shorthelp__ + "\n\nbcfg2-admin xcmd command\n") - __usage__ = ("bcfg2-admin xcmd ") - - def __call__(self, args): - optinfo = { - 'server': Bcfg2.Options.SERVER_LOCATION, - 'user': Bcfg2.Options.CLIENT_USER, - 'password': Bcfg2.Options.SERVER_PASSWORD, - 'key': Bcfg2.Options.SERVER_KEY, - 'certificate': Bcfg2.Options.CLIENT_CERT, - 'ca': Bcfg2.Options.CLIENT_CA, - 'timeout': Bcfg2.Options.CLIENT_TIMEOUT, - } - setup = Bcfg2.Options.OptionParser(optinfo) - setup.parse(args) - Bcfg2.Proxy.RetryMethod.max_retries = 1 - proxy = Bcfg2.Proxy.ComponentProxy(setup['server'], - setup['user'], - setup['password'], - key=setup['key'], - cert=setup['certificate'], - ca=setup['ca'], - timeout=setup['timeout']) - if len(setup['args']) == 0: - print("Usage: xcmd ") - return - cmd = setup['args'][0] - args = () - if len(setup['args']) > 1: - args = tuple(setup['args'][1:]) - try: - data = getattr(proxy, cmd)(*args) - except xmlrpclib.Fault: - flt = sys.exc_info()[1] - if flt.faultCode == 7: - print("Unknown method %s" % cmd) - return - elif flt.faultCode == 20: - return - else: - raise - except Bcfg2.Proxy.ProxyError: - err = sys.exc_info()[1] - print("Proxy Error: %s" % err) - return - - if data != None: - print(data) diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py deleted file mode 100644 index 5031392d0..000000000 --- a/src/lib/Server/Admin/__init__.py +++ /dev/null @@ -1,136 +0,0 @@ -__all__ = [ - 'Backup', - 'Bundle', - 'Client', - 'Compare', - 'Group', - 'Init', - 'Minestruct', - 'Perf', - 'Pull', - 'Query', - 'Reports', - 'Snapshots', - 'Tidy', - 'Viz', - 'Xcmd' - ] - -import logging -import lxml.etree -import sys - -import Bcfg2.Server.Core -import Bcfg2.Options -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - - -class ModeOperationError(Exception): - pass - - -class Mode(object): - """Help message has not yet been added for mode.""" - __shorthelp__ = 'Shorthelp not defined yet' - __longhelp__ = 'Longhelp not defined yet' - __usage__ = None - __args__ = [] - - def __init__(self, setup): - self.setup = setup - self.configfile = setup['configfile'] - self.__cfp = False - self.log = logging.getLogger('Bcfg2.Server.Admin.Mode') - if self.__usage__ is not None: - setup.hm = self.__usage__ - - def getCFP(self): - if not self.__cfp: - self.__cfp = ConfigParser.ConfigParser() - self.__cfp.read(self.configfile) - return self.__cfp - - cfp = property(getCFP) - - def __call__(self, args): - pass - - def errExit(self, emsg): - print(emsg) - raise SystemExit(1) - - def load_stats(self, client): - stats = lxml.etree.parse("%s/etc/statistics.xml" % self.setup['repo']) - hostent = stats.xpath('//Node[@name="%s"]' % client) - if not hostent: - self.errExit("Could not find stats for client %s" % (client)) - return hostent[0] - - def print_table(self, rows, justify='left', hdr=True, vdelim=" ", padding=1): - """Pretty print a table - - rows - list of rows ([[row 1], [row 2], ..., [row n]]) - hdr - if True the first row is treated as a table header - vdelim - vertical delimiter between columns - padding - # of spaces around the longest element in the column - justify - may be left,center,right - - """ - hdelim = "=" - justify = {'left': str.ljust, - 'center': str.center, - 'right': str.rjust}[justify.lower()] - - """ - Calculate column widths (longest item in each column - plus padding on both sides) - - """ - cols = list(zip(*rows)) - colWidths = [max([len(str(item)) + 2 * padding for \ - item in col]) for col in cols] - borderline = vdelim.join([w * hdelim for w in colWidths]) - - # Print out the table - print(borderline) - for row in rows: - print(vdelim.join([justify(str(item), width) for \ - (item, width) in zip(row, colWidths)])) - if hdr: - print(borderline) - hdr = False - - -class MetadataCore(Mode): - """Base class for admin-modes that handle metadata.""" - __plugin_whitelist__ = None - __plugin_blacklist__ = None - - def __init__(self, setup): - Mode.__init__(self, setup) - if self.__plugin_whitelist__ is not None: - setup['plugins'] = [p for p in setup['plugins'] - if p in self.__plugin_whitelist__] - elif self.__plugin_blacklist__ is not None: - setup['plugins'] = [p for p in setup['plugins'] - if p not in self.__plugin_blacklist__] - - try: - self.bcore = \ - Bcfg2.Server.Core.Core(setup['repo'], - setup['plugins'], - setup['password'], - setup['encoding'], - filemonitor=setup['filemonitor']) - if setup['event debug']: - self.bcore.fam.debug = True - except Bcfg2.Server.Core.CoreInitError: - msg = sys.exc_info()[1] - self.errExit("Core load failed: %s" % msg) - self.bcore.fam.handle_events_in_interval(5) - self.metadata = self.bcore.metadata - - -class StructureMode(MetadataCore): - pass diff --git a/src/lib/Server/Core.py b/src/lib/Server/Core.py deleted file mode 100644 index cb95e3e9c..000000000 --- a/src/lib/Server/Core.py +++ /dev/null @@ -1,479 +0,0 @@ -"""Bcfg2.Server.Core provides the runtime support for Bcfg2 modules.""" - -import atexit -import logging -import select -import sys -import threading -import time -try: - import lxml.etree -except ImportError: - print("Failed to import lxml dependency. Shutting down server.") - raise SystemExit(1) - -from Bcfg2.Component import Component, exposed -from Bcfg2.Server.Plugin import PluginInitError, PluginExecutionError -import Bcfg2.Server.FileMonitor -import Bcfg2.Server.Plugins.Metadata -# Compatibility imports -from Bcfg2.Bcfg2Py3k import xmlrpclib -if sys.hexversion >= 0x03000000: - from functools import reduce - -logger = logging.getLogger('Bcfg2.Server.Core') - - -def critical_error(operation): - """Log and err, traceback and return an xmlrpc fault to client.""" - logger.error(operation, exc_info=1) - raise xmlrpclib.Fault(7, "Critical unexpected failure: %s" % (operation)) - -try: - import psyco - psyco.full() -except: - pass - - -def sort_xml(node, key=None): - for child in node: - sort_xml(child, key) - - sorted_children = sorted(node, key=key) - node[:] = sorted_children - - -class CoreInitError(Exception): - """This error is raised when the core cannot be initialized.""" - pass - - -class Core(Component): - """The Core object is the container for all - Bcfg2 Server logic and modules. - """ - name = 'bcfg2-server' - implementation = 'bcfg2-server' - - def __init__(self, repo, plugins, password, encoding, - cfile='/etc/bcfg2.conf', ca=None, - filemonitor='default', start_fam_thread=False): - Component.__init__(self) - self.datastore = repo - if filemonitor not in Bcfg2.Server.FileMonitor.available: - logger.error("File monitor driver %s not available; " - "forcing to default" % filemonitor) - filemonitor = 'default' - try: - self.fam = Bcfg2.Server.FileMonitor.available[filemonitor]() - except IOError: - logger.error("Failed to instantiate fam driver %s" % filemonitor, - exc_info=1) - raise CoreInitError("failed to instantiate fam driver (used %s)" % \ - filemonitor) - self.pubspace = {} - self.cfile = cfile - self.cron = {} - self.plugins = {} - self.plugin_blacklist = {} - self.revision = '-1' - self.password = password - self.encoding = encoding - atexit.register(self.shutdown) - # Create an event to signal worker threads to shutdown - self.terminate = threading.Event() - - if '' in plugins: - plugins.remove('') - - for plugin in plugins: - if not plugin in self.plugins: - self.init_plugins(plugin) - # Remove blacklisted plugins - for p, bl in list(self.plugin_blacklist.items()): - if len(bl) > 0: - logger.error("The following plugins conflict with %s;" - "Unloading %s" % (p, bl)) - for plug in bl: - del self.plugins[plug] - # This section logs the experimental plugins - expl = [plug for (name, plug) in list(self.plugins.items()) - if plug.experimental] - if expl: - logger.info("Loading experimental plugin(s): %s" % \ - (" ".join([x.name for x in expl]))) - logger.info("NOTE: Interfaces subject to change") - # This section logs the deprecated plugins - depr = [plug for (name, plug) in list(self.plugins.items()) - if plug.deprecated] - if depr: - logger.info("Loading deprecated plugin(s): %s" % \ - (" ".join([x.name for x in depr]))) - - mlist = self.plugins_by_type(Bcfg2.Server.Plugin.Metadata) - if len(mlist) == 1: - self.metadata = mlist[0] - else: - logger.error("No Metadata Plugin loaded; failed to instantiate Core") - raise CoreInitError("No Metadata Plugin") - self.statistics = self.plugins_by_type(Bcfg2.Server.Plugin.Statistics) - self.pull_sources = self.plugins_by_type(Bcfg2.Server.Plugin.PullSource) - self.generators = self.plugins_by_type(Bcfg2.Server.Plugin.Generator) - self.structures = self.plugins_by_type(Bcfg2.Server.Plugin.Structure) - self.connectors = self.plugins_by_type(Bcfg2.Server.Plugin.Connector) - self.ca = ca - self.fam_thread = threading.Thread(target=self._file_monitor_thread) - if start_fam_thread: - self.fam_thread.start() - - def plugins_by_type(self, base_cls): - """Return a list of loaded plugins that match the passed type. - - The returned list is sorted in ascending order by the Plugins' - sort_order value. The sort_order defaults to 500 in Plugin.py, - but can be overridden by individual plugins. Plugins with the - same numerical sort_order value are sorted in alphabetical - order by their name. - """ - return sorted([plugin for plugin in self.plugins.values() - if isinstance(plugin, base_cls)], - key=lambda p: (p.sort_order, p.name)) - - def _file_monitor_thread(self): - """The thread for monitor the files.""" - famfd = self.fam.fileno() - terminate = self.terminate - while not terminate.isSet(): - try: - if famfd: - select.select([famfd], [], [], 2) - else: - if not self.fam.pending(): - terminate.wait(15) - self.fam.handle_event_set(self.lock) - except: - continue - # VCS plugin periodic updates - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Version): - self.revision = plugin.get_revision() - - def init_plugins(self, plugin): - """Handling for the plugins.""" - try: - mod = getattr(__import__("Bcfg2.Server.Plugins.%s" % - (plugin)).Server.Plugins, plugin) - except ImportError: - try: - mod = __import__(plugin) - except: - logger.error("Failed to load plugin %s" % (plugin)) - return - plug = getattr(mod, plugin) - # Blacklist conflicting plugins - cplugs = [conflict for conflict in plug.conflicts - if conflict in self.plugins] - self.plugin_blacklist[plug.name] = cplugs - try: - self.plugins[plugin] = plug(self, self.datastore) - except PluginInitError: - logger.error("Failed to instantiate plugin %s" % (plugin)) - except: - logger.error("Unexpected instantiation failure for plugin %s" % - (plugin), exc_info=1) - - def shutdown(self): - """Shutting down the plugins.""" - if not self.terminate.isSet(): - self.terminate.set() - for plugin in list(self.plugins.values()): - plugin.shutdown() - - def validate_structures(self, metadata, data): - """Checks the data structure.""" - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.StructureValidator): - try: - plugin.validate_structures(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: - err = sys.exc_info()[1] - logger.error("Plugin %s structure validation failed: %s" \ - % (plugin.name, err.message)) - raise - except: - logger.error("Plugin %s: unexpected structure validation failure" \ - % (plugin.name), exc_info=1) - - def validate_goals(self, metadata, data): - """Checks that the config matches the goals enforced by the plugins.""" - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.GoalValidator): - try: - plugin.validate_goals(metadata, data) - except Bcfg2.Server.Plugin.ValidationError: - err = sys.exc_info()[1] - logger.error("Plugin %s goal validation failed: %s" \ - % (plugin.name, err.message)) - raise - except: - logger.error("Plugin %s: unexpected goal validation failure" \ - % (plugin.name), exc_info=1) - - def GetStructures(self, metadata): - """Get all structures for client specified by metadata.""" - structures = reduce(lambda x, y: x + y, - [struct.BuildStructures(metadata) for struct \ - in self.structures], []) - sbundles = [b.get('name') for b in structures if b.tag == 'Bundle'] - missing = [b for b in metadata.bundles if b not in sbundles] - if missing: - logger.error("Client %s configuration missing bundles: %s" \ - % (metadata.hostname, ':'.join(missing))) - return structures - - def BindStructure(self, structure, metadata): - """Bind a complete structure.""" - for entry in structure.getchildren(): - if entry.tag.startswith("Bound"): - entry.tag = entry.tag[5:] - continue - try: - self.Bind(entry, metadata) - except PluginExecutionError, exc: - if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) - logger.error("Failed to bind entry: %s %s" % \ - (entry.tag, entry.get('name'))) - except Exception, exc: - if 'failure' not in entry.attrib: - entry.set('failure', 'bind error: %s' % exc) - logger.error("Unexpected failure in BindStructure: %s %s" \ - % (entry.tag, entry.get('name')), exc_info=1) - - def Bind(self, entry, metadata): - """Bind an entry using the appropriate generator.""" - if 'altsrc' in entry.attrib: - oldname = entry.get('name') - entry.set('name', entry.get('altsrc')) - entry.set('realname', oldname) - del entry.attrib['altsrc'] - try: - ret = self.Bind(entry, metadata) - entry.set('name', oldname) - del entry.attrib['realname'] - return ret - except: - entry.set('name', oldname) - logger.error("Failed binding entry %s:%s with altsrc %s" \ - % (entry.tag, entry.get('name'), - entry.get('altsrc'))) - logger.error("Falling back to %s:%s" % (entry.tag, - entry.get('name'))) - - glist = [gen for gen in self.generators if - entry.get('name') in gen.Entries.get(entry.tag, {})] - if len(glist) == 1: - return glist[0].Entries[entry.tag][entry.get('name')](entry, - metadata) - elif len(glist) > 1: - generators = ", ".join([gen.name for gen in glist]) - logger.error("%s %s served by multiple generators: %s" % \ - (entry.tag, entry.get('name'), generators)) - g2list = [gen for gen in self.generators if - gen.HandlesEntry(entry, metadata)] - if len(g2list) == 1: - return g2list[0].HandleEntry(entry, metadata) - entry.set('failure', 'no matching generator') - raise PluginExecutionError(entry.tag, entry.get('name')) - - def BuildConfiguration(self, client): - """Build configuration for clients.""" - start = time.time() - config = lxml.etree.Element("Configuration", version='2.0', \ - revision=self.revision) - try: - meta = self.build_metadata(client) - except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - logger.error("Metadata consistency error for client %s" % client) - return lxml.etree.Element("error", type='metadata error') - - try: - structures = self.GetStructures(meta) - except: - logger.error("error in GetStructures", exc_info=1) - return lxml.etree.Element("error", type='structure error') - - self.validate_structures(meta, structures) - - # Perform altsrc consistency checking - esrcs = {} - for struct in structures: - for entry in struct: - key = (entry.tag, entry.get('name')) - if key in esrcs: - if esrcs[key] != entry.get('altsrc'): - logger.error("Found inconsistent altsrc mapping for entry %s:%s" % key) - else: - esrcs[key] = entry.get('altsrc', None) - del esrcs - - for astruct in structures: - try: - self.BindStructure(astruct, meta) - config.append(astruct) - except: - logger.error("error in BindStructure", exc_info=1) - self.validate_goals(meta, config) - - sort_xml(config, key=lambda e: e.get('name')) - - logger.info("Generated config for %s in %.03f seconds" % \ - (client, time.time() - start)) - return config - - def GetDecisions(self, metadata, mode): - """Get data for the decision list.""" - result = [] - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Decision): - try: - result += plugin.GetDecisions(metadata, mode) - except: - logger.error("Plugin: %s failed to generate decision list" \ - % plugin.name, exc_info=1) - return result - - def build_metadata(self, client_name): - """Build the metadata structure.""" - if not hasattr(self, 'metadata'): - # some threads start before metadata is even loaded - raise Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError - imd = self.metadata.get_initial_metadata(client_name) - for conn in self.connectors: - grps = conn.get_additional_groups(imd) - self.metadata.merge_additional_groups(imd, grps) - for conn in self.connectors: - data = conn.get_additional_data(imd) - self.metadata.merge_additional_data(imd, conn.name, data) - imd.query.by_name = self.build_metadata - return imd - - def process_statistics(self, client_name, statistics): - """Proceed statistics for client.""" - meta = self.build_metadata(client_name) - state = statistics.find(".//Statistics") - if state.get('version') >= '2.0': - for plugin in self.statistics: - try: - plugin.process_statistics(meta, statistics) - except: - logger.error("Plugin %s failed to process stats from %s" \ - % (plugin.name, meta.hostname), - exc_info=1) - - logger.info("Client %s reported state %s" % (client_name, - state.get('state'))) - # XMLRPC handlers start here - - @exposed - def GetProbes(self, address): - """Fetch probes for a particular client.""" - resp = lxml.etree.Element('probes') - try: - name = self.metadata.resolve_client(address, cleanup_cache=True) - meta = self.build_metadata(name) - - for plugin in self.plugins_by_type(Bcfg2.Server.Plugin.Probing): - for probe in plugin.GetProbes(meta): - resp.append(probe) - return lxml.etree.tostring(resp, encoding='UTF-8', - xml_declaration=True) - except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - warning = 'Client metadata resolution error for %s; check server log' % address[0] - self.logger.warning(warning) - raise xmlrpclib.Fault(6, warning) - except Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError: - err_msg = 'Metadata system runtime failure' - self.logger.error(err_msg) - raise xmlrpclib.Fault(6, err_msg) - except: - critical_error("Error determining client probes") - - @exposed - def RecvProbeData(self, address, probedata): - """Receive probe data from clients.""" - try: - name = self.metadata.resolve_client(address) - meta = self.build_metadata(name) - except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - warning = 'Metadata consistency error' - self.logger.warning(warning) - raise xmlrpclib.Fault(6, warning) - # clear dynamic groups - self.metadata.cgroups[meta.hostname] = [] - try: - xpdata = lxml.etree.XML(probedata.encode('utf-8')) - except: - self.logger.error("Failed to parse probe data from client %s" % \ - (address[0])) - return False - - sources = [] - [sources.append(data.get('source')) for data in xpdata - if data.get('source') not in sources] - for source in sources: - if source not in self.plugins: - self.logger.warning("Failed to locate plugin %s" % (source)) - continue - dl = [data for data in xpdata if data.get('source') == source] - try: - self.plugins[source].ReceiveData(meta, dl) - except: - logger.error("Failed to process probe data from client %s" % \ - (address[0]), exc_info=1) - return True - - @exposed - def AssertProfile(self, address, profile): - """Set profile for a client.""" - try: - client = self.metadata.resolve_client(address) - self.metadata.set_profile(client, profile, address) - except (Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError, - Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError): - warning = 'Metadata consistency error' - self.logger.warning(warning) - raise xmlrpclib.Fault(6, warning) - return True - - @exposed - def GetConfig(self, address, checksum=False): - """Build config for a client.""" - try: - client = self.metadata.resolve_client(address) - config = self.BuildConfiguration(client) - return lxml.etree.tostring(config, encoding='UTF-8', - xml_declaration=True) - except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - self.logger.warning("Metadata consistency failure for %s" % (address)) - raise xmlrpclib.Fault(6, "Metadata consistency failure") - - @exposed - def RecvStats(self, address, stats): - """Act on statistics upload.""" - sdata = lxml.etree.XML(stats.encode('utf-8')) - client = self.metadata.resolve_client(address) - self.process_statistics(client, sdata) - return "" - - def authenticate(self, cert, user, password, address): - if self.ca: - acert = cert - else: - # No ca, so no cert validation can be done - acert = None - return self.metadata.AuthenticateConnection(acert, user, password, address) - - @exposed - def GetDecisionList(self, address, mode): - """Get the data of the decision list.""" - client = self.metadata.resolve_client(address) - meta = self.build_metadata(client) - return self.GetDecisions(meta, mode) diff --git a/src/lib/Server/FileMonitor.py b/src/lib/Server/FileMonitor.py deleted file mode 100644 index d6b313e6b..000000000 --- a/src/lib/Server/FileMonitor.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Bcfg2.Server.FileMonitor provides the support for monitorung files.""" - -import logging -import os -import stat -from time import sleep, time - -logger = logging.getLogger('Bcfg2.Server.FileMonitor') - - -def ShouldIgnore(event): - """Test if the event should be suppresed.""" - # FIXME should move event suppression out of the core - if event.filename.split('/')[-1] == '.svn': - return True - if event.filename.endswith('~') or \ - event.filename.startswith('#') or event.filename.startswith('.#'): - #logger.error("Suppressing event for file %s" % (event.filename)) - return True - return False - - -class Event(object): - def __init__(self, request_id, filename, code): - self.requestID = request_id - self.filename = filename - self.action = code - - def code2str(self): - """return static code for event""" - return self.action - -available = {} - - -class FileMonitor(object): - """File Monitor baseclass.""" - def __init__(self, debug=False): - object.__init__(self) - self.debug = debug - self.handles = dict() - - def get_event(self): - return None - - def pending(self): - return False - - def fileno(self): - return 0 - - def handle_one_event(self, event): - if ShouldIgnore(event): - return - if event.requestID not in self.handles: - logger.info("Got event for unexpected id %s, file %s" % - (event.requestID, event.filename)) - return - if self.debug: - logger.info("Dispatching event %s %s to obj %s" \ - % (event.code2str(), event.filename, - self.handles[event.requestID])) - try: - self.handles[event.requestID].HandleEvent(event) - except: - logger.error("error in handling of gamin event for %s" % \ - (event.filename), exc_info=1) - - def handle_event_set(self, lock=None): - count = 1 - event = self.get_event() - start = time() - if lock: - lock.acquire() - try: - self.handle_one_event(event) - while self.pending(): - self.handle_one_event(self.get_event()) - count += 1 - except: - pass - if lock: - lock.release() - end = time() - logger.info("Handled %d events in %.03fs" % (count, (end - start))) - - def handle_events_in_interval(self, interval): - end = time() + interval - while time() < end: - if self.pending(): - self.handle_event_set() - end = time() + interval - else: - sleep(0.5) - - -class FamFam(object): - """The fam object is a set of callbacks for - file alteration events (FAM support). - """ - - def __init__(self): - object.__init__(self) - self.fm = _fam.open() - self.users = {} - self.handles = {} - self.debug = False - - def fileno(self): - """Return fam file handle number.""" - return self.fm.fileno() - - def handle_event_set(self, _): - self.Service() - - def handle_events_in_interval(self, interval): - now = time() - while (time() - now) < interval: - if self.Service(): - now = time() - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - mode = os.stat(path)[stat.ST_MODE] - if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) - else: - handle = self.fm.monitorFile(path, None) - self.handles[handle.requestID()] = handle - if obj != None: - self.users[handle.requestID()] = obj - return handle.requestID() - - def Service(self, interval=0.50): - """Handle all fam work.""" - count = 0 - collapsed = 0 - rawevents = [] - start = time() - now = time() - while (time() - now) < interval: - if self.fm.pending(): - while self.fm.pending(): - count += 1 - rawevents.append(self.fm.nextEvent()) - now = time() - unique = [] - bookkeeping = [] - for event in rawevents: - if ShouldIgnore(event): - continue - if event.code2str() != 'changed': - # process all non-change events - unique.append(event) - else: - if (event.filename, event.requestID) not in bookkeeping: - bookkeeping.append((event.filename, event.requestID)) - unique.append(event) - else: - collapsed += 1 - for event in unique: - if event.requestID in self.users: - try: - self.users[event.requestID].HandleEvent(event) - except: - logger.error("handling event for file %s" % (event.filename), exc_info=1) - end = time() - logger.info("Processed %s fam events in %03.03f seconds. %s coalesced" % - (count, (end - start), collapsed)) - return count - - -class Fam(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (FAM support). - """ - - def __init__(self, debug=False): - FileMonitor.__init__(self, debug) - self.fm = _fam.open() - - def fileno(self): - return self.fm.fileno() - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - mode = os.stat(path)[stat.ST_MODE] - if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) - else: - handle = self.fm.monitorFile(path, None) - if obj != None: - self.handles[handle.requestID()] = obj - return handle.requestID() - - def pending(self): - return self.fm.pending() - - def get_event(self): - return self.fm.nextEvent() - - -class Pseudo(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (static monitor support). - """ - - def __init__(self, debug=False): - FileMonitor.__init__(self, debug=False) - self.pending_events = [] - - def pending(self): - return len(self.pending_events) != 0 - - def get_event(self): - return self.pending_events.pop() - - def AddMonitor(self, path, obj): - """add a monitor to path, installing a callback to obj.HandleEvent""" - handleID = len(list(self.handles.keys())) - mode = os.stat(path)[stat.ST_MODE] - handle = Event(handleID, path, 'exists') - if stat.S_ISDIR(mode): - dirList = os.listdir(path) - self.pending_events.append(handle) - for includedFile in dirList: - self.pending_events.append(Event(handleID, - includedFile, - 'exists')) - self.pending_events.append(Event(handleID, path, 'endExist')) - else: - self.pending_events.append(Event(handleID, path, 'exists')) - if obj != None: - self.handles[handleID] = obj - return handleID - - -try: - from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ - GAMChanged, GAMDeleted, GAMMoved - - class GaminEvent(Event): - """ - This class provides an event analogous to - python-fam events based on gamin sources. - """ - def __init__(self, request_id, filename, code): - Event.__init__(self, request_id, filename, code) - action_map = {GAMCreated: 'created', GAMExists: 'exists', - GAMChanged: 'changed', GAMDeleted: 'deleted', - GAMEndExist: 'endExist', GAMMoved: 'moved'} - if code in action_map: - self.action = action_map[code] - - class Gamin(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (Gamin support) - """ - def __init__(self, debug=False): - FileMonitor.__init__(self, debug) - self.mon = WatchMonitor() - self.counter = 0 - self.events = [] - - def fileno(self): - return self.mon.get_fd() - - def queue(self, path, action, request_id): - """queue up the event for later handling""" - self.events.append(GaminEvent(request_id, path, action)) - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - handle = self.counter - self.counter += 1 - mode = os.stat(path)[stat.ST_MODE] - - # Flush queued gamin events - while self.mon.event_pending(): - self.mon.handle_one_event() - - if stat.S_ISDIR(mode): - self.mon.watch_directory(path, self.queue, handle) - else: - self.mon.watch_file(path, self.queue, handle) - self.handles[handle] = obj - return handle - - def pending(self): - return len(self.events) > 0 or self.mon.event_pending() - - def get_event(self): - if self.mon.event_pending(): - self.mon.handle_one_event() - return self.events.pop(0) - - available['gamin'] = Gamin -except ImportError: - # fall back to _fam - pass - -try: - import _fam - available['fam'] = FamFam -except ImportError: - pass -available['pseudo'] = Pseudo - -for fdrv in ['gamin', 'fam', 'pseudo']: - if fdrv in available: - available['default'] = available[fdrv] - break diff --git a/src/lib/Server/Hostbase/.gitignore b/src/lib/Server/Hostbase/.gitignore deleted file mode 100644 index 8e15b5395..000000000 --- a/src/lib/Server/Hostbase/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.pyc -dev.db -bcfg2.conf diff --git a/src/lib/Server/Hostbase/__init__.py b/src/lib/Server/Hostbase/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lib/Server/Hostbase/backends.py b/src/lib/Server/Hostbase/backends.py deleted file mode 100644 index ecaf3c109..000000000 --- a/src/lib/Server/Hostbase/backends.py +++ /dev/null @@ -1,68 +0,0 @@ -from django.contrib.auth.models import User -#from ldapauth import * -from nisauth import * - -## class LDAPBackend(object): - -## def authenticate(self,username=None,password=None): -## try: - -## l = ldapauth(username,password) -## temp_pass = User.objects.make_random_password(100) -## ldap_user = dict(username=l.sAMAccountName, -## ) -## user_session_obj = dict( -## email=l.email, -## first_name=l.name_f, -## last_name=l.name_l, -## uid=l.badge_no -## ) -## #fixme: need to add this user session obj to session -## #print str(ldap_user) -## user,created = User.objects.get_or_create(username=username) -## #print user -## #print "created " + str(created) -## return user - -## except LDAPAUTHError,e: -## #print str(e) -## return None - -## def get_user(self,user_id): -## try: -## return User.objects.get(pk=user_id) -## except User.DoesNotExist, e: -## print str(e) -## return None - - -class NISBackend(object): - - def authenticate(self, username=None, password=None): - try: - n = nisauth(username, password) - temp_pass = User.objects.make_random_password(100) - nis_user = dict(username=username, - ) - - user_session_obj = dict( - email = username + "@mcs.anl.gov", - first_name = None, - last_name = None, - uid = n.uid - ) - user, created = User.objects.get_or_create(username=username) - - return user - - except NISAUTHError: - e = sys.exc_info()[1] - return None - - - def get_user(self, user_id): - try: - return User.objects.get(pk=user_id) - except User.DoesNotExist: - e = sys.exc_info()[1] - return None diff --git a/src/lib/Server/Hostbase/hostbase/__init__.py b/src/lib/Server/Hostbase/hostbase/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lib/Server/Hostbase/hostbase/admin.py b/src/lib/Server/Hostbase/hostbase/admin.py deleted file mode 100644 index 70a2233cc..000000000 --- a/src/lib/Server/Hostbase/hostbase/admin.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.contrib import admin - -from models import Host, Interface, IP, MX, Name, CName, Nameserver, ZoneAddress, Zone, Log, ZoneLog - -admin.site.register(Host) -admin.site.register(Interface) -admin.site.register(IP) -admin.site.register(MX) -admin.site.register(Name) -admin.site.register(CName) -admin.site.register(Nameserver) -admin.site.register(ZoneAddress) -admin.site.register(Zone) -admin.site.register(Log) -admin.site.register(ZoneLog) diff --git a/src/lib/Server/Hostbase/hostbase/models.py b/src/lib/Server/Hostbase/hostbase/models.py deleted file mode 100644 index 3f08a09a0..000000000 --- a/src/lib/Server/Hostbase/hostbase/models.py +++ /dev/null @@ -1,210 +0,0 @@ -from django.db import models - -# Create your models here. -class Host(models.Model): - NETGROUP_CHOICES = ( - ('none', 'none'),('cave', 'cave'),('ccst', 'ccst'),('mcs', 'mcs'), - ('mmlab', 'mmlab'),('sp', 'sp'),('red', 'red'),('virtual', 'virtual'), - ('win', 'win'),('xterm', 'xterm'),('lcrc', 'lcrc'),('anlext', 'anlext'), - ('teragrid', 'teragrid') - ) - STATUS_CHOICES = ( - ('active','active'),('dormant','dormant') - ) - SUPPORT_CHOICES = ( - ('green','green'),('yellow','yellow'),('red','red') - ) - CLASS_CHOICES = ( - ('scientific','scientific'), - ('operations','operations'),('guest','guest'), - ('confidential','confidential'),('public','public') - ) - WHATAMI_CHOICES = ( - ('aix-3', 'aix-3'), ('aix-4', 'aix-4'), - ('aix-5', 'aix-5'), ('baytech', 'baytech'), - ('decserver', 'decserver'), ('dialup', 'dialup'), - ('dos', 'dos'), ('freebsd', 'freebsd'), - ('hpux', 'hpux'), ('irix-5', 'irix-5'), - ('irix-6', 'irix-6'), ('linux', 'linux'), - ('linux-2', 'linux-2'), ('linux-rh73', 'linux-rh73'), - ('linux-rh8', 'linux-rh8'), ('linux-sles8', 'linux-sles8'), - ('linux-sles8-64', 'linux-sles8-64'), ('linux-sles8-ia32', 'linux-sles8-ia32'), - ('linux-sles8-ia64', 'linux-sles8-ia64'), ('mac', 'mac'), - ('network', 'network'), ('next', 'next'), - ('none', 'none'), ('osf', 'osf'), ('printer', 'printer'), - ('robot', 'robot'), ('solaris-2', 'solaris-2'), - ('sun4', 'sun4'), ('unknown', 'unknown'), ('virtual', 'virtual'), - ('win31', 'win31'), ('win95', 'win95'), - ('winNTs', 'winNTs'), ('winNTw', 'winNTw'), - ('win2k', 'win2k'), ('winXP', 'winXP'), ('xterm', 'xterm') - ) - hostname = models.CharField(max_length=64) - whatami = models.CharField(max_length=16) - netgroup = models.CharField(max_length=32, choices=NETGROUP_CHOICES) - security_class = models.CharField('class', max_length=16) - support = models.CharField(max_length=8, choices=SUPPORT_CHOICES) - csi = models.CharField(max_length=32, blank=True) - printq = models.CharField(max_length=32, blank=True) - outbound_smtp = models.BooleanField() - primary_user = models.EmailField() - administrator = models.EmailField(blank=True) - location = models.CharField(max_length=16) - comments = models.TextField(blank=True) - expiration_date = models.DateField(null=True, blank=True) - last = models.DateField(auto_now=True, auto_now_add=True) - status = models.CharField(max_length=7, choices=STATUS_CHOICES) - dirty = models.BooleanField() - - class Admin: - list_display = ('hostname', 'last') - search_fields = ['hostname'] - - def __str__(self): - return self.hostname - - def get_logs(self): - """ - Get host's log. - """ - return Log.objects.filter(hostname=self.hostname) - -class Interface(models.Model): - TYPE_CHOICES = ( - ('eth', 'ethernet'), ('wl', 'wireless'), ('virtual', 'virtual'), ('myr', 'myr'), - ('mgmt', 'mgmt'), ('tape', 'tape'), ('fe', 'fe'), ('ge', 'ge'), - ) - # FIXME: The new admin interface has change a lot. - #host = models.ForeignKey(Host, edit_inline=models.TABULAR, num_in_admin=2) - host = models.ForeignKey(Host) - # FIXME: The new admin interface has change a lot. - #mac_addr = models.CharField(max_length=32, core=True) - mac_addr = models.CharField(max_length=32) - hdwr_type = models.CharField('type', max_length=16, choices=TYPE_CHOICES, blank=True) - # FIXME: The new admin interface has change a lot. - # radio_admin=True, blank=True) - dhcp = models.BooleanField() - - def __str__(self): - return self.mac_addr - - class Admin: - list_display = ('mac_addr', 'host') - search_fields = ['mac_addr'] - -class IP(models.Model): - interface = models.ForeignKey(Interface) - # FIXME: The new admin interface has change a lot. - # edit_inline=models.TABULAR, num_in_admin=1) - #ip_addr = models.IPAddressField(core=True) - ip_addr = models.IPAddressField() - - def __str__(self): - return self.ip_addr - - class Admin: - pass - - class Meta: - ordering = ('ip_addr', ) - -class MX(models.Model): - priority = models.IntegerField(blank=True) - # FIXME: The new admin interface has change a lot. - #mx = models.CharField(max_length=64, blank=True, core=True) - mx = models.CharField(max_length=64, blank=True) - - def __str__(self): - return (" ".join([str(self.priority), self.mx])) - - class Admin: - pass - -class Name(models.Model): - DNS_CHOICES = ( - ('global','global'),('internal','ANL internal'), - ('private','private') - ) - # FIXME: The new admin interface has change a lot. - #ip = models.ForeignKey(IP, edit_inline=models.TABULAR, num_in_admin=1) - ip = models.ForeignKey(IP) - # FIXME: The new admin interface has change a lot. - #name = models.CharField(max_length=64, core=True) - name = models.CharField(max_length=64) - dns_view = models.CharField(max_length=16, choices=DNS_CHOICES) - only = models.BooleanField(blank=True) - mxs = models.ManyToManyField(MX) - - def __str__(self): - return self.name - - class Admin: - pass - -class CName(models.Model): - # FIXME: The new admin interface has change a lot. - #name = models.ForeignKey(Name, edit_inline=models.TABULAR, num_in_admin=1) - name = models.ForeignKey(Name) - # FIXME: The new admin interface has change a lot. - #cname = models.CharField(max_length=64, core=True) - cname = models.CharField(max_length=64) - - def __str__(self): - return self.cname - - class Admin: - pass - -class Nameserver(models.Model): - name = models.CharField(max_length=64, blank=True) - - def __str__(self): - return self.name - - class Admin: - pass - -class ZoneAddress(models.Model): - ip_addr = models.IPAddressField(blank=True) - - def __str__(self): - return self.ip_addr - - class Admin: - pass - -class Zone(models.Model): - zone = models.CharField(max_length=64) - serial = models.IntegerField() - admin = models.CharField(max_length=64) - primary_master = models.CharField(max_length=64) - expire = models.IntegerField() - retry = models.IntegerField() - refresh = models.IntegerField() - ttl = models.IntegerField() - nameservers = models.ManyToManyField(Nameserver, blank=True) - mxs = models.ManyToManyField(MX, blank=True) - addresses = models.ManyToManyField(ZoneAddress, blank=True) - aux = models.TextField(blank=True) - - def __str__(self): - return self.zone - - class Admin: - pass - -class Log(models.Model): - # FIXME: Proposal hostname = models.ForeignKey(Host) - hostname = models.CharField(max_length=64) - date = models.DateTimeField(auto_now=True, auto_now_add=True) - log = models.TextField() - - def __str__(self): - return self.hostname - -class ZoneLog(models.Model): - zone = models.CharField(max_length=64) - date = models.DateTimeField(auto_now=True, auto_now_add=True) - log = models.TextField() - - def __str__(self): - return self.zone diff --git a/src/lib/Server/Hostbase/hostbase/sql/zone.sql b/src/lib/Server/Hostbase/hostbase/sql/zone.sql deleted file mode 100644 index b78187ab2..000000000 --- a/src/lib/Server/Hostbase/hostbase/sql/zone.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT INTO hostbase_zone (zone, serial, admin, primary_master, expire, retry, refresh, ttl, aux) -VALUES ('.rev', 0, '', '', 1209600, 1800, 7200, 7200, ''); \ No newline at end of file diff --git a/src/lib/Server/Hostbase/hostbase/urls.py b/src/lib/Server/Hostbase/hostbase/urls.py deleted file mode 100644 index 0ee204abe..000000000 --- a/src/lib/Server/Hostbase/hostbase/urls.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -from django.conf.urls.defaults import * -from django.contrib.auth.decorators import login_required -from django.core.urlresolvers import reverse -from django.views.generic.create_update import create_object, update_object, delete_object -from django.views.generic.list_detail import object_detail, object_list - -from models import Host, Zone, Log - -host_detail_dict = { - 'queryset':Host.objects.all(), - 'template_name':'host.html', - 'template_object_name':'host', -} - -host_delete_dict = { - 'model':Host, - 'post_delete_redirect':'/', -} - -host_log_detail_dict = host_detail_dict.copy() -host_log_detail_dict['template_name'] = 'logviewer.html' - -host_dns_detail_dict = host_detail_dict.copy() -host_dns_detail_dict['template_name'] = 'dns.html' - -zone_new_dict = { - 'model':Zone, - 'template_name':'zonenew.html', - 'post_save_redirect':'../%(id)s', -} - -zones_list_dict = { - 'queryset':Zone.objects.all(), - 'template_name':'zones.html', - 'template_object_name':'zone', -} - -zone_detail_dict = { - 'queryset':Zone.objects.all(), - 'template_name':'zoneview.html', - 'template_object_name':'zone', -} - -urlpatterns = patterns('', - (r'^(?P\d+)/$', object_detail, host_detail_dict, 'host_detail'), - (r'^zones/new/$', login_required(create_object), zone_new_dict, 'zone_new'), - (r'^zones/(?P\d+)/edit', login_required(update_object), zone_new_dict, 'zone_edit'), - (r'^zones/$', object_list, zones_list_dict, 'zone_list'), - (r'^zones/(?P\d+)/$', object_detail, zone_detail_dict, 'zone_detail'), - (r'^zones/(?P\d+)/$', object_detail, zone_detail_dict, 'zone_detail'), - (r'^\d+/logs/(?P\d+)/', object_detail, { 'queryset':Log.objects.all() }, 'log_detail'), - (r'^(?P\d+)/logs/', object_detail, host_log_detail_dict, 'host_log_list'), - (r'^(?P\d+)/dns', object_detail, host_dns_detail_dict, 'host_dns_list'), - (r'^(?P\d+)/remove', login_required(delete_object), host_delete_dict, 'host_delete'), -) - -urlpatterns += patterns('Bcfg2.Server.Hostbase.hostbase.views', - (r'^$', 'search'), - (r'^(?P\d+)/edit', 'edit'), - (r'^(?P\d+)/(?P\D+)/(?P\d+)/confirm', 'confirm'), - (r'^(?P\d+)/(?P\D+)/(?P\d+)/(?P\d+)/confirm', 'confirm'), - (r'^(?P\d+)/dns/edit', 'dnsedit'), - (r'^new', 'new'), - (r'^(?P\d+)/copy', 'copy'), -# (r'^hostinfo', 'hostinfo'), - (r'^zones/(?P\d+)/(?P\D+)/(?P\d+)/confirm', 'confirm'), -) diff --git a/src/lib/Server/Hostbase/hostbase/views.py b/src/lib/Server/Hostbase/hostbase/views.py deleted file mode 100644 index 57ef5eff8..000000000 --- a/src/lib/Server/Hostbase/hostbase/views.py +++ /dev/null @@ -1,970 +0,0 @@ -"""Views.py -Contains all the views associated with the hostbase app -Also has does form validation -""" -from django.http import HttpResponse, HttpResponseRedirect - -from django.contrib.auth.decorators import login_required -from django.contrib.auth import logout -from django.template import RequestContext -from Bcfg2.Server.Hostbase.hostbase.models import * -from datetime import date -from django.db import connection -from django.shortcuts import render_to_response -from django import forms -from Bcfg2.Server.Hostbase import settings, regex -import re, copy - -attribs = ['hostname', 'whatami', 'netgroup', 'security_class', 'support', - 'csi', 'printq', 'primary_user', 'administrator', 'location', - 'status', 'comments'] - -zoneattribs = ['zone', 'admin', 'primary_master', 'expire', 'retry', - 'refresh', 'ttl', 'aux'] - -dispatch = {'mac_addr':'i.mac_addr LIKE \'%%%%%s%%%%\'', - 'ip_addr':'p.ip_addr LIKE \'%%%%%s%%%%\'', - 'name':'n.name LIKE \'%%%%%s%%%%\'', -## 'hostname':'n.name LIKE \'%%%%%s%%%%\'', -## 'cname':'n.name LIKE \'%%%%%s%%%%\'', - 'mx':'m.mx LIKE \'%%%%%s%%%%\'', - 'dns_view':'n.dns_view = \'%s\'', - 'hdwr_type':'i.hdwr_type = \'%s\'', - 'dhcp':'i.dhcp = \'%s\''} - -def search(request): - """Search for hosts in the database - If more than one field is entered, logical AND is used - """ - if 'sub' in request.GET: - querystring = """SELECT DISTINCT h.hostname, h.id, h.status - FROM (((((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE """ - - _and = False - for field in request.POST: - if request.POST[field] and field == 'hostname': - if _and: - querystring += ' AND ' - querystring += 'n.name LIKE \'%%%%%s%%%%\' or c.cname LIKE \'%%%%%s%%%%\'' % (request.POST[field], request.POST[field]) - _and = True - elif request.POST[field] and field in dispatch: - if _and: - querystring += ' AND ' - querystring += dispatch[field] % request.POST[field] - _and = True - elif request.POST[field]: - if _and: - querystring += ' AND ' - querystring += "h.%s LIKE \'%%%%%s%%%%\'" % (field, request.POST[field]) - _and = True - - if not _and: - cursor = connection.cursor() - cursor.execute("""SELECT hostname, id, status - FROM hostbase_host ORDER BY hostname""") - results = cursor.fetchall() - else: - querystring += " ORDER BY h.hostname" - cursor = connection.cursor() - cursor.execute(querystring) - results = cursor.fetchall() - - return render_to_response('results.html', - {'hosts': results, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - else: - return render_to_response('search.html', - {'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'DNS_CHOICES': Name.DNS_CHOICES, - 'yesno': [(1, 'yes'), (0, 'no')], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - - -def gethostdata(host_id, dnsdata=False): - """Grabs the necessary data about a host - Replaces a lot of repeated code""" - hostdata = {} - hostdata['ips'] = {} - hostdata['names'] = {} - hostdata['cnames'] = {} - hostdata['mxs'] = {} - hostdata['host'] = Host.objects.get(id=host_id) - hostdata['interfaces'] = hostdata['host'].interface_set.all() - for interface in hostdata['interfaces']: - hostdata['ips'][interface.id] = interface.ip_set.all() - if dnsdata: - for ip in hostdata['ips'][interface.id]: - hostdata['names'][ip.id] = ip.name_set.all() - for name in hostdata['names'][ip.id]: - hostdata['cnames'][name.id] = name.cname_set.all() - hostdata['mxs'][name.id] = name.mxs.all() - return hostdata - -def fill(template, hostdata, dnsdata=False): - """Fills a generic template - Replaces a lot of repeated code""" - if dnsdata: - template.names = hostdata['names'] - template.cnames = hostdata['cnames'] - template.mxs = hostdata['mxs'] - template.host = hostdata['host'] - template.interfaces = hostdata['interfaces'] - template.ips = hostdata['ips'] - return template - -def edit(request, host_id): - """edit general host information""" - manipulator = Host.ChangeManipulator(host_id) - changename = False - if request.method == 'POST': - host = Host.objects.get(id=host_id) - before = host.__dict__.copy() - if request.POST['hostname'] != host.hostname: - oldhostname = host.hostname.split(".")[0] - changename = True - interfaces = host.interface_set.all() - old_interfaces = [interface.__dict__.copy() for interface in interfaces] - - new_data = request.POST.copy() - - errors = manipulator.get_validation_errors(new_data) - if not errors: - - # somehow keep track of multiple interface change manipulators - # as well as multiple ip chnage manipulators??? (add manipulators???) - # change to many-to-many?????? - - # dynamically look up mx records? - text = '' - - for attrib in attribs: - if host.__dict__[attrib] != request.POST[attrib]: - text = do_log(text, attrib, host.__dict__[attrib], request.POST[attrib]) - host.__dict__[attrib] = request.POST[attrib] - - if 'expiration_date' in request.POST: - ymd = request.POST['expiration_date'].split("-") - if date(int(ymd[0]), int(ymd[1]), int(ymd[2])) != host.__dict__['expiration_date']: - text = do_log(text, 'expiration_date', host.__dict__['expiration_date'], - request.POST['expiration_date']) - host.__dict__['expiration_date'] = date(int(ymd[0]), int(ymd[1]), int(ymd[2])) - - for inter in interfaces: - changetype = False - ips = IP.objects.filter(interface=inter.id) - if inter.mac_addr != request.POST['mac_addr%d' % inter.id]: - text = do_log(text, 'mac_addr', inter.mac_addr, request.POST['mac_addr%d' % inter.id]) - inter.mac_addr = request.POST['mac_addr%d' % inter.id].lower().replace('-',':') - if inter.hdwr_type != request.POST['hdwr_type%d' % inter.id]: - oldtype = inter.hdwr_type - text = do_log(text, 'hdwr_type', oldtype, request.POST['hdwr_type%d' % inter.id]) - inter.hdwr_type = request.POST['hdwr_type%d' % inter.id] - changetype = True - if (('dhcp%d' % inter.id) in request.POST and not inter.dhcp or - not ('dhcp%d' % inter.id) in request.POST and inter.dhcp): - text = do_log(text, 'dhcp', inter.dhcp, int(not inter.dhcp)) - inter.dhcp = not inter.dhcp - for ip in ips: - names = ip.name_set.all() - if not ip.ip_addr == request.POST['ip_addr%d' % ip.id]: - oldip = ip.ip_addr - oldsubnet = oldip.split(".")[2] - ip.ip_addr = request.POST['ip_addr%d' % ip.id] - ip.save() - text = do_log(text, 'ip_addr', oldip, ip.ip_addr) - for name in names: - if name.name.split(".")[0].endswith('-%s' % oldsubnet): - name.name = name.name.replace('-%s' % oldsubnet, '-%s' % ip.ip_addr.split(".")[2]) - name.save() - if changetype: - for name in names: - if name.name.split(".")[0].endswith('-%s' % oldtype): - name.name = name.name.replace('-%s' % oldtype, '-%s' % inter.hdwr_type) - name.save() - if changename: - for name in names: - if name.name.startswith(oldhostname): - name.name = name.name.replace(oldhostname, host.hostname.split(".")[0]) - name.save() - if request.POST['%dip_addr' % inter.id]: - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_ip = IP(interface=inter, ip_addr=request.POST['%dip_addr' % inter.id]) - new_ip.save() - text = do_log(text, '*new*', 'ip_addr', new_ip.ip_addr) - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - inter.save() - if request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr=request.POST['mac_addr_new'].lower().replace('-',':'), - hdwr_type=request.POST['hdwr_type_new'], - dhcp=request.POST['dhcp_new']) - text = do_log(text, '*new*', 'mac_addr', new_inter.mac_addr) - new_inter.save() - if request.POST['mac_addr_new'] and request.POST['ip_addr_new']: - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - text = do_log(text, '*new*', 'ip_addr', new_ip.ip_addr) - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new'] and not request.POST['mac_addr_new']: - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_inter = Interface(host=host, mac_addr="", - hdwr_type=request.POST['hdwr_type_new'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - text = do_log(text, '*new*', 'ip_addr', new_ip.ip_addr) - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if text: - log = Log(hostname=host.hostname, log=text) - log.save() - host.save() - return HttpResponseRedirect('/hostbase/%s/' % host.id) - else: - return render_to_response('errors.html', - {'failures': errors, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - else: - host = Host.objects.get(id=host_id) - interfaces = [] - for interface in host.interface_set.all(): - interfaces.append([interface, interface.ip_set.all()]) - return render_to_response('edit.html', - {'host': host, - 'interfaces': interfaces, - 'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def confirm(request, item, item_id, host_id=None, name_id=None, zone_id=None): - """Asks if the user is sure he/she wants to remove an item""" - if 'sub' in request.GET: - if item == 'interface': - for ip in Interface.objects.get(id=item_id).ip_set.all(): - for name in ip.name_set.all(): - name.cname_set.all().delete() - ip.name_set.all().delete() - Interface.objects.get(id=item_id).ip_set.all().delete() - Interface.objects.get(id=item_id).delete() - elif item=='ip': - for name in IP.objects.get(id=item_id).name_set.all(): - name.cname_set.all().delete() - IP.objects.get(id=item_id).name_set.all().delete() - IP.objects.get(id=item_id).delete() - elif item=='cname': - CName.objects.get(id=item_id).delete() - elif item=='mx': - mx = MX.objects.get(id=item_id) - Name.objects.get(id=name_id).mxs.remove(mx) - elif item=='name': - Name.objects.get(id=item_id).cname_set.all().delete() - Name.objects.get(id=item_id).delete() - elif item=='nameserver': - nameserver = Nameserver.objects.get(id=item_id) - Zone.objects.get(id=zone_id).nameservers.remove(nameserver) - elif item=='zonemx': - mx = MX.objects.get(id=item_id) - Zone.objects.get(id=zone_id).mxs.remove(mx) - elif item=='address': - address = ZoneAddress.objects.get(id=item_id) - Zone.objects.get(id=zone_id).addresses.remove(address) - if item == 'cname' or item == 'mx' or item == 'name': - return HttpResponseRedirect('/hostbase/%s/dns/edit' % host_id) - elif item == 'nameserver' or item == 'zonemx' or item == 'address': - return HttpResponseRedirect('/hostbase/zones/%s/edit' % zone_id) - else: - return HttpResponseRedirect('/hostbase/%s/edit' % host_id) - else: - interface = None - ips = [] - names = [] - cnames = [] - mxs = [] - zonemx = None - nameserver = None - address = None - if item == 'interface': - interface = Interface.objects.get(id=item_id) - ips = interface.ip_set.all() - for ip in ips: - for name in ip.name_set.all(): - names.append((ip.id, name)) - for cname in name.cname_set.all(): - cnames.append((name.id, cname)) - for mx in name.mxs.all(): - mxs.append((name.id, mx)) - elif item=='ip': - ips = [IP.objects.get(id=item_id)] - for name in ips[0].name_set.all(): - names.append((ips[0].id, name)) - for cname in name.cname_set.all(): - cnames.append((name.id, cname)) - for mx in name.mxs.all(): - mxs.append((name.id, mx)) - elif item=='name': - names = [Name.objects.get(id=item_id)] - for cname in names[0].cname_set.all(): - cnames.append((names[0].id, cname)) - for mx in names[0].mxs.all(): - mxs.append((names[0].id, mx)) - elif item=='cname': - cnames = [CName.objects.get(id=item_id)] - elif item=='mx': - mxs = [MX.objects.get(id=item_id)] - elif item=='zonemx': - zonemx = MX.objects.get(id=item_id) - elif item=='nameserver': - nameserver = Nameserver.objects.get(id=item_id) - elif item=='address': - address = ZoneAddress.objects.get(id=item_id) - return render_to_response('confirm.html', - {'interface': interface, - 'ips': ips, - 'names': names, - 'cnames': cnames, - 'id': item_id, - 'type': item, - 'host_id': host_id, - 'mxs': mxs, - 'zonemx': zonemx, - 'nameserver': nameserver, - 'address': address, - 'zone_id': zone_id, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def dnsedit(request, host_id): - """Edits specific DNS information - Data is validated before committed to the database""" - text = '' - if 'sub' in request.GET: - hostdata = gethostdata(host_id, True) - for ip in hostdata['names']: - ipaddr = IP.objects.get(id=ip) - ipaddrstr = ipaddr.__str__() - for name in hostdata['cnames']: - for cname in hostdata['cnames'][name]: - if regex.host.match(request.POST['cname%d' % cname.id]): - text = do_log(text, 'cname', cname.cname, request.POST['cname%d' % cname.id]) - cname.cname = request.POST['cname%d' % cname.id] - cname.save() - for name in hostdata['mxs']: - for mx in hostdata['mxs'][name]: - if (mx.priority != request.POST['priority%d' % mx.id] and mx.mx != request.POST['mx%d' % mx.id]): - text = do_log(text, 'mx', ' '.join([str(mx.priority), str(mx.mx)]), - ' '.join([request.POST['priority%d' % mx.id], request.POST['mx%d' % mx.id]])) - nameobject = Name.objects.get(id=name) - nameobject.mxs.remove(mx) - newmx, created = MX.objects.get_or_create(priority=request.POST['priority%d' % mx.id], mx=request.POST['mx%d' % mx.id]) - if created: - newmx.save() - nameobject.mxs.add(newmx) - nameobject.save() - for name in hostdata['names'][ip]: - name.name = request.POST['name%d' % name.id] - name.dns_view = request.POST['dns_view%d' % name.id] - if (request.POST['%dcname' % name.id] and - regex.host.match(request.POST['%dcname' % name.id])): - cname = CName(name=name, - cname=request.POST['%dcname' % name.id]) - text = do_log(text, '*new*', 'cname', cname.cname) - cname.save() - if (request.POST['%dpriority' % name.id] and - request.POST['%dmx' % name.id]): - mx, created = MX.objects.get_or_create(priority=request.POST['%dpriority' % name.id], - mx=request.POST['%dmx' % name.id]) - if created: - mx.save() - text = do_log(text, '*new*', 'mx', - ' '.join([request.POST['%dpriority' % name.id], - request.POST['%dmx' % name.id]])) - name.mxs.add(mx) - name.save() - if request.POST['%sname' % ipaddrstr]: - name = Name(ip=ipaddr, - dns_view=request.POST['%sdns_view' % ipaddrstr], - name=request.POST['%sname' % ipaddrstr], only=False) - text = do_log(text, '*new*', 'name', name.name) - name.save() - if (request.POST['%scname' % ipaddrstr] and - regex.host.match(request.POST['%scname' % ipaddrstr])): - cname = CName(name=name, - cname=request.POST['%scname' % ipaddrstr]) - text = do_log(text, '*new*', 'cname', cname.cname) - cname.save() - if (request.POST['%smx' % ipaddrstr] and - request.POST['%spriority' % ipaddrstr]): - mx, created = MX.objects.get_or_create(priority=request.POST['%spriority' % ipaddrstr], - mx=request.POST['%smx' % ipaddrstr]) - if created: - mx.save() - text = do_log(text, '*new*', 'mx', - ' '.join([request.POST['%spriority' % ipaddrstr], request.POST['%smx' % ipaddrstr]])) - name.mxs.add(mx) - if text: - log = Log(hostname=hostdata['host'].hostname, log=text) - log.save() - return HttpResponseRedirect('/hostbase/%s/dns' % host_id) - else: - host = Host.objects.get(id=host_id) - ips = [] - info = [] - cnames = [] - mxs = [] - interfaces = host.interface_set.all() - for interface in host.interface_set.all(): - ips.extend(interface.ip_set.all()) - for ip in ips: - info.append([ip, ip.name_set.all()]) - for name in ip.name_set.all(): - cnames.extend(name.cname_set.all()) - mxs.append((name.id, name.mxs.all())) - return render_to_response('dnsedit.html', - {'host': host, - 'info': info, - 'cnames': cnames, - 'mxs': mxs, - 'request': request, - 'interfaces': interfaces, - 'DNS_CHOICES': Name.DNS_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def new(request): - """Function for creating a new host in hostbase - Data is validated before committed to the database""" - if 'sub' in request.GET: - try: - Host.objects.get(hostname=request.POST['hostname'].lower()) - return render_to_response('errors.html', - {'failures': ['%s already exists in hostbase' % request.POST['hostname']], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - except: - pass - if not validate(request, True): - if not request.POST['ip_addr_new'] and not request.POST['ip_addr_new2']: - return render_to_response('errors.html', - {'failures': ['ip_addr: You must enter an ip address'], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - host = Host() - # this is the stuff that validate() should take care of - # examine the check boxes for any changes - host.outbound_smtp = 'outbound_smtp' in request.POST - for attrib in attribs: - if attrib in request.POST: - host.__dict__[attrib] = request.POST[attrib].lower() - if 'comments' in request.POST: - host.comments = request.POST['comments'] - if 'expiration_date' in request.POST: -# ymd = request.POST['expiration_date'].split("-") -# host.__dict__['expiration_date'] = date(int(ymd[0]), int(ymd[1]), int(ymd[2])) - host.__dict__['expiration_date'] = date(2000, 1, 1) - host.status = 'active' - host.save() - else: - return render_to_response('errors.html', - {'failures': validate(request, True), - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - - if request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new'], - dhcp = 'dhcp_new' in request.POST) - new_inter.save() - if request.POST['mac_addr_new'] and request.POST['ip_addr_new']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) -# Change all this things. Use a "post_save" signal handler for model Host to create all sociate models -# and use a generi view. - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new'] and not request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new2'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new2'], - dhcp = 'dhcp_new2' in request.POST) - new_inter.save() - if request.POST['mac_addr_new2'] and request.POST['ip_addr_new2']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new2'] and not request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new2'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - host.save() - return HttpResponseRedirect('/hostbase/%s/' % host.id) - else: - return render_to_response('new.html', - {'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'NETGROUP_CHOICES': Host.NETGROUP_CHOICES, - 'CLASS_CHOICES': Host.CLASS_CHOICES, - 'SUPPORT_CHOICES': Host.SUPPORT_CHOICES, - 'WHATAMI_CHOICES': Host.WHATAMI_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -def copy(request, host_id): - """Function for creating a new host in hostbase - Data is validated before committed to the database""" - if 'sub' in request.GET: - try: - Host.objects.get(hostname=request.POST['hostname'].lower()) - return render_to_response('errors.html', - {'failures': ['%s already exists in hostbase' % request.POST['hostname']], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - except: - pass - if not validate(request, True): - if not request.POST['ip_addr_new'] and not request.POST['ip_addr_new2']: - return render_to_response('errors.html', - {'failures': ['ip_addr: You must enter an ip address'], - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - host = Host() - # this is the stuff that validate() should take care of - # examine the check boxes for any changes - host.outbound_smtp = 'outbound_smtp' in request.POST - for attrib in attribs: - if attrib in request.POST: - host.__dict__[attrib] = request.POST[attrib].lower() - if 'comments' in request.POST: - host.comments = request.POST['comments'] - if 'expiration_date' in request.POST: -# ymd = request.POST['expiration_date'].split("-") -# host.__dict__['expiration_date'] = date(int(ymd[0]), int(ymd[1]), int(ymd[2])) - host.__dict__['expiration_date'] = date(2000, 1, 1) - host.status = 'active' - host.save() - else: - return render_to_response('errors.html', - {'failures': validate(request, True), - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - - if request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new'], - dhcp = 'dhcp_new' in request.POST) - new_inter.save() - if request.POST['mac_addr_new'] and request.POST['ip_addr_new']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new'] and not request.POST['mac_addr_new']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr = request.POST['mac_addr_new2'].lower().replace('-',':'), - hdwr_type = request.POST['hdwr_type_new2'], - dhcp = 'dhcp_new2' in request.POST) - new_inter.save() - if request.POST['mac_addr_new2'] and request.POST['ip_addr_new2']: - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - if request.POST['ip_addr_new2'] and not request.POST['mac_addr_new2']: - new_inter = Interface(host=host, - mac_addr="", - hdwr_type=request.POST['hdwr_type_new2'], - dhcp=False) - new_inter.save() - new_ip = IP(interface=new_inter, ip_addr=request.POST['ip_addr_new2']) - new_ip.save() - mx, created = MX.objects.get_or_create(priority=settings.PRIORITY, mx=settings.DEFAULT_MX) - if created: - mx.save() - new_name = "-".join([host.hostname.split(".")[0], - new_ip.ip_addr.split(".")[2]]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - new_name = "-".join([host.hostname.split(".")[0], - new_inter.hdwr_type]) - new_name += "." + host.hostname.split(".", 1)[1] - name = Name(ip=new_ip, name=new_name, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - name = Name(ip=new_ip, name=host.hostname, - dns_view='global', only=False) - name.save() - name.mxs.add(mx) - host.save() - return HttpResponseRedirect('/hostbase/%s/' % host.id) - else: - host = Host.objects.get(id=host_id) - return render_to_response('copy.html', - {'host': host, - 'TYPE_CHOICES': Interface.TYPE_CHOICES, - 'NETGROUP_CHOICES': Host.NETGROUP_CHOICES, - 'CLASS_CHOICES': Host.CLASS_CHOICES, - 'SUPPORT_CHOICES': Host.SUPPORT_CHOICES, - 'WHATAMI_CHOICES': Host.WHATAMI_CHOICES, - 'logged_in': request.session.get('_auth_user_id', False)}, - context_instance = RequestContext(request)) - -# FIXME: delete all this things in a signal handler "pre_delete" -#def remove(request, host_id): -# host = Host.objects.get(id=host_id) -# if 'sub' in request: -# for interface in host.interface_set.all(): -# for ip in interface.ip_set.all(): -# for name in ip.name_set.all(): -# name.cname_set.all().delete() -# ip.name_set.all().delete() -# interface.ip_set.all().delete() -# interface.delete() -# host.delete() - -def validate(request, new=False, host_id=None): - """Function for checking form data""" - failures = [] - if (request.POST['expiration_date'] - and regex.date.match(request.POST['expiration_date'])): - try: - (year, month, day) = request.POST['expiration_date'].split("-") - date(int(year), int(month), int(day)) - except (ValueError): - failures.append('expiration_date') - elif request.POST['expiration_date']: - failures.append('expiration_date') - - if not (request.POST['hostname'] - and regex.host.match(request.POST['hostname'])): - failures.append('hostname') - -## if not regex.printq.match(request.POST['printq']) and request.POST['printq']: -## failures.append('printq') - -## if not regex.user.match(request.POST['primary_user']): -## failures.append('primary_user') - -## if (not regex.user.match(request.POST['administrator']) -## and request.POST['administrator']): -## failures.append('administrator') - -## if not (request.POST['location'] -## and regex.location.match(request.POST['location'])): -## failures.append('location') - - if new: - if (not regex.macaddr.match(request.POST['mac_addr_new']) - and request.POST['mac_addr_new']): - failures.append('mac_addr (#1)') - if ((request.POST['mac_addr_new'] or request.POST['ip_addr_new']) and - not 'hdwr_type_new' in request.REQUEST): - failures.append('hdwr_type (#1)') - if ((request.POST['mac_addr_new2'] or request.POST['ip_addr_new2']) and - not 'hdwr_type_new2' in request.REQUEST): - failures.append('hdwr_type (#2)') - - if (not regex.macaddr.match(request.POST['mac_addr_new2']) - and request.POST['mac_addr_new2']): - failures.append('mac_addr (#2)') - - if (not regex.ipaddr.match(request.POST['ip_addr_new']) - and request.POST['ip_addr_new']): - failures.append('ip_addr (#1)') - if (not regex. ipaddr.match(request.POST['ip_addr_new2']) - and request.POST['ip_addr_new2']): - failures.append('ip_addr (#2)') - - [failures.append('ip_addr (#1)') for number in - request.POST['ip_addr_new'].split(".") - if number.isdigit() and int(number) > 255 - and 'ip_addr (#1)' not in failures] - [failures.append('ip_addr (#2)') for number in - request.POST['ip_addr_new2'].split(".") - if number.isdigit() and int(number) > 255 - and 'ip_addr (#2)' not in failures] - - elif host_id: - interfaces = Interface.objects.filter(host=host_id) - for interface in interfaces: - if (not regex.macaddr.match(request.POST['mac_addr%d' % interface.id]) - and request.POST['mac_addr%d' % interface.id]): - failures.append('mac_addr (%s)' % request.POST['mac_addr%d' % interface.id]) - for ip in interface.ip_set.all(): - if not regex.ipaddr.match(request.POST['ip_addr%d' % ip.id]): - failures.append('ip_addr (%s)' % request.POST['ip_addr%d' % ip.id]) - [failures.append('ip_addr (%s)' % request.POST['ip_addr%d' % ip.id]) - for number in request.POST['ip_addr%d' % ip.id].split(".") - if (number.isdigit() and int(number) > 255 and - 'ip_addr (%s)' % request.POST['ip_addr%d' % ip.id] not in failures)] - if (request.POST['%dip_addr' % interface.id] - and not regex.ipaddr.match(request.POST['%dip_addr' % interface.id])): - failures.append('ip_addr (%s)' % request.POST['%dip_addr' % interface.id]) - if (request.POST['mac_addr_new'] - and not regex.macaddr.match(request.POST['mac_addr_new'])): - failures.append('mac_addr (%s)' % request.POST['mac_addr_new']) - if (request.POST['ip_addr_new'] - and not regex.ipaddr.match(request.POST['ip_addr_new'])): - failures.append('ip_addr (%s)' % request.POST['ip_addr_new']) - - if not failures: - return 0 - return failures - -def do_log(text, attribute, previous, new): - if previous != new: - text += "%-20s%-20s -> %s\n" % (attribute, previous, new) - return text - -## login required stuff -## uncomment the views below that you would like to restrict access to - -## uncomment the lines below this point to restrict access to pages that modify the database -## anonymous users can still view data in Hostbase - -edit = login_required(edit) -confirm = login_required(confirm) -dnsedit = login_required(dnsedit) -new = login_required(new) -copy = login_required(copy) -#remove = login_required(remove) -#zoneedit = login_required(zoneedit) -#zonenew = login_required(zonenew) - -## uncomment the lines below this point to restrict access to all of hostbase - -## search = login_required(search) -## look = login_required(look) -## dns = login_required(dns) -## zones = login_required(zones) -## zoneview = login_required(zoneview) - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/base.html b/src/lib/Server/Hostbase/hostbase/webtemplates/base.html deleted file mode 100644 index 1d7c5565b..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/base.html +++ /dev/null @@ -1,34 +0,0 @@ - - - - {% block title %}BCFG2 - Hostbase{% endblock %} - - - - {% block extra_header_info %}{% endblock %} - - - - - - -
-
- {% block pagebanner %}{% endblock %} - {% block content %}{% endblock %} - -
-
- - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html b/src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html deleted file mode 100644 index ca8b0cc07..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/confirm.html +++ /dev/null @@ -1,117 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Confirm Removal

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -
-Are you sure you want to remove these items? - -{% if interface %} -
    -
  • interface: {{ interface.mac_addr }}
    • -{% endif %} - - -{% if ips %} -
      -{% for ip in ips %} -
    • ip: {{ ip.ip_addr }}
      • -
        -{% for name in names %} -{% ifequal name.0 ip.id %} -
      • name: {{ name.1.name }}
        • -
          -{% endifequal %} -{% for cname in cnames %} -{% ifequal cname.0 name.1.id %} -
        • cname: {{ cname.1.name }}
          • -{% endifequal %} -{% endfor %} -
        -
          -{% for mx in mxs %} -{% ifequal mx.0 name.1.id %} -
        • mx: {{ mx.1.priority }} {{ mx.1.mx }}
          • -{% endifequal %} -{% endfor %} -
        -{% endfor %} -
      -{% endfor %} -
    -{% endif %} - -{% if names and not ips %} -
      -{% for name in names %} -
    • name: {{ name.name }}
      • -
        -{% for cname in cnames %} -{% ifequal cname.0 name.id %} -
      • cname: {{ cname.1.cname }}
        • -{% endifequal %} -{% endfor %} -
      -
        -{% for mx in mxs %} -{% ifequal mx.0 name.id %} -
      • mx: {{ mx.1.priority }} {{ mx.1.mx }}
        • -{% endifequal %} -{% endfor %} -
      -{% endfor %} -
    -{% endif %} - -{% if cnames and not names %} -
      -{% for cname in cnames %} -
    • cname: {{ cname.cname }}
      • -{% endfor %} -
    -{% endif %} - -{% if mxs and not names %} -
      -{% for mx in mxs %} -
    • mx: {{ mx.priority }} {{ mx.mx }}
      • -{% endfor %} -
    -{% endif %} - -{% if interface %} -
-{% endif %} - -{% if zone_id %} -
    -{% ifequal type 'zonemx' %} -
  • mx: {{ zonemx.priority }} {{ zonemx.mx }}
    • -{% endifequal %} - -{% ifequal type 'nameserver' %} -
  • nameserver: {{ nameserver.name }}
    • -{% endifequal %} - -{% ifequal type 'address' %} -
  • address: {{ address.ip_addr }}
    • -{% endifequal %} -
-{% endif %} - - - -
- -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/copy.html b/src/lib/Server/Hostbase/hostbase/webtemplates/copy.html deleted file mode 100644 index 400ef58f2..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/copy.html +++ /dev/null @@ -1,122 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

new host information

-
-
-{% endblock %} - -{% block sidebar %} -search hostbase -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -
- - - - - - - - - - - - - - - - - - - - - - {% else %} - - {% endif %} - - - - - - - - - - - - - - - - - - - - - - - - -
hostname
whatami - -
netgroup - -
class -
support -
csi
printq
outbound_smtp - {% if host.outbound_smtp %} -
primary_user (email address)
administrator (email address)
location
expiration_date YYYY-MM-DD

Interface
- {% for choice in TYPE_CHOICES %} - {{ choice.1 }} - {% endfor %} -
dhcp -
mac_addr
ip_addr

Interface
- {% for choice in TYPE_CHOICES %} - {{ choice.1 }} - {% endfor %} -
dhcp -
mac_addr
ip_addr
comments
-
-

-

- -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/dns.html b/src/lib/Server/Hostbase/hostbase/webtemplates/dns.html deleted file mode 100644 index da179e5a1..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/dns.html +++ /dev/null @@ -1,40 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

dns info for {{ host.hostname }}

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% for interface in host.interface_set.all %} - {% for ip in interface.ip_set.all %} -
  • ip_addr: {{ ip.ip_addr }}
    • - {% for name in ip.name_set.all %} -
    • name: {{ name.name }}
        • - {% for cname in name.cname_set.all %} -
      • cname: {{ cname.cname }}
        • - {% endfor %} - {% for mx in name.mxs.all %} -
      • mx: {{ mx.priority }} {{ mx.mx }}
        • - {% endfor %} -
    - {% endfor %} -
- {% endfor %} -{% endfor %} -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html b/src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html deleted file mode 100644 index 18bd7ab83..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/dnsedit.html +++ /dev/null @@ -1,98 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

dns info for {{ host.hostname }}

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -
- - - - - - {% for interface in interfaces %} - - - - - - - {% for ip in info %} - {% ifequal ip.0.interface interface %} - - - {% for name in ip.1 %} - - - {% for cname in cnames %} - {% ifequal name cname.name %} - - - {% endifequal %} - {% endfor %} - - - {% for mx in mxs %} - {% ifequal mx.0 name.id %} - {% for record in mx.1 %} - - - {% endfor %} - {% endifequal %} - {% endfor %} - - - {% endfor %} - - - - - - - - - {% endifequal %} - {% endfor %} - {% endfor %} -

interface type {{ interface.hdwr_type }}
mac_addr {{ interface.mac_addr }}
ip_addr{{ ip.0.ip_addr }}
name(dns) - - remove
cname - remove
cname
mx - - remove
mx -
name -
cname
mx -
- -

-

- -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/edit.html b/src/lib/Server/Hostbase/hostbase/webtemplates/edit.html deleted file mode 100644 index 961c9d143..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/edit.html +++ /dev/null @@ -1,191 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

{{ host.hostname }}

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - - - - - -
-
- - -
- -
- -
- -
- -
- -
- -
- - {% if host.outbound_smtp %} - - {% else %} - - {% endif %}
- -
- -
- -
- - YYYY-MM-DD
- {% for interface in interfaces %} - -
- - {% if interface.0.dhcp %} - - {% else %} - - {% endif %}
- - - remove
- {% for ip in interface.1 %} - - - remove
- {% endfor %} - - - -
- -
-
- Add a New IP Address
- {% endfor %} - - - - -
- -
- - {% if host.dhcp %} - - {% else %} - - {% endif %}
- -
- -
-
-Add a New Interface
- - - - -
-edit detailed DNS information for this host -
-this host is -
-last update on {{ host.last }}
- - - - -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/errors.html b/src/lib/Server/Hostbase/hostbase/webtemplates/errors.html deleted file mode 100644 index e5429b86c..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/errors.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Search Results

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% if failures %} -There were errors in the following fields

-{% for failure in failures %} - -{{ failure }}
-{% comment %} -{{ failure.1|join:", " }} -{% endcomment %} - -{% endfor %} -{% endif %} -
-Press the back button on your browser and edit those field(s) - -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/host.html b/src/lib/Server/Hostbase/hostbase/webtemplates/host.html deleted file mode 100644 index d6b8873bc..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/host.html +++ /dev/null @@ -1,80 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

{{ host.hostname }}

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - - - - - - - - - - - - - - - - - - - - - {% if host.outbound_smtp %} - - {% else %} - - {% endif %} - - - - - - - - - {% for interface in host.inserface_set.all %} - - {% ifnotequal interface.0.hdwr_type 'no' %} - - {% endifnotequal %} - {% if interface.0.dhcp %} - - - {% endif %} - {% for ip in interface.1 %} - - - {% endfor %} - {% endfor %} - - - -
hostname {{ host.hostname }}
whatami {{ host.whatami }}
netgroup {{ host.netgroup }}
class {{ host.security_class }}
support {{ host.support }}
csi {{ host.csi }}
printq {{ host.printq }}
outbound_smtp y
n
primary_user {{ host.primary_user }}
administrator {{ host.administrator }}
location {{ host.location }}
expiration_date {{ host.expiration_date }}

Interface
{{ interface.0.hdwr_type }}
mac_addr {{ interface.0.mac_addr }}
ip_addr {{ ip.ip_addr }}
comments - {{ host.comments|linebreaksbr }}
-
-see detailed DNS information for this host -

-this host is {{ host.status }}
-last update on {{ host.last }}
- -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html b/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html deleted file mode 100644 index 551bf3254..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/host_confirm_delete.html +++ /dev/null @@ -1,89 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Are you sure you want to remove {{ object.hostname }}?

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - - - - - - - - - - - - - - - - - - - - - {% if host.dhcp %} - - {% else %} - - {% endif %} - - {% if host.outbound_smtp %} - - {% else %} - - {% endif %} - - - - - - - - - {% for interface in interfaces %} - - {% ifnotequal interface.0.hdwr_type 'no' %} - - {% endifnotequal %} - - - {% for ip in interface.1 %} - - - {% endfor %} - {% endfor %} - - - -
hostname {{ object.hostname }}
whatami {{ object.whatami }}
netgroup {{ object.netgroup }}
class {{ object.security_class }}
support {{ object.support }}
csi {{ object.csi }}
printq {{ object.printq }}
dhcp y
n
outbound_smtp y
n
primary_user {{ object.primary_user }}
administrator {{ object.administrator }}
location {{ object.location }}
expiration_date {{ object.expiration_date }}

Interface
{{ interface.0.hdwr_type }}
mac_addr {{ interface.0.mac_addr }}
ip_addr {{ ip.ip_addr }}
comments - {{ object.comments|linebreaksbr }}
-
-see detailed DNS information for this host -

-this host is {{ object.status }}
-last update on {{ object.last }}
- -
- - -
- -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html b/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html deleted file mode 100644 index aa9679cbd..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/hostbase/log_detail.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Change Logs for {{ object.hostname }}

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -
    -
  • Hostname:{{ object.hostname }}
    • -
  • Date:{{ object.date }}
    • -
  • Log:{{ object.log }}
    • -
- -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/index.html b/src/lib/Server/Hostbase/hostbase/webtemplates/index.html deleted file mode 100644 index 92258b648..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/index.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "base.html" %} -{% block pagebanner %} -
-

Welcome to Hostbase!

-

Hostbase is a web based management tools for Bcfg2 Hosts

-
-
-{% endblock %} -{% block sidebar %} -login to hostbase
-search for hosts
-zone file information -{% endblock %} -{% block content %} -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/login.html b/src/lib/Server/Hostbase/hostbase/webtemplates/login.html deleted file mode 100644 index ec24a0fc0..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/login.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "base.html" %} -{% block pagebanner %} -
-

Login to Hostbase!

-

You must login to manage hosts

-
-
-{% endblock %} -{% block sidebar %} -search for hosts
-add a new host
-zone file information -{% endblock %} -{% block content %} - {% if form.has_errors %} - {{ form.username.errors|join:", " }} -

Login Failed.

- {% endif %} - {% if user.is_authenticated %} -

Welcome, {{ user.username }}. Thanks for logging in.

- {% else %} -

Welcome, user. Please log in.

-
- -
- -
- - {% if next %} - - {% else %} - - {% endif %} - -
- {% endif %} -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.html b/src/lib/Server/Hostbase/hostbase/webtemplates/logout.html deleted file mode 100644 index 994f631a8..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.html +++ /dev/null @@ -1,13 +0,0 @@ -{% extends "base.html" %} -{% block pagebanner %} -
-

You are logged out of Hostbase!

-
-
-{% endblock %} -{% block sidebar %} -Login to Hostbase -{% endblock %} -{% block content %} -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl b/src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl deleted file mode 100644 index e71e90e76..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logout.tmpl +++ /dev/null @@ -1,6 +0,0 @@ -

-{% if logged_in %} -logout -{% else %} -login -{% endif %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html b/src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html deleted file mode 100644 index 806ccd63d..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/logviewer.html +++ /dev/null @@ -1,27 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -

-

Change Logs for {{ hostname }}

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% if host.get_logs %} - -{% else %} -There are no logs for this host
-{% endif %} - -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl b/src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl deleted file mode 100644 index 877d427d0..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/navbar.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -host search
-add a new host
-zone file information
-add a zone
- diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/new.html b/src/lib/Server/Hostbase/hostbase/webtemplates/new.html deleted file mode 100644 index 2dcd6271f..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/new.html +++ /dev/null @@ -1,102 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

new host information

-
-
-{% endblock %} - -{% block sidebar %} -search hostbase -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
hostname
whatami - -
netgroup - -
class -
support -
csi
printq
outbound_smtp -
primary_user (email address)
administrator (email address)
location
expiration_date YYYY-MM-DD

Interface
- {% for choice in TYPE_CHOICES %} - {{ choice.1 }} - {% endfor %} -
dhcp -
mac_addr
ip_addr

Interface
- {% for choice in TYPE_CHOICES %} - {{ choice.1 }} - {% endfor %} -
dhcp -
mac_addr
ip_addr
comments
-
-

-

- -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/remove.html b/src/lib/Server/Hostbase/hostbase/webtemplates/remove.html deleted file mode 100644 index 4329200dd..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/remove.html +++ /dev/null @@ -1,89 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Are you sure you want to remove {{ host.hostname }}?

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - - - - - - - - - - - - - - - - - - - - - {% if host.dhcp %} - - {% else %} - - {% endif %} - - {% if host.outbound_smtp %} - - {% else %} - - {% endif %} - - - - - - - - - {% for interface in interfaces %} - - {% ifnotequal interface.0.hdwr_type 'no' %} - - {% endifnotequal %} - - - {% for ip in interface.1 %} - - - {% endfor %} - {% endfor %} - - - -
hostname {{ host.hostname }}
whatami {{ host.whatami }}
netgroup {{ host.netgroup }}
class {{ host.security_class }}
support {{ host.support }}
csi {{ host.csi }}
printq {{ host.printq }}
dhcp y
n
outbound_smtp y
n
primary_user {{ host.primary_user }}
administrator {{ host.administrator }}
location {{ host.location }}
expiration_date {{ host.expiration_date }}

Interface
{{ interface.0.hdwr_type }}
mac_addr {{ interface.0.mac_addr }}
ip_addr {{ ip.ip_addr }}
comments - {{ host.comments|linebreaksbr }}
-
-see detailed DNS information for this host -

-this host is {{ host.status }}
-last update on {{ host.last }}
- -
- - -
- -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/results.html b/src/lib/Server/Hostbase/hostbase/webtemplates/results.html deleted file mode 100644 index 45b22058d..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/results.html +++ /dev/null @@ -1,45 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Search Results

-
-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - -{% if hosts %} - - - - - - - - - - - - {% for host in hosts %} - - - - - - - - - {% endfor %} -
hostname status
{{ host.0 }} {{ host.2 }} view edit copy logs
-{% else %} -No hosts matched your query
-Click the back button on your browser to edit your search -{% endif %} - -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/search.html b/src/lib/Server/Hostbase/hostbase/webtemplates/search.html deleted file mode 100644 index 409d418fe..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/search.html +++ /dev/null @@ -1,57 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Welcome to Hostbase!

-

search for hosts using one or more of the fields below -

-
-{% endblock %} - -{% block sidebar %} -add a new host
-zone file information
-{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} -{% comment %} - ...or go to this - page to enter hostinfo-like queries

-{% endcomment %} - -
-
-
-
-
-
-
-
- - {% for choice in yesno %} - {{ choice.1 }} - {% endfor %}
-
-
-
-
-
- {% for choice in TYPE_CHOICES %} - {{ choice.1 }} - {% endfor %}
- - {% for choice in yesno %} - {{ choice.1 }} - {% endfor %}
-
-
- - {% for choice in DNS_CHOICES %} - {{ choice.1 }} - {% endfor %}
-
-

- -

-{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html b/src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html deleted file mode 100644 index ee355ee87..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneedit.html +++ /dev/null @@ -1,81 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Zones

-

Edit information for {{ zone }} -

-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - - - - - -
-
- {{ form.zone }}
- {{ form.admin }}
- {{ form.primary_master }}
- {{ form.expire }}
- {{ form.retry }}
- {{ form.refresh }}
- {{ form.ttl }}
-{% for ns in nsforms %} - {{ ns.name }}
-{% endfor %} - -
- {{ nsadd.name }}
- {{ nsadd.name }}
-
-Add NS records
-{% for mx in mxforms %} - {{ mx.priority }} {{ mx.mx }}
-{% endfor %} -
- {{ mxadd.priority }} {{ mxadd.mx }}
- {{ mxadd.priority }} {{ mxadd.mx }}
-
-Add MX records
-{% for a in aforms %} - {{ a.ip_addr }}
-{% endfor %} -
- {{ addadd.ip_addr }}
- {{ addadd.ip_addr }}
-
-Add A records
- {{ form.aux }}
-

-

- -{% endblock %} diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html b/src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html deleted file mode 100644 index b59fa9e3c..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zonenew.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Zones

-

Enter information for a new zone to be generated by Hostbase -

-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} -
-
- {{ form.as_p}} - -

-

-
-{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zones.html b/src/lib/Server/Hostbase/hostbase/webtemplates/zones.html deleted file mode 100644 index c773e7922..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zones.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Zones

-

Hostbase generates DNS zone files for the following zones. -

-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} -{% if zone_list %} - - - - - - - - - {% for zone in zone_list|dictsort:"zone" %} - - - - - {% endfor %} -
zone
{{ zone.zone }} view edit
-{% else %} -There is no zone data currently in the database
-{% endif %} -{% endblock %} - diff --git a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html b/src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html deleted file mode 100644 index fa12e3ec5..000000000 --- a/src/lib/Server/Hostbase/hostbase/webtemplates/zoneview.html +++ /dev/null @@ -1,71 +0,0 @@ -{% extends "base.html" %} - -{% block pagebanner %} -
-

Zones

-

Hostbase generates DNS zone files for the following zones. -

-
-{% endblock %} - -{% block sidebar %} -{% include "navbar.tmpl" %} - -{% include "logout.tmpl" %} -{% endblock %} - -{% block content %} - - - - - - - - - - - - - - - - - - - - - - - - - - {% if addresses %} - - - {% endif %} - - - - -
zone {{ zone.zone }}
serial {{ zone.serial }}
admin {{ zone.admin }}
primary_master {{ zone.primary_master }}
expire {{ zone.expire }}
retry {{ zone.retry }}
refresh {{ zone.refresh }}
ttl {{ zone.ttl }}
nameservers - {% for nameserver in zone.nameservers.all %} - {{ nameserver.name }}
- {% endfor %} -
mxs - {% for mx in zone.mxs.all %} - {{ mx.priority }} {{ mx.mx }}
- {% endfor %} -
A records - {% for address in sof.addresses.all %} - {{ address.ip_addr }}
- {% endfor %} -
aux - {{ zone.aux|linebreaksbr }} -
-

-{% endblock %} - diff --git a/src/lib/Server/Hostbase/ldapauth.py b/src/lib/Server/Hostbase/ldapauth.py deleted file mode 100644 index f3db26f67..000000000 --- a/src/lib/Server/Hostbase/ldapauth.py +++ /dev/null @@ -1,180 +0,0 @@ -""" -Checks with LDAP (ActiveDirectory) to see if the current user is an LDAP(AD) -user, and returns a subset of the user's profile that is needed by Argonne/CIS -to set user level privleges in Django -""" - -import os -import ldap - - -class LDAPAUTHError(Exception): - """LDAPAUTHError is raised when somehting goes boom.""" - pass - - -class ldapauth(object): - group_test = False - check_member_of = os.environ['LDAP_CHECK_MBR_OF_GRP'] - securitylevel = 0 - distinguishedName = None - sAMAccountName = None - telephoneNumber = None - title = None - memberOf = None - department = None # this will be a list - mail = None - extensionAttribute1 = None # badgenumber - badge_no = None - - def __init__(self, login, passwd): - """get username (if using ldap as auth the - apache env var REMOTE_USER should be used) - from username get user profile from AD/LDAP - """ - #p = self.user_profile(login,passwd) - d = self.user_dn(login) # success, distname - print(d[1]) - if d[0] == 'success': - pass - p = self.user_bind(d[1], passwd) - if p[0] == 'success': - #parse results - parsed = self.parse_results(p[2]) - print(self.department) - self.group_test = self.member_of() - securitylevel = self.security_level() - print("ACCESS LEVEL: " + str(securitylevel)) - else: - raise LDAPAUTHError(p[2]) - else: - raise LDAPAUTHError(p[2]) - - def user_profile(self, login, passwd=None): - """NOT USED RIGHT NOW""" - ldap_login = "CN=%s" % login - svc_acct = os.environ['LDAP_SVC_ACCT_NAME'] - svc_pass = os.environ['LDAP_SVC_ACCT_PASS'] - #svc_acct = 'CN=%s,DC=anl,DC=gov' % login - #svc_pass = passwd - - search_pth = os.environ['LDAP_SEARCH_PTH'] - - try: - conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(svc_acct, svc_pass, ldap.AUTH_SIMPLE) - result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - ldap_login, - None) - result_type, result_data = conn.result(result_id, 0) - return ('success', 'User profile found', result_data,) - except ldap.LDAPError: - e = sys.exc_info()[1] - #connection failed - return ('error', 'LDAP connect failed', e,) - - def user_bind(self, distinguishedName, passwd): - """Binds to LDAP Server""" - search_pth = os.environ['LDAP_SEARCH_PTH'] - try: - conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(distinguishedName, passwd, ldap.AUTH_SIMPLE) - cn = distinguishedName.split(",") - result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - cn[0], - None) - result_type, result_data = conn.result(result_id, 0) - return ('success', 'User profile found', result_data,) - except ldap.LDAPError: - e = sys.exc_info()[1] - #connection failed - return ('error', 'LDAP connect failed', e,) - - def user_dn(self, cn): - """Uses Service Account to get distinguishedName""" - ldap_login = "CN=%s" % cn - svc_acct = os.environ['LDAP_SVC_ACCT_NAME'] - svc_pass = os.environ['LDAP_SVC_ACCT_PASS'] - search_pth = os.environ['LDAP_SEARCH_PTH'] - - try: - conn = ldap.initialize(os.environ['LDAP_URI']) - conn.bind(svc_acct, svc_pass, ldap.AUTH_SIMPLE) - result_id = conn.search(search_pth, - ldap.SCOPE_SUBTREE, - ldap_login, - None) - result_type, result_data = conn.result(result_id, 0) - raw_obj = result_data[0][1] - distinguishedName = raw_obj['distinguishedName'] - return ('success', distinguishedName[0],) - except ldap.LDAPError: - e = sys.exc_info()[1] - #connection failed - return ('error', 'LDAP connect failed', e,) - - def parse_results(self, user_obj): - """Clean up the huge ugly object handed to us in the LDAP query""" - #user_obj is a list formatted like this: - #[('LDAP_DN',{user_dict},),] - try: - raw_obj = user_obj[0][1] - self.memberOf = raw_obj['memberOf'] - self.sAMAccountName = raw_obj['sAMAccountName'][0] - self.distinguishedName = raw_obj['distinguishedName'][0] - self.telephoneNumber = raw_obj['telephoneNumber'][0] - self.title = raw_obj['title'][0] - self.department = raw_obj['department'][0] - self.mail = raw_obj['mail'][0] - self.badge_no = raw_obj['extensionAttribute1'][0] - self.email = raw_obj['extensionAttribute2'][0] - display_name = raw_obj['displayName'][0].split(",") - self.name_f = raw_obj['givenName'][0] - self.name_l = display_name[0] - self.is_staff = False - self.is_superuser = False - - return - except KeyError: - e = sys.exc_info()[1] - raise LDAPAUTHError("Portions of the LDAP User profile not present") - - def member_of(self): - """See if this user is in our group that is allowed to login""" - m = [g for g in self.memberOf if g == self.check_member_of] - #print m - if len(m) == 1: - return True - else: - return False - - def security_level(self): - level = self.securitylevel - - user = os.environ['LDAP_GROUP_USER'] - m = [g for g in self.memberOf if g == user] - if len(m) == 1: - if level < 1: - level = 1 - - cspr = os.environ['LDAP_GROUP_SECURITY_LOW'] - m = [g for g in self.memberOf if g == cspr] - if len(m) == 1: - if level < 2: - level = 2 - - cspo = os.environ['LDAP_GROUP_SECURITY_HIGH'] - m = [g for g in self.memberOf if g == cspo] - if len(m) == 1: - if level < 3: - level = 3 - - admin = os.environ['LDAP_GROUP_ADMIN'] - m = [g for g in self.memberOf if g == admin] - if len(m) == 1: - if level < 4: - level = 4 - - return level diff --git a/src/lib/Server/Hostbase/manage.py b/src/lib/Server/Hostbase/manage.py deleted file mode 100755 index 5e78ea979..000000000 --- a/src/lib/Server/Hostbase/manage.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python -from django.core.management import execute_manager -try: - import settings # Assumed to be in the same directory. -except ImportError: - import sys - sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) - sys.exit(1) - -if __name__ == "__main__": - execute_manager(settings) diff --git a/src/lib/Server/Hostbase/media/base.css b/src/lib/Server/Hostbase/media/base.css deleted file mode 100644 index ddbf02165..000000000 --- a/src/lib/Server/Hostbase/media/base.css +++ /dev/null @@ -1,5 +0,0 @@ - -/* Import other styles */ -@import url('global.css'); -@import url('layout.css'); -@import url('boxypastel.css'); diff --git a/src/lib/Server/Hostbase/media/boxypastel.css b/src/lib/Server/Hostbase/media/boxypastel.css deleted file mode 100644 index 7ae0684ef..000000000 --- a/src/lib/Server/Hostbase/media/boxypastel.css +++ /dev/null @@ -1,179 +0,0 @@ -body { - background-color: #fff; - color: #000; - font: 12px 'Lucida Grande', Arial, Helvetica, sans-serif; - margin-left:0px; - margin-right:100px; -} -/* links */ -a:link { - color: #00f; - text-decoration: none; -} -a:visited { - color: #00a; - text-decoration: none; -} -a:hover { - color: #00a; - text-decoration: underline; -} -a:active { - color: #00a; - text-decoration: underline; -} -/* divs*/ -div.bad { - border: 1px solid #660000; - background: #FF6A6A; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.modified { - border: 1px solid #CC9900; - background: #FFEC8B; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.clean { - border: 1px solid #006600; - background: #9AFF9A; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.extra { - border: 1px solid #006600; - background: #6699CC; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.warning { - border: 1px - solid #CC3300; - background: #FF9933; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.all-warning { - border: 1px solid #DD5544; - background: #FFD9A2; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.down { - border: 1px - solid #999; - background-color: #DDD; - margin: 10px 0; - padding: 8px; - text-align: left; - margin-left:50px; - margin-right:50px; -} -div.items{ - display: none; -} -div.nodebox { - border: 1px solid #c7cfd5; - background: #f1f5f9; - margin: 20px 0; - padding: 8px 8px 16px 8px; - text-align: left; - position:relative; -} -div.header { - background-color: #DDD; - padding: 8px; - text-indent:50px; - position:relative; -} - -/*Spans*/ -.nodename { - font-style: italic; -} -.nodelisttitle { - font-size: 14px; -} - -h2{ - font-size: 16px; - color: #000; -} - -ul.plain { - list-style-type:none; - text-align: left; -} - -.notebox { - position: absolute; - top: 0px; - right: 0px; - padding: 1px; - text-indent:0px; - border: 1px solid #FFF; - background: #999; - color: #FFF; -} - -.configbox { - position: absolute; - bottom: 0px; - right: 0px; - padding: 1px; - text-indent:0px; - border: 1px solid #999; - background: #FFF; - color: #999; -} - -p.indented{ - text-indent: 50px -} - -/* - Sortable tables */ -table.sortable a.sortheader { - background-color:#dfd; - font-weight: bold; - text-decoration: none; - display: block; -} -table.sortable { - padding: 2px 4px 2px 4px; - border: 1px solid #000000; - border-spacing: 0px -} -td.sortable{ - padding: 2px 8px 2px 8px; -} - -th.sortable{ - background-color:#F3DD91; - border: 1px solid #FFFFFF; -} -tr.tablelist { - background-color:#EDF3FE; -} -tr.tablelist-alt{ - background-color:#FFFFFF; -} diff --git a/src/lib/Server/Hostbase/media/global.css b/src/lib/Server/Hostbase/media/global.css deleted file mode 100644 index 73451e1bc..000000000 --- a/src/lib/Server/Hostbase/media/global.css +++ /dev/null @@ -1,8 +0,0 @@ -body { - margin:0; - padding:0; - font-size:12px; - font-family:"Lucida Grande","Bitstream Vera Sans",Verdana,Arial,sans-serif; - color:#000; - background:#fff; - } diff --git a/src/lib/Server/Hostbase/media/layout.css b/src/lib/Server/Hostbase/media/layout.css deleted file mode 100644 index 9085cc220..000000000 --- a/src/lib/Server/Hostbase/media/layout.css +++ /dev/null @@ -1,62 +0,0 @@ -/* Page Structure */ -#container { position:absolute; top: 3em; margin-left:1em; margin-right:2em; padding:0; margin-top:1.5em; min-width: - 650px; } -#header { width:100%; } -#content-main { float:left; } - -/* HEADER */ -#header { -background:#000; -color:#ffc; -position:absolute; -} -#header a:link, #header a:visited { color:white; } -#header a:hover { text-decoration:underline; } -#branding h1 { padding:0 10px; font-size:18px; margin:8px 0; font-weight:normal; color:#f4f379; } -#branding h2 { padding:0 10px; font-size:14px; margin:-8px 0 8px 0; font-weight:normal; color:#ffc; } -#user-tools { position:absolute; top:0; right:0; padding:1.2em 10px; font-size:11px; text-align:right; } - -/*SIDEBAR*/ -#sidebar { - float:left; - position: relative; - width: auto; - height: 100%; - margin-top: 3em; - padding-right: 1.5em; - padding-left: 1.5em; - padding-top: 1em; - padding-bottom:3em; - background: #000; - color:ffc; -} - -a.sidebar:link {color: #fff;} -a.sidebar:active {color: #fff;} -a.sidebar:visited {color: #fff;} -a.sidebar:hover {color: #fff;} - -ul.sidebar { - color: #ffc; - text-decoration: none; - list-style-type: none; - text-indent: -1em; -} -ul.sidebar-level2 { - text-indent: -2em; - list-style-type: none; - font-size: 11px; -} - -/* ALIGNED FIELDSETS */ -.aligned label { display:block; padding:0 1em 3px 0; float:left; width:8em; } -.aligned label.inline { display:inline; float:none; } -.colMS .aligned .vLargeTextField, .colMS .aligned .vXMLLargeTextField { width:350px; } -form .aligned p, form .aligned ul { margin-left:7em; padding-left:30px; } -form .aligned table p { margin-left:0; padding-left:0; } -form .aligned p.help { padding-left:38px; } -.aligned .vCheckboxLabel { float:none !important; display:inline; padding-left:4px; } -.colM .aligned .vLargeTextField, colM .aligned .vXMLLargeTextField { width:610px; } -.checkbox-row p.help { margin-left:0; padding-left:0 !important; } - - diff --git a/src/lib/Server/Hostbase/nisauth.py b/src/lib/Server/Hostbase/nisauth.py deleted file mode 100644 index ae4c6c021..000000000 --- a/src/lib/Server/Hostbase/nisauth.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Checks with NIS to see if the current user is in the support group""" -import os -import crypt, nis -from Bcfg2.Server.Hostbase.settings import AUTHORIZED_GROUP - - -class NISAUTHError(Exception): - """NISAUTHError is raised when somehting goes boom.""" - pass - -class nisauth(object): - group_test = False -# check_member_of = os.environ['LDAP_CHECK_MBR_OF_GRP'] - samAcctName = None - distinguishedName = None - sAMAccountName = None - telephoneNumber = None - title = None - memberOf = None - department = None #this will be a list - mail = None - extensionAttribute1 = None #badgenumber - badge_no = None - uid = None - - def __init__(self,login,passwd=None): - """get user profile from NIS""" - try: - p = nis.match(login, 'passwd.byname').split(":") - except: - raise NISAUTHError('username') - # check user password using crypt and 2 character salt from passwd file - if p[1] == crypt.crypt(passwd, p[1][:2]): - # check to see if user is in valid support groups - # will have to include these groups in a settings file eventually - if not login in nis.match(AUTHORIZED_GROUP, 'group.byname').split(':')[-1].split(',') and p[3] != nis.match(AUTHORIZED_GROUP, 'group.byname').split(':')[2]: - raise NISAUTHError('group') - self.uid = p[2] - else: - raise NISAUTHError('password') diff --git a/src/lib/Server/Hostbase/regex.py b/src/lib/Server/Hostbase/regex.py deleted file mode 100644 index 41cc0f6f0..000000000 --- a/src/lib/Server/Hostbase/regex.py +++ /dev/null @@ -1,6 +0,0 @@ -import re - -date = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}$') -host = re.compile('^[a-z0-9-_]+(\.[a-z0-9-_]+)+$') -macaddr = re.compile('^[0-9abcdefABCDEF]{2}(:[0-9abcdefABCDEF]{2}){5}$|virtual') -ipaddr = re.compile('^[0-9]{1,3}(\.[0-9]{1,3}){3}$') diff --git a/src/lib/Server/Hostbase/settings.py b/src/lib/Server/Hostbase/settings.py deleted file mode 100644 index 4e641f13c..000000000 --- a/src/lib/Server/Hostbase/settings.py +++ /dev/null @@ -1,143 +0,0 @@ -import os.path -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - -PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) - -c = ConfigParser.ConfigParser() -#This needs to be configurable one day somehow -c.read(['./bcfg2.conf']) - -defaults = {'database_engine':'sqlite3', - 'database_name':'./dev.db', - 'database_user':'', - 'database_password':'', - 'database_host':'', - 'database_port':3306, - 'default_mx':'localhost', - 'priority':10, - 'authorized_group':'admins', - } - -if c.has_section('hostbase'): - options = dict(c.items('hostbase')) -else: - options = defaults - -# Django settings for Hostbase project. -DEBUG = True -TEMPLATE_DEBUG = DEBUG -ADMINS = ( - ('Root', 'root'), -) -MANAGERS = ADMINS - -# 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'. -DATABASE_ENGINE = options['database_engine'] -# Or path to database file if using sqlite3. -DATABASE_NAME = options['database_name'] -# Not used with sqlite3. -DATABASE_USER = options['database_user'] -# Not used with sqlite3. -DATABASE_PASSWORD = options['database_password'] -# Set to empty string for localhost. Not used with sqlite3. -DATABASE_HOST = options['database_host'] -# Set to empty string for default. Not used with sqlite3. -DATABASE_PORT = int(options['database_port']) -# Local time zone for this installation. All choices can be found here: -# http://docs.djangoproject.com/en/dev/ref/settings/#time-zone -try: - TIME_ZONE = c.get('statistics', 'time_zone') -except: - TIME_ZONE = None - -# enter the defauly MX record machines will get in Hostbase -# this setting may move elsewhere eventually -DEFAULT_MX = options['default_mx'] -PRIORITY = int(options['priority']) - -SESSION_EXPIRE_AT_BROWSER_CLOSE = True - -# Uncomment a backend below if you would like to use it for authentication -AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend', - 'Bcfg2.Server.Hostbase.backends.NISBackend', - #'Bcfg2.Server.Hostbase.backends.LDAPBacken', - ) -# enter an NIS group name you'd like to give access to edit hostbase records -AUTHORIZED_GROUP = options['authorized_group'] - -#create login url area: -import django.contrib.auth -django.contrib.auth.LOGIN_URL = '/login' -# Absolute path to the directory that holds media. -# Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media') -# Just for development -SERVE_MEDIA = DEBUG - -# Language code for this installation. All choices can be found here: -# http://www.w3.org/TR/REC-html40/struct/dirlang.html#langcodes -# http://blogs.law.harvard.edu/tech/stories/storyReader$15 -LANGUAGE_CODE = 'en-us' -SITE_ID = 1 -# URL that handles the media served from MEDIA_ROOT. -# Example: "http://media.lawrence.com" -MEDIA_URL = '/site_media/' -# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a -# trailing slash. -# Examples: "http://foo.com/media/", "/media/". -ADMIN_MEDIA_PREFIX = '/media/' -# Make this unique, and don't share it with anybody. -SECRET_KEY = '*%=fv=yh9zur&gvt4&*d#84o(cy^-*$ox-v1e9%32pzf2*qu#s' -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.load_template_source', - 'django.template.loaders.app_directories.load_template_source', -# 'django.template.loaders.eggs.load_template_source', -) - -TEMPLATE_CONTEXT_PROCESSORS = ( - "django.core.context_processors.auth", - "django.core.context_processors.debug", - "django.core.context_processors.i18n", - "django.core.context_processors.request", - "django.core.context_processors.media", -# Django development version. -# "django.core.context_processors.csrf", -) - - -MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.locale.LocaleMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.middleware.doc.XViewMiddleware', -) - -ROOT_URLCONF = 'Bcfg2.Server.Hostbase.urls' - -TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates". - # Always use forward slashes, even on Windows. - '/usr/lib/python2.3/site-packages/Bcfg2/Server/Hostbase/hostbase/webtemplates', - '/usr/lib/python2.4/site-packages/Bcfg2/Server/Hostbase/hostbase/webtemplates', - '/usr/lib/python2.3/site-packages/Bcfg2/Server/Hostbase/templates', - '/usr/lib/python2.4/site-packages/Bcfg2/Server/Hostbase/templates', - '/usr/share/bcfg2/Hostbase/templates', - os.path.join(PROJECT_ROOT, 'templates'), - os.path.join(PROJECT_ROOT, 'hostbase/webtemplates'), -) - -INSTALLED_APPS = ( - 'django.contrib.admin', - 'django.contrib.admindocs', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.humanize', - 'Bcfg2.Server.Hostbase.hostbase', -) - -LOGIN_URL = '/login/' diff --git a/src/lib/Server/Hostbase/templates/batchadd.tmpl b/src/lib/Server/Hostbase/templates/batchadd.tmpl deleted file mode 100644 index 74ea3c047..000000000 --- a/src/lib/Server/Hostbase/templates/batchadd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#mx -> -#priority -> - -hostname -> -whatami -> -netgroup -> -security_class -> -support -> -csi -> -printq -> -dhcp -> -outbound_smtp -> -primary_user -> -administrator -> -location -> -expiration_date -> YYYY-MM-DD -comments -> - -mac_addr -> -hdwr_type -> -ip_addr -> -#ip_addr -> -cname -> -#cname -> - -#mac_addr -> -#hdwr_type -> -#ip_addr -> -#cname -> diff --git a/src/lib/Server/Hostbase/templates/dhcpd.conf.head b/src/lib/Server/Hostbase/templates/dhcpd.conf.head deleted file mode 100644 index a3d19547e..000000000 --- a/src/lib/Server/Hostbase/templates/dhcpd.conf.head +++ /dev/null @@ -1,5 +0,0 @@ -# -# dhcpd.conf -# -# Configuration file for ISC dhcpd -# diff --git a/src/lib/Server/Hostbase/templates/dhcpd.tmpl b/src/lib/Server/Hostbase/templates/dhcpd.tmpl deleted file mode 100644 index 757b263cd..000000000 --- a/src/lib/Server/Hostbase/templates/dhcpd.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# -# This file is automatically generated. -# DO NOT EDIT IT BY HAND! -# -# This file contains {{ numips }} IP addresses -# Generated on: {% now "r" %} -# - -{% include "dhcpd.conf.head" %} - -# Hosts which require special configuration options can be listed in -# host statements. If no address is specified, the address will be -# allocated dynamically (if possible), but the host-specific information -# will still come from the host declaration. - -{% for host in hosts %}host {{ host.0 }} {hardware ethernet {{ host.1 }};fixed-address {{ host.2 }};} -{% endfor %} diff --git a/src/lib/Server/Hostbase/templates/hosts.tmpl b/src/lib/Server/Hostbase/templates/hosts.tmpl deleted file mode 100644 index 251cb5a79..000000000 --- a/src/lib/Server/Hostbase/templates/hosts.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -############################################################################## -# MCS hosts file -# -# This file is generated automatically - DO NOT EDIT IT. -# -# Generated on: {% now "r" %} -# - -127.0.0.1 localhost.mcs.anl.gov localhost - -# This file lists hosts in these domains: -{% for domain in domain_data %}# {{ domain.0 }}: {{ domain.1 }} -{% endfor %} -# -# This file lists hosts on these networks: -# -# Network Hosts -# --------------------------------------------------------------------- -{% for octet in two_octets_data %}# {{ octet.0 }} {{octet.1 }} -{% endfor %} -# -{% for octet in three_octets_data %}# {{ octet.0 }} {{ octet.1 }} -{% endfor %} -# -# Total host interfaces (ip addresses) in this file: {{ num_ips }} - diff --git a/src/lib/Server/Hostbase/templates/hostsappend.tmpl b/src/lib/Server/Hostbase/templates/hostsappend.tmpl deleted file mode 100644 index 00e0d5d04..000000000 --- a/src/lib/Server/Hostbase/templates/hostsappend.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -########################################################################## -# Hosts on subnet: {{ subnet.0 }} -# total hosts: {{ subnet.1 }} -{% for ip in ips %}{{ ip.0 }} {{ ip.1 }}{% if ip.4 and not ip.3 %} # {{ ip.5 }}{% else %}{% for name in ip.2 %} {{ name }}{% endfor %}{% for cname in ip.3 %} {{ cname }}{% endfor %} # {{ ip.5 }}{% endif %} -{% endfor %} diff --git a/src/lib/Server/Hostbase/templates/named.tmpl b/src/lib/Server/Hostbase/templates/named.tmpl deleted file mode 100644 index 03e054198..000000000 --- a/src/lib/Server/Hostbase/templates/named.tmpl +++ /dev/null @@ -1,69 +0,0 @@ -// This is the primary configuration file for the BIND DNS server named. -// -// Please read /usr/share/doc/bind9/README.Debian.gz for information on the -// structure of BIND configuration files in Debian, *BEFORE* you customize -// this configuration file. -// - -include "/etc/bind/named.conf.options"; - -include "/etc/bind/rndc.key"; - -// prime the server with knowledge of the root servers -zone "." { - type hint; - file "/etc/bind/db.root"; -}; - -// be authoritative for the localhost forward and reverse zones, and for -// broadcast zones as per RFC 1912 -{% for zone in zones %} -zone "{{ zone.1 }}" { - type master; - file "/etc/bind/hostbase/{{ zone.1 }}"; - notify no; - also-notify { 140.221.9.6;140.221.8.10; }; -};{% endfor %} - -zone "localhost" { - type master; - file "/etc/bind/db.local"; -}; - -zone "127.in-addr.arpa" { - type master; - file "/etc/bind/db.127"; -}; - -zone "0.in-addr.arpa" { - type master; - file "/etc/bind/db.0"; -}; - -zone "255.in-addr.arpa" { - type master; - file "/etc/bind/db.255"; -}; -{% for reverse in reverses %} -zone "{{ reverse.0 }}.in-addr.arpa" { - type master; - file "/etc/bind/hostbase/{{ reverse.0 }}.rev"; - notify no; - also-notify { 140.221.9.6;140.221.8.10; }; -};{% endfor %} - -// zone "com" { type delegation-only; }; -// zone "net" { type delegation-only; }; - -// From the release notes: -// Because many of our users are uncomfortable receiving undelegated answers -// from root or top level domains, other than a few for whom that behaviour -// has been trusted and expected for quite some length of time, we have now -// introduced the "root-delegations-only" feature which applies delegation-only -// logic to all top level domains, and to the root domain. An exception list -// should be specified, including "MUSEUM" and "DE", and any other top level -// domains from whom undelegated responses are expected and trusted. -// root-delegation-only exclude { "DE"; "MUSEUM"; }; - -include "/etc/bind/named.conf.local"; -include "/etc/bind/named.conf.static"; diff --git a/src/lib/Server/Hostbase/templates/namedviews.tmpl b/src/lib/Server/Hostbase/templates/namedviews.tmpl deleted file mode 100644 index 52021620e..000000000 --- a/src/lib/Server/Hostbase/templates/namedviews.tmpl +++ /dev/null @@ -1,92 +0,0 @@ -// This is the primary configuration file for the BIND DNS server named. -// -// Please read /usr/share/doc/bind9/README.Debian.gz for information on the -// structure of BIND configuration files in Debian, *BEFORE* you customize -// this configuration file. -// - -include "/etc/bind/named.conf.options"; - -include "/etc/bind/rndc.key"; - -view "internal" { - match-clients { 140.221.9.6;140.221.8.10;140.221.8.88;140.221.8.15; }; - recursion yes; - // prime the server with knowledge of the root servers - zone "." { - type hint; - file "/etc/bind/db.root"; - }; - {% for zone in zones %} - zone "{{ zone.1 }}" { - type master; - file "/etc/bind/hostbase/{{ zone.1 }}"; - notify no; - also-notify { 140.221.9.6;140.221.8.10;140.221.8.88;140.221.8.15; }; - };{% endfor %} - // be authoritative for the localhost forward and reverse zones, and for - // broadcast zones as per RFC 1912 - - zone "localhost" { - type master; - file "/etc/bind/db.local"; - }; - - zone "127.in-addr.arpa" { - type master; - file "/etc/bind/db.127"; - }; - - zone "0.in-addr.arpa" { - type master; - file "/etc/bind/db.0"; - }; - - zone "255.in-addr.arpa" { - type master; - file "/etc/bind/db.255"; - }; - {% for reverse in reverses %} - zone "{{ reverse.0 }}.in-addr.arpa" { - type master; - file "/etc/bind/hostbase/{{ reverse.0 }}.rev"; - notify no; - also-notify { 140.221.9.6;140.221.8.10;140.221.8.88; }; - };{% endfor %} - include "/etc/bind/named.conf.static"; -}; - -view "external" { - match-clients { any; }; - recursion no; - {% for zone in zones %} - zone "{{ zone.1 }}" { - type master; - file "/etc/bind/hostbase/{{ zone.1 }}.external"; - notify no; - };{% endfor %} - - {% for reverse in reverses %} - zone "{{ reverse.0 }}.in-addr.arpa" { - type master; - file "/etc/bind/hostbase/{{ reverse.0 }}.rev.external"; - notify no; - };{% endfor %} - include "/etc/bind/named.conf.static"; -}; - - -// zone "com" { type delegation-only; }; -// zone "net" { type delegation-only; }; - -// From the release notes: -// Because many of our users are uncomfortable receiving undelegated answers -// from root or top level domains, other than a few for whom that behaviour -// has been trusted and expected for quite some length of time, we have now -// introduced the "root-delegations-only" feature which applies delegation-only -// logic to all top level domains, and to the root domain. An exception list -// should be specified, including "MUSEUM" and "DE", and any other top level -// domains from whom undelegated responses are expected and trusted. -// root-delegation-only exclude { "DE"; "MUSEUM"; }; - -include "/etc/bind/named.conf.local"; diff --git a/src/lib/Server/Hostbase/templates/reverseappend.tmpl b/src/lib/Server/Hostbase/templates/reverseappend.tmpl deleted file mode 100644 index 6ed520c98..000000000 --- a/src/lib/Server/Hostbase/templates/reverseappend.tmpl +++ /dev/null @@ -1,4 +0,0 @@ -{% if fileorigin %}$ORIGIN {{ fileorigin }}.in-addr.arpa.{% endif %} -$ORIGIN {{ inaddr }}.in-addr.arpa. -{% for host in hosts %}{{ host.0.3 }} PTR {{ host.1 }}. -{% endfor %} diff --git a/src/lib/Server/Hostbase/templates/reversesoa.tmpl b/src/lib/Server/Hostbase/templates/reversesoa.tmpl deleted file mode 100644 index d142eaf7f..000000000 --- a/src/lib/Server/Hostbase/templates/reversesoa.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -$ORIGIN . -$TTL {{ zone.8 }} -{{ inaddr }}.in-addr.arpa IN SOA {{ zone.4 }}. {{ zone.3 }} ( - {{ zone.2 }} ; serial - {{ zone.7 }} ; refresh interval - {{ zone.6 }} ; retry interval - {{ zone.5 }} ; expire interval - {{ zone.8 }} ; min ttl - ) - - {% for ns in nameservers %}NS {{ ns.0 }} - {% endfor %} - diff --git a/src/lib/Server/Hostbase/templates/zone.tmpl b/src/lib/Server/Hostbase/templates/zone.tmpl deleted file mode 100644 index aad48d179..000000000 --- a/src/lib/Server/Hostbase/templates/zone.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -$ORIGIN . -$TTL {{ zone.8 }} -{{ zone.1 }}. IN SOA {{ zone.4 }}. {{ zone.3 }}. ( - {{ zone.2 }} ; serial - {{ zone.7 }} ; refresh interval - {{ zone.6 }} ; retry interval - {{ zone.5 }} ; expire interval - {{ zone.8 }} ; min ttl - ) - - {% for ns in nameservers %}NS {{ ns.0 }} - {% endfor %} - {% for a in addresses %}A {{ a.0 }} - {% endfor %} - {% for mx in mxs %}MX {{ mx.0 }} {{ mx.1 }} - {% endfor %} -$ORIGIN {{ zone.1 }}. -localhost A 127.0.0.1 diff --git a/src/lib/Server/Hostbase/test/harness.py b/src/lib/Server/Hostbase/test/harness.py deleted file mode 100644 index befcff5c0..000000000 --- a/src/lib/Server/Hostbase/test/harness.py +++ /dev/null @@ -1,11 +0,0 @@ -import os, sys - -here = os.path.realpath('harness') - -server_hostbase = os.path.realpath(here + '../../../../') - -sys.path.insert(0,server_hostbase) -sys.path.insert(0,server_hostbase + '../') -#commented this out, but might be needed for now until the harness is figured out -#if so, use your actual path to the Hostbase module -#sys.path.insert(0,'/home/dahl/Code/bcfg2/src/lib/Server/Hostbase') diff --git a/src/lib/Server/Hostbase/test/test_environ_settings.py b/src/lib/Server/Hostbase/test/test_environ_settings.py deleted file mode 100644 index ad35c624e..000000000 --- a/src/lib/Server/Hostbase/test/test_environ_settings.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys -import os - - -def env_setup(): - os.environ['bcfg_db_engine'] = 'foo' - os.environ['bcfg_db_name'] = 'bar' - os.environ['bcfg_db_user'] = 'baz' - os.environ['bcfg_db_password'] = 'pass' - os.environ['bcfg_db_host'] = 'biff' - os.environ['bcfg_db_port'] = '3306' - os.environ['bcfg_time_zone'] = 'CHI' - -def teardown(): - pass - -def test_environ_settings(): - - os.environ['bcfg_db_engine'] = 'foo' - os.environ['bcfg_db_name'] = 'bar' - os.environ['bcfg_db_user'] = 'baz' - os.environ['bcfg_db_password'] = 'pass' - os.environ['bcfg_db_host'] = 'biff' - os.environ['bcfg_db_port'] = '3306' - os.environ['bcfg_time_zone'] = 'CHI' - import Hostbase.settings - s = Hostbase.settings - s.CFG_TYPE = 'environ' - assert s.DATABASE_ENGINE == 'mysql' - assert s.DATABASE_PASSWORD == 'pass' - assert s.DATABASE_NAME == 'bar' - assert s.DATABASE_USER == 'baz' - assert s.DATABASE_HOST == 'biff' - assert s.DATABASE_PORT == '3306' - assert s.TIME_ZONE == 'CHI' diff --git a/src/lib/Server/Hostbase/test/test_ldapauth.py b/src/lib/Server/Hostbase/test/test_ldapauth.py deleted file mode 100644 index 7fc009ad2..000000000 --- a/src/lib/Server/Hostbase/test/test_ldapauth.py +++ /dev/null @@ -1,10 +0,0 @@ -import os,sys -import harness - -from Hostbase.ldapauth import * - -def test_it(): - l = ldapauth(os.environ['LDAP_SVC_ACCT_NAME'], - os.environ['LDAP_SVC_ACCT_PASS']) - - assert l.department == 'foo' diff --git a/src/lib/Server/Hostbase/test/test_settings.py b/src/lib/Server/Hostbase/test/test_settings.py deleted file mode 100644 index 0dfc30f38..000000000 --- a/src/lib/Server/Hostbase/test/test_settings.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys -import os -import Hostbase.settings - -def setup(): - pass - -def teardown(): - pass - -def test_mcs_settings(): - pass diff --git a/src/lib/Server/Hostbase/urls.py b/src/lib/Server/Hostbase/urls.py deleted file mode 100644 index 01fe97d4f..000000000 --- a/src/lib/Server/Hostbase/urls.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.conf.urls.defaults import * -from django.conf import settings -from django.views.generic.simple import direct_to_template -from django.contrib import admin - - -admin.autodiscover() - - -urlpatterns = patterns('', - # Uncomment the admin/doc line below and add 'django.contrib.admindocs' - # to INSTALLED_APPS to enable admin documentation: - (r'^admin/doc/', include('django.contrib.admindocs.urls')), - - # Uncomment the next line to enable the admin: - (r'^admin/', include(admin.site.urls)), - - (r'^$',direct_to_template, {'template':'index.html'}, 'index'), - (r'^hostbase/', include('hostbase.urls')), - (r'^login/$', 'django.contrib.auth.views.login', {'template_name': 'login.html'}), - (r'^logout/$', 'django.contrib.auth.views.logout', {'template_name': 'logout.html'}) -) - -if settings.SERVE_MEDIA: - urlpatterns += patterns('', - (r'^site_media/(?P.*)$', 'django.views.static.serve', - dict(document_root=settings.MEDIA_ROOT)),) diff --git a/src/lib/Server/Lint/Bundles.py b/src/lib/Server/Lint/Bundles.py deleted file mode 100644 index 472915cfd..000000000 --- a/src/lib/Server/Lint/Bundles.py +++ /dev/null @@ -1,61 +0,0 @@ -import lxml.etree -import Bcfg2.Server.Lint - -class Bundles(Bcfg2.Server.Lint.ServerPlugin): - """ Perform various bundle checks """ - - def Run(self): - """ run plugin """ - if 'Bundler' in self.core.plugins: - self.missing_bundles() - for bundle in self.core.plugins['Bundler'].entries.values(): - if self.HandlesFile(bundle.name): - if (not Bcfg2.Server.Plugins.Bundler.have_genshi or - type(bundle) is not - Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile): - self.bundle_names(bundle) - - def missing_bundles(self): - """ find bundles listed in Metadata but not implemented in Bundler """ - if self.files is None: - # when given a list of files on stdin, this check is - # useless, so skip it - groupdata = self.metadata.groups_xml.xdata - ref_bundles = set([b.get("name") - for b in groupdata.findall("//Bundle")]) - - allbundles = self.core.plugins['Bundler'].entries.keys() - for bundle in ref_bundles: - xmlbundle = "%s.xml" % bundle - genshibundle = "%s.genshi" % bundle - if (xmlbundle not in allbundles and - genshibundle not in allbundles): - self.LintError("bundle-not-found", - "Bundle %s referenced, but does not exist" % - bundle) - - def bundle_names(self, bundle): - """ verify bundle name attribute matches filename """ - try: - xdata = lxml.etree.XML(bundle.data) - except AttributeError: - # genshi template - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - - fname = bundle.name.split('Bundler/')[1].split('.')[0] - bname = xdata.get('name') - if fname != bname: - self.LintError("inconsistent-bundle-name", - "Inconsistent bundle name: filename is %s, bundle name is %s" % - (fname, bname)) - - def sgenshi_groups(self, bundle): - """ ensure that Genshi Bundles do not include tags, - which are not supported """ - xdata = lxml.etree.parse(bundle.name) - groups = [self.RenderXML(g) - for g in xdata.getroottree().findall("//Group")] - if groups: - self.LintError("group-tag-not-allowed", - " tag is not allowed in SGenshi Bundle:\n%s" % - "\n".join(groups)) diff --git a/src/lib/Server/Lint/Comments.py b/src/lib/Server/Lint/Comments.py deleted file mode 100644 index 19fae1b08..000000000 --- a/src/lib/Server/Lint/Comments.py +++ /dev/null @@ -1,187 +0,0 @@ -import os.path -import lxml.etree -import Bcfg2.Server.Lint - -class Comments(Bcfg2.Server.Lint.ServerPlugin): - """ check files for various required headers """ - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.config_cache = {} - - def Run(self): - self.check_bundles() - self.check_properties() - self.check_metadata() - self.check_cfg() - self.check_infoxml() - self.check_probes() - - def required_keywords(self, rtype): - """ given a file type, fetch the list of required VCS keywords - from the bcfg2-lint config """ - return self.required_items(rtype, "keyword") - - def required_comments(self, rtype): - """ given a file type, fetch the list of required comments - from the bcfg2-lint config """ - return self.required_items(rtype, "comment") - - def required_items(self, rtype, itype): - """ given a file type and item type (comment or keyword), - fetch the list of required items from the bcfg2-lint config """ - if itype not in self.config_cache: - self.config_cache[itype] = {} - - if rtype not in self.config_cache[itype]: - rv = [] - global_item = "global_%ss" % itype - if global_item in self.config: - rv.extend(self.config[global_item].split(",")) - - item = "%s_%ss" % (rtype.lower(), itype) - if item in self.config: - if self.config[item]: - rv.extend(self.config[item].split(",")) - else: - # config explicitly specifies nothing - rv = [] - self.config_cache[itype][rtype] = rv - return self.config_cache[itype][rtype] - - def check_bundles(self): - """ check bundle files for required headers """ - if 'Bundler' in self.core.plugins: - for bundle in self.core.plugins['Bundler'].entries.values(): - xdata = None - rtype = "" - try: - xdata = lxml.etree.XML(bundle.data) - rtype = "bundler" - except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - rtype = "sgenshi" - - self.check_xml(bundle.name, xdata, rtype) - - def check_properties(self): - """ check properties files for required headers """ - if 'Properties' in self.core.plugins: - props = self.core.plugins['Properties'] - for propfile, pdata in props.store.entries.items(): - if os.path.splitext(propfile)[1] == ".xml": - self.check_xml(pdata.name, pdata.xdata, 'properties') - - def check_metadata(self): - """ check metadata files for required headers """ - if self.has_all_xincludes("groups.xml"): - self.check_xml(os.path.join(self.metadata.data, "groups.xml"), - self.metadata.groups_xml.data, - "metadata") - if self.has_all_xincludes("clients.xml"): - self.check_xml(os.path.join(self.metadata.data, "clients.xml"), - self.metadata.clients_xml.data, - "metadata") - - def check_cfg(self): - """ check Cfg files for required headers """ - if 'Cfg' in self.core.plugins: - for entryset in self.core.plugins['Cfg'].entries.values(): - for entry in entryset.entries.values(): - if entry.name.endswith(".genshi"): - rtype = "tgenshi" - else: - rtype = "cfg" - self.check_plaintext(entry.name, entry.data, rtype) - - def check_infoxml(self): - """ check info.xml files for required headers """ - if 'Cfg' in self.core.plugins: - for entryset in self.core.plugins['Cfg'].entries.items(): - if (hasattr(entryset, "infoxml") and - entryset.infoxml is not None): - self.check_xml(entryset.infoxml.name, - entryset.infoxml.pnode.data, - "infoxml") - - def check_probes(self): - """ check probes for required headers """ - if 'Probes' in self.core.plugins: - for probe in self.core.plugins['Probes'].probes.entries.values(): - self.check_plaintext(probe.name, probe.data, "probes") - - def check_xml(self, filename, xdata, rtype): - """ check generic XML files for required headers """ - self.check_lines(filename, - [str(el) - for el in xdata.getiterator(lxml.etree.Comment)], - rtype) - - def check_plaintext(self, filename, data, rtype): - """ check generic plaintex files for required headers """ - self.check_lines(filename, data.splitlines(), rtype) - - def check_lines(self, filename, lines, rtype): - """ generic header check for a set of lines """ - if self.HandlesFile(filename): - # found is trivalent: - # False == not found - # None == found but not expanded - # True == found and expanded - found = dict((k, False) for k in self.required_keywords(rtype)) - - for line in lines: - # we check for both '$:' and '$$' to see - # if the keyword just hasn't been expanded - for (keyword, status) in found.items(): - if not status: - if '$%s:' % keyword in line: - found[keyword] = True - elif '$%s$' % keyword in line: - found[keyword] = None - - unexpanded = [keyword for (keyword, status) in found.items() - if status is None] - if unexpanded: - self.LintError("unexpanded-keywords", - "%s: Required keywords(s) found but not expanded: %s" % - (filename, ", ".join(unexpanded))) - missing = [keyword for (keyword, status) in found.items() - if status is False] - if missing: - self.LintError("keywords-not-found", - "%s: Required keywords(s) not found: $%s$" % - (filename, "$, $".join(missing))) - - # next, check for required comments. found is just - # boolean - found = dict((k, False) for k in self.required_comments(rtype)) - - for line in lines: - for (comment, status) in found.items(): - if not status: - found[comment] = comment in line - - missing = [comment for (comment, status) in found.items() - if status is False] - if missing: - self.LintError("comments-not-found", - "%s: Required comments(s) not found: %s" % - (filename, ", ".join(missing))) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True - diff --git a/src/lib/Server/Lint/Duplicates.py b/src/lib/Server/Lint/Duplicates.py deleted file mode 100644 index 75f620603..000000000 --- a/src/lib/Server/Lint/Duplicates.py +++ /dev/null @@ -1,81 +0,0 @@ -import os.path -import lxml.etree -import Bcfg2.Server.Lint - -class Duplicates(Bcfg2.Server.Lint.ServerPlugin): - """ Find duplicate clients, groups, etc. """ - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.groups_xdata = None - self.clients_xdata = None - self.load_xdata() - - def Run(self): - """ run plugin """ - # only run this plugin if we were not given a list of files. - # not only is it marginally silly to run this plugin with a - # partial list of files, it turns out to be really freaking - # hard to get only a fragment of group or client metadata - if self.groups_xdata is not None: - self.duplicate_groups() - self.duplicate_defaults() - if self.clients_xdata is not None: - self.duplicate_clients() - - def load_xdata(self): - """ attempt to load XML data for groups and clients. only - actually load data if all documents reference in XIncludes can - be found in self.files""" - if self.has_all_xincludes("groups.xml"): - self.groups_xdata = self.metadata.clients_xml.xdata - if self.has_all_xincludes("clients.xml"): - self.clients_xdata = self.metadata.clients_xml.xdata - - def duplicate_groups(self): - """ find duplicate groups """ - self.duplicate_entries(self.clients_xdata.xpath('//Groups/Group'), - 'group') - - def duplicate_clients(self): - """ find duplicate clients """ - self.duplicate_entries(self.clients_xdata.xpath('//Clients/Client'), - 'client') - - def duplicate_entries(self, data, etype): - """ generic duplicate entry finder """ - seen = {} - for el in data: - if el.get('name') not in seen: - seen[el.get('name')] = el - else: - self.LintError("duplicate-%s" % etype, - "Duplicate %s '%s':\n%s\n%s" % - (etype, el.get('name'), - self.RenderXML(seen[el.get('name')]), - self.RenderXML(el))) - - def duplicate_defaults(self): - """ check for multiple default group definitions """ - default_groups = [g for g in self.groups_xdata.findall('.//Group') - if g.get('default') == 'true'] - if len(default_groups) > 1: - self.LintError("multiple-default-groups", - "Multiple default groups defined: %s" % - ",".join(default_groups)) - - def has_all_xincludes(self, mfile): - """ return true if self.files includes all XIncludes listed in - the specified metadata type, false otherwise""" - if self.files is None: - return True - else: - path = os.path.join(self.metadata.data, mfile) - if path in self.files: - xdata = lxml.etree.parse(path) - for el in xdata.findall('./{http://www.w3.org/2001/XInclude}include'): - if not self.has_all_xincludes(el.get('href')): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: could not include %s" % path) - return False - - return True diff --git a/src/lib/Server/Lint/Genshi.py b/src/lib/Server/Lint/Genshi.py deleted file mode 100755 index 552c495b2..000000000 --- a/src/lib/Server/Lint/Genshi.py +++ /dev/null @@ -1,27 +0,0 @@ -import genshi.template -import Bcfg2.Server.Lint - -class Genshi(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ - - def Run(self): - """ run plugin """ - loader = genshi.template.TemplateLoader() - for plugin in ['Cfg', 'TGenshi']: - if plugin in self.core.plugins: - self.check_files(self.core.plugins[plugin].entries, - loader=loader) - - def check_files(self, entries, loader=None): - if loader is None: - loader = genshi.template.TemplateLoader() - - for eset in entries.values(): - for fname, sdata in list(eset.entries.items()): - if fname.endswith(".genshi") or fname.endswith(".newtxt"): - try: - loader.load(sdata.name, - cls=genshi.template.NewTextTemplate) - except genshi.template.TemplateSyntaxError, err: - self.LintError("genshi-syntax-error", - "Genshi syntax error: %s" % err) diff --git a/src/lib/Server/Lint/GroupPatterns.py b/src/lib/Server/Lint/GroupPatterns.py deleted file mode 100644 index b69d7a5d8..000000000 --- a/src/lib/Server/Lint/GroupPatterns.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -import Bcfg2.Server.Lint -from Bcfg2.Server.Plugins.GroupPatterns import PatternMap - -class GroupPatterns(Bcfg2.Server.Lint.ServerPlugin): - """ Check Genshi templates for syntax errors """ - - def Run(self): - """ run plugin """ - if 'GroupPatterns' in self.core.plugins: - cfg = self.core.plugins['GroupPatterns'].config - for entry in cfg.xdata.xpath('//GroupPattern'): - groups = [g.text for g in entry.findall('Group')] - self.check(entry, groups, ptype='NamePattern') - self.check(entry, groups, ptype='NameRange') - - def check(self, entry, groups, ptype="NamePattern"): - if ptype == "NamePattern": - pmap = lambda p: PatternMap(p, None, groups) - else: - pmap = lambda p: PatternMap(None, p, groups) - - for el in entry.findall(ptype): - pat = el.text - try: - pmap(pat) - except: - err = sys.exc_info()[1] - self.LintError("pattern-fails-to-initialize", - "Failed to initialize %s %s for %s: %s" % - (ptype, pat, entry.get('pattern'), err)) diff --git a/src/lib/Server/Lint/InfoXML.py b/src/lib/Server/Lint/InfoXML.py deleted file mode 100644 index 2054e23bf..000000000 --- a/src/lib/Server/Lint/InfoXML.py +++ /dev/null @@ -1,42 +0,0 @@ -import os.path -import Bcfg2.Options -import Bcfg2.Server.Lint - -class InfoXML(Bcfg2.Server.Lint.ServerPlugin): - """ ensure that all config files have an info.xml file""" - - def Run(self): - if 'Cfg' in self.core.plugins: - for filename, entryset in self.core.plugins['Cfg'].entries.items(): - infoxml_fname = os.path.join(entryset.path, "info.xml") - if self.HandlesFile(infoxml_fname): - if (hasattr(entryset, "infoxml") and - entryset.infoxml is not None): - self.check_infoxml(infoxml_fname, - entryset.infoxml.pnode.data) - else: - self.LintError("no-infoxml", - "No info.xml found for %s" % filename) - - def check_infoxml(self, fname, xdata): - for info in xdata.getroottree().findall("//Info"): - required = [] - if "required_attrs" in self.config: - required = self.config["required_attrs"].split(",") - - missing = [attr for attr in required if info.get(attr) is None] - if missing: - self.LintError("required-infoxml-attrs-missing", - "Required attribute(s) %s not found in %s:%s" % - (",".join(missing), fname, self.RenderXML(info))) - - if ((Bcfg2.Options.MDATA_PARANOID.value and - info.get("paranoid") is not None and - info.get("paranoid").lower() == "false") or - (not Bcfg2.Options.MDATA_PARANOID.value and - (info.get("paranoid") is None or - info.get("paranoid").lower() != "true"))): - self.LintError("paranoid-false", - "Paranoid must be true in %s:%s" % - (fname, self.RenderXML(info))) - diff --git a/src/lib/Server/Lint/MergeFiles.py b/src/lib/Server/Lint/MergeFiles.py deleted file mode 100644 index ff6e3449a..000000000 --- a/src/lib/Server/Lint/MergeFiles.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import copy -from difflib import SequenceMatcher -import Bcfg2.Server.Lint - -class MergeFiles(Bcfg2.Server.Lint.ServerPlugin): - """ find Probes or Cfg files with multiple similar files that - might be merged into one """ - - def Run(self): - if 'Cfg' in self.core.plugins: - self.check_cfg() - if 'Probes' in self.core.plugins: - self.check_probes() - - def check_cfg(self): - for filename, entryset in self.core.plugins['Cfg'].entries.items(): - for mset in self.get_similar(entryset.entries): - self.LintError("merge-cfg", - "The following files are similar: %s. " - "Consider merging them into a single Genshi " - "template." % - ", ".join([os.path.join(filename, p) - for p in mset])) - - def check_probes(self): - probes = self.core.plugins['Probes'].probes.entries - for mset in self.get_similar(probes): - self.LintError("merge-cfg", - "The following probes are similar: %s. " - "Consider merging them into a single probe." % - ", ".join([p for p in mset])) - - def get_similar(self, entries): - if "threshold" in self.config: - # accept threshold either as a percent (e.g., "threshold=75") or - # as a ratio (e.g., "threshold=.75") - threshold = float(self.config['threshold']) - if threshold > 1: - threshold /= 100 - else: - threshold = 0.75 - rv = [] - elist = entries.items() - while elist: - result = self._find_similar(elist.pop(0), copy.copy(elist), - threshold) - if len(result) > 1: - elist = [(fname, fdata) - for fname, fdata in elist - if fname not in result] - rv.append(result) - return rv - - def _find_similar(self, ftuple, others, threshold): - fname, fdata = ftuple - rv = [fname] - while others: - cname, cdata = others.pop(0) - sm = SequenceMatcher(None, fdata.data, cdata.data) - # perform progressively more expensive comparisons - if (sm.real_quick_ratio() > threshold and - sm.quick_ratio() > threshold and - sm.ratio() > threshold): - rv.extend(self._find_similar((cname, cdata), copy.copy(others), - threshold)) - return rv - - diff --git a/src/lib/Server/Lint/Pkgmgr.py b/src/lib/Server/Lint/Pkgmgr.py deleted file mode 100644 index 8f099163a..000000000 --- a/src/lib/Server/Lint/Pkgmgr.py +++ /dev/null @@ -1,35 +0,0 @@ -import glob -import lxml.etree -import Bcfg2.Server.Lint - -class Pkgmgr(Bcfg2.Server.Lint.ServerlessPlugin): - """ find duplicate Pkgmgr entries with the same priority """ - - def Run(self): - pset = set() - for pfile in glob.glob("%s/Pkgmgr/*.xml" % self.config['repo']): - if self.HandlesFile(pfile): - xdata = lxml.etree.parse(pfile).getroot() - # get priority, type, group - priority = xdata.get('priority') - ptype = xdata.get('type') - for pkg in xdata.xpath("//Package"): - if pkg.getparent().tag == 'Group': - grp = pkg.getparent().get('name') - if (type(grp) is not str and - grp.getparent().tag == 'Group'): - pgrp = grp.getparent().get('name') - else: - pgrp = 'none' - else: - grp = 'none' - pgrp = 'none' - ptuple = (pkg.get('name'), priority, ptype, grp, pgrp) - # check if package is already listed with same - # priority, type, grp - if ptuple in pset: - self.LintError("duplicate-package", - "Duplicate Package %s, priority:%s, type:%s" % - (pkg.get('name'), priority, ptype)) - else: - pset.add(ptuple) diff --git a/src/lib/Server/Lint/RequiredAttrs.py b/src/lib/Server/Lint/RequiredAttrs.py deleted file mode 100644 index 55206d2ba..000000000 --- a/src/lib/Server/Lint/RequiredAttrs.py +++ /dev/null @@ -1,138 +0,0 @@ -import os.path -import lxml.etree -import Bcfg2.Server.Lint -from Bcfg2.Server.Plugins.Packages import Apt, Yum - -class RequiredAttrs(Bcfg2.Server.Lint.ServerPlugin): - """ verify attributes for configuration entries (as defined in - doc/server/configurationentries) """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerPlugin.__init__(self, *args, **kwargs) - self.required_attrs = { - 'Path': { - 'device': ['name', 'owner', 'group', 'dev_type'], - 'directory': ['name', 'owner', 'group', 'perms'], - 'file': ['name', 'owner', 'group', 'perms', '__text__'], - 'hardlink': ['name', 'to'], - 'symlink': ['name', 'to'], - 'ignore': ['name'], - 'nonexistent': ['name'], - 'permissions': ['name', 'owner', 'group', 'perms'], - 'vcs': ['vcstype', 'revision', 'sourceurl']}, - 'Service': { - 'chkconfig': ['name'], - 'deb': ['name'], - 'rc-update': ['name'], - 'smf': ['name', 'FMRI'], - 'upstart': ['name']}, - 'Action': ['name', 'timing', 'when', 'status', 'command'], - 'Package': ['name']} - - def Run(self): - self.check_packages() - if "Defaults" in self.core.plugins: - self.logger.info("Defaults plugin enabled; skipping required " - "attribute checks") - else: - self.check_rules() - self.check_bundles() - - def check_packages(self): - """ check package sources for Source entries with missing attrs """ - if 'Packages' in self.core.plugins: - for source in self.core.plugins['Packages'].sources: - if isinstance(source, Yum.YumSource): - if (not source.pulp_id and not source.url and - not source.rawurl): - self.LintError("required-attrs-missing", - "A %s source must have either a url, " - "rawurl, or pulp_id attribute: %s" % - (source.ptype, - self.RenderXML(source.xsource))) - elif not source.url and not source.rawurl: - self.LintError("required-attrs-missing", - "A %s source must have either a url or " - "rawurl attribute: %s" % - (source.ptype, - self.RenderXML(source.xsource))) - - if (not isinstance(source, Apt.AptSource) and - source.recommended): - self.LintError("extra-attrs", - "The recommended attribute is not " - "supported on %s sources: %s" % - (source.ptype, - self.RenderXML(source.xsource))) - - def check_rules(self): - """ check Rules for Path entries with missing attrs """ - if 'Rules' in self.core.plugins: - for rules in self.core.plugins['Rules'].entries.values(): - xdata = rules.pnode.data - for path in xdata.xpath("//Path"): - self.check_entry(path, os.path.join(self.config['repo'], - rules.name)) - - def check_bundles(self): - """ check bundles for BoundPath entries with missing attrs """ - if 'Bundler' in self.core.plugins: - for bundle in self.core.plugins['Bundler'].entries.values(): - try: - xdata = lxml.etree.XML(bundle.data) - except (lxml.etree.XMLSyntaxError, AttributeError): - xdata = lxml.etree.parse(bundle.template.filepath).getroot() - - for path in xdata.xpath("//*[substring(name(), 1, 5) = 'Bound']"): - self.check_entry(path, bundle.name) - - def check_entry(self, entry, filename): - """ generic entry check """ - if self.HandlesFile(filename): - name = entry.get('name') - tag = entry.tag - if tag.startswith("Bound"): - tag = tag[5:] - if tag not in self.required_attrs: - self.LintError("unknown-entry-tag", - "Unknown entry tag '%s': %s" % - (entry.tag, self.RenderXML(entry))) - - if isinstance(self.required_attrs[tag], dict): - etype = entry.get('type') - if etype in self.required_attrs[tag]: - required_attrs = set(self.required_attrs[tag][etype] + - ['type']) - else: - self.LintError("unknown-entry-type", - "Unknown %s type %s: %s" % - (tag, etype, self.RenderXML(entry))) - return - else: - required_attrs = set(self.required_attrs[tag]) - attrs = set(entry.attrib.keys()) - - if 'dev_type' in required_attrs: - dev_type = entry.get('dev_type') - if dev_type in ['block', 'char']: - # check if major/minor are specified - required_attrs |= set(['major', 'minor']) - - if '__text__' in required_attrs: - required_attrs.remove('__text__') - if (not entry.text and - not entry.get('empty', 'false').lower() == 'true'): - self.LintError("required-attrs-missing", - "Text missing for %s %s in %s: %s" % - (entry.tag, name, filename, - self.RenderXML(entry))) - - if not attrs.issuperset(required_attrs): - self.LintError("required-attrs-missing", - "The following required attribute(s) are " - "missing for %s %s in %s: %s\n%s" % - (entry.tag, name, filename, - ", ".join([attr - for attr in - required_attrs.difference(attrs)]), - self.RenderXML(entry))) diff --git a/src/lib/Server/Lint/Validate.py b/src/lib/Server/Lint/Validate.py deleted file mode 100644 index 952a65365..000000000 --- a/src/lib/Server/Lint/Validate.py +++ /dev/null @@ -1,200 +0,0 @@ -import fnmatch -import glob -import lxml.etree -import os -from subprocess import Popen, PIPE, STDOUT -import sys - -import Bcfg2.Server.Lint - -class Validate(Bcfg2.Server.Lint.ServerlessPlugin): - """ Ensure that the repo validates """ - - def __init__(self, *args, **kwargs): - Bcfg2.Server.Lint.ServerlessPlugin.__init__(self, *args, **kwargs) - self.filesets = {"metadata:groups":"%s/metadata.xsd", - "metadata:clients":"%s/clients.xsd", - "info":"%s/info.xsd", - "%s/Bundler/*.xml":"%s/bundle.xsd", - "%s/Bundler/*.genshi":"%s/bundle.xsd", - "%s/Pkgmgr/*.xml":"%s/pkglist.xsd", - "%s/Base/*.xml":"%s/base.xsd", - "%s/Rules/*.xml":"%s/rules.xsd", - "%s/Defaults/*.xml":"%s/defaults.xsd", - "%s/etc/report-configuration.xml":"%s/report-configuration.xsd", - "%s/Svcmgr/*.xml":"%s/services.xsd", - "%s/Deps/*.xml":"%s/deps.xsd", - "%s/Decisions/*.xml":"%s/decisions.xsd", - "%s/Packages/sources.xml":"%s/packages.xsd", - "%s/GroupPatterns/config.xml":"%s/grouppatterns.xsd", - "%s/NagiosGen/config.xml":"%s/nagiosgen.xsd", - "%s/FileProbes/config.xml":"%s/fileprobes.xsd", - } - - self.filelists = {} - self.get_filelists() - - def Run(self): - schemadir = self.config['schema'] - - for path, schemaname in self.filesets.items(): - try: - filelist = self.filelists[path] - except KeyError: - filelist = [] - - if filelist: - # avoid loading schemas for empty file lists - schemafile = schemaname % schemadir - try: - schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) - except IOError: - e = sys.exc_info()[1] - self.LintError("input-output-error", str(e)) - continue - except lxml.etree.XMLSchemaParseError: - e = sys.exc_info()[1] - self.LintError("schema-failed-to-parse", - "Failed to process schema %s: %s" % - (schemafile, e)) - continue - for filename in filelist: - self.validate(filename, schemafile, schema=schema) - - self.check_properties() - - def check_properties(self): - """ check Properties files against their schemas """ - for filename in self.filelists['props']: - schemafile = "%s.xsd" % os.path.splitext(filename)[0] - if os.path.exists(schemafile): - self.validate(filename, schemafile) - else: - self.LintError("properties-schema-not-found", - "No schema found for %s" % filename) - - def validate(self, filename, schemafile, schema=None): - """validate a file against the given lxml.etree.Schema. - return True on success, False on failure """ - if schema is None: - # if no schema object was provided, instantiate one - try: - schema = lxml.etree.XMLSchema(lxml.etree.parse(schemafile)) - except: - self.LintError("schema-failed-to-parse", - "Failed to process schema %s" % schemafile) - return False - - try: - datafile = lxml.etree.parse(filename) - except SyntaxError: - lint = Popen(["xmllint", filename], stdout=PIPE, stderr=STDOUT) - self.LintError("xml-failed-to-parse", - "%s fails to parse:\n%s" % (filename, - lint.communicate()[0])) - lint.wait() - return False - except IOError: - self.LintError("xml-failed-to-read", - "Failed to open file %s" % filename) - return False - - if not schema.validate(datafile): - cmd = ["xmllint"] - if self.files is None: - cmd.append("--xinclude") - cmd.extend(["--noout", "--schema", schemafile, filename]) - lint = Popen(cmd, stdout=PIPE, stderr=STDOUT) - output = lint.communicate()[0] - if lint.wait(): - self.LintError("xml-failed-to-verify", - "%s fails to verify:\n%s" % (filename, output)) - return False - return True - - def get_filelists(self): - """ get lists of different kinds of files to validate """ - if self.files is not None: - listfiles = lambda p: fnmatch.filter(self.files, p % "*") - else: - listfiles = lambda p: glob.glob(p % self.config['repo']) - - for path in self.filesets.keys(): - if path.startswith("metadata:"): - mtype = path.split(":")[1] - self.filelists[path] = self.get_metadata_list(mtype) - elif path == "info": - if self.files is not None: - self.filelists[path] = \ - [f for f in self.files - if os.path.basename(f) == 'info.xml'] - else: # self.files is None - self.filelists[path] = [] - for infodir in ['Cfg', 'TGenshi', 'TCheetah']: - for root, dirs, files in os.walk('%s/%s' % - (self.config['repo'], - infodir)): - self.filelists[path].extend([os.path.join(root, f) - for f in files - if f == 'info.xml']) - else: - self.filelists[path] = listfiles(path) - - self.filelists['props'] = listfiles("%s/Properties/*.xml") - all_metadata = listfiles("%s/Metadata/*.xml") - - # if there are other files in Metadata that aren't xincluded - # from clients.xml or groups.xml, we can't verify them. warn - # about those. - for fname in all_metadata: - if (fname not in self.filelists['metadata:groups'] and - fname not in self.filelists['metadata:clients']): - self.LintError("broken-xinclude-chain", - "Broken XInclude chain: Could not determine file type of %s" % fname) - - def get_metadata_list(self, mtype): - """ get all metadata files for the specified type (clients or - group) """ - if self.files is not None: - rv = fnmatch.filter(self.files, "*/Metadata/%s.xml" % mtype) - else: - rv = glob.glob("%s/Metadata/%s.xml" % (self.config['repo'], mtype)) - - # attempt to follow XIncludes. if the top-level files aren't - # listed in self.files, though, there's really nothing we can - # do to guess what a file in Metadata is - if rv: - try: - rv.extend(self.follow_xinclude(rv[0])) - except lxml.etree.XMLSyntaxError: - e = sys.exc_info()[1] - self.LintError("xml-failed-to-parse", - "%s fails to parse:\n%s" % (rv[0], e)) - - - return rv - - def follow_xinclude(self, xfile): - """ follow xincludes in the given file """ - xdata = lxml.etree.parse(xfile) - included = set([ent.get('href') for ent in - xdata.findall('./{http://www.w3.org/2001/XInclude}include')]) - rv = [] - - while included: - try: - filename = included.pop() - except KeyError: - continue - - path = os.path.join(os.path.dirname(xfile), filename) - if self.HandlesFile(path): - rv.append(path) - groupdata = lxml.etree.parse(path) - [included.add(el.get('href')) - for el in - groupdata.findall('./{http://www.w3.org/2001/XInclude}include')] - included.discard(filename) - - return rv - diff --git a/src/lib/Server/Lint/__init__.py b/src/lib/Server/Lint/__init__.py deleted file mode 100644 index 326f5b1dc..000000000 --- a/src/lib/Server/Lint/__init__.py +++ /dev/null @@ -1,215 +0,0 @@ -__all__ = ['Bundles', - 'Comments', - 'Duplicates', - 'InfoXML', - 'MergeFiles', - 'Pkgmgr', - 'RequiredAttrs', - 'Validate', - 'Genshi'] - -import logging -import os -import sys -from copy import copy -import textwrap -import lxml.etree -import Bcfg2.Logger -import fcntl -import termios -import struct - -def _ioctl_GWINSZ(fd): - try: - cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - except: - return None - return cr - -def get_termsize(): - """ get a tuple of (width, height) giving the size of the terminal """ - if not sys.stdout.isatty(): - return None - cr = _ioctl_GWINSZ(0) or _ioctl_GWINSZ(1) or _ioctl_GWINSZ(2) - if not cr: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) - cr = _ioctl_GWINSZ(fd) - os.close(fd) - except: - pass - if not cr: - try: - cr = (os.environ['LINES'], os.environ['COLUMNS']) - except KeyError: - return None - return int(cr[1]), int(cr[0]) - -class Plugin (object): - """ base class for ServerlessPlugin and ServerPlugin """ - - def __init__(self, config, errorhandler=None, files=None): - self.files = files - self.config = config - self.logger = logging.getLogger('bcfg2-lint') - if errorhandler is None: - self.errorHandler = ErrorHandler() - else: - self.errorHandler = errorhandler - - def Run(self): - """ run the plugin. must be overloaded by child classes """ - pass - - def HandlesFile(self, fname): - """ returns true if the given file should be handled by the - plugin according to the files list, false otherwise """ - return (self.files is None or - fname in self.files or - os.path.join(self.config['repo'], fname) in self.files or - os.path.abspath(fname) in self.files or - os.path.abspath(os.path.join(self.config['repo'], - fname)) in self.files) - - def LintError(self, err, msg): - self.errorHandler.dispatch(err, msg) - - def RenderXML(self, element): - """render an XML element for error output -- line number - prefixed, no children""" - xml = None - if len(element) or element.text: - el = copy(element) - if el.text: - el.text = '...' - [el.remove(c) for c in el.iterchildren()] - xml = lxml.etree.tostring(el).strip() - else: - xml = lxml.etree.tostring(element).strip() - return " line %s: %s" % (element.sourceline, xml) - - -class ErrorHandler (object): - # how to handle different errors by default - _errors = {"no-infoxml":"warning", - "paranoid-false":"warning", - "bundle-not-found":"error", - "inconsistent-bundle-name":"warning", - "group-tag-not-allowed":"error", - "unexpanded-keywords":"warning", - "keywords-not-found":"warning", - "comments-not-found":"warning", - "broken-xinclude-chain":"warning", - "duplicate-client":"error", - "duplicate-group":"error", - "duplicate-package":"error", - "multiple-default-groups":"error", - "required-infoxml-attrs-missing":"error", - "unknown-entry-type":"error", - "required-attrs-missing":"error", - "extra-attrs":"warning", - "schema-failed-to-parse":"warning", - "properties-schema-not-found":"warning", - "xml-failed-to-parse":"error", - "xml-failed-to-read":"error", - "xml-failed-to-verify":"error", - "merge-cfg":"warning", - "merge-probes":"warning", - "input-output-error":"error", - "genshi-syntax-error":"error", - "pattern-fails-to-initialize":"error"} - - def __init__(self, config=None): - self.errors = 0 - self.warnings = 0 - - self.logger = logging.getLogger('bcfg2-lint') - - termsize = get_termsize() - if termsize is not None: - self._wrapper = textwrap.TextWrapper(initial_indent=" ", - subsequent_indent=" ", - width=termsize[0]) - else: - self._wrapper = None - - self._handlers = {} - if config is not None: - for err, action in config.items(): - if "warn" in action: - self._handlers[err] = self.warn - elif "err" in action: - self._handlers[err] = self.error - else: - self._handlers[err] = self.debug - - for err, action in self._errors.items(): - if err not in self._handlers: - if "warn" in action: - self._handlers[err] = self.warn - elif "err" in action: - self._handlers[err] = self.error - else: - self._handlers[err] = self.debug - - def dispatch(self, err, msg): - if err in self._handlers: - self._handlers[err](msg) - self.logger.debug(" (%s)" % err) - else: - # assume that it's an error, but complain - self.error(msg) - self.logger.warning("Unknown error %s" % err) - - def error(self, msg): - """ log an error condition """ - self.errors += 1 - self._log(msg, self.logger.error, prefix="ERROR: ") - - def warn(self, msg): - """ log a warning condition """ - self.warnings += 1 - self._log(msg, self.logger.warning, prefix="WARNING: ") - - def debug(self, msg): - """ log a silent/debug condition """ - self._log(msg, self.logger.debug) - - def _log(self, msg, logfunc, prefix=""): - # a message may itself consist of multiple lines. wrap() will - # elide them all into a single paragraph, which we don't want. - # so we split the message into its paragraphs and wrap each - # paragraph individually. this means, unfortunately, that we - # lose textwrap's built-in initial indent functionality, - # because we want to only treat the very first line of the - # first paragraph specially. so we do some silliness. - rawlines = msg.splitlines() - firstline = True - for rawline in rawlines: - if self._wrapper: - lines = self._wrapper.wrap(rawline) - else: - lines = [rawline] - for line in lines: - if firstline: - logfunc("%s%s" % (prefix, line.lstrip())) - firstline = False - else: - logfunc(line) - - -class ServerlessPlugin (Plugin): - """ base class for plugins that are run before the server starts - up (i.e., plugins that check things that may prevent the server - from starting up) """ - pass - - -class ServerPlugin (Plugin): - """ base class for plugins that check things that require the - running Bcfg2 server """ - def __init__(self, lintCore, config, **kwargs): - Plugin.__init__(self, config, **kwargs) - self.core = lintCore - self.logger = self.core.logger - self.metadata = self.core.metadata diff --git a/src/lib/Server/Plugin.py b/src/lib/Server/Plugin.py deleted file mode 100644 index c48859a44..000000000 --- a/src/lib/Server/Plugin.py +++ /dev/null @@ -1,1232 +0,0 @@ -"""This module provides the baseclass for Bcfg2 Server Plugins.""" - -import copy -import logging -import lxml.etree -import os -import os.path -import pickle -import posixpath -import re -import sys -import threading -from Bcfg2.Bcfg2Py3k import ConfigParser - -from lxml.etree import XML, XMLSyntaxError - -import Bcfg2.Options - -# py3k compatibility -if sys.hexversion >= 0x03000000: - from functools import reduce - from io import FileIO as BUILTIN_FILE_TYPE -else: - BUILTIN_FILE_TYPE = file -from Bcfg2.Bcfg2Py3k import Queue -from Bcfg2.Bcfg2Py3k import Empty -from Bcfg2.Bcfg2Py3k import Full - -# grab default metadata info from bcfg2.conf -opts = {'owner': Bcfg2.Options.MDATA_OWNER, - 'group': Bcfg2.Options.MDATA_GROUP, - 'important': Bcfg2.Options.MDATA_IMPORTANT, - 'perms': Bcfg2.Options.MDATA_PERMS, - 'paranoid': Bcfg2.Options.MDATA_PARANOID, - 'sensitive': Bcfg2.Options.MDATA_SENSITIVE} -mdata_setup = Bcfg2.Options.OptionParser(opts) -mdata_setup.parse([]) -del mdata_setup['args'] - -logger = logging.getLogger('Bcfg2.Server.Plugin') - -default_file_metadata = mdata_setup - -info_regex = re.compile( \ - 'encoding:(\s)*(?P\w+)|' + - 'group:(\s)*(?P\S+)|' + - 'important:(\s)*(?P\S+)|' + - 'mtime:(\s)*(?P\w+)|' + - 'owner:(\s)*(?P\S+)|' + - 'paranoid:(\s)*(?P\S+)|' + - 'perms:(\s)*(?P\w+)|' + - 'sensitive:(\s)*(?P\S+)|') - - -class PluginInitError(Exception): - """Error raised in cases of Plugin initialization errors.""" - pass - - -class PluginExecutionError(Exception): - """Error raised in case of Plugin execution errors.""" - pass - - -class Debuggable(object): - __rmi__ = ['toggle_debug'] - - def __init__(self, name=None): - if name is None: - name = "%s.%s" % (self.__class__.__module__, - self.__class__.__name__) - self.debug_flag = False - self.logger = logging.getLogger(name) - - def toggle_debug(self): - self.debug_flag = not self.debug_flag - - def debug_log(self, message, flag=None): - if (flag is None and self.debug_flag) or flag: - self.logger.error(message) - - -class Plugin(Debuggable): - """This is the base class for all Bcfg2 Server plugins. - Several attributes must be defined in the subclass: - name : the name of the plugin - __author__ : the author/contact for the plugin - - Plugins can provide three basic types of functionality: - - Structure creation (overloading BuildStructures) - - Configuration entry binding (overloading HandlesEntry, or loads the Entries table) - - Data collection (overloading GetProbes/ReceiveData) - """ - name = 'Plugin' - __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = False - deprecated = False - conflicts = [] - - # Default sort_order to 500. Plugins of the same type are - # processed in order of ascending sort_order value. Plugins with - # the same sort_order are sorted alphabetically by their name. - sort_order = 500 - - def __init__(self, core, datastore): - """Initialize the plugin. - - :param core: the Bcfg2.Server.Core initializing the plugin - :param datastore: the filesystem path of Bcfg2's repository - """ - object.__init__(self) - self.Entries = {} - self.core = core - self.data = os.path.join(datastore, self.name) - self.running = True - Debuggable.__init__(self, name=self.name) - - @classmethod - def init_repo(cls, repo): - path = "%s/%s" % (repo, cls.name) - os.makedirs(path) - - def shutdown(self): - self.running = False - - -class Generator(object): - """Generator plugins contribute to literal client configurations.""" - def HandlesEntry(self, entry, metadata): - """This is the slow path method for routing configuration binding requests.""" - return False - - def HandleEntry(self, entry, metadata): - """This is the slow-path handler for configuration entry binding.""" - raise PluginExecutionError - - -class Structure(object): - """Structure Plugins contribute to abstract client configurations.""" - def BuildStructures(self, metadata): - """Return a list of abstract goal structures for client.""" - raise PluginExecutionError - - -class Metadata(object): - """Signal metadata capabilities for this plugin""" - def add_client(self, client_name, attribs): - """Add client.""" - pass - - def remove_client(self, client_name): - """Remove client.""" - pass - - def viz(self, hosts, bundles, key, colors): - """Create viz str for viz admin mode.""" - pass - - def get_initial_metadata(self, client_name): - raise PluginExecutionError - - def merge_additional_data(self, imd, source, groups, data): - raise PluginExecutionError - - -class Connector(object): - """Connector Plugins augment client metadata instances.""" - def get_additional_groups(self, metadata): - """Determine additional groups for metadata.""" - return list() - - def get_additional_data(self, metadata): - """Determine additional data for metadata instances.""" - return dict() - - -class Probing(object): - """Signal probe capability for this plugin.""" - def GetProbes(self, _): - """Return a set of probes for execution on client.""" - return [] - - def ReceiveData(self, _, dummy): - """Receive probe results pertaining to client.""" - pass - - -class Statistics(object): - """Signal statistics handling capability.""" - def process_statistics(self, client, xdata): - pass - - -class ThreadedStatistics(Statistics, - threading.Thread): - """Threaded statistics handling capability.""" - def __init__(self, core, datastore): - Statistics.__init__(self) - threading.Thread.__init__(self) - # Event from the core signaling an exit - self.terminate = core.terminate - self.work_queue = Queue(100000) - self.pending_file = "%s/etc/%s.pending" % (datastore, self.__class__.__name__) - self.daemon = True - self.start() - - def save(self): - """Save any pending data to a file.""" - pending_data = [] - try: - while not self.work_queue.empty(): - (metadata, data) = self.work_queue.get_nowait() - try: - pending_data.append((metadata.hostname, lxml.etree.tostring(data))) - except: - self.logger.warning("Dropping interaction for %s" % metadata.hostname) - except Empty: - pass - - try: - savefile = open(self.pending_file, 'w') - pickle.dump(pending_data, savefile) - savefile.close() - self.logger.info("Saved pending %s data" % self.__class__.__name__) - except: - self.logger.warning("Failed to save pending data") - - def load(self): - """Load any pending data to a file.""" - if not os.path.exists(self.pending_file): - return True - pending_data = [] - try: - savefile = open(self.pending_file, 'r') - pending_data = pickle.load(savefile) - savefile.close() - except Exception: - e = sys.exc_info()[1] - self.logger.warning("Failed to load pending data: %s" % e) - for (pmetadata, pdata) in pending_data: - # check that shutdown wasnt called early - if self.terminate.isSet(): - return False - - try: - while True: - try: - metadata = self.core.build_metadata(pmetadata) - break - except Bcfg2.Server.Plugins.Metadata.MetadataRuntimeError: - pass - - self.terminate.wait(5) - if self.terminate.isSet(): - return False - - self.work_queue.put_nowait((metadata, lxml.etree.fromstring(pdata))) - except Full: - self.logger.warning("Queue.Full: Failed to load queue data") - break - except lxml.etree.LxmlError: - lxml_error = sys.exc_info()[1] - self.logger.error("Unable to load save interaction: %s" % lxml_error) - except Bcfg2.Server.Plugins.Metadata.MetadataConsistencyError: - self.logger.error("Unable to load metadata for save interaction: %s" % pmetadata) - try: - os.unlink(self.pending_file) - except: - self.logger.error("Failed to unlink save file: %s" % self.pending_file) - self.logger.info("Loaded pending %s data" % self.__class__.__name__) - return True - - def run(self): - if not self.load(): - return - while not self.terminate.isSet(): - try: - (xdata, client) = self.work_queue.get(block=True, timeout=2) - except Empty: - continue - except Exception: - e = sys.exc_info()[1] - self.logger.error("ThreadedStatistics: %s" % e) - continue - self.handle_statistic(xdata, client) - if not self.work_queue.empty(): - self.save() - - def process_statistics(self, metadata, data): - warned = False - try: - self.work_queue.put_nowait((metadata, copy.copy(data))) - warned = False - except Full: - if not warned: - self.logger.warning("%s: Queue is full. Dropping interactions." % self.__class__.__name__) - warned = True - - def handle_statistics(self, metadata, data): - """Handle stats here.""" - pass - - -class PullSource(object): - def GetExtra(self, client): - return [] - - def GetCurrentEntry(self, client, e_type, e_name): - raise PluginExecutionError - - -class PullTarget(object): - def AcceptChoices(self, entry, metadata): - raise PluginExecutionError - - def AcceptPullData(self, specific, new_entry, verbose): - """This is the null per-plugin implementation - of bcfg2-admin pull.""" - raise PluginExecutionError - - -class Decision(object): - """Signal decision handling capability.""" - def GetDecisions(self, metadata, mode): - return [] - - -class ValidationError(Exception): - pass - - -class StructureValidator(object): - """Validate/modify goal structures.""" - def validate_structures(self, metadata, structures): - raise ValidationError("not implemented") - - -class GoalValidator(object): - """Validate/modify configuration goals.""" - def validate_goals(self, metadata, goals): - raise ValidationError("not implemented") - - -class Version(object): - """Interact with various version control systems.""" - def get_revision(self): - return [] - - def commit_data(self, file_list, comment=None): - pass - - -# the rest of the file contains classes for coherent file caching - -class FileBacked(object): - """This object caches file data in memory. - HandleEvent is called whenever fam registers an event. - Index can parse the data into member data as required. - This object is meant to be used as a part of DirectoryBacked. - """ - - def __init__(self, name): - object.__init__(self) - self.data = '' - self.name = name - - def HandleEvent(self, event=None): - """Read file upon update.""" - if event and event.code2str() not in ['exists', 'changed', 'created']: - return - try: - self.data = BUILTIN_FILE_TYPE(self.name).read() - self.Index() - except IOError: - err = sys.exc_info()[1] - logger.error("Failed to read file %s: %s" % (self.name, err)) - - def Index(self): - """Update local data structures based on current file state""" - pass - - def __repr__(self): - return "%s: %s" % (self.__class__.__name__, str(self)) - - def __str__(self): - return "%s: %s" % (self.name, self.data) - - -class DirectoryBacked(object): - """This object is a coherent cache for a filesystem hierarchy of files.""" - __child__ = FileBacked - patterns = re.compile('.*') - - def __init__(self, data, fam): - """Initialize the DirectoryBacked object. - - :param self: the object being initialized. - :param data: the path to the data directory that will be - monitored. - :param fam: The FileMonitor object used to receive - notifications of changes. - """ - object.__init__(self) - - self.data = os.path.normpath(data) - self.fam = fam - - # self.entries contains information about the files monitored - # by this object.... The keys of the dict are the relative - # paths to the files. The values are the objects (of type - # __child__) that handle their contents. - self.entries = {} - - # self.handles contains information about the directories - # monitored by this object. The keys of the dict are the - # values returned by the initial fam.AddMonitor() call (which - # appear to be integers). The values are the relative paths of - # the directories. - self.handles = {} - - # Monitor everything in the plugin's directory - self.add_directory_monitor('') - - def __getitem__(self, key): - return self.entries[key] - - def __iter__(self): - return iter(list(self.entries.items())) - - def add_directory_monitor(self, relative): - """Add a new directory to FAM structures for monitoring. - - :param relative: Path name to monitor. This must be relative - to the plugin's directory. An empty string value ("") will - cause the plugin directory itself to be monitored. - """ - dirpathname = os.path.join(self.data, relative) - if relative not in self.handles.values(): - if not posixpath.isdir(dirpathname): - logger.error("Failed to open directory %s" % (dirpathname)) - return - reqid = self.fam.AddMonitor(dirpathname, self) - self.handles[reqid] = relative - - def add_entry(self, relative, event): - """Add a new file to our structures for monitoring. - - :param relative: Path name to monitor. This must be relative - to the plugin's directory. - :param event: File Monitor event that caused this entry to be - added. - """ - self.entries[relative] = self.__child__(os.path.join(self.data, - relative)) - self.entries[relative].HandleEvent(event) - - def HandleEvent(self, event): - """Handle FAM/Gamin events. - - This method is invoked by FAM/Gamin when it detects a change - to a filesystem object we have requsted to be monitored. - - This method manages the lifecycle of events related to the - monitored objects, adding them to our indiciess and creating - objects of type __child__ that actually do the domain-specific - processing. When appropriate, it propogates events those - objects by invoking their HandleEvent in turn. - """ - action = event.code2str() - - # Clean up the absolute path names passed in - event.filename = os.path.normpath(event.filename) - if event.filename.startswith(self.data): - event.filename = event.filename[len(self.data)+1:] - - # Exclude events for actions we don't care about - if action == 'endExist': - return - - if event.requestID not in self.handles: - logger.warn("Got %s event with unknown handle (%s) for %s" - % (action, event.requestID, abspath)) - return - - # Calculate the absolute and relative paths this event refers to - abspath = os.path.join(self.data, self.handles[event.requestID], - event.filename) - relpath = os.path.join(self.handles[event.requestID], event.filename) - - if action == 'deleted': - for key in self.entries.keys(): - if key.startswith(relpath): - del self.entries[key] - # We remove values from self.entries, but not - # self.handles, because the FileMonitor doesn't stop - # watching a directory just because it gets deleted. If it - # is recreated, we will start getting notifications for it - # again without having to add a new monitor. - elif posixpath.isdir(abspath): - # Deal with events for directories - if action in ['exists', 'created']: - self.add_directory_monitor(relpath) - elif action == 'changed': - if relpath in self.entries: - # Ownerships, permissions or timestamps changed on - # the directory. None of these should affect the - # contents of the files, though it could change - # our ability to access them. - # - # It seems like the right thing to do is to cancel - # monitoring the directory and then begin - # monitoring it again. But the current FileMonitor - # class doesn't support canceling, so at least let - # the user know that a restart might be a good - # idea. - logger.warn("Directory properties for %s changed, please " + - " consider restarting the server" % (abspath)) - else: - # Got a "changed" event for a directory that we - # didn't know about. Go ahead and treat it like a - # "created" event, but log a warning, because this - # is unexpected. - logger.warn("Got %s event for unexpected dir %s" % (action, - abspath)) - self.add_directory_monitor(relpath) - else: - logger.warn("Got unknown dir event %s %s %s" % (event.requestID, - event.code2str(), - abspath)) - else: - # Deal with events for non-directories - if ((event.filename[-1] == '~') or - (event.filename[:2] == '.#') or - (event.filename[-4:] == '.swp') or - (event.filename in ['SCCS', '.svn', '4913']) or - (not self.patterns.match(event.filename))): - return - if action in ['exists', 'created']: - self.add_entry(relpath, event) - elif action == 'changed': - if relpath in self.entries: - self.entries[relpath].HandleEvent(event) - else: - # Got a "changed" event for a file that we didn't - # know about. Go ahead and treat it like a - # "created" event, but log a warning, because this - # is unexpected. - logger.warn("Got %s event for unexpected file %s" % (action, - abspath)) - self.add_entry(relpath, event) - else: - logger.warn("Got unknown file event %s %s %s" % (event.requestID, - event.code2str(), - abspath)) - - -class XMLFileBacked(FileBacked): - """ - This object is a coherent cache for an XML file to be used as a - part of DirectoryBacked. - """ - __identifier__ = 'name' - - def __init__(self, filename): - self.label = "dummy" - self.entries = [] - FileBacked.__init__(self, filename) - - def Index(self): - """Build local data structures.""" - try: - self.xdata = XML(self.data) - except XMLSyntaxError: - logger.error("Failed to parse %s" % (self.name)) - return - self.entries = self.xdata.getchildren() - if self.__identifier__ is not None: - self.label = self.xdata.attrib[self.__identifier__] - - def __iter__(self): - return iter(self.entries) - - def __str__(self): - return "%s: %s" % (self.name, lxml.etree.tostring(self.xdata)) - - -class SingleXMLFileBacked(XMLFileBacked): - """This object is a coherent cache for an independent XML file.""" - def __init__(self, filename, fam): - XMLFileBacked.__init__(self, filename) - self.extras = [] - self.fam = fam - self.fam.AddMonitor(filename, self) - - def Index(self): - """Build local data structures.""" - try: - self.xdata = lxml.etree.XML(self.data, base_url=self.name) - except lxml.etree.XMLSyntaxError: - err = sys.exc_info()[1] - logger.error("Failed to parse %s: %s" % (self.name, err)) - raise Bcfg2.Server.Plugin.PluginInitError - - included = [ent.get('href') - for ent in self.xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.fam.AddMonitor(os.path.join(os.path.dirname(self.name), - name), - self) - self.extras.append(name) - try: - self.xdata.getroottree().xinclude() - except lxml.etree.XIncludeError: - err = sys.exc_info()[1] - logger.error("XInclude failed on %s: %s" % (self.name, err)) - - - self.entries = self.xdata.getchildren() - if self.__identifier__ is not None: - self.label = self.xdata.attrib[self.__identifier__] - - -class StructFile(XMLFileBacked): - """This file contains a set of structure file formatting logic.""" - __identifier__ = None - - def __init__(self, name): - XMLFileBacked.__init__(self, name) - - def _match(self, item, metadata): - """ recursive helper for Match() """ - if isinstance(item, lxml.etree._Comment): - return [] - elif item.tag == 'Group': - rv = [] - if ((item.get('negate', 'false').lower() == 'true' and - item.get('name') not in metadata.groups) or - (item.get('negate', 'false').lower() == 'false' and - item.get('name') in metadata.groups)): - for child in item.iterchildren(): - rv.extend(self._match(child, metadata)) - return rv - elif item.tag == 'Client': - rv = [] - if ((item.get('negate', 'false').lower() == 'true' and - item.get('name') != metadata.hostname) or - (item.get('negate', 'false').lower() == 'false' and - item.get('name') == metadata.hostname)): - for child in item.iterchildren(): - rv.extend(self._match(child, metadata)) - return rv - else: - rv = copy.copy(item) - for child in rv.iterchildren(): - rv.remove(child) - for child in item.iterchildren(): - rv.extend(self._match(child, metadata)) - return [rv] - - def Match(self, metadata): - """Return matching fragments of independent.""" - rv = [] - for child in self.entries: - rv.extend(self._match(child, metadata)) - return rv - - -class INode: - """ - LNodes provide lists of things available at a particular - group intersection. - """ - raw = {'Client': "lambda m, e:'%(name)s' == m.hostname and predicate(m, e)", - 'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)"} - nraw = {'Client': "lambda m, e:'%(name)s' != m.hostname and predicate(m, e)", - 'Group': "lambda m, e:'%(name)s' not in m.groups and predicate(m, e)"} - containers = ['Group', 'Client'] - ignore = [] - - def __init__(self, data, idict, parent=None): - self.data = data - self.contents = {} - if parent == None: - self.predicate = lambda m, d: True - else: - predicate = parent.predicate - if data.get('negate', 'false') in ['true', 'True']: - psrc = self.nraw - else: - psrc = self.raw - if data.tag in list(psrc.keys()): - self.predicate = eval(psrc[data.tag] % - {'name': data.get('name')}, - {'predicate': predicate}) - else: - raise Exception - mytype = self.__class__ - self.children = [] - for item in data.getchildren(): - if item.tag in self.ignore: - continue - elif item.tag in self.containers: - self.children.append(mytype(item, idict, self)) - else: - try: - self.contents[item.tag][item.get('name')] = item.attrib - except KeyError: - self.contents[item.tag] = {item.get('name'): item.attrib} - if item.text: - self.contents[item.tag]['__text__'] = item.text - try: - idict[item.tag].append(item.get('name')) - except KeyError: - idict[item.tag] = [item.get('name')] - - def Match(self, metadata, data, entry=lxml.etree.Element("None")): - """Return a dictionary of package mappings.""" - if self.predicate(metadata, entry): - for key in self.contents: - try: - data[key].update(self.contents[key]) - except: - data[key] = {} - data[key].update(self.contents[key]) - for child in self.children: - child.Match(metadata, data, entry=entry) - - -class InfoNode (INode): - """ INode implementation that includes tags """ - raw = {'Client': "lambda m, e:'%(name)s' == m.hostname and predicate(m, e)", - 'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)", - 'Path': "lambda m, e:('%(name)s' == e.get('name') or '%(name)s' == e.get('realname')) and predicate(m, e)"} - nraw = {'Client': "lambda m, e:'%(name)s' != m.hostname and predicate(m, e)", - 'Group': "lambda m, e:'%(name)s' not in m.groups and predicate(m, e)", - 'Path': "lambda m, e:('%(name)s' != e.get('name') and '%(name)s' != e.get('realname')) and predicate(m, e)"} - containers = ['Group', 'Client', 'Path'] - - -class XMLSrc(XMLFileBacked): - """XMLSrc files contain a LNode hierarchy that returns matching entries.""" - __node__ = INode - __cacheobj__ = dict - - def __init__(self, filename, noprio=False): - XMLFileBacked.__init__(self, filename) - self.items = {} - self.cache = None - self.pnode = None - self.priority = -1 - self.noprio = noprio - - def HandleEvent(self, _=None): - """Read file upon update.""" - try: - data = BUILTIN_FILE_TYPE(self.name).read() - except IOError: - logger.error("Failed to read file %s" % (self.name)) - return - self.items = {} - try: - xdata = lxml.etree.XML(data) - except lxml.etree.XMLSyntaxError: - logger.error("Failed to parse file %s" % (self.name)) - return - self.pnode = self.__node__(xdata, self.items) - self.cache = None - try: - self.priority = int(xdata.get('priority')) - except (ValueError, TypeError): - if not self.noprio: - logger.error("Got bogus priority %s for file %s" % - (xdata.get('priority'), self.name)) - del xdata, data - - def Cache(self, metadata): - """Build a package dict for a given host.""" - if self.cache == None or self.cache[0] != metadata: - cache = (metadata, self.__cacheobj__()) - if self.pnode == None: - logger.error("Cache method called early for %s; forcing data load" % (self.name)) - self.HandleEvent() - return - self.pnode.Match(metadata, cache[1]) - self.cache = cache - - def __str__(self): - return str(self.items) - - -class InfoXML (XMLSrc): - __node__ = InfoNode - - -class XMLDirectoryBacked(DirectoryBacked): - """Directorybacked for *.xml.""" - patterns = re.compile('.*\.xml') - - -class PrioDir(Plugin, Generator, XMLDirectoryBacked): - """This is a generator that handles package assignments.""" - name = 'PrioDir' - __child__ = XMLSrc - - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - Generator.__init__(self) - try: - XMLDirectoryBacked.__init__(self, self.data, self.core.fam) - except OSError: - self.logger.error("Failed to load %s indices" % (self.name)) - raise PluginInitError - - def HandleEvent(self, event): - """Handle events and update dispatch table.""" - XMLDirectoryBacked.HandleEvent(self, event) - self.Entries = {} - for src in list(self.entries.values()): - for itype, children in list(src.items.items()): - for child in children: - try: - self.Entries[itype][child] = self.BindEntry - except KeyError: - self.Entries[itype] = {child: self.BindEntry} - - def _matches(self, entry, metadata, rules): - return entry.get('name') in rules - - def BindEntry(self, entry, metadata): - attrs = self.get_attrs(entry, metadata) - for key, val in list(attrs.items()): - entry.attrib[key] = val - - def get_attrs(self, entry, metadata): - """ get a list of attributes to add to the entry during the bind """ - for src in self.entries.values(): - src.Cache(metadata) - - matching = [src for src in list(self.entries.values()) - if (src.cache and - entry.tag in src.cache[1] and - self._matches(entry, metadata, - src.cache[1][entry.tag]))] - if len(matching) == 0: - raise PluginExecutionError - elif len(matching) == 1: - index = 0 - else: - prio = [int(src.priority) for src in matching] - if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting sources with " - "same priority for %s, %s %s" % - (metadata.hostname, - entry.tag.lower(), entry.get('name'))) - self.logger.error([item.name for item in matching]) - self.logger.error("Priority was %s" % max(prio)) - raise PluginExecutionError - index = prio.index(max(prio)) - - for rname in list(matching[index].cache[1][entry.tag].keys()): - if self._matches(entry, metadata, [rname]): - data = matching[index].cache[1][entry.tag][rname] - break - if '__text__' in data: - entry.text = data['__text__'] - if '__children__' in data: - [entry.append(copy.copy(item)) for item in data['__children__']] - - return dict([(key, data[key]) - for key in list(data.keys()) - if not key.startswith('__')]) - - -# new unified EntrySet backend -class SpecificityError(Exception): - """Thrown in case of filename parse failure.""" - pass - - -class Specificity: - - def __init__(self, all=False, group=False, hostname=False, prio=0, delta=False): - self.hostname = hostname - self.all = all - self.group = group - self.prio = prio - self.delta = delta - - def __lt__(self, other): - return self.__cmp__(other) < 0 - - def matches(self, metadata): - return self.all or \ - self.hostname == metadata.hostname or \ - self.group in metadata.groups - - def __cmp__(self, other): - """Sort most to least specific.""" - if self.all: - return 1 - if self.group: - if other.hostname: - return 1 - if other.group and other.prio > self.prio: - return 1 - if other.group and other.prio == self.prio: - return 0 - return -1 - - def more_specific(self, other): - """Test if self is more specific than other.""" - if self.all: - True - elif self.group: - if other.hostname: - return True - elif other.group and other.prio > self.prio: - return True - return False - - -class SpecificData(object): - def __init__(self, name, specific, encoding): - self.name = name - self.specific = specific - - def handle_event(self, event): - if event.code2str() == 'deleted': - return - try: - self.data = open(self.name).read() - except: - logger.error("Failed to read file %s" % self.name) - - -class EntrySet(object): - """Entry sets deal with the host- and group-specific entries.""" - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(sw[px])|.*\\.genshi_include)$") - - def __init__(self, basename, path, entry_type, encoding): - self.path = path - self.entry_type = entry_type - self.entries = {} - self.metadata = default_file_metadata.copy() - self.infoxml = None - self.encoding = encoding - pattern = '(.*/)?%s(\.((H_(?P\S+))|' % basename - pattern += '(G(?P\d+)_(?P\S+))))?$' - self.specific = re.compile(pattern) - - def get_matching(self, metadata): - return [item for item in list(self.entries.values()) - if item.specific.matches(metadata)] - - def best_matching(self, metadata): - """ Return the appropriate interpreted template from the set of - available templates. """ - matching = self.get_matching(metadata) - - hspec = [ent for ent in matching if ent.specific.hostname] - if hspec: - return hspec[0] - - gspec = [ent for ent in matching if ent.specific.group] - if gspec: - gspec.sort(self.group_sortfunc) - return gspec[-1] - - aspec = [ent for ent in matching if ent.specific.all] - if aspec: - return aspec[0] - - raise PluginExecutionError - - def handle_event(self, event): - """Handle FAM events for the TemplateSet.""" - action = event.code2str() - - if event.filename in ['info', 'info.xml', ':info']: - if action in ['exists', 'created', 'changed']: - self.update_metadata(event) - elif action == 'deleted': - self.reset_metadata(event) - return - - if action in ['exists', 'created']: - self.entry_init(event) - else: - if event.filename not in self.entries: - logger.warning("Got %s event for unknown file %s" % - (action, event.filename)) - if action == 'changed': - # received a bogus changed event; warn, but treat - # it like a created event - self.entry_init(event) - return - if action == 'changed': - self.entries[event.filename].handle_event(event) - elif action == 'deleted': - del self.entries[event.filename] - - def entry_init(self, event): - """Handle template and info file creation.""" - if event.filename in self.entries: - logger.warn("Got duplicate add for %s" % event.filename) - else: - fpath = "%s/%s" % (self.path, event.filename) - try: - spec = self.specificity_from_filename(event.filename) - except SpecificityError: - if not self.ignore.match(event.filename): - logger.error("Could not process filename %s; ignoring" % - fpath) - return - self.entries[event.filename] = self.entry_type(fpath, - spec, self.encoding) - self.entries[event.filename].handle_event(event) - - def specificity_from_filename(self, fname): - """Construct a specificity instance from a filename and regex.""" - data = self.specific.match(fname) - if not data: - raise SpecificityError(fname) - kwargs = {} - if data.group('hostname'): - kwargs['hostname'] = data.group('hostname') - elif data.group('group'): - kwargs['group'] = data.group('group') - kwargs['prio'] = int(data.group('prio')) - else: - kwargs['all'] = True - if 'delta' in data.groupdict(): - kwargs['delta'] = data.group('delta') - return Specificity(**kwargs) - - def update_metadata(self, event): - """Process info and info.xml files for the templates.""" - fpath = "%s/%s" % (self.path, event.filename) - if event.filename == 'info.xml': - if not self.infoxml: - self.infoxml = InfoXML(fpath, True) - self.infoxml.HandleEvent(event) - elif event.filename in [':info', 'info']: - for line in open(fpath).readlines(): - match = info_regex.match(line) - if not match: - logger.warning("Failed to match line in %s: %s" % (fpath, - line)) - continue - else: - mgd = match.groupdict() - for key, value in list(mgd.items()): - if value: - self.metadata[key] = value - if len(self.metadata['perms']) == 3: - self.metadata['perms'] = "0%s" % \ - (self.metadata['perms']) - - def reset_metadata(self, event): - """Reset metadata to defaults if info or info.xml removed.""" - if event.filename == 'info.xml': - self.infoxml = None - elif event.filename in [':info', 'info']: - self.metadata = default_file_metadata.copy() - - def group_sortfunc(self, x, y): - """sort groups by their priority""" - return cmp(x.specific.prio, y.specific.prio) - - def bind_info_to_entry(self, entry, metadata): - # first set defaults from global metadata/:info - for key in self.metadata: - entry.set(key, self.metadata[key]) - if self.infoxml: - mdata = {} - self.infoxml.pnode.Match(metadata, mdata, entry=entry) - if 'Info' not in mdata: - logger.error("Failed to set metadata for file %s" % \ - (entry.get('name'))) - raise PluginExecutionError - [entry.attrib.__setitem__(key, value) \ - for (key, value) in list(mdata['Info'][None].items())] - - def bind_entry(self, entry, metadata): - """Return the appropriate interpreted template from the set of available templates.""" - self.bind_info_to_entry(entry, metadata) - return self.best_matching(metadata).bind_entry(entry, metadata) - - -class GroupSpool(Plugin, Generator): - """Unified interface for handling group-specific data (e.g. .G## files).""" - name = 'GroupSpool' - __author__ = 'bcfg-dev@mcs.anl.gov' - filename_pattern = "" - es_child_cls = object - es_cls = EntrySet - - def __init__(self, core, datastore): - Plugin.__init__(self, core, datastore) - Generator.__init__(self) - if self.data[-1] == '/': - self.data = self.data[:-1] - self.Entries['Path'] = {} - self.entries = {} - self.handles = {} - self.AddDirectoryMonitor('') - self.encoding = core.encoding - - def add_entry(self, event): - epath = self.event_path(event) - ident = self.event_id(event) - if posixpath.isdir(epath): - self.AddDirectoryMonitor(epath[len(self.data):]) - if ident not in self.entries and posixpath.isfile(epath): - dirpath = "".join([self.data, ident]) - self.entries[ident] = self.es_cls(self.filename_pattern, - dirpath, - self.es_child_cls, - self.encoding) - self.Entries['Path'][ident] = self.entries[ident].bind_entry - if not posixpath.isdir(epath): - # do not pass through directory events - self.entries[ident].handle_event(event) - - def event_path(self, event): - return "".join([self.data, self.handles[event.requestID], - event.filename]) - - def event_id(self, event): - epath = self.event_path(event) - if posixpath.isdir(epath): - return self.handles[event.requestID] + event.filename - else: - return self.handles[event.requestID][:-1] - - def HandleEvent(self, event): - """Unified FAM event handler for GroupSpool.""" - action = event.code2str() - if event.filename[0] == '/': - return - ident = self.event_id(event) - - if action in ['exists', 'created']: - self.add_entry(event) - if action == 'changed': - if ident in self.entries: - self.entries[ident].handle_event(event) - else: - # got a changed event for a file we didn't know - # about. go ahead and process this as a 'created', but - # warn - self.logger.warning("Got changed event for unknown file %s" % - ident) - self.add_entry(event) - elif action == 'deleted': - fbase = self.handles[event.requestID] + event.filename - if fbase in self.entries: - # a directory was deleted - del self.entries[fbase] - del self.Entries['Path'][fbase] - elif ident in self.entries: - self.entries[ident].handle_event(event) - elif ident not in self.entries: - self.logger.warning("Got deleted event for unknown file %s" % - ident) - - def AddDirectoryMonitor(self, relative): - """Add new directory to FAM structures.""" - if not relative.endswith('/'): - relative += '/' - name = self.data + relative - if relative not in list(self.handles.values()): - if not posixpath.isdir(name): - print("Failed to open directory %s" % (name)) - return - reqid = self.core.fam.AddMonitor(name, self) - self.handles[reqid] = relative - -class SimpleConfig(FileBacked, - ConfigParser.SafeConfigParser): - ''' a simple plugin config using ConfigParser ''' - _required = True - - def __init__(self, plugin): - filename = os.path.join(plugin.data, plugin.name.lower() + ".conf") - self.plugin = plugin - self.fam = self.plugin.core.fam - Bcfg2.Server.Plugin.FileBacked.__init__(self, filename) - ConfigParser.SafeConfigParser.__init__(self) - - if (self._required or - (not self._required and os.path.exists(self.name))): - self.fam.AddMonitor(self.name, self) - - def Index(self): - """ Build local data structures """ - for section in self.sections(): - self.remove_section(section) - self.read(self.name) - - def get(self, section, option, **kwargs): - """ convenience method for getting config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.SafeConfigParser.get(self, section, option, - **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - if default is not None: - return default - else: - raise - - def getboolean(self, section, option, **kwargs): - """ convenience method for getting boolean config items """ - default = None - if 'default' in kwargs: - default = kwargs['default'] - del kwargs['default'] - try: - return ConfigParser.SafeConfigParser.getboolean(self, section, - option, **kwargs) - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError, - ValueError): - if default is not None: - return default - else: - raise diff --git a/src/lib/Server/Plugins/Account.py b/src/lib/Server/Plugins/Account.py deleted file mode 100644 index f2703dccb..000000000 --- a/src/lib/Server/Plugins/Account.py +++ /dev/null @@ -1,101 +0,0 @@ -"""This handles authentication setup.""" - -import Bcfg2.Server.Plugin - - -class Account(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): - """This module generates account config files, - based on an internal data repo: - static.(passwd|group|limits.conf) -> static entries - dyn.(passwd|group) -> dynamic entries (usually acquired from yp or somesuch) - useraccess -> users to be granted login access on some hosts - superusers -> users to be granted root privs on all hosts - rootlike -> users to be granted root privs on some hosts - - """ - name = 'Account' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - self.Entries = {'ConfigFile': {'/etc/passwd': self.from_yp_cb, - '/etc/group': self.from_yp_cb, - '/etc/security/limits.conf': self.gen_limits_cb, - '/root/.ssh/authorized_keys': self.gen_root_keys_cb, - '/etc/sudoers': self.gen_sudoers}} - try: - self.repository = Bcfg2.Server.Plugin.DirectoryBacked(self.data, - self.core.fam) - except: - self.logger.error("Failed to load repos: %s, %s" % \ - (self.data, "%s/ssh" % (self.data))) - raise Bcfg2.Server.Plugin.PluginInitError - - def from_yp_cb(self, entry, metadata): - """Build password file from cached yp data.""" - fname = entry.attrib['name'].split('/')[-1] - entry.text = self.repository.entries["static.%s" % (fname)].data - entry.text += self.repository.entries["dyn.%s" % (fname)].data - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0644'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - - def gen_limits_cb(self, entry, metadata): - """Build limits entries based on current ACLs.""" - entry.text = self.repository.entries["static.limits.conf"].data - superusers = self.repository.entries["superusers"].data.split() - useraccess = [line.split(':') for line in \ - self.repository.entries["useraccess"].data.split()] - users = [user for (user, host) in \ - useraccess if host == metadata.hostname.split('.')[0]] - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) in \ - list(perms.items())] - entry.text += "".join(["%s hard maxlogins 1024\n" % uname for uname in superusers + users]) - if "*" not in users: - entry.text += "* hard maxlogins 0\n" - - def gen_root_keys_cb(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - rdata = self.repository.entries - entry.text = "".join([rdata["%s.key" % user].data for user \ - in superusers if \ - ("%s.key" % user) in rdata]) - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0600'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] - - def gen_sudoers(self, entry, metadata): - """Build root authorized keys file based on current ACLs.""" - superusers = self.repository.entries['superusers'].data.split() - try: - rootlike = [line.split(':', 1) for line in \ - self.repository.entries['rootlike'].data.split()] - superusers += [user for (user, host) in rootlike \ - if host == metadata.hostname.split('.')[0]] - except: - pass - entry.text = self.repository.entries['static.sudoers'].data - entry.text += "".join(["%s ALL=(ALL) ALL\n" % uname \ - for uname in superusers]) - perms = {'owner': 'root', - 'group': 'root', - 'perms': '0440'} - [entry.attrib.__setitem__(key, value) for (key, value) \ - in list(perms.items())] diff --git a/src/lib/Server/Plugins/BB.py b/src/lib/Server/Plugins/BB.py deleted file mode 100644 index c015ec47c..000000000 --- a/src/lib/Server/Plugins/BB.py +++ /dev/null @@ -1,83 +0,0 @@ -import lxml.etree -import Bcfg2.Server.Plugin -import glob -import os -import socket - -#manage boot symlinks - #add statistics check to do build->boot mods - -#map profiles: first array is not empty we replace the -p with a determined profile. -logger = Bcfg2.Server.Plugin.logger - -class BBfile(Bcfg2.Server.Plugin.XMLFileBacked): - """Class for bb files.""" - def Index(self): - """Build data into an xml object.""" - - try: - self.data = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - Bcfg2.Server.Plugin.logger.error("Failed to parse %s" % self.name) - return - self.tftppath = self.data.get('tftp', '/tftpboot') - self.macs = {} - self.users = {} - self.actions = {} - self.bootlinks = [] - - for node in self.data.findall('Node'): - iface = node.find('Interface') - if iface != None: - mac = "01-%s" % (iface.get('mac'.replace(':','-').lower())) - self.actions[node.get('name')] = node.get('action') - self.bootlinks.append((mac, node.get('action'))) - try: - ip = socket.gethostbyname(node.get('name')) - except: - logger.error("failed host resolution for %s" % node.get('name')) - - self.macs[node.get('name')] = (iface.get('mac'), ip) - else: - logger.error("%s" % lxml.etree.tostring(node)) - self.users[node.get('name')] = node.get('user',"").split(':') - - def enforce_bootlinks(self): - for mac, target in self.bootlinks: - path = self.tftppath + '/' + mac - if not os.path.islink(path): - logger.error("Boot file %s not a link" % path) - if target != os.readlink(path): - try: - os.unlink(path) - os.symlink(target, path) - except: - logger.error("Failed to modify link %s" % path) - -class BBDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): - __child__ = BBfile - - -class BB(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - """The BB plugin maps users to machines and metadata to machines.""" - name = 'BB' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.store = BBDirectoryBacked(self.data, core.fam) - - def get_additional_data(self, metadata): - - users = {} - for user in self.store.entries['bb.xml'].users.get(metadata.hostname.split(".")[0], []): - pubkeys = [] - for fname in glob.glob('/home/%s/.ssh/*.pub'%user): - pubkeys.append(open(fname).read()) - - users[user] = pubkeys - - return dict([('users', users), - ('macs', self.store.entries['bb.xml'].macs)]) diff --git a/src/lib/Server/Plugins/Base.py b/src/lib/Server/Plugins/Base.py deleted file mode 100644 index 389ca7a95..000000000 --- a/src/lib/Server/Plugins/Base.py +++ /dev/null @@ -1,43 +0,0 @@ -"""This module sets up a base list of configuration entries.""" - -import copy -import lxml.etree -import sys -# py3k compatibility -if sys.hexversion >= 0x03000000: - from functools import reduce - -import Bcfg2.Server.Plugin - - -class Base(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): - """This Structure is good for the pile of independent configs - needed for most actual systems. - """ - name = 'Base' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = Bcfg2.Server.Plugin.StructFile - deprecated = True - - """Base creates independent clauses based on client metadata.""" - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - self.logger.error("Failed to load Base repository") - raise Bcfg2.Server.Plugin.PluginInitError - - def BuildStructures(self, metadata): - """Build structures for client described by metadata.""" - ret = lxml.etree.Element("Independent", version='2.0') - fragments = reduce(lambda x, y: x + y, - [base.Match(metadata) for base - in list(self.entries.values())], []) - [ret.append(copy.copy(frag)) for frag in fragments] - return [ret] diff --git a/src/lib/Server/Plugins/Bundler.py b/src/lib/Server/Plugins/Bundler.py deleted file mode 100644 index ccb99481e..000000000 --- a/src/lib/Server/Plugins/Bundler.py +++ /dev/null @@ -1,99 +0,0 @@ -"""This provides bundle clauses with translation functionality.""" - -import copy -import lxml.etree -import os -import os.path -import re -import sys - -import Bcfg2.Server.Plugin - -try: - import genshi.template - import genshi.template.base - import Bcfg2.Server.Plugins.SGenshi - have_genshi = True -except: - have_genshi = False - - -class BundleFile(Bcfg2.Server.Plugin.StructFile): - - def get_xml_value(self, metadata): - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - [bundle.append(copy.copy(item)) for item in self.Match(metadata)] - return bundle - - -class Bundler(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.XMLDirectoryBacked): - """The bundler creates dependent clauses based on the - bundle/translation scheme from Bcfg1. - """ - name = 'Bundler' - __author__ = 'bcfg-dev@mcs.anl.gov' - patterns = re.compile('^(?P.*)\.(xml|genshi)$') - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - self.encoding = core.encoding - self.__child__ = self.template_dispatch - try: - Bcfg2.Server.Plugin.XMLDirectoryBacked.__init__(self, - self.data, - self.core.fam) - except OSError: - self.logger.error("Failed to load Bundle repository") - raise Bcfg2.Server.Plugin.PluginInitError - - def template_dispatch(self, name): - bundle = lxml.etree.parse(name) - nsmap = bundle.getroot().nsmap - if name.endswith('.xml'): - if have_genshi and \ - (nsmap == {'py': 'http://genshi.edgewall.org/'}): - # allow for genshi bundles with .xml extensions - spec = Bcfg2.Server.Plugin.Specificity() - return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name, - spec, - self.encoding) - else: - return BundleFile(name) - elif name.endswith('.genshi'): - if have_genshi: - spec = Bcfg2.Server.Plugin.Specificity() - return Bcfg2.Server.Plugins.SGenshi.SGenshiTemplateFile(name, - spec, - self.encoding) - - def BuildStructures(self, metadata): - """Build all structures for client (metadata).""" - bundleset = [] - - bundle_entries = {} - for key, item in self.entries.items(): - bundle_entries.setdefault(self.patterns.match(os.path.basename(key)).group('name'), - []).append(item) - - for bundlename in metadata.bundles: - try: - entries = bundle_entries[bundlename] - except KeyError: - self.logger.error("Bundler: Bundle %s does not exist" % - bundlename) - continue - try: - bundleset.append(entries[0].get_xml_value(metadata)) - except genshi.template.base.TemplateError: - t = sys.exc_info()[1] - self.logger.error("Bundler: Failed to template genshi bundle %s" - % bundlename) - self.logger.error(t) - except: - self.logger.error("Bundler: Unexpected bundler error for %s" % - bundlename, exc_info=1) - return bundleset diff --git a/src/lib/Server/Plugins/Bzr.py b/src/lib/Server/Plugins/Bzr.py deleted file mode 100644 index a71021cb5..000000000 --- a/src/lib/Server/Plugins/Bzr.py +++ /dev/null @@ -1,35 +0,0 @@ -import Bcfg2.Server.Plugin -from bzrlib.workingtree import WorkingTree -from bzrlib import errors - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Bzr') - -class Bzr(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Bzr is a version plugin for dealing with Bcfg2 repos.""" - name = 'Bzr' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # Read revision from bcfg2 repo - revision = self.get_revision() - - logger.debug("Initialized Bazaar plugin with directory = %(dir)s at revision = %(rev)s" % {'dir': datastore, 'rev': revision}) - - def get_revision(self): - """Read Bazaar revision information for the Bcfg2 repository.""" - try: - working_tree = WorkingTree.open(self.datastore) - revision = str(working_tree.branch.revno()) - if working_tree.has_changes(working_tree.basis_tree()) or working_tree.unknowns(): - revision += "+" - except errors.NotBranchError: - logger.error("Failed to read Bazaar branch; disabling Bazaar support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision diff --git a/src/lib/Server/Plugins/Cfg.py b/src/lib/Server/Plugins/Cfg.py deleted file mode 100644 index c3e807d7a..000000000 --- a/src/lib/Server/Plugins/Cfg.py +++ /dev/null @@ -1,283 +0,0 @@ -"""This module implements a config file repository.""" - -import binascii -import logging -import lxml -import operator -import os -import os.path -import re -import stat -import sys -import tempfile -from subprocess import Popen, PIPE -from Bcfg2.Bcfg2Py3k import u_str - -import Bcfg2.Server.Plugin - -try: - import genshi.core - import genshi.input - from genshi.template import TemplateLoader, NewTextTemplate - have_genshi = True -except: - have_genshi = False - -try: - import Cheetah.Template - import Cheetah.Parser - have_cheetah = True -except: - have_cheetah = False - -# setup logging -logger = logging.getLogger('Bcfg2.Plugins.Cfg') - - -# snipped from TGenshi -def removecomment(stream): - """A genshi filter that removes comments from the stream.""" - for kind, data, pos in stream: - if kind is genshi.core.COMMENT: - continue - yield kind, data, pos - - -def process_delta(data, delta): - if not delta.specific.delta: - return data - if delta.specific.delta == 'cat': - datalines = data.strip().split('\n') - for line in delta.data.split('\n'): - if not line: - continue - if line[0] == '+': - datalines.append(line[1:]) - elif line[0] == '-': - if line[1:] in datalines: - datalines.remove(line[1:]) - return "\n".join(datalines) + "\n" - elif delta.specific.delta == 'diff': - basehandle, basename = tempfile.mkstemp() - basefile = open(basename, 'w') - basefile.write(data) - basefile.close() - os.close(basehandle) - - cmd = ["patch", "-u", "-f", basefile.name] - patch = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - stderr = patch.communicate(input=delta.data)[1] - ret = patch.wait() - output = open(basefile.name, 'r').read() - os.unlink(basefile.name) - if ret >> 8 != 0: - logger.error("Error applying diff %s: %s" % (delta.name, stderr)) - raise Bcfg2.Server.Plugin.PluginExecutionError('delta', delta) - return output - - -class CfgMatcher: - - def __init__(self, fname): - name = re.escape(fname) - self.basefile_reg = re.compile('^(?P%s)(|\\.H_(?P\S+?)|.G(?P\d+)_(?P\S+?))((?P\\.genshi)|(?P\\.cheetah))?$' % name) - self.delta_reg = re.compile('^(?P%s)(|\\.H_(?P\S+)|\\.G(?P\d+)_(?P\S+))\\.(?P(cat|diff))$' % name) - self.cat_count = fname.count(".cat") - self.diff_count = fname.count(".diff") - - def match(self, fname): - if fname.count(".cat") > self.cat_count \ - or fname.count('.diff') > self.diff_count: - return self.delta_reg.match(fname) - return self.basefile_reg.match(fname) - - -class CfgEntrySet(Bcfg2.Server.Plugin.EntrySet): - - def __init__(self, basename, path, entry_type, encoding): - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, - entry_type, encoding) - self.specific = CfgMatcher(path.split('/')[-1]) - path = path - - def debug_log(self, message, flag=None): - if (flag is None and self.debug_flag) or flag: - logger.error(message) - - def sort_by_specific(self, one, other): - return cmp(one.specific, other.specific) - - def get_pertinent_entries(self, entry, metadata): - """return a list of all entries pertinent - to a client => [base, delta1, delta2] - """ - matching = [ent for ent in list(self.entries.values()) if \ - ent.specific.matches(metadata)] - matching.sort(key=operator.attrgetter('specific')) - # base entries which apply to a client - # (e.g. foo, foo.G##_groupname, foo.H_hostname) - base_files = [matching.index(m) for m in matching - if not m.specific.delta] - if not base_files: - logger.error("No base file found for %s" % entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError - base = min(base_files) - used = matching[:base + 1] - used.reverse() - return used - - def bind_entry(self, entry, metadata): - self.bind_info_to_entry(entry, metadata) - used = self.get_pertinent_entries(entry, metadata) - basefile = used.pop(0) - if entry.get('perms').lower() == 'inherit': - # use on-disk permissions - fname = "%s/%s" % (self.path, entry.get('name')) - entry.set('perms', - str(oct(stat.S_IMODE(os.stat(fname).st_mode)))) - if entry.tag == 'Path': - entry.set('type', 'file') - if basefile.name.endswith(".genshi"): - if not have_genshi: - logger.error("Cfg: Genshi is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - template_cls = NewTextTemplate - loader = TemplateLoader() - template = loader.load(basefile.name, cls=template_cls, - encoding=self.encoding) - fname = entry.get('realname', entry.get('name')) - stream = template.generate(name=fname, - metadata=metadata, - path=basefile.name).filter(removecomment) - try: - data = stream.render('text', encoding=self.encoding, - strip_whitespace=False) - except TypeError: - data = stream.render('text', encoding=self.encoding) - if data == '': - entry.set('empty', 'true') - except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: genshi exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError - elif basefile.name.endswith(".cheetah"): - if not have_cheetah: - logger.error("Cfg: Cheetah is not available") - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - fname = entry.get('realname', entry.get('name')) - s = {'useStackFrames': False} - template = Cheetah.Template.Template(open(basefile.name).read(), - compilerSettings=s) - template.metadata = metadata - template.path = fname - template.source_path = basefile.name - data = template.respond() - if data == '': - entry.set('empty', 'true') - except Exception: - e = sys.exc_info()[1] - logger.error("Cfg: cheetah exception: %s" % e) - raise Bcfg2.Server.Plugin.PluginExecutionError - else: - data = basefile.data - for delta in used: - data = process_delta(data, delta) - if entry.get('encoding') == 'base64': - entry.text = binascii.b2a_base64(data) - else: - try: - entry.text = u_str(data, self.encoding) - except UnicodeDecodeError: - e = sys.exc_info()[1] - logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) - logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError - except ValueError: - e = sys.exc_info()[1] - logger.error("Error in specification for %s" % entry.get('name')) - logger.error("%s" % e) - logger.error("You need to specify base64 encoding for %s." % - entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError - if entry.text in ['', None]: - entry.set('empty', 'true') - - def list_accept_choices(self, entry, metadata): - '''return a list of candidate pull locations''' - used = self.get_pertinent_entries(entry, metadata) - ret = [] - if used: - ret.append(used[0].specific) - if not ret[0].hostname: - ret.append(Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)) - return ret - - def build_filename(self, specific): - bfname = self.path + '/' + self.path.split('/')[-1] - if specific.all: - return bfname - elif specific.group: - return "%s.G%02d_%s" % (bfname, specific.prio, specific.group) - elif specific.hostname: - return "%s.H_%s" % (bfname, specific.hostname) - - def write_update(self, specific, new_entry, log): - if 'text' in new_entry: - name = self.build_filename(specific) - if os.path.exists("%s.genshi" % name): - logger.error("Cfg: Unable to pull data for genshi types") - raise Bcfg2.Server.Plugin.PluginExecutionError - elif os.path.exists("%s.cheetah" % name): - logger.error("Cfg: Unable to pull data for cheetah types") - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - etext = new_entry['text'].encode(self.encoding) - except: - logger.error("Cfg: Cannot encode content of %s as %s" % (name, self.encoding)) - raise Bcfg2.Server.Plugin.PluginExecutionError - open(name, 'w').write(etext) - self.debug_log("Wrote file %s" % name, flag=log) - badattr = [attr for attr in ['owner', 'group', 'perms'] - if attr in new_entry] - if badattr: - # check for info files and inform user of their removal - if os.path.exists(self.path + "/:info"): - logger.info("Removing :info file and replacing with " - "info.xml") - os.remove(self.path + "/:info") - if os.path.exists(self.path + "/info"): - logger.info("Removing info file and replacing with " - "info.xml") - os.remove(self.path + "/info") - metadata_updates = {} - metadata_updates.update(self.metadata) - for attr in badattr: - metadata_updates[attr] = new_entry.get(attr) - infoxml = lxml.etree.Element('FileInfo') - infotag = lxml.etree.SubElement(infoxml, 'Info') - [infotag.attrib.__setitem__(attr, metadata_updates[attr]) \ - for attr in metadata_updates] - ofile = open(self.path + "/info.xml", "w") - ofile.write(lxml.etree.tostring(infoxml, pretty_print=True)) - ofile.close() - self.debug_log("Wrote file %s" % (self.path + "/info.xml"), - flag=log) - - -class Cfg(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.PullTarget): - """This generator in the configuration file repository for Bcfg2.""" - name = 'Cfg' - __author__ = 'bcfg-dev@mcs.anl.gov' - es_cls = CfgEntrySet - es_child_cls = Bcfg2.Server.Plugin.SpecificData - - def AcceptChoices(self, entry, metadata): - return self.entries[entry.get('name')].list_accept_choices(entry, metadata) - - def AcceptPullData(self, specific, new_entry, log): - return self.entries[new_entry.get('name')].write_update(specific, - new_entry, - log) diff --git a/src/lib/Server/Plugins/Cvs.py b/src/lib/Server/Plugins/Cvs.py deleted file mode 100644 index 6ce72acd2..000000000 --- a/src/lib/Server/Plugins/Cvs.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Cvs') - -class Cvs(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """CVS is a version plugin for dealing with Bcfg2 repository.""" - name = 'Cvs' - __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to cvs directory for Bcfg2 repo - cvs_dir = "%s/CVSROOT" % datastore - - # Read revision from Bcfg2 repo - if os.path.isdir(cvs_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % cvs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized cvs plugin with cvs directory = %s" % cvs_dir) - - def get_revision(self): - """Read cvs revision information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C cvs log", - shell=True, - cwd=self.datastore, - stdout=PIPE).stdout.readlines() - revision = data[3].strip('\n') - except IndexError: - logger.error("Failed to read cvs log; disabling cvs support") - logger.error('''Ran command "cvs log %s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Server/Plugins/DBStats.py b/src/lib/Server/Plugins/DBStats.py deleted file mode 100644 index 95395f74e..000000000 --- a/src/lib/Server/Plugins/DBStats.py +++ /dev/null @@ -1,113 +0,0 @@ -import binascii -import difflib -import logging -import lxml.etree -import platform -import time - -try: - from django.core.exceptions import MultipleObjectsReturned -except ImportError: - pass - -import Bcfg2.Server.Plugin -import Bcfg2.Server.Reports.importscript -from Bcfg2.Server.Reports.reports.models import Client -import Bcfg2.Server.Reports.settings -from Bcfg2.Server.Reports.updatefix import update_database -# for debugging output only -logger = logging.getLogger('Bcfg2.Plugins.DBStats') - -class DBStats(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.ThreadedStatistics, - Bcfg2.Server.Plugin.PullSource): - name = 'DBStats' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.ThreadedStatistics.__init__(self, core, datastore) - Bcfg2.Server.Plugin.PullSource.__init__(self) - self.cpath = "%s/Metadata/clients.xml" % datastore - self.core = core - logger.debug("Searching for new models to add to the statistics database") - try: - update_database() - except Exception: - inst = sys.exc_info()[1] - logger.debug(str(inst)) - logger.debug(str(type(inst))) - - def handle_statistic(self, metadata, data): - newstats = data.find("Statistics") - newstats.set('time', time.asctime(time.localtime())) - # ick - data = lxml.etree.tostring(newstats) - ndx = lxml.etree.XML(data) - e = lxml.etree.Element('Node', name=metadata.hostname) - e.append(ndx) - container = lxml.etree.Element("ConfigStatistics") - container.append(e) - - # FIXME need to build a metadata interface to expose a list of clients - start = time.time() - for i in [1, 2, 3]: - try: - Bcfg2.Server.Reports.importscript.load_stats(self.core.metadata.clients_xml.xdata, - container, - self.core.encoding, - 0, - logger, - True, - platform.node()) - logger.info("Imported data for %s in %s seconds" \ - % (metadata.hostname, time.time() - start)) - return - except MultipleObjectsReturned: - e = sys.exc_info()[1] - logger.error("DBStats: MultipleObjectsReturned while handling %s: %s" % \ - (metadata.hostname, e)) - logger.error("DBStats: Data is inconsistent") - break - except: - logger.error("DBStats: Failed to write to db (lock); retrying", - exc_info=1) - logger.error("DBStats: Retry limit failed for %s; aborting operation" \ - % metadata.hostname) - - def GetExtra(self, client): - c_inst = Client.objects.filter(name=client)[0] - return [(a.entry.kind, a.entry.name) for a in - c_inst.current_interaction.extra()] - - def GetCurrentEntry(self, client, e_type, e_name): - try: - c_inst = Client.objects.filter(name=client)[0] - except IndexError: - self.logger.error("Unknown client: %s" % client) - raise Bcfg2.Server.Plugin.PluginExecutionError - result = c_inst.current_interaction.bad().filter(entry__kind=e_type, - entry__name=e_name) - if not result: - raise Bcfg2.Server.Plugin.PluginExecutionError - entry = result[0] - ret = [] - data = ('owner', 'group', 'perms') - for t in data: - if getattr(entry.reason, "current_%s" % t) == '': - ret.append(getattr(entry.reason, t)) - else: - ret.append(getattr(entry.reason, "current_%s" % t)) - if entry.reason.is_sensitive: - raise Bcfg2.Server.Plugin.PluginExecutionError - elif entry.reason.current_diff != '': - if entry.reason.is_binary: - ret.append(binascii.a2b_base64(entry.reason.current_diff)) - else: - ret.append('\n'.join(difflib.restore(\ - entry.reason.current_diff.split('\n'), 1))) - elif entry.reason.is_binary: - # If len is zero the object was too large to store - raise Bcfg2.Server.Plugin.PluginExecutionError - else: - ret.append(None) - return ret diff --git a/src/lib/Server/Plugins/Darcs.py b/src/lib/Server/Plugins/Darcs.py deleted file mode 100644 index 9fb9ff4f1..000000000 --- a/src/lib/Server/Plugins/Darcs.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Darcs') - -class Darcs(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Darcs is a version plugin for dealing with Bcfg2 repos.""" - name = 'Darcs' - __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to darcs directory for bcfg2 repo - darcs_dir = "%s/_darcs" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(darcs_dir): - self.get_revision() - else: - logger.error("%s is not present." % darcs_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Darcs plugin with darcs directory = %s" % darcs_dir) - - def get_revision(self): - """Read Darcs changeset information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C darcs changes", - shell=True, - cwd=self.datastore, - stdout=PIPE).stdout.readlines() - revision = data[0].strip('\n') - except: - logger.error("Failed to read darcs repository; disabling Darcs support") - logger.error('''Ran command "darcs changes" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - return revision - diff --git a/src/lib/Server/Plugins/Decisions.py b/src/lib/Server/Plugins/Decisions.py deleted file mode 100644 index b432474f2..000000000 --- a/src/lib/Server/Plugins/Decisions.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -import lxml.etree -import sys - -import Bcfg2.Server.Plugin -logger = logging.getLogger('Bcfg2.Plugins.Decisions') - -class DecisionFile(Bcfg2.Server.Plugin.SpecificData): - def handle_event(self, event): - Bcfg2.Server.Plugin.SpecificData.handle_event(self, event) - self.contents = lxml.etree.XML(self.data) - - def get_decisions(self): - return [(x.get('type'), x.get('name')) for x in self.contents.xpath('.//Decision')] - -class DecisionSet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, path, fam, encoding): - """Container for decision specification files. - - Arguments: - - `path`: repository path - - `fam`: reference to the file monitor - - `encoding`: XML character encoding - - """ - pattern = '(white|black)list' - Bcfg2.Server.Plugin.EntrySet.__init__(self, pattern, path, \ - DecisionFile, encoding) - try: - fam.AddMonitor(path, self) - except OSError: - e = sys.exc_info()[1] - logger.error('Adding filemonitor for %s failed. ' - 'Make sure directory exists' % path) - raise Bcfg2.Server.Plugin.PluginInitError(e) - - def HandleEvent(self, event): - if event.filename != self.path: - return self.handle_event(event) - - def GetDecisions(self, metadata, mode): - ret = [] - candidates = [c for c in self.get_matching(metadata) - if c.name.split('/')[-1].startswith(mode)] - for c in candidates: - ret += c.get_decisions() - return ret - -class Decisions(DecisionSet, - Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Decision): - name = 'Decisions' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - """Decisions plugins - - Arguments: - - `core`: Bcfg2.Core instance - - `datastore`: File repository location - - """ - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Decision.__init__(self) - DecisionSet.__init__(self, self.data, core.fam, core.encoding) - diff --git a/src/lib/Server/Plugins/Defaults.py b/src/lib/Server/Plugins/Defaults.py deleted file mode 100644 index 718192e2a..000000000 --- a/src/lib/Server/Plugins/Defaults.py +++ /dev/null @@ -1,53 +0,0 @@ -"""This generator provides rule-based entry mappings.""" - -import re -import Bcfg2.Server.Plugin -import Bcfg2.Server.Plugins.Rules - -class Defaults(Bcfg2.Server.Plugins.Rules.Rules, - Bcfg2.Server.Plugin.StructureValidator): - """Set default attributes on bound entries""" - name = 'Defaults' - __author__ = 'bcfg-dev@mcs.anl.gov' - - # Rules is a Generator that happens to implement all of the - # functionality we want, so we overload it, but Defaults should - # _not_ handle any entries; it does its stuff in the structure - # validation phase. so we overload Handle(s)Entry and HandleEvent - # to ensure that Defaults handles no entries, even though it's a - # Generator. - - def HandlesEntry(self, entry, metadata): - return False - - def HandleEntry(self, entry, metadata): - raise PluginExecutionError - - def HandleEvent(self, event): - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - - def validate_structures(self, metadata, structures): - """ Apply defaults """ - for struct in structures: - for entry in struct.iter(): - if entry.tag.startswith("Bound"): - is_bound = True - entry.tag = entry.tag[5:] - else: - is_bound = False - try: - try: - self.BindEntry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - # either no matching defaults (which is okay), - # or multiple matching defaults (which is not - # okay, but is logged). either way, we don't - # care about the error. - pass - finally: - if is_bound: - entry.tag = "Bound" + entry.tag - - def _regex_enabled(self): - """ Defaults depends on regex matching, so force it enabled """ - return True diff --git a/src/lib/Server/Plugins/Deps.py b/src/lib/Server/Plugins/Deps.py deleted file mode 100644 index 9b848baae..000000000 --- a/src/lib/Server/Plugins/Deps.py +++ /dev/null @@ -1,134 +0,0 @@ -"""This plugin provides automatic dependency handling.""" - -import lxml.etree - -import Bcfg2.Server.Plugin - - -class DNode(Bcfg2.Server.Plugin.INode): - """DNode provides supports for single predicate types for dependencies.""" - raw = {'Group': "lambda m, e:'%(name)s' in m.groups and predicate(m, e)"} - containers = ['Group'] - - def __init__(self, data, idict, parent=None): - self.data = data - self.contents = {} - if parent == None: - self.predicate = lambda x, d: True - else: - predicate = parent.predicate - if data.tag in list(self.raw.keys()): - self.predicate = eval(self.raw[data.tag] % - {'name': data.get('name')}, - {'predicate': predicate}) - else: - raise Exception - mytype = self.__class__ - self.children = [] - for item in data.getchildren(): - if item.tag in self.containers: - self.children.append(mytype(item, idict, self)) - else: - data = [(child.tag, child.get('name')) - for child in item.getchildren()] - try: - self.contents[item.tag][item.get('name')] = data - except KeyError: - self.contents[item.tag] = {item.get('name'): data} - - -class DepXMLSrc(Bcfg2.Server.Plugin.XMLSrc): - __node__ = DNode - - -class Deps(Bcfg2.Server.Plugin.PrioDir, - Bcfg2.Server.Plugin.StructureValidator): - name = 'Deps' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = DepXMLSrc - - # Override the default sort_order (of 500) so that this plugin - # gets handled after others running at the default. In particular, - # we want to run after Packages, so we can see the final set of - # packages that will be installed on the client. - sort_order = 750 - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) - Bcfg2.Server.Plugin.StructureValidator.__init__(self) - self.cache = {} - - def HandleEvent(self, event): - self.cache = {} - Bcfg2.Server.Plugin.PrioDir.HandleEvent(self, event) - - def validate_structures(self, metadata, structures): - """Examine the passed structures and append any additional - prerequisite entries as defined by the files in Deps. - """ - entries = [] - for structure in structures: - for entry in structure.getchildren(): - tag = entry.tag - if tag.startswith('Bound'): - tag = tag[5:] - if (tag, entry.get('name')) not in entries \ - and not isinstance(entry, lxml.etree._Comment): - entries.append((tag, entry.get('name'))) - entries.sort() - entries = tuple(entries) - gdata = list(metadata.groups) - gdata.sort() - gdata = tuple(gdata) - - # Check to see if we have cached the prereqs already - if (entries, gdata) in self.cache: - prereqs = self.cache[(entries, gdata)] - else: - prereqs = self.calculate_prereqs(metadata, entries) - self.cache[(entries, gdata)] = prereqs - - newstruct = lxml.etree.Element("Independent") - for tag, name in prereqs: - try: - lxml.etree.SubElement(newstruct, tag, name=name) - except: - self.logger.error("Failed to add dep entry for %s:%s" % (tag, name)) - structures.append(newstruct) - - - def calculate_prereqs(self, metadata, entries): - """Calculate the prerequisites defined in Deps for the passed - set of entries. - """ - prereqs = [] - [src.Cache(metadata) for src in self.entries.values()] - - toexamine = list(entries[:]) - while toexamine: - entry = toexamine.pop() - matching = [src for src in list(self.entries.values()) - if src.cache and entry[0] in src.cache[1] - and entry[1] in src.cache[1][entry[0]]] - if len(matching) > 1: - prio = [int(src.priority) for src in matching] - if prio.count(max(prio)) > 1: - self.logger.error("Found conflicting %s sources with same priority for %s, pkg %s" % - (entry[0].lower(), metadata.hostname, entry[1])) - raise Bcfg2.Server.Plugin.PluginExecutionError - index = prio.index(max(prio)) - matching = [matching[index]] - elif len(matching) == 1: - for prq in matching[0].cache[1][entry[0]][entry[1]]: - # XML comments seem to show up in the cache as a - # tuple with item 0 being callable. The logic - # below filters them out. Would be better to - # exclude them when we load the cache in the first - # place. - if prq not in prereqs and prq not in entries and not callable(prq[0]): - toexamine.append(prq) - prereqs.append(prq) - else: - continue - - return prereqs diff --git a/src/lib/Server/Plugins/Editor.py b/src/lib/Server/Plugins/Editor.py deleted file mode 100644 index c0d2cfbad..000000000 --- a/src/lib/Server/Plugins/Editor.py +++ /dev/null @@ -1,80 +0,0 @@ -import Bcfg2.Server.Plugin -import re -import lxml.etree - - -def linesub(pattern, repl, filestring): - """Substitutes instances of pattern with repl in filestring.""" - if filestring == None: - filestring = '' - output = list() - fileread = filestring.split('\n') - for line in fileread: - output.append(re.sub(pattern, repl, filestring)) - return '\n'.join(output) - - -class EditDirectives(Bcfg2.Server.Plugin.SpecificData): - """This object handles the editing directives.""" - def ProcessDirectives(self, input): - """Processes a list of edit directives on input.""" - temp = input - for directive in self.data.split('\n'): - directive = directive.split(',') - temp = linesub(directive[0], directive[1], temp) - return temp - - -class EditEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, basename, path, entry_type, encoding): - self.ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|%s\.H_.*)$" % path.split('/')[-1]) - Bcfg2.Server.Plugin.EntrySet.__init__(self, - basename, - path, - entry_type, - encoding) - self.inputs = dict() - - def bind_entry(self, entry, metadata): - client = metadata.hostname - filename = entry.get('name') - permdata = {'owner': 'root', - 'group': 'root', - 'perms': '0644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - entry.text = self.entries['edits'].ProcessDirectives(self.get_client_data(client)) - if not entry.text: - entry.set('empty', 'true') - try: - f = open('%s/%s.H_%s' % (self.path, filename.split('/')[-1], client), 'w') - f.write(entry.text) - f.close() - except: - pass - - def get_client_data(self, client): - return self.inputs[client] - - -class Editor(Bcfg2.Server.Plugin.GroupSpool, - Bcfg2.Server.Plugin.Probing): - name = 'Editor' - __author__ = 'bcfg2-dev@mcs.anl.gov' - filename_pattern = 'edits' - es_child_cls = EditDirectives - es_cls = EditEntrySet - - def GetProbes(self, _): - '''Return a set of probes for execution on client''' - probelist = list() - for name in list(self.entries.keys()): - probe = lxml.etree.Element('probe') - probe.set('name', name) - probe.set('source', "Editor") - probe.text = "cat %s" % name - probelist.append(probe) - return probelist - - def ReceiveData(self, client, datalist): - for data in datalist: - self.entries[data.get('name')].inputs[client.hostname] = data.text diff --git a/src/lib/Server/Plugins/FileProbes.py b/src/lib/Server/Plugins/FileProbes.py deleted file mode 100644 index a76d7cac4..000000000 --- a/src/lib/Server/Plugins/FileProbes.py +++ /dev/null @@ -1,230 +0,0 @@ -""" This module allows you to probe a client for a file, which is then -added to the specification. On subsequent runs, the file will be -replaced on the client if it is missing; if it has changed on the -client, it can either be updated in the specification or replaced on -the client """ - -import os -import sys -import errno -import binascii -import lxml.etree -import Bcfg2.Options -import Bcfg2.Server.Plugin - -probecode = """#!/usr/bin/env python - -import os -import pwd -import grp -import binascii -import lxml.etree - -path = "%s" - -if not os.path.exists(path): - print "%%s does not exist" %% path - raise SystemExit(1) - -stat = os.stat(path) -data = lxml.etree.Element("ProbedFileData", - name=path, - owner=pwd.getpwuid(stat[4])[0], - group=grp.getgrgid(stat[5])[0], - perms=oct(stat[0] & 07777)) -data.text = binascii.b2a_base64(open(path).read()) -print lxml.etree.tostring(data) -""" - -class FileProbesConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile): - """ Config file handler for FileProbes """ - def __init__(self, filename, fam): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - - -class FileProbes(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Probing): - """ This module allows you to probe a client for a file, which is then - added to the specification. On subsequent runs, the file will be - replaced on the client if it is missing; if it has changed on the - client, it can either be updated in the specification or replaced on - the client """ - - name = 'FileProbes' - experimental = True - __author__ = 'chris.a.st.pierre@gmail.com' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Probing.__init__(self) - self.config = FileProbesConfig(os.path.join(self.data, 'config.xml'), - core.fam) - self.entries = dict() - self.probes = dict() - - def GetProbes(self, metadata): - """Return a set of probes for execution on client.""" - if metadata.hostname not in self.probes: - cfg = self.core.plugins['Cfg'] - self.entries[metadata.hostname] = dict() - self.probes[metadata.hostname] = [] - for entry in self.config.Match(metadata): - path = entry.get("name") - # do not probe for files that are already in Cfg and - # for which update is false; we can't possibly do - # anything with the data we get from such a probe - try: - if (entry.get('update', 'false').lower() == "false" and - cfg.entries[path].get_pertinent_entries(entry, - metadata)): - continue - except (KeyError, Bcfg2.Server.Plugin.PluginExecutionError): - pass - self.entries[metadata.hostname][path] = entry - probe = lxml.etree.Element('probe', name=path, - source=self.name, - interpreter="/usr/bin/env python") - probe.text = probecode % path - self.probes[metadata.hostname].append(probe) - self.debug_log("Adding file probe for %s to %s" % - (path, metadata.hostname)) - return self.probes[metadata.hostname] - - def ReceiveData(self, metadata, datalist): - """Receive data from probe.""" - self.debug_log("Receiving file probe data from %s" % metadata.hostname) - - for data in datalist: - if data.text is None: - self.logger.error("Got null response to %s file probe from %s" % - (data.get('name'), metadata.hostname)) - else: - try: - self.write_data(lxml.etree.XML(data.text), metadata) - except lxml.etree.XMLSyntaxError: - # if we didn't get XML back from the probe, assume - # it's an error message - self.logger.error(data.text) - - def write_data(self, data, metadata): - """Write the probed file data to the bcfg2 specification.""" - filename = data.get("name") - contents = binascii.a2b_base64(data.text) - entry = self.entries[metadata.hostname][filename] - cfg = self.core.plugins['Cfg'] - specific = "%s.H_%s" % (os.path.basename(filename), metadata.hostname) - # we can't use os.path.join() for this because specific - # already has a leading /, which confuses os.path.join() - fileloc = "%s%s" % (cfg.data, os.path.join(filename, specific)) - - create = False - try: - cfg.entries[filename].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - create = True - - # get current entry data - if entry.text and entry.get("encoding") == "base64": - entrydata = binascii.a2b_base64(entry.text) - else: - entrydata = entry.text - - if create: - self.logger.info("Writing new probed file %s" % fileloc) - self.write_file(fileloc, contents) - self.verify_file(filename, contents, metadata) - infoxml = os.path.join("%s%s" % (cfg.data, filename), "info.xml") - self.write_infoxml(infoxml, entry, data) - elif entrydata == contents: - self.debug_log("Existing %s contents match probed contents" % - filename) - return - elif (entry.get('update', 'false').lower() == "true"): - self.logger.info("Writing updated probed file %s" % fileloc) - self.write_file(fileloc, contents) - self.verify_file(filename, contents, metadata) - else: - self.logger.info("Skipping updated probed file %s" % fileloc) - return - - def write_file(self, fileloc, contents): - try: - os.makedirs(os.path.dirname(fileloc)) - except OSError: - err = sys.exc_info()[1] - if err.errno == errno.EEXIST: - pass - else: - self.logger.error("Could not create parent directories for %s: " - "%s" % (fileloc, err)) - return - - try: - open(fileloc, 'wb').write(contents) - except IOError: - err = sys.exc_info()[1] - self.logger.error("Could not write %s: %s" % (fileloc, err)) - return - - def verify_file(self, filename, contents, metadata): - # Service the FAM events queued up by the key generation so - # the data structure entries will be available for binding. - # - # NOTE: We wait for up to ten seconds. There is some potential - # for race condition, because if the file monitor doesn't get - # notified about the new key files in time, those entries - # won't be available for binding. In practice, this seems - # "good enough". - entry = self.entries[metadata.hostname][filename] - cfg = self.core.plugins['Cfg'] - tries = 0 - updated = False - while not updated: - if tries >= 10: - self.logger.error("%s still not registered" % filename) - return - self.core.fam.handle_events_in_interval(1) - try: - cfg.entries[filename].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - tries += 1 - continue - - # get current entry data - if entry.get("encoding") == "base64": - entrydata = binascii.a2b_base64(entry.text) - else: - entrydata = entry.text - if entrydata == contents: - updated = True - tries += 1 - - def write_infoxml(self, infoxml, entry, data): - """ write an info.xml for the file """ - if os.path.exists(infoxml): - return - - self.logger.info("Writing info.xml at %s for %s" % - (infoxml, data.get("name"))) - info = \ - lxml.etree.Element("Info", - owner=data.get("owner", - Bcfg2.Options.MDATA_OWNER.value), - group=data.get("group", - Bcfg2.Options.MDATA_GROUP.value), - perms=data.get("perms", - Bcfg2.Options.MDATA_PERMS.value), - encoding=entry.get("encoding", - Bcfg2.Options.ENCODING.value)) - - root = lxml.etree.Element("FileInfo") - root.append(info) - try: - open(infoxml, "w").write(lxml.etree.tostring(root, - pretty_print=True)) - except IOError: - err = sys.exc_info()[1] - self.logger.error("Could not write %s: %s" % (fileloc, err)) - return diff --git a/src/lib/Server/Plugins/Fossil.py b/src/lib/Server/Plugins/Fossil.py deleted file mode 100644 index 1b1627688..000000000 --- a/src/lib/Server/Plugins/Fossil.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Fossil') - -class Fossil(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Fossil is a version plugin for dealing with Bcfg2 repos.""" - name = 'Fossil' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - self.core = core - self.datastore = datastore - - # path to fossil file for bcfg2 repo - fossil_file = "%s/_FOSSIL_" % datastore - - # Read revision from bcfg2 repo - if os.path.isfile(fossil_file): - revision = self.get_revision() - elif not os.path.isdir(datastore): - logger.error("%s is not a directory" % datastore) - raise Bcfg2.Server.Plugin.PluginInitError - else: - logger.error("%s is not a file" % fossil_file) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized Fossil.py plugin with %(ffile)s at revision %(frev)s" \ - % {'ffile': fossil_file, 'frev': revision}) - - def get_revision(self): - """Read fossil revision information for the Bcfg2 repository.""" - try: - data = Popen("env LC_ALL=C fossil info", - shell=True, - cwd=self.datastore, - stdout=PIPE).stdout.readlines() - revline = [line.split(': ')[1].strip() for line in data if \ - line.split(': ')[0].strip() == 'checkout'][-1] - revision = revline.split(' ')[0] - except IndexError: - logger.error("Failed to read fossil info; disabling fossil support") - logger.error('''Ran command "fossil info" from directory "%s"''' % (self.datastore)) - logger.error("Got output: %s" % data) - raise Bcfg2.Server.Plugin.PluginInitError - return revision diff --git a/src/lib/Server/Plugins/Git.py b/src/lib/Server/Plugins/Git.py deleted file mode 100644 index 8f8ea87f1..000000000 --- a/src/lib/Server/Plugins/Git.py +++ /dev/null @@ -1,44 +0,0 @@ -"""The Git plugin provides a revision interface for Bcfg2 repos using git.""" - -import os -from dulwich.repo import Repo -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Git') - - -class Git(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Git is a version plugin for dealing with Bcfg2 repos.""" - name = 'Git' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to git directory for bcfg2 repo - git_dir = "%s/.git" % datastore - - # Read revision from bcfg2 repo - if os.path.isdir(git_dir): - self.get_revision() - else: - logger.error("%s is not a directory" % git_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized git plugin with git directory %s" % git_dir) - - def get_revision(self): - """Read git revision information for the Bcfg2 repository.""" - try: - repo = Repo(self.datastore) - revision = repo.head() - except: - logger.error("Failed to read git repository; disabling git support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision diff --git a/src/lib/Server/Plugins/GroupPatterns.py b/src/lib/Server/Plugins/GroupPatterns.py deleted file mode 100644 index 58b4d4afb..000000000 --- a/src/lib/Server/Plugins/GroupPatterns.py +++ /dev/null @@ -1,124 +0,0 @@ -import re -import logging -import lxml.etree -import Bcfg2.Server.Plugin - -class PackedDigitRange(object): - def __init__(self, digit_range): - self.sparse = list() - self.ranges = list() - for item in digit_range.split(','): - if '-' in item: - self.ranges.append(tuple([int(x) for x in item.split('-')])) - else: - self.sparse.append(int(item)) - - def includes(self, other): - iother = int(other) - if iother in self.sparse: - return True - for (start, end) in self.ranges: - if iother in range(start, end + 1): - return True - return False - - -class PatternMap(object): - range_finder = r'\[\[[\d\-,]+\]\]' - - def __init__(self, pattern, rangestr, groups): - self.pattern = pattern - self.rangestr = rangestr - self.groups = groups - if pattern != None: - self.re = re.compile(pattern) - self.process = self.process_re - elif rangestr != None: - if '\\' in rangestr: - raise Exception("Backslashes are not allowed in NameRanges") - self.process = self.process_range - self.re = re.compile('^' + re.sub(self.range_finder, '(\d+)', - rangestr)) - dmatcher = re.compile(re.sub(self.range_finder, - r'\[\[([\d\-,]+)\]\]', - rangestr)) - self.dranges = [PackedDigitRange(x) - for x in dmatcher.match(rangestr).groups()] - else: - raise Exception("No pattern or range given") - - def process_range(self, name): - match = self.re.match(name) - if not match: - return None - digits = match.groups() - for i in range(len(digits)): - if not self.dranges[i].includes(digits[i]): - return None - return self.groups - - def process_re(self, name): - match = self.re.match(name) - if not match: - return None - ret = list() - sub = match.groups() - for group in self.groups: - newg = group - for idx in range(len(sub)): - newg = newg.replace('$%s' % (idx + 1), sub[idx]) - ret.append(newg) - return ret - - -class PatternFile(Bcfg2.Server.Plugin.SingleXMLFileBacked): - __identifier__ = None - - def __init__(self, filename, fam): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) - self.patterns = [] - self.logger = logging.getLogger(self.__class__.__name__) - - def Index(self): - Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) - self.patterns = [] - for entry in self.xdata.xpath('//GroupPattern'): - try: - groups = [g.text for g in entry.findall('Group')] - for pat_ent in entry.findall('NamePattern'): - pat = pat_ent.text - self.patterns.append(PatternMap(pat, None, groups)) - for range_ent in entry.findall('NameRange'): - rng = range_ent.text - self.patterns.append(PatternMap(None, rng, groups)) - except: - self.logger.error("GroupPatterns: Failed to initialize pattern " - "%s" % entry.get('pattern')) - - def process_patterns(self, hostname): - ret = [] - for pattern in self.patterns: - try: - gn = pattern.process(hostname) - if gn is not None: - ret.extend(gn) - except: - self.logger.error("GroupPatterns: Failed to process pattern %s " - "for %s" % (pattern.pattern, hostname), - exc_info=1) - return ret - - -class GroupPatterns(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - name = "GroupPatterns" - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.config = PatternFile(self.data + '/config.xml', - core.fam) - - def get_additional_groups(self, metadata): - return self.config.process_patterns(metadata.hostname) diff --git a/src/lib/Server/Plugins/Guppy.py b/src/lib/Server/Plugins/Guppy.py deleted file mode 100644 index 046aedc0b..000000000 --- a/src/lib/Server/Plugins/Guppy.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -This plugin is used to trace memory leaks within the bcfg2-server -process using Guppy. By default the remote debugger is started -when this plugin is enabled. The debugger can be shutoff in a running -process using "bcfg2-admin xcmd Guppy.Disable" and reenabled using -"bcfg2-admin xcmd Guppy.Enable". - -To attach the console run: - -python -c "from guppy import hpy;hpy().monitor()" - -For example: - -# python -c "from guppy import hpy;hpy().monitor()" - -*** Connection 1 opened *** - lc -CID PID ARGV - 1 25063 ['/usr/sbin/bcfg2-server', '-D', '/var/run/bcfg2-server.pid'] - sc 1 -Remote connection 1. To return to Monitor, type or . - int -Remote interactive console. To return to Annex, type '-'. ->>> hp.heap() -... - - -""" -import re -import Bcfg2.Server.Plugin - -class Guppy(Bcfg2.Server.Plugin.Plugin): - """Guppy is a debugging plugin to help trace memory leaks""" - name = 'Guppy' - __author__ = 'bcfg-dev@mcs.anl.gov' - - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Enable','Disable'] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - - self.Enable() - - def Enable(self): - """Enable remote debugging""" - try: - from guppy.heapy import Remote - Remote.on() - except: - self.logger.error("Failed to create Heapy context") - raise Bcfg2.Server.Plugin.PluginInitError - - def Disable(self): - """Disable remote debugging""" - try: - from guppy.heapy import Remote - Remote.off() - except: - self.logger.error("Failed to disable Heapy") - raise Bcfg2.Server.Plugin.PluginInitError - diff --git a/src/lib/Server/Plugins/Hg.py b/src/lib/Server/Plugins/Hg.py deleted file mode 100644 index 0c3537613..000000000 --- a/src/lib/Server/Plugins/Hg.py +++ /dev/null @@ -1,45 +0,0 @@ -import os -from mercurial import ui, hg -import Bcfg2.Server.Plugin - -# for debugging output only -import logging -logger = logging.getLogger('Bcfg2.Plugins.Mercurial') - -class Hg(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Version): - """Mercurial is a version plugin for dealing with Bcfg2 repository.""" - name = 'Mercurial' - __author__ = 'bcfg-dev@mcs.anl.gov' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Version.__init__(self) - self.core = core - self.datastore = datastore - - # path to hg directory for Bcfg2 repo - hg_dir = "%s/.hg" % datastore - - # Read changeset from bcfg2 repo - if os.path.isdir(hg_dir): - self.get_revision() - else: - logger.error("%s is not present." % hg_dir) - raise Bcfg2.Server.Plugin.PluginInitError - - logger.debug("Initialized hg plugin with hg directory = %s" % hg_dir) - - def get_revision(self): - """Read hg revision information for the Bcfg2 repository.""" - try: - repo_path = "%s/" % self.datastore - repo = hg.repository(ui.ui(), repo_path) - tip = repo.changelog.tip() - revision = repo.changelog.rev(tip) - except: - logger.error("Failed to read hg repository; disabling mercurial support") - raise Bcfg2.Server.Plugin.PluginInitError - return revision - diff --git a/src/lib/Server/Plugins/Hostbase.py b/src/lib/Server/Plugins/Hostbase.py deleted file mode 100644 index e9c1c1cff..000000000 --- a/src/lib/Server/Plugins/Hostbase.py +++ /dev/null @@ -1,593 +0,0 @@ -""" -This file provides the Hostbase plugin. -It manages dns/dhcp/nis host information -""" - -import os -os.environ['DJANGO_SETTINGS_MODULE'] = 'Bcfg2.Server.Hostbase.settings' -from lxml.etree import Element, SubElement -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugin import PluginExecutionError, PluginInitError -from time import strftime -from sets import Set -from django.template import Context, loader -from django.db import connection -import re -# Compatibility imports -from Bcfg2.Bcfg2Py3k import StringIO - - -class Hostbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure, - Bcfg2.Server.Plugin.Generator): - """The Hostbase plugin handles host/network info.""" - name = 'Hostbase' - __author__ = 'bcfg-dev@mcs.anl.gov' - filepath = '/my/adm/hostbase/files/bind' - - def __init__(self, core, datastore): - - self.ready = False - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - Bcfg2.Server.Plugin.Generator.__init__(self) - files = ['zone.tmpl', - 'reversesoa.tmpl', - 'named.tmpl', - 'reverseappend.tmpl', - 'dhcpd.tmpl', - 'hosts.tmpl', - 'hostsappend.tmpl'] - self.filedata = {} - self.dnsservers = [] - self.dhcpservers = [] - self.templates = {'zone': loader.get_template('zone.tmpl'), - 'reversesoa': loader.get_template('reversesoa.tmpl'), - 'named': loader.get_template('named.tmpl'), - 'namedviews': loader.get_template('namedviews.tmpl'), - 'reverseapp': loader.get_template('reverseappend.tmpl'), - 'dhcp': loader.get_template('dhcpd.tmpl'), - 'hosts': loader.get_template('hosts.tmpl'), - 'hostsapp': loader.get_template('hostsappend.tmpl'), - } - self.Entries['ConfigFile'] = {} - self.__rmi__ = ['rebuildState'] - try: - self.rebuildState(None) - except: - raise PluginInitError - - def FetchFile(self, entry, metadata): - """Return prebuilt file data.""" - fname = entry.get('name').split('/')[-1] - if not fname in self.filedata: - raise PluginExecutionError - perms = {'owner': 'root', - 'group': 'root', - 'perms': '644'} - [entry.attrib.__setitem__(key, value) - for (key, value) in list(perms.items())] - entry.text = self.filedata[fname] - - def BuildStructures(self, metadata): - """Build hostbase bundle.""" - if metadata.hostname not in self.dnsservers or metadata.hostname not in self.dhcpservers: - return [] - output = Element("Bundle", name='hostbase') - if metadata.hostname in self.dnsservers: - for configfile in self.Entries['ConfigFile']: - if re.search('/etc/bind/', configfile): - SubElement(output, "ConfigFile", name=configfile) - if metadata.hostname in self.dhcpservers: - SubElement(output, "ConfigFile", name="/etc/dhcp3/dhcpd.conf") - return [output] - - def rebuildState(self, _): - """Pre-cache all state information for hostbase config files - callable as an XMLRPC function. - - """ - self.buildZones() - self.buildDHCP() - self.buildHosts() - self.buildHostsLPD() - self.buildPrinters() - self.buildNetgroups() - return True - - def buildZones(self): - """Pre-build and stash zone files.""" - cursor = connection.cursor() - - cursor.execute("SELECT id, serial FROM hostbase_zone") - zones = cursor.fetchall() - - for zone in zones: - # update the serial number for all zone files - todaydate = (strftime('%Y%m%d')) - try: - if todaydate == str(zone[1])[:8]: - serial = zone[1] + 1 - else: - serial = int(todaydate) * 100 - except (KeyError): - serial = int(todaydate) * 100 - cursor.execute("""UPDATE hostbase_zone SET serial = \'%s\' WHERE id = \'%s\'""" % (str(serial), zone[0])) - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone NOT LIKE \'%%.rev\'") - zones = cursor.fetchall() - - iplist = [] - hosts = {} - - for zone in zones: - zonefile = StringIO() - externalzonefile = StringIO() - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - nameservers = cursor.fetchall() - cursor.execute("""SELECT i.ip_addr FROM hostbase_zone_addresses z - INNER JOIN hostbase_zoneaddress i ON z.zoneaddress_id = i.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - addresses = cursor.fetchall() - cursor.execute("""SELECT m.priority, m.mx FROM hostbase_zone_mxs z - INNER JOIN hostbase_mx m ON z.mx_id = m.id - WHERE z.zone_id = \'%s\'""" % zone[0]) - mxs = cursor.fetchall() - context = Context({ - 'zone': zone, - 'nameservers': nameservers, - 'addresses': addresses, - 'mxs': mxs - }) - zonefile.write(self.templates['zone'].render(context)) - externalzonefile.write(self.templates['zone'].render(context)) - - querystring = """SELECT h.hostname, p.ip_addr, - n.name, c.cname, m.priority, m.mx, n.dns_view - FROM (((((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - INNER JOIN hostbase_name_mxs x ON n.id = x.name_id) - INNER JOIN hostbase_mx m ON m.id = x.mx_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE n.name LIKE '%%%%%s' - AND h.status = 'active' - ORDER BY h.hostname, n.name, p.ip_addr - """ % zone[1] - cursor.execute(querystring) - zonehosts = cursor.fetchall() - prevhost = (None, None, None, None) - cnames = StringIO() - cnamesexternal = StringIO() - for host in zonehosts: - if not host[2].split(".", 1)[1] == zone[1]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - continue - if not prevhost[1] == host[1] or not prevhost[2] == host[2]: - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - cnames = StringIO() - cnamesexternal = StringIO() - zonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-32s\n" % - (host[2].split(".", 1)[0], 'A', host[1])) - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - elif not prevhost[5] == host[5]: - zonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - if host[6] == 'global': - externalzonefile.write("%-32s%-10s%-3s%s.\n" % - ('', 'MX', host[4], host[5])) - - if host[3]: - try: - if host[3].split(".", 1)[1] == zone[1]: - cnames.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3].split(".", 1)[0], - 'CNAME', host[2].split(".", 1)[0])) - else: - cnames.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - if host[6] == 'global': - cnamesexternal.write("%-32s%-10s%-32s\n" % - (host[3] + ".", - 'CNAME', - host[2].split(".", 1)[0])) - - except: - pass - prevhost = host - zonefile.write(cnames.getvalue()) - externalzonefile.write(cnamesexternal.getvalue()) - zonefile.write("\n\n%s" % zone[9]) - externalzonefile.write("\n\n%s" % zone[9]) - self.filedata[zone[1]] = zonefile.getvalue() - self.filedata[zone[1] + ".external"] = externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']["%s/%s" % (self.filepath, zone[1])] = self.FetchFile - self.Entries['ConfigFile']["%s/%s.external" % (self.filepath, zone[1])] = self.FetchFile - - cursor.execute("SELECT * FROM hostbase_zone WHERE zone LIKE \'%%.rev\' AND zone <> \'.rev\'") - reversezones = cursor.fetchall() - - reversenames = [] - for reversezone in reversezones: - cursor.execute("""SELECT n.name FROM hostbase_zone_nameservers z - INNER JOIN hostbase_nameserver n ON z.nameserver_id = n.id - WHERE z.zone_id = \'%s\'""" % reversezone[0]) - reverse_nameservers = cursor.fetchall() - - context = Context({ - 'inaddr': reversezone[1].rstrip('.rev'), - 'zone': reversezone, - 'nameservers': reverse_nameservers, - }) - - self.filedata[reversezone[1]] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1] + '.external'] = self.templates['reversesoa'].render(context) - self.filedata[reversezone[1]] += reversezone[9] - self.filedata[reversezone[1] + '.external'] += reversezone[9] - - subnet = reversezone[1].split(".") - subnet.reverse() - reversenames.append((reversezone[1].rstrip('.rev'), ".".join(subnet[1:]))) - - for filename in reversenames: - cursor.execute(""" - SELECT DISTINCT h.hostname, p.ip_addr, n.dns_view FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON n.ip_id = p.id - WHERE p.ip_addr LIKE '%s%%%%' AND h.status = 'active' ORDER BY p.ip_addr - """ % filename[1]) - reversehosts = cursor.fetchall() - zonefile = StringIO() - externalzonefile = StringIO() - if len(filename[0].split(".")) == 2: - originlist = [] - [originlist.append((".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]]))) - for ip in reversehosts - if (".".join([ip[1].split(".")[2], filename[0]]), - ".".join([filename[1], ip[1].split(".")[2]])) not in originlist] - for origin in originlist: - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if host[1].rstrip('0123456789').rstrip('.') == origin[1]] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].rstrip('0123456789').rstrip('.') == origin[1] - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': origin[0], - 'fileorigin': filename[0], - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - else: - originlist = [filename[0]] - hosts = [(host[1].split("."), host[0]) - for host in reversehosts - if (host[1].split("."), host[0]) not in hosts] - hosts_external = [(host[1].split("."), host[0]) - for host in reversehosts - if ((host[1].split("."), host[0]) not in hosts_external - and host[2] == 'global')] - context = Context({ - 'hosts': hosts, - 'inaddr': filename[0], - 'fileorigin': None, - }) - zonefile.write(self.templates['reverseapp'].render(context)) - context = Context({ - 'hosts': hosts_external, - 'inaddr': filename[0], - 'fileorigin': None, - }) - externalzonefile.write(self.templates['reverseapp'].render(context)) - self.filedata['%s.rev' % filename[0]] += zonefile.getvalue() - self.filedata['%s.rev.external' % filename[0]] += externalzonefile.getvalue() - zonefile.close() - externalzonefile.close() - self.Entries['ConfigFile']['%s/%s.rev' % (self.filepath, filename[0])] = self.FetchFile - self.Entries['ConfigFile']['%s/%s.rev.external' % (self.filepath, filename[0])] = self.FetchFile - - ## here's where the named.conf file gets written - context = Context({ - 'zones': zones, - 'reverses': reversenames, - }) - self.filedata['named.conf'] = self.templates['named'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf'] = self.FetchFile - self.filedata['named.conf.views'] = self.templates['namedviews'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/named.conf.views'] = self.FetchFile - - def buildDHCP(self): - """Pre-build dhcpd.conf and stash in the filedata table.""" - - # fetches all the hosts with DHCP == True - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname, mac_addr, ip_addr - FROM (hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip ip ON i.id = ip.interface_id - WHERE i.dhcp=1 AND h.status='active' AND i.mac_addr <> '' - AND i.mac_addr <> 'float' AND i.mac_addr <> 'unknown' - ORDER BY h.hostname, i.mac_addr - """) - - dhcphosts = cursor.fetchall() - count = 0 - hosts = [] - hostdata = [dhcphosts[0][0], dhcphosts[0][1], dhcphosts[0][2]] - if len(dhcphosts) > 1: - for x in range(1, len(dhcphosts)): - # if an interface has 2 or more ip addresses - # adds the ip to the current interface - if hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0] and hostdata[1] == dhcphosts[x][1]: - hostdata[2] = ", ".join([hostdata[2], dhcphosts[x][2]]) - # if a host has 2 or more interfaces - # writes the current one and grabs the next - elif hostdata[0].split(".")[0] == dhcphosts[x][0].split(".")[0]: - hosts.append(hostdata) - count += 1 - hostdata = ["-".join([dhcphosts[x][0], str(count)]), dhcphosts[x][1], dhcphosts[x][2]] - # new host found, writes current data to the template - else: - hosts.append(hostdata) - count = 0 - hostdata = [dhcphosts[x][0], dhcphosts[x][1], dhcphosts[x][2]] - #makes sure the last of the data gets written out - if hostdata not in hosts: - hosts.append(hostdata) - - context = Context({ - 'hosts': hosts, - 'numips': len(hosts), - }) - - self.filedata['dhcpd.conf'] = self.templates['dhcp'].render(context) - self.Entries['ConfigFile']['/my/adm/hostbase/files/dhcpd.conf'] = self.FetchFile - - def buildHosts(self): - """Pre-build and stash /etc/hosts file.""" - - append_data = [] - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host ORDER BY hostname - """) - hostbase = cursor.fetchall() - domains = [host[0].split(".", 1)[1] for host in hostbase] - domains_set = Set(domains) - domain_data = [(domain, domains.count(domain)) for domain in domains_set] - domain_data.sort() - - cursor.execute(""" - SELECT ip_addr FROM hostbase_ip ORDER BY ip_addr - """) - ips = cursor.fetchall() - three_octets = [ip[0].rstrip('0123456789').rstrip('.') \ - for ip in ips] - three_octets_set = Set(three_octets) - three_octets_data = [(octet, three_octets.count(octet)) \ - for octet in three_octets_set] - three_octets_data.sort() - - for three_octet in three_octets_data: - querystring = """SELECT h.hostname, h.primary_user, - p.ip_addr, n.name, c.cname - FROM (((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id) - LEFT JOIN hostbase_cname c ON n.id = c.name_id - WHERE p.ip_addr LIKE \'%s.%%%%\' AND h.status = 'active'""" % three_octet[0] - cursor.execute(querystring) - tosort = list(cursor.fetchall()) - tosort.sort(lambda x, y: cmp(int(x[2].split(".")[-1]), int(y[2].split(".")[-1]))) - append_data.append((three_octet, tuple(tosort))) - - two_octets = [ip.rstrip('0123456789').rstrip('.') for ip in three_octets] - two_octets_set = Set(two_octets) - two_octets_data = [(octet, two_octets.count(octet)) - for octet in two_octets_set] - two_octets_data.sort() - - context = Context({ - 'domain_data': domain_data, - 'three_octets_data': three_octets_data, - 'two_octets_data': two_octets_data, - 'three_octets': three_octets, - 'num_ips': len(three_octets), - }) - - self.filedata['hosts'] = self.templates['hosts'].render(context) - - for subnet in append_data: - ips = [] - simple = True - namelist = [name.split('.', 1)[0] for name in [subnet[1][0][3]]] - cnamelist = [] - if subnet[1][0][4]: - cnamelist.append(subnet[1][0][4].split('.', 1)[0]) - simple = False - appenddata = subnet[1][0] - for ip in subnet[1][1:]: - if appenddata[2] == ip[2]: - namelist.append(ip[3].split('.', 1)[0]) - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - appenddata = ip - else: - if appenddata[0] == ip[0]: - simple = False - ips.append((appenddata[2], appenddata[0], Set(namelist), - cnamelist, simple, appenddata[1])) - appenddata = ip - simple = True - namelist = [ip[3].split('.', 1)[0]] - cnamelist = [] - if ip[4]: - cnamelist.append(ip[4].split('.', 1)[0]) - simple = False - ips.append((appenddata[2], appenddata[0], Set(namelist), - cnamelist, simple, appenddata[1])) - context = Context({ - 'subnet': subnet[0], - 'ips': ips, - }) - self.filedata['hosts'] += self.templates['hostsapp'].render(context) - self.Entries['ConfigFile']['/mcs/etc/hosts'] = self.FetchFile - - def buildPrinters(self): - """The /mcs/etc/printers.data file""" - header = """# This file is automatically generated. DO NOT EDIT IT! -# -Name Room User Type Notes -============== ========== ============================== ======================== ==================== -""" - - cursor = connection.cursor() - # fetches all the printers from the database - cursor.execute(""" - SELECT printq, location, primary_user, comments - FROM hostbase_host - WHERE whatami='printer' AND printq <> '' AND status = 'active' - ORDER BY printq - """) - printers = cursor.fetchall() - - printersfile = header - for printer in printers: - # splits up the printq line and gets the - # correct description out of the comments section - temp = printer[3].split('\n') - for printq in re.split(',[ ]*', printer[0]): - if len(temp) > 1: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], temp[1], temp[0])) - else: - printersfile += ("%-16s%-12s%-32s%-26s%s\n" % - (printq, printer[1], printer[2], '', printer[3])) - self.filedata['printers.data'] = printersfile - self.Entries['ConfigFile']['/mcs/etc/printers.data'] = self.FetchFile - - def buildHostsLPD(self): - """Creates the /mcs/etc/hosts.lpd file""" - - # this header needs to be changed to be more generic - header = """+@machines -+@all-machines -achilles.ctd.anl.gov -raven.ops.anl.gov -seagull.hr.anl.gov -parrot.ops.anl.gov -condor.ops.anl.gov -delphi.esh.anl.gov -anlcv1.ctd.anl.gov -anlvms.ctd.anl.gov -olivia.ctd.anl.gov\n\n""" - - cursor = connection.cursor() - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"red\" AND status = 'active' - ORDER BY hostname""") - redmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"red\" AND n.only=1 AND h.status = 'active' - """) - redmachines.extend(list(cursor.fetchall())) - cursor.execute(""" - SELECT hostname FROM hostbase_host WHERE netgroup=\"win\" AND status = 'active' - ORDER BY hostname""") - winmachines = list(cursor.fetchall()) - cursor.execute(""" - SELECT n.name FROM ((hostbase_host h INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE netgroup=\"win\" AND n.only=1 AND h.status = 'active' - """) - winmachines.__add__(list(cursor.fetchall())) - hostslpdfile = header - for machine in redmachines: - hostslpdfile += machine[0] + "\n" - hostslpdfile += "\n" - for machine in winmachines: - hostslpdfile += machine[0] + "\n" - self.filedata['hosts.lpd'] = hostslpdfile - self.Entries['ConfigFile']['/mcs/etc/hosts.lpd'] = self.FetchFile - - def buildNetgroups(self): - """Makes the *-machine files""" - header = """################################################################### -# This file lists hosts in the '%s' machine netgroup, it is -# automatically generated. DO NOT EDIT THIS FILE! -# -# Number of hosts in '%s' machine netgroup: %i -#\n\n""" - - cursor = connection.cursor() - # fetches all the hosts that with valid netgroup entries - cursor.execute(""" - SELECT h.hostname, n.name, h.netgroup, n.only FROM ((hostbase_host h - INNER JOIN hostbase_interface i ON h.id = i.host_id) - INNER JOIN hostbase_ip p ON i.id = p.interface_id) - INNER JOIN hostbase_name n ON p.id = n.ip_id - WHERE h.netgroup <> '' AND h.netgroup <> 'none' AND h.status = 'active' - ORDER BY h.netgroup, h.hostname - """) - nameslist = cursor.fetchall() - # gets the first host and initializes the hash - hostdata = nameslist[0] - netgroups = {hostdata[2]: [hostdata[0]]} - for row in nameslist: - # if new netgroup, create it - if row[2] not in netgroups: - netgroups.update({row[2]: []}) - # if it belongs in the netgroup and has multiple interfaces, put them in - if hostdata[0] == row[0] and row[3]: - netgroups[row[2]].append(row[1]) - hostdata = row - # if its a new host, write the old one to the hash - elif hostdata[0] != row[0]: - netgroups[row[2]].append(row[0]) - hostdata = row - - for netgroup in netgroups: - fileoutput = StringIO() - fileoutput.write(header % (netgroup, netgroup, len(netgroups[netgroup]))) - for each in netgroups[netgroup]: - fileoutput.write(each + "\n") - self.filedata['%s-machines' % netgroup] = fileoutput.getvalue() - fileoutput.close() - self.Entries['ConfigFile']['/my/adm/hostbase/makenets/machines/%s-machines' % netgroup] = self.FetchFile - - cursor.execute(""" - UPDATE hostbase_host SET dirty=0 - """) diff --git a/src/lib/Server/Plugins/Ldap.py b/src/lib/Server/Plugins/Ldap.py deleted file mode 100644 index 04417339b..000000000 --- a/src/lib/Server/Plugins/Ldap.py +++ /dev/null @@ -1,245 +0,0 @@ -import imp -import logging -import sys -import time -import traceback -import Bcfg2.Options -import Bcfg2.Server.Plugin - -logger = logging.getLogger('Bcfg2.Plugins.Ldap') - -try: - import ldap -except: - logger.error("Unable to load ldap module. Is python-ldap installed?") - raise ImportError - -# time in seconds between retries after failed LDAP connection -RETRY_DELAY = 5 -# how many times to try reaching the LDAP server if a connection is broken -# at the very minimum, one retry is needed to handle a restarted LDAP daemon -RETRY_COUNT = 3 - -SCOPE_MAP = { - "base" : ldap.SCOPE_BASE, - "one" : ldap.SCOPE_ONELEVEL, - "sub" : ldap.SCOPE_SUBTREE, -} - -LDAP_QUERIES = [] - -def register_query(query): - LDAP_QUERIES.append(query) - -class ConfigFile(Bcfg2.Server.Plugin.FileBacked): - """ - Config file for the Ldap plugin - - The config file cannot be 'parsed' in the traditional sense as we would - need some serious type checking ugliness to just get the LdapQuery - subclasses. The alternative would be to have the user create a list with - a predefined name that contains all queries. - The approach implemented here is having the user call a registering - decorator that updates a global variable in this module. - """ - def __init__(self, filename, fam): - self.filename = filename - Bcfg2.Server.Plugin.FileBacked.__init__(self, self.filename) - fam.AddMonitor(self.filename, self) - - def Index(self): - """ - Reregisters the queries in the config file - - The config will take care of actually registering the queries, - so we just load it once and don't keep it. - """ - global LDAP_QUERIES - LDAP_QUERIES = [] - imp.load_source("ldap_cfg", self.filename) - -class Ldap(Bcfg2.Server.Plugin.Plugin, Bcfg2.Server.Plugin.Connector): - """ - The Ldap plugin allows adding data from an LDAP server to your metadata. - """ - name = "Ldap" - experimental = True - debug_flag = False - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.config = ConfigFile(self.data + "/config.py", core.fam) - - def debug_log(self, message, flag = None): - if (flag is None) and self.debug_flag or flag: - self.logger.error(message) - - def get_additional_data(self, metadata): - query = None - try: - data = {} - self.debug_log("LdapPlugin debug: found queries " + - str(LDAP_QUERIES)) - for QueryClass in LDAP_QUERIES: - query = QueryClass() - if query.is_applicable(metadata): - self.debug_log("LdapPlugin debug: processing query '" + - query.name + "'") - data[query.name] = query.get_result(metadata) - else: - self.debug_log("LdapPlugin debug: query '" + query.name + - "' not applicable to host '" + metadata.hostname + "'") - return data - except Exception: - if hasattr(query, "name"): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "Exception during processing of query named '" + - str(query.name) + - "', query results will be empty" + - " and may cause bind failures") - for line in traceback.format_exception(sys.exc_info()[0], - sys.exc_info()[1], - sys.exc_info()[2]): - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - line.replace("\n", "")) - return {} - -class LdapConnection(object): - """ - Connection to an LDAP server. - """ - def __init__(self, host = "localhost", port = 389, - binddn = None, bindpw = None): - self.host = host - self.port = port - self.binddn = binddn - self.bindpw = bindpw - self.conn = None - - def __del__(self): - if self.conn: - self.conn.unbind() - - def init_conn(self): - self.conn = ldap.initialize(self.url) - if self.binddn is not None and self.bindpw is not None: - self.conn.simple_bind_s(self.binddn, self.bindpw) - - def run_query(self, query): - result = None - for attempt in range(RETRY_COUNT + 1): - if attempt >= 1: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "LDAP server down (retry " + str(attempt) + "/" + - str(RETRY_COUNT) + ")") - try: - if not self.conn: - self.init_conn() - result = self.conn.search_s( - query.base, - SCOPE_MAP[query.scope], - query.filter, - query.attrs, - ) - break - except ldap.SERVER_DOWN: - self.conn = None - time.sleep(RETRY_DELAY) - return result - - @property - def url(self): - return "ldap://" + self.host + ":" + str(self.port) - -class LdapQuery(object): - """ - Query referencing an LdapConnection and providing several - methods for query manipulation. - """ - - name = "unknown" - base = "" - scope = "sub" - filter = "(objectClass=*)" - attrs = None - connection = None - result = None - - def __unicode__(self): - return "LdapQuery:" + self.name - - def is_applicable(self, metadata): - """ - Overrideable method to determine if the query is to be executed for - the given metadata object. - Defaults to true. - """ - return True - - def prepare_query(self, metadata): - """ - Overrideable method to alter the query based on metadata. - Defaults to doing nothing. - - In most cases, you will do something like - - self.filter = "(cn=" + metadata.hostname + ")" - - here. - """ - pass - - def process_result(self, metadata): - """ - Overrideable method to post-process the query result. - Defaults to returning the unaltered result. - """ - return self.result - - def get_result(self, metadata): - """ - Method to handle preparing, executing and processing the query. - """ - if isinstance(self.connection, LdapConnection): - self.prepare_query(metadata) - self.result = self.connection.run_query(self) - self.result = self.process_result(metadata) - return self.result - else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) - return None - -class LdapSubQuery(LdapQuery): - """ - SubQueries are meant for internal use only and are not added - to the metadata object. They are useful for situations where - you need to run more than one query to obtain some data. - """ - def prepare_query(self, metadata, **kwargs): - """ - Overrideable method to alter the query based on metadata. - Defaults to doing nothing. - """ - pass - - def process_result(self, metadata, **kwargs): - """ - Overrideable method to post-process the query result. - Defaults to returning the unaltered result. - """ - return self.result - - def get_result(self, metadata, **kwargs): - """ - Method to handle preparing, executing and processing the query. - """ - if isinstance(self.connection, LdapConnection): - self.prepare_query(metadata, **kwargs) - self.result = self.connection.run_query(self) - return self.process_result(metadata, **kwargs) - else: - Bcfg2.Server.Plugin.logger.error("LdapPlugin error: " + - "No valid connection defined for query " + str(self)) - return None diff --git a/src/lib/Server/Plugins/Metadata.py b/src/lib/Server/Plugins/Metadata.py deleted file mode 100644 index fa2cf0a78..000000000 --- a/src/lib/Server/Plugins/Metadata.py +++ /dev/null @@ -1,866 +0,0 @@ -""" -This file stores persistent metadata for the Bcfg2 Configuration Repository. -""" - -import copy -import fcntl -import lxml.etree -import os -import os.path -import socket -import sys -import time - -import Bcfg2.Server.FileMonitor -import Bcfg2.Server.Plugin - - -def locked(fd): - """Aquire a lock on a file""" - try: - fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - return True - return False - - -class MetadataConsistencyError(Exception): - """This error gets raised when metadata is internally inconsistent.""" - pass - - -class MetadataRuntimeError(Exception): - """This error is raised when the metadata engine - is called prior to reading enough data. - """ - pass - - -class XMLMetadataConfig(object): - """Handles xml config files and all XInclude statements""" - def __init__(self, metadata, watch_clients, basefile): - self.metadata = metadata - self.basefile = basefile - self.should_monitor = watch_clients - self.extras = [] - self.data = None - self.basedata = None - self.basedir = metadata.data - self.logger = metadata.logger - self.pseudo_monitor = isinstance(metadata.core.fam, - Bcfg2.Server.FileMonitor.Pseudo) - - @property - def xdata(self): - if not self.data: - raise MetadataRuntimeError - return self.data - - @property - def base_xdata(self): - if not self.basedata: - raise MetadataRuntimeError - return self.basedata - - def add_monitor(self, fname): - """Add a fam monitor for an included file""" - if self.should_monitor: - self.metadata.core.fam.AddMonitor(os.path.join(self.basedir, fname), - self.metadata) - self.extras.append(fname) - - def load_xml(self): - """Load changes from XML""" - try: - xdata = lxml.etree.parse(os.path.join(self.basedir, self.basefile)) - except lxml.etree.XMLSyntaxError: - self.logger.error('Failed to parse %s' % self.basefile) - return - self.basedata = copy.copy(xdata) - included = [ent.get('href') for ent in \ - xdata.findall('./{http://www.w3.org/2001/XInclude}include')] - if included: - for name in included: - if name not in self.extras: - self.add_monitor(name) - try: - xdata.xinclude() - except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % - self.basefile) - self.data = xdata - - def write(self): - """Write changes to xml back to disk.""" - self.write_xml(os.path.join(self.basedir, self.basefile), - self.basedata) - - def write_xml(self, fname, xmltree): - """Write changes to xml back to disk.""" - tmpfile = "%s.new" % fname - try: - datafile = open(tmpfile, 'w') - except IOError: - e = sys.exc_info()[1] - self.logger.error("Failed to write %s: %s" % (tmpfile, e)) - raise MetadataRuntimeError - # prep data - dataroot = xmltree.getroot() - newcontents = lxml.etree.tostring(dataroot, pretty_print=True) - - fd = datafile.fileno() - while locked(fd) == True: - pass - try: - datafile.write(newcontents) - except: - fcntl.lockf(fd, fcntl.LOCK_UN) - self.logger.error("Metadata: Failed to write new xml data to %s" % - tmpfile, exc_info=1) - os.unlink(tmpfile) - raise MetadataRuntimeError - datafile.close() - - # check if clients.xml is a symlink - if os.path.islink(fname): - fname = os.readlink(fname) - - try: - os.rename(tmpfile, fname) - except: - self.logger.error("Metadata: Failed to rename %s" % tmpfile) - raise MetadataRuntimeError - - def find_xml_for_xpath(self, xpath): - """Find and load xml file containing the xpath query""" - if self.pseudo_monitor: - # Reload xml if we don't have a real monitor - self.load_xml() - cli = self.basedata.xpath(xpath) - if len(cli) > 0: - return {'filename': os.path.join(self.basedir, self.basefile), - 'xmltree': self.basedata, - 'xquery': cli} - else: - """Try to find the data in included files""" - for included in self.extras: - try: - xdata = lxml.etree.parse(os.path.join(self.basedir, - included)) - cli = xdata.xpath(xpath) - if len(cli) > 0: - return {'filename': os.path.join(self.basedir, - included), - 'xmltree': xdata, - 'xquery': cli} - except lxml.etree.XMLSyntaxError: - self.logger.error('Failed to parse %s' % (included)) - return {} - - def HandleEvent(self, event): - """Handle fam events""" - filename = event.filename.split('/')[-1] - if filename in self.extras: - if event.code2str() == 'exists': - return False - elif filename != self.basefile: - return False - if event.code2str() == 'endExist': - return False - self.load_xml() - return True - - -class ClientMetadata(object): - """This object contains client metadata.""" - def __init__(self, client, profile, groups, bundles, - aliases, addresses, categories, uuid, password, query): - self.hostname = client - self.profile = profile - self.bundles = bundles - self.aliases = aliases - self.addresses = addresses - self.groups = groups - self.categories = categories - self.uuid = uuid - self.password = password - self.connectors = [] - self.query = query - - def inGroup(self, group): - """Test to see if client is a member of group.""" - return group in self.groups - - def group_in_category(self, category): - for grp in self.query.all_groups_in_category(category): - if grp in self.groups: - return grp - return '' - - -class MetadataQuery(object): - def __init__(self, by_name, get_clients, by_groups, by_profiles, all_groups, all_groups_in_category): - # resolver is set later - self.by_name = by_name - self.names_by_groups = by_groups - self.names_by_profiles = by_profiles - self.all_clients = get_clients - self.all_groups = all_groups - self.all_groups_in_category = all_groups_in_category - - def by_groups(self, groups): - return [self.by_name(name) for name in self.names_by_groups(groups)] - - def by_profiles(self, profiles): - return [self.by_name(name) for name in self.names_by_profiles(profiles)] - - def all(self): - return [self.by_name(name) for name in self.all_clients()] - - -class Metadata(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Metadata, - Bcfg2.Server.Plugin.Statistics): - """This class contains data for bcfg2 server metadata.""" - __author__ = 'bcfg-dev@mcs.anl.gov' - name = "Metadata" - sort_order = 500 - - def __init__(self, core, datastore, watch_clients=True): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Metadata.__init__(self) - Bcfg2.Server.Plugin.Statistics.__init__(self) - if watch_clients: - try: - core.fam.AddMonitor(os.path.join(self.data, "groups.xml"), self) - core.fam.AddMonitor(os.path.join(self.data, "clients.xml"), self) - except: - print("Unable to add file monitor for groups.xml or clients.xml") - raise Bcfg2.Server.Plugin.PluginInitError - - self.clients_xml = XMLMetadataConfig(self, watch_clients, 'clients.xml') - self.groups_xml = XMLMetadataConfig(self, watch_clients, 'groups.xml') - self.states = {} - if watch_clients: - self.states = {"groups.xml": False, - "clients.xml": False} - self.addresses = {} - self.auth = dict() - self.clients = {} - self.aliases = {} - self.groups = {} - self.cgroups = {} - self.public = [] - self.private = [] - self.profiles = [] - self.categories = {} - self.bad_clients = {} - self.uuid = {} - self.secure = [] - self.floating = [] - self.passwords = {} - self.session_cache = {} - self.default = None - self.pdirty = False - self.extra = {'groups.xml': [], - 'clients.xml': []} - self.password = core.password - self.query = MetadataQuery(core.build_metadata, - lambda: list(self.clients.keys()), - self.get_client_names_by_groups, - self.get_client_names_by_profiles, - self.get_all_group_names, - self.get_all_groups_in_category) - - @classmethod - def init_repo(cls, repo, groups, os_selection, clients): - path = os.path.join(repo, cls.name) - os.makedirs(path) - open(os.path.join(repo, "Metadata", "groups.xml"), - "w").write(groups % os_selection) - open(os.path.join(repo, "Metadata", "clients.xml"), - "w").write(clients % socket.getfqdn()) - - def get_groups(self): - '''return groups xml tree''' - groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml")) - root = groups_tree.getroot() - return root - - def _search_xdata(self, tag, name, tree, alias=False): - for node in tree.findall("//%s" % tag): - if node.get("name") == name: - return node - elif alias: - for child in node: - if (child.tag == "Alias" and - child.attrib["name"] == name): - return node - return None - - def search_group(self, group_name, tree): - """Find a group.""" - return self._search_xdata("Group", group_name, tree) - - def search_bundle(self, bundle_name, tree): - """Find a bundle.""" - return self._search_xdata("Bundle", bundle_name, tree) - - def search_client(self, client_name, tree): - return self._search_xdata("Client", client_name, tree, alias=True) - - def _add_xdata(self, config, tag, name, attribs=None, alias=False): - node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node != None: - self.logger.error("%s \"%s\" already exists" % (tag, name)) - raise MetadataConsistencyError - element = lxml.etree.SubElement(config.base_xdata.getroot(), - tag, name=name) - if attribs: - for key, val in list(attribs.items()): - element.set(key, val) - config.write() - - def add_group(self, group_name, attribs): - """Add group to groups.xml.""" - return self._add_xdata(self.groups_xml, "Group", group_name, - attribs=attribs) - - def add_bundle(self, bundle_name): - """Add bundle to groups.xml.""" - return self._add_xdata(self.groups_xml, "Bundle", bundle_name) - - def add_client(self, client_name, attribs): - """Add client to clients.xml.""" - return self._add_xdata(self.clients_xml, "Client", client_name, - attribs=attribs, alias=True) - - def _update_xdata(self, config, tag, name, attribs, alias=False): - node = self._search_xdata(tag, name, config.xdata, alias=alias) - if node == None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise MetadataConsistencyError - xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % - (tag, node.get('name'))) - if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise MetadataConsistencyError - for key, val in list(attribs.items()): - xdict['xquery'][0].set(key, val) - config.write_xml(xdict['filename'], xdict['xmltree']) - - def update_group(self, group_name, attribs): - """Update a groups attributes.""" - return self._update_xdata(self.groups_xml, "Group", group_name, attribs) - - def update_client(self, client_name, attribs): - """Update a clients attributes.""" - return self._update_xdata(self.clients_xml, "Client", client_name, - attribs, alias=True) - - def _remove_xdata(self, config, tag, name, alias=False): - node = self._search_xdata(tag, name, config.xdata) - if node == None: - self.logger.error("%s \"%s\" does not exist" % (tag, name)) - raise MetadataConsistencyError - xdict = config.find_xml_for_xpath('.//%s[@name="%s"]' % - (tag, node.get('name'))) - if not xdict: - self.logger.error("Unexpected error finding %s \"%s\"" % - (tag, name)) - raise MetadataConsistencyError - xdict['xquery'][0].getparent().remove(xdict['xquery'][0]) - self.groups_xml.write_xml(xdict['filename'], xdict['xmltree']) - - def remove_group(self, group_name): - """Remove a group.""" - return self._remove_xdata(self.groups_xml, "Group", group_name) - - def remove_bundle(self, bundle_name): - """Remove a bundle.""" - return self._remove_xdata(self.groups_xml, "Bundle", bundle_name) - - def _handle_clients_xml_event(self, event): - xdata = self.clients_xml.xdata - self.clients = {} - self.aliases = {} - self.raliases = {} - self.bad_clients = {} - self.secure = [] - self.floating = [] - self.addresses = {} - self.raddresses = {} - for client in xdata.findall('.//Client'): - clname = client.get('name').lower() - if 'address' in client.attrib: - caddr = client.get('address') - if caddr in self.addresses: - self.addresses[caddr].append(clname) - else: - self.addresses[caddr] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(caddr) - if 'auth' in client.attrib: - self.auth[client.get('name')] = client.get('auth', - 'cert+password') - if 'uuid' in client.attrib: - self.uuid[client.get('uuid')] = clname - if client.get('secure', 'false') == 'true': - self.secure.append(clname) - if client.get('location', 'fixed') == 'floating': - self.floating.append(clname) - if 'password' in client.attrib: - self.passwords[clname] = client.get('password') - - self.raliases[clname] = set() - for alias in client.findall('Alias'): - self.aliases.update({alias.get('name'): clname}) - self.raliases[clname].add(alias.get('name')) - if 'address' not in alias.attrib: - continue - if alias.get('address') in self.addresses: - self.addresses[alias.get('address')].append(clname) - else: - self.addresses[alias.get('address')] = [clname] - if clname not in self.raddresses: - self.raddresses[clname] = set() - self.raddresses[clname].add(alias.get('address')) - self.clients.update({clname: client.get('profile')}) - self.states['clients.xml'] = True - - def _handle_groups_xml_event(self, event): - xdata = self.groups_xml.xdata - self.public = [] - self.private = [] - self.profiles = [] - self.groups = {} - grouptmp = {} - self.categories = {} - groupseen = list() - for group in xdata.xpath('//Groups/Group'): - if group.get('name') not in groupseen: - groupseen.append(group.get('name')) - else: - self.logger.error("Metadata: Group %s defined multiply" % - group.get('name')) - grouptmp[group.get('name')] = \ - ([item.get('name') for item in group.findall('./Bundle')], - [item.get('name') for item in group.findall('./Group')]) - grouptmp[group.get('name')][1].append(group.get('name')) - if group.get('default', 'false') == 'true': - self.default = group.get('name') - if group.get('profile', 'false') == 'true': - self.profiles.append(group.get('name')) - if group.get('public', 'false') == 'true': - self.public.append(group.get('name')) - elif group.get('public', 'true') == 'false': - self.private.append(group.get('name')) - if 'category' in group.attrib: - self.categories[group.get('name')] = group.get('category') - - for group in grouptmp: - # self.groups[group] => (bundles, groups, categories) - self.groups[group] = (set(), set(), {}) - tocheck = [group] - group_cat = self.groups[group][2] - while tocheck: - now = tocheck.pop() - self.groups[group][1].add(now) - if now in grouptmp: - (bundles, groups) = grouptmp[now] - for ggg in groups: - if ggg in self.groups[group][1]: - continue - if (ggg not in self.categories or \ - self.categories[ggg] not in self.groups[group][2]): - self.groups[group][1].add(ggg) - tocheck.append(ggg) - if ggg in self.categories: - group_cat[self.categories[ggg]] = ggg - elif ggg in self.categories: - self.logger.info("Group %s: %s cat-suppressed %s" % \ - (group, - group_cat[self.categories[ggg]], - ggg)) - [self.groups[group][0].add(bund) for bund in bundles] - self.states['groups.xml'] = True - - def HandleEvent(self, event): - """Handle update events for data files.""" - if self.clients_xml.HandleEvent(event): - self._handle_clients_xml_event(event) - elif self.groups_xml.HandleEvent(event): - self._handle_groups_xml_event(event) - - if False not in list(self.states.values()): - # check that all client groups are real and complete - real = list(self.groups.keys()) - for client in list(self.clients.keys()): - if self.clients[client] not in self.profiles: - self.logger.error("Client %s set as nonexistent or " - "incomplete group %s" % - (client, self.clients[client])) - self.logger.error("Removing client mapping for %s" % client) - self.bad_clients[client] = self.clients[client] - del self.clients[client] - for bclient in list(self.bad_clients.keys()): - if self.bad_clients[bclient] in self.profiles: - self.logger.info("Restored profile mapping for client %s" % - bclient) - self.clients[bclient] = self.bad_clients[bclient] - del self.bad_clients[bclient] - - def set_profile(self, client, profile, addresspair): - """Set group parameter for provided client.""" - self.logger.info("Asserting client %s profile to %s" % (client, profile)) - if False in list(self.states.values()): - raise MetadataRuntimeError - if profile not in self.public: - self.logger.error("Failed to set client %s to private group %s" % - (client, profile)) - raise MetadataConsistencyError - if client in self.clients: - self.logger.info("Changing %s group from %s to %s" % - (client, self.clients[client], profile)) - self.update_client(client, dict(profile=profile)) - else: - self.logger.info("Creating new client: %s, profile %s" % - (client, profile)) - if addresspair in self.session_cache: - # we are working with a uuid'd client - self.add_client(self.session_cache[addresspair][1], - dict(uuid=client, profile=profile, - address=addresspair[0])) - else: - self.add_client(client, dict(profile=profile)) - self.clients[client] = profile - self.clients_xml.write() - - def resolve_client(self, addresspair, cleanup_cache=False): - """Lookup address locally or in DNS to get a hostname.""" - if addresspair in self.session_cache: - # client _was_ cached, so there can be some expired - # entries. we need to clean them up to avoid potentially - # infinite memory swell - cache_ttl = 90 - if cleanup_cache: - # remove entries for this client's IP address with - # _any_ port numbers - perhaps a priority queue could - # be faster? - curtime = time.time() - for addrpair in self.session_cache.keys(): - if addresspair[0] == addrpair[0]: - (stamp, _) = self.session_cache[addrpair] - if curtime - stamp > cache_ttl: - del self.session_cache[addrpair] - # return the cached data - try: - (stamp, uuid) = self.session_cache[addresspair] - if time.time() - stamp < cache_ttl: - return self.session_cache[addresspair][1] - except KeyError: - # we cleaned all cached data for this client in cleanup_cache - pass - address = addresspair[0] - if address in self.addresses: - if len(self.addresses[address]) != 1: - self.logger.error("Address %s has multiple reverse assignments; " - "a uuid must be used" % (address)) - raise MetadataConsistencyError - return self.addresses[address][0] - try: - cname = socket.gethostbyaddr(address)[0].lower() - if cname in self.aliases: - return self.aliases[cname] - return cname - except socket.herror: - warning = "address resolution error for %s" % (address) - self.logger.warning(warning) - raise MetadataConsistencyError - - def get_initial_metadata(self, client): - """Return the metadata for a given client.""" - if False in list(self.states.values()): - raise MetadataRuntimeError - client = client.lower() - if client in self.aliases: - client = self.aliases[client] - if client in self.clients: - profile = self.clients[client] - (bundles, groups, categories) = self.groups[profile] - else: - if self.default == None: - self.logger.error("Cannot set group for client %s; " - "no default group set" % client) - raise MetadataConsistencyError - self.set_profile(client, self.default, (None, None)) - profile = self.default - [bundles, groups, categories] = self.groups[self.default] - aliases = self.raliases.get(client, set()) - addresses = self.raddresses.get(client, set()) - newgroups = set(groups) - newbundles = set(bundles) - newcategories = {} - newcategories.update(categories) - if client in self.passwords: - password = self.passwords[client] - else: - password = None - uuids = [item for item, value in list(self.uuid.items()) - if value == client] - if uuids: - uuid = uuids[0] - else: - uuid = None - for group in self.cgroups.get(client, []): - if group in self.groups: - nbundles, ngroups, ncategories = self.groups[group] - else: - nbundles, ngroups, ncategories = ([], [group], {}) - [newbundles.add(b) for b in nbundles if b not in newbundles] - [newgroups.add(g) for g in ngroups if g not in newgroups] - newcategories.update(ncategories) - return ClientMetadata(client, profile, newgroups, newbundles, aliases, - addresses, newcategories, uuid, password, - self.query) - - def get_all_group_names(self): - all_groups = set() - [all_groups.update(g[1]) for g in list(self.groups.values())] - return all_groups - - def get_all_groups_in_category(self, category): - all_groups = set() - [all_groups.add(g) for g in self.categories \ - if self.categories[g] == category] - return all_groups - - def get_client_names_by_profiles(self, profiles): - return [client for client, profile in list(self.clients.items()) \ - if profile in profiles] - - def get_client_names_by_groups(self, groups): - mdata = [self.core.build_metadata(client) - for client in list(self.clients.keys())] - return [md.hostname for md in mdata if md.groups.issuperset(groups)] - - def merge_additional_groups(self, imd, groups): - for group in groups: - if (group in self.categories and - self.categories[group] in imd.categories): - continue - newbundles, newgroups, _ = self.groups.get(group, - (list(), - [group], - dict())) - for newbundle in newbundles: - if newbundle not in imd.bundles: - imd.bundles.add(newbundle) - for newgroup in newgroups: - if newgroup not in imd.groups: - if (newgroup in self.categories and - self.categories[newgroup] in imd.categories): - continue - if newgroup in self.private: - self.logger.error("Refusing to add dynamic membership " - "in private group %s for client %s" % - (newgroup, imd.hostname)) - continue - imd.groups.add(newgroup) - - def merge_additional_data(self, imd, source, data): - if not hasattr(imd, source): - setattr(imd, source, data) - imd.connectors.append(source) - - def validate_client_address(self, client, addresspair): - """Check address against client.""" - address = addresspair[0] - if client in self.floating: - self.debug_log("Client %s is floating" % client) - return True - if address in self.addresses: - if client in self.addresses[address]: - self.debug_log("Client %s matches address %s" % - (client, address)) - return True - else: - self.logger.error("Got request for non-float client %s from %s" % - (client, address)) - return False - resolved = self.resolve_client(addresspair) - if resolved.lower() == client.lower(): - return True - else: - self.logger.error("Got request for %s from incorrect address %s" % - (client, address)) - self.logger.error("Resolved to %s" % resolved) - return False - - def AuthenticateConnection(self, cert, user, password, address): - """This function checks auth creds.""" - if cert: - id_method = 'cert' - certinfo = dict([x[0] for x in cert['subject']]) - # look at cert.cN - client = certinfo['commonName'] - self.debug_log("Got cN %s; using as client name" % client) - auth_type = self.auth.get(client, 'cert+password') - elif user == 'root': - id_method = 'address' - try: - client = self.resolve_client(address) - except MetadataConsistencyError: - self.logger.error("Client %s failed to resolve; metadata problem" - % address[0]) - return False - else: - id_method = 'uuid' - # user maps to client - if user not in self.uuid: - client = user - self.uuid[user] = user - else: - client = self.uuid[user] - - # we have the client name - self.debug_log("Authenticating client %s" % client) - - # next we validate the address - if id_method == 'uuid': - addr_is_valid = True - else: - addr_is_valid = self.validate_client_address(client, address) - - if not addr_is_valid: - return False - - if id_method == 'cert' and auth_type != 'cert+password': - # remember the cert-derived client name for this connection - if client in self.floating: - self.session_cache[address] = (time.time(), client) - # we are done if cert+password not required - return True - - if client not in self.passwords: - if client in self.secure: - self.logger.error("Client %s in secure mode but has no password" - % address[0]) - return False - if password != self.password: - self.logger.error("Client %s used incorrect global password" % - address[0]) - return False - if client not in self.secure: - if client in self.passwords: - plist = [self.password, self.passwords[client]] - else: - plist = [self.password] - if password not in plist: - self.logger.error("Client %s failed to use either allowed " - "password" % address[0]) - return False - else: - # client in secure mode and has a client password - if password != self.passwords[client]: - self.logger.error("Client %s failed to use client password in " - "secure mode" % address[0]) - return False - # populate the session cache - if user.decode('utf-8') != 'root': - self.session_cache[address] = (time.time(), client) - return True - - def process_statistics(self, meta, _): - """Hook into statistics interface to toggle clients in bootstrap mode.""" - client = meta.hostname - if client in self.auth and self.auth[client] == 'bootstrap': - self.update_client(client, dict(auth='cert')) - - def viz(self, hosts, bundles, key, only_client, colors): - """Admin mode viz support.""" - if only_client: - clientmeta = self.core.build_metadata(only_client) - - def include_client(client): - return not only_client or client != only_client - - def include_bundle(bundle): - return not only_client or bundle in clientmeta.bundles - - def include_group(group): - return not only_client or group in clientmeta.groups - - groups_tree = lxml.etree.parse(os.path.join(self.data, "groups.xml")) - try: - groups_tree.xinclude() - except lxml.etree.XIncludeError: - self.logger.error("Failed to process XInclude for file %s" % dest) - groups = groups_tree.getroot() - categories = {'default': 'grey83'} - viz_str = [] - egroups = groups.findall("Group") + groups.findall('.//Groups/Group') - for group in egroups: - if not group.get('category') in categories: - categories[group.get('category')] = colors.pop() - group.set('color', categories[group.get('category')]) - if None in categories: - del categories[None] - if hosts: - instances = {} - clients = self.clients - for client, profile in list(clients.items()): - if include_client(client): - continue - if profile in instances: - instances[profile].append(client) - else: - instances[profile] = [client] - for profile, clist in list(instances.items()): - clist.sort() - viz_str.append('"%s-instances" [ label="%s", shape="record" ];' % - (profile, '|'.join(clist))) - viz_str.append('"%s-instances" -> "group-%s";' % - (profile, profile)) - if bundles: - bundles = [] - [bundles.append(bund.get('name')) \ - for bund in groups.findall('.//Bundle') \ - if bund.get('name') not in bundles \ - and include_bundle(bund.get('name'))] - bundles.sort() - for bundle in bundles: - viz_str.append('"bundle-%s" [ label="%s", shape="septagon"];' % - (bundle, bundle)) - gseen = [] - for group in egroups: - if group.get('profile', 'false') == 'true': - style = "filled, bold" - else: - style = "filled" - gseen.append(group.get('name')) - if include_group(group.get('name')): - viz_str.append('"group-%s" [label="%s", style="%s", fillcolor=%s];' % - (group.get('name'), group.get('name'), style, - group.get('color'))) - if bundles: - for bundle in group.findall('Bundle'): - viz_str.append('"group-%s" -> "bundle-%s";' % - (group.get('name'), bundle.get('name'))) - gfmt = '"group-%s" [label="%s", style="filled", fillcolor="grey83"];' - for group in egroups: - for parent in group.findall('Group'): - if parent.get('name') not in gseen and include_group(parent.get('name')): - viz_str.append(gfmt % (parent.get('name'), - parent.get('name'))) - gseen.append(parent.get("name")) - if include_group(group.get('name')): - viz_str.append('"group-%s" -> "group-%s";' % - (group.get('name'), parent.get('name'))) - if key: - for category in categories: - viz_str.append('"%s" [label="%s", shape="record", style="filled", fillcolor="%s"];' % - (category, category, categories[category])) - return "\n".join("\t" + s for s in viz_str) diff --git a/src/lib/Server/Plugins/NagiosGen.py b/src/lib/Server/Plugins/NagiosGen.py deleted file mode 100644 index d67dac6a6..000000000 --- a/src/lib/Server/Plugins/NagiosGen.py +++ /dev/null @@ -1,151 +0,0 @@ -'''This module implements a Nagios configuration generator''' - -import os -import re -import sys -import glob -import socket -import logging -import lxml.etree - -import Bcfg2.Server.Plugin - -LOGGER = logging.getLogger('Bcfg2.Plugins.NagiosGen') - -line_fmt = '\t%-32s %s' - -class NagiosGenConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile): - def __init__(self, filename, fam): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, filename, fam) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - - -class NagiosGen(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator): - """NagiosGen is a Bcfg2 plugin that dynamically generates - Nagios configuration file based on Bcfg2 data. - """ - name = 'NagiosGen' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - self.config = NagiosGenConfig(os.path.join(self.data, 'config.xml'), - core.fam) - self.Entries = {'Path': - {'/etc/nagiosgen.status': self.createhostconfig, - '/etc/nagios/nagiosgen.cfg': self.createserverconfig}} - - self.client_attrib = {'encoding': 'ascii', - 'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0400'} - self.server_attrib = {'encoding': 'ascii', - 'owner': 'nagios', - 'group': 'nagios', - 'type': 'file', - 'perms': '0440'} - - def createhostconfig(self, entry, metadata): - """Build host specific configuration file.""" - host_address = socket.gethostbyname(metadata.hostname) - host_groups = [grp for grp in metadata.groups - if os.path.isfile('%s/%s-group.cfg' % (self.data, grp))] - host_config = ['define host {', - line_fmt % ('host_name', metadata.hostname), - line_fmt % ('alias', metadata.hostname), - line_fmt % ('address', host_address)] - - if host_groups: - host_config.append(line_fmt % ("hostgroups", - ",".join(host_groups))) - - # read the old-style Properties config, but emit a warning. - xtra = dict() - props = None - if (hasattr(metadata, 'Properties') and - 'NagiosGen.xml' in metadata.Properties): - props = metadata.Properties['NagiosGen.xml'].data - if props is not None: - LOGGER.warn("Parsing deprecated Properties/NagiosGen.xml. " - "Update to the new-style config with " - "nagiosgen-convert.py.") - xtra = dict((el.tag, el.text) - for el in props.find(metadata.hostname)) - # hold off on parsing the defaults until we've checked for - # a new-style config - - # read the old-style parents.xml, but emit a warning - pfile = os.path.join(self.data, "parents.xml") - if os.path.exists(pfile): - LOGGER.warn("Parsing deprecated NagiosGen/parents.xml. " - "Update to the new-style config with " - "nagiosgen-convert.py.") - parents = lxml.etree.parse(pfile) - for el in parents.xpath("//Depend[@name='%s']" % metadata.hostname): - if 'parent' in xtra: - xtra['parent'] += "," + el.get("on") - else: - xtra['parent'] = el.get("on") - - # read the new-style config and overwrite the old-style config - for el in self.config.Match(metadata): - if el.tag == 'Option': - xtra[el.get("name")] = el.text - - # if we haven't found anything in the new- or old-style - # configs, finally read defaults from old-style config - if not xtra and props is not None: - xtra = dict((el.tag, el.text) for el in props.find('default')) - - if xtra: - host_config.extend([line_fmt % (opt, val) - for opt, val in list(xtra.items())]) - else: - host_config.append(line_fmt % ('use', 'default')) - - host_config.append('}') - entry.text = "%s\n" % "\n".join(host_config) - [entry.attrib.__setitem__(key, value) - for (key, value) in list(self.client_attrib.items())] - try: - fileh = open("%s/%s-host.cfg" % - (self.data, metadata.hostname), 'w') - fileh.write(entry.text) - fileh.close() - except OSError: - ioerr = sys.exc_info()[1] - LOGGER.error("Failed to write %s/%s-host.cfg" % - (self.data, metadata.hostname)) - LOGGER.error(ioerr) - - def createserverconfig(self, entry, _): - """Build monolithic server configuration file.""" - host_configs = glob.glob('%s/*-host.cfg' % self.data) - group_configs = glob.glob('%s/*-group.cfg' % self.data) - host_data = [] - group_data = [] - for host in host_configs: - host_data.append(open(host, 'r').read()) - - for group in group_configs: - group_name = re.sub("(-group.cfg|.*/(?=[^/]+))", "", group) - if "\n".join(host_data).find(group_name) != -1: - groupfile = open(group, 'r') - group_data.append(groupfile.read()) - groupfile.close() - - entry.text = "%s\n\n%s" % ("\n".join(group_data), "\n".join(host_data)) - [entry.attrib.__setitem__(key, value) - for (key, value) in list(self.server_attrib.items())] - try: - fileh = open("%s/nagiosgen.cfg" % self.data, 'w') - fileh.write(entry.text) - fileh.close() - except OSError: - ioerr = sys.exc_info()[1] - LOGGER.error("Failed to write %s/nagiosgen.cfg" % self.data) - LOGGER.error(ioerr) diff --git a/src/lib/Server/Plugins/Ohai.py b/src/lib/Server/Plugins/Ohai.py deleted file mode 100644 index 5fff20d98..000000000 --- a/src/lib/Server/Plugins/Ohai.py +++ /dev/null @@ -1,92 +0,0 @@ -import lxml.etree -import os - -import logging -logger = logging.getLogger('Bcfg2.Plugins.Ohai') - -import Bcfg2.Server.Plugin - -try: - import json -except: - # FIXME: can be removed when server prereq is >= python 2.6 - # necessary for clients without the in-tree json module - try: - import simplejson as json - except: - logger.error("Unable to load any json modules. Make sure " - "python-simplejson is installed.") - raise ImportError - - -probecode = """#!/bin/sh - -export PATH=$PATH:/sbin:/usr/sbin - -if type ohai >& /dev/null; then - ohai -else - # an empty dict, so "'foo' in metadata.Ohai" tests succeed - echo '{}' -fi -""" - -class OhaiCache(object): - def __init__(self, dirname): - self.dirname = dirname - self.cache = dict() - - def __setitem__(self, item, value): - if value == None: - # simply return if the client returned nothing - return - self.cache[item] = json.loads(value) - file("%s/%s.json" % (self.dirname, item), 'w').write(value) - - def __getitem__(self, item): - if item not in self.cache: - try: - data = open("%s/%s.json" % (self.dirname, item)).read() - except: - raise KeyError(item) - self.cache[item] = json.loads(data) - return self.cache[item] - - def __iter__(self): - data = list(self.cache.keys()) - data.extend([x[:-5] for x in os.listdir(self.dirname)]) - return data.__iter__() - - -class Ohai(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Connector): - """The Ohai plugin is used to detect information - about the client operating system. - """ - name = 'Ohai' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Probing.__init__(self) - Bcfg2.Server.Plugin.Connector.__init__(self) - self.probe = lxml.etree.Element('probe', name='Ohai', source='Ohai', - interpreter='/bin/sh') - self.probe.text = probecode - try: - os.stat(self.data) - except: - os.makedirs(self.data) - self.cache = OhaiCache(self.data) - - def GetProbes(self, meta, force=False): - return [self.probe] - - def ReceiveData(self, meta, datalist): - self.cache[meta.hostname] = datalist[0].text - - def get_additional_data(self, meta): - if meta.hostname in self.cache: - return self.cache[meta.hostname] - return dict() diff --git a/src/lib/Server/Plugins/Packages/Apt.py b/src/lib/Server/Plugins/Packages/Apt.py deleted file mode 100644 index f76bf7fa1..000000000 --- a/src/lib/Server/Plugins/Packages/Apt.py +++ /dev/null @@ -1,139 +0,0 @@ -import re -import gzip -from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import Source -from Bcfg2.Bcfg2Py3k import cPickle, file - -class AptCollection(Collection): - def get_group(self, group): - self.logger.warning("Packages: Package groups are not supported by APT") - return [] - -class AptSource(Source): - basegroups = ['apt', 'debian', 'ubuntu', 'nexenta'] - ptype = 'deb' - - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) - self.pkgnames = set() - - self.url_map = [{'rawurl': self.rawurl, 'url': self.url, - 'version': self.version, - 'components': self.components, 'arches': self.arches}] - - def save_state(self): - cache = file(self.cachefile, 'wb') - cPickle.dump((self.pkgnames, self.deps, self.provides), - cache, 2) - cache.close() - - def load_state(self): - data = file(self.cachefile) - self.pkgnames, self.deps, self.provides = cPickle.load(data) - - def filter_unknown(self, unknown): - filtered = set([u for u in unknown if u.startswith('choice')]) - unknown.difference_update(filtered) - - def get_urls(self): - if not self.rawurl: - rv = [] - for part in self.components: - for arch in self.arches: - rv.append("%sdists/%s/%s/binary-%s/Packages.gz" % - (self.url, self.version, part, arch)) - return rv - else: - return ["%sPackages.gz" % self.rawurl] - urls = property(get_urls) - - def read_files(self): - bdeps = dict() - bprov = dict() - if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] - for fname in self.files: - if not self.rawurl: - barch = [x - for x in fname.split('@') - if x.startswith('binary-')][0][7:] - else: - # RawURL entries assume that they only have one - # element and that it is the architecture of the source. - barch = self.arches[0] - if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() - try: - reader = gzip.GzipFile(fname) - except: - self.logger.error("Packages: Failed to read file %s" % fname) - raise - for line in reader.readlines(): - words = str(line.strip()).split(':', 1) - if words[0] == 'Package': - pkgname = words[1].strip().rstrip() - self.pkgnames.add(pkgname) - bdeps[barch][pkgname] = [] - elif words[0] in depfnames: - vindex = 0 - for dep in words[1].split(','): - if '|' in dep: - cdeps = [re.sub('\s+', '', - re.sub('\(.*\)', '', cdep)) - for cdep in dep.split('|')] - dyn_dname = "choice-%s-%s-%s" % (pkgname, - barch, - vindex) - vindex += 1 - bdeps[barch][pkgname].append(dyn_dname) - bprov[barch][dyn_dname] = set(cdeps) - else: - raw_dep = re.sub('\(.*\)', '', dep) - raw_dep = raw_dep.rstrip().strip() - bdeps[barch][pkgname].append(raw_dep) - elif words[0] == 'Provides': - for pkg in words[1].split(','): - dname = pkg.rstrip().strip() - if dname not in bprov[barch]: - bprov[barch][dname] = set() - bprov[barch][dname].add(pkgname) - - self.deps['global'] = dict() - self.provides['global'] = dict() - for barch in bdeps: - self.deps[barch] = dict() - self.provides[barch] = dict() - for pkgname in self.pkgnames: - pset = set() - for barch in bdeps: - if pkgname not in bdeps[barch]: - bdeps[barch][pkgname] = [] - pset.add(tuple(bdeps[barch][pkgname])) - if len(pset) == 1: - self.deps['global'][pkgname] = pset.pop() - else: - for barch in bdeps: - self.deps[barch][pkgname] = bdeps[barch][pkgname] - provided = set() - for bprovided in list(bprov.values()): - provided.update(set(bprovided)) - for prov in provided: - prset = set() - for barch in bprov: - if prov not in bprov[barch]: - continue - prset.add(tuple(bprov[barch].get(prov, ()))) - if len(prset) == 1: - self.provides['global'][prov] = prset.pop() - else: - for barch in bprov: - self.provides[barch][prov] = bprov[barch].get(prov, ()) - self.save_state() - - def is_package(self, _, pkg): - return (pkg in self.pkgnames and - pkg not in self.blacklist and - (len(self.whitelist) == 0 or pkg in self.whitelist)) diff --git a/src/lib/Server/Plugins/Packages/Collection.py b/src/lib/Server/Plugins/Packages/Collection.py deleted file mode 100644 index 32eeda1ec..000000000 --- a/src/lib/Server/Plugins/Packages/Collection.py +++ /dev/null @@ -1,343 +0,0 @@ -import copy -import logging -import Bcfg2.Server.Plugin - -logger = logging.getLogger(__name__) - -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - -# we have to cache Collection objects so that calling Packages.Refresh -# or .Reload can tell the collection objects to clean up their cache, -# but we don't actually use the cache to return a Collection object -# when one is requested, because that prevents new machines from -# working, since a Collection object gets created by -# get_additional_data(), which is called for all clients at server -# startup. (It would also prevent machines that change groups from -# working properly; e.g., if you reinstall a machine with a new OS, -# then returning a cached Collection object would give the wrong -# sources to that client.) -collections = dict() - -class Collection(Bcfg2.Server.Plugin.Debuggable): - def __init__(self, metadata, sources, basepath, debug=False): - """ don't call this directly; use the factory function """ - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.debug_flag = debug - self.metadata = metadata - self.sources = sources - self.basepath = basepath - self.virt_pkgs = dict() - - try: - self.config = sources[0].config - self.cachepath = sources[0].basepath - self.ptype = sources[0].ptype - except IndexError: - self.config = None - self.cachepath = None - self.ptype = "unknown" - - self.cachefile = None - - @property - def cachekey(self): - return md5(self.get_config()).hexdigest() - - def get_config(self): - self.logger.error("Packages: Cannot generate config for host with " - "multiple source types (%s)" % self.metadata.hostname) - return "" - - def get_relevant_groups(self): - groups = [] - for source in self.sources: - groups.extend(source.get_relevant_groups(self.metadata)) - return sorted(list(set(groups))) - - @property - def basegroups(self): - groups = set() - for source in self.sources: - groups.update(source.basegroups) - return list(groups) - - @property - def cachefiles(self): - cachefiles = set([self.cachefile]) - for source in self.sources: - cachefiles.add(source.cachefile) - return list(cachefiles) - - def get_group(self, group, ptype=None): - for source in self.sources: - pkgs = source.get_group(self.metadata, group, ptype=ptype) - if pkgs: - return pkgs - self.logger.warning("Packages: '%s' is not a valid group" % group) - return [] - - def is_package(self, package): - for source in self.sources: - if source.is_package(self.metadata, package): - return True - return False - - def is_virtual_package(self, package): - for source in self.sources: - if source.is_virtual_package(self.metadata, package): - return True - return False - - def get_deps(self, package): - for source in self.sources: - if source.is_package(self.metadata, package): - return source.get_deps(self.metadata, package) - return [] - - def get_provides(self, package): - for source in self.sources: - providers = source.get_provides(self.metadata, package) - if providers: - return providers - return [] - - def get_vpkgs(self): - """ get virtual packages """ - vpkgs = dict() - for source in self.sources: - s_vpkgs = source.get_vpkgs(self.metadata) - for name, prov_set in list(s_vpkgs.items()): - if name not in vpkgs: - vpkgs[name] = set(prov_set) - else: - vpkgs[name].update(prov_set) - return vpkgs - - def filter_unknown(self, unknown): - for source in self.sources: - source.filter_unknown(unknown) - - def magic_groups_match(self): - for source in self.sources: - if source.magic_groups_match(self.metadata): - return True - - def build_extra_structures(self, independent): - pass - - def get_additional_data(self): - sdata = [] - for source in self.sources: - sdata.extend(copy.deepcopy(source.url_map)) - return sdata - - def setup_data(self, force_update=False): - """ do any collection-level data setup tasks """ - pass - - def complete(self, packagelist): - '''Build the transitive closure of all package dependencies - - Arguments: - packageslist - set of package names - returns => (set(packages), set(unsatisfied requirements)) - ''' - - # setup vpkg cache - pgrps = tuple(self.get_relevant_groups()) - if pgrps not in self.virt_pkgs: - self.virt_pkgs[pgrps] = self.get_vpkgs() - vpkg_cache = self.virt_pkgs[pgrps] - - # unclassified is set of unsatisfied requirements (may be pkg - # for vpkg) - unclassified = set(packagelist) - vpkgs = set() - both = set() - pkgs = set(packagelist) - - packages = set() - examined = set() - unknown = set() - - final_pass = False - really_done = False - # do while unclassified or vpkgs or both or pkgs - while unclassified or pkgs or both or final_pass: - if really_done: - break - if len(unclassified) + len(pkgs) + len(both) == 0: - # one more pass then exit - really_done = True - - while unclassified: - current = unclassified.pop() - examined.add(current) - is_pkg = False - if self.is_package(current): - is_pkg = True - - is_vpkg = current in vpkg_cache - - if is_pkg and is_vpkg: - both.add(current) - elif is_pkg and not is_vpkg: - pkgs.add(current) - elif is_vpkg and not is_pkg: - vpkgs.add(current) - elif not is_vpkg and not is_pkg: - unknown.add(current) - - while pkgs: - # direct packages; current can be added, and all deps - # should be resolved - current = pkgs.pop() - self.debug_log("Packages: handling package requirement %s" % - current) - packages.add(current) - deps = self.get_deps(current) - newdeps = set(deps).difference(examined) - if newdeps: - self.debug_log("Packages: Package %s added requirements %s" - % (current, newdeps)) - unclassified.update(newdeps) - - satisfied_vpkgs = set() - for current in vpkgs: - # virtual dependencies, satisfied if one of N in the - # config, or can be forced if only one provider - if len(vpkg_cache[current]) == 1: - self.debug_log("Packages: requirement %s satisfied by %s" % - (current, vpkg_cache[current])) - unclassified.update(vpkg_cache[current].difference(examined)) - satisfied_vpkgs.add(current) - else: - satisfiers = [item for item in vpkg_cache[current] - if item in packages] - self.debug_log("Packages: requirement %s satisfied by %s" % - (current, satisfiers)) - satisfied_vpkgs.add(current) - vpkgs.difference_update(satisfied_vpkgs) - - satisfied_both = set() - for current in both: - # packages that are both have virtual providers as - # well as a package with that name. allow use of virt - # through explicit specification, then fall back to - # forcing current on last pass - satisfiers = [item for item in vpkg_cache[current] - if item in packages] - if satisfiers: - self.debug_log("Packages: requirement %s satisfied by %s" % - (current, satisfiers)) - satisfied_both.add(current) - elif current in packagelist or final_pass: - pkgs.add(current) - satisfied_both.add(current) - both.difference_update(satisfied_both) - - if len(unclassified) + len(pkgs) == 0: - final_pass = True - else: - final_pass = False - - self.filter_unknown(unknown) - - return packages, unknown - - def __len__(self): - return len(self.sources) - - def __getitem__(self, item): - return self.sources[item] - - def __setitem__(self, item, value): - self.sources[item] = value - - def __delitem__(self, item): - del self.sources[item] - - def append(self, item): - self.sources.append(item) - - def count(self): - return self.sources.count() - - def index(self, item): - return self.sources.index(item) - - def extend(self, items): - self.sources.extend(items) - - def insert(self, index, item): - self.sources.insert(index, item) - - def pop(self, index=None): - self.sources.pop(index) - - def remove(self, item): - self.sources.remove(item) - - def sort(self, cmp=None, key=None, reverse=False): - self.sources.sort(cmp, key, reverse) - -def clear_cache(): - global collections - collections = dict() - -def factory(metadata, sources, basepath, debug=False): - global collections - - if not sources.loaded: - # if sources.xml has not received a FAM event yet, defer; - # instantiate a dummy Collection object - return Collection(metadata, [], basepath) - - sclasses = set() - relevant = list() - - for source in sources: - if source.applies(metadata): - relevant.append(source) - sclasses.update([source.__class__]) - - if len(sclasses) > 1: - logger.warning("Packages: Multiple source types found for %s: %s" % - ",".join([s.__name__ for s in sclasses])) - cclass = Collection - elif len(sclasses) == 0: - # you'd think this should be a warning, but it happens all the - # freaking time if you have a) machines in your clients.xml - # that do not have the proper groups set up yet (e.g., if you - # have multiple Bcfg2 servers and Packages-relevant groups set - # by probes); and b) templates that query all or multiple - # machines (e.g., with metadata.query.all_clients()) - if debug: - logger.error("Packages: No sources found for %s" % - metadata.hostname) - cclass = Collection - else: - stype = sclasses.pop().__name__.replace("Source", "") - try: - module = \ - getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cclass = getattr(module, "%sCollection" % stype.title()) - except ImportError: - logger.error("Packages: Unknown source type %s" % stype) - except AttributeError: - logger.warning("Packages: No collection class found for %s sources" - % stype) - - if debug: - logger.error("Packages: Using %s for Collection of sources for %s" % - (cclass.__name__, metadata.hostname)) - - collection = cclass(metadata, relevant, basepath, debug=debug) - collections[metadata.hostname] = collection - return collection - diff --git a/src/lib/Server/Plugins/Packages/Pac.py b/src/lib/Server/Plugins/Packages/Pac.py deleted file mode 100644 index 9db6b0535..000000000 --- a/src/lib/Server/Plugins/Packages/Pac.py +++ /dev/null @@ -1,120 +0,0 @@ -import gzip -import tarfile -from Bcfg2.Bcfg2Py3k import cPickle, file -from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import Source - -class PacCollection(Collection): - def get_group(self, group): - self.logger.warning("Packages: Package groups are not supported by APT") - return [] - -class PacSource(Source): - basegroups = ['arch', 'parabola'] - ptype = 'pacman' - - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) - self.pkgnames = set() - - self.url_map = [{'rawurl': self.rawurl, 'url': self.url, - 'version': self.version, - 'components': self.components, 'arches': self.arches}] - - def save_state(self): - cache = file(self.cachefile, 'wb') - cPickle.dump((self.pkgnames, self.deps, self.provides), - cache, 2) - cache.close() - - def load_state(self): - data = file(self.cachefile) - self.pkgnames, self.deps, self.provides = cPickle.load(data) - - def filter_unknown(self, unknown): - filtered = set([u for u in unknown if u.startswith('choice')]) - unknown.difference_update(filtered) - - def get_urls(self): - if not self.rawurl: - rv = [] - for part in self.components: - for arch in self.arches: - rv.append("%s%s/os/%s/%s.db.tar.gz" % - (self.url, part, arch, part)) - return rv - else: - raise Exception("PacSource : RAWUrl not supported (yet)") - urls = property(get_urls) - - def read_files(self): - bdeps = dict() - bprov = dict() - - if self.recommended: - depfnames = ['Depends', 'Pre-Depends', 'Recommends'] - else: - depfnames = ['Depends', 'Pre-Depends'] - - for fname in self.files: - if not self.rawurl: - barch = [x for x in fname.split('@') if x in self.arches][0] - else: - # RawURL entries assume that they only have one - # element and that it is the architecture of the source. - barch = self.arches[0] - - if barch not in bdeps: - bdeps[barch] = dict() - bprov[barch] = dict() - try: - self.debug_log("Packages: try to read %s" % fname) - tar = tarfile.open(fname, "r") - reader = gzip.GzipFile(fname) - except: - self.logger.error("Packages: Failed to read file %s" % fname) - raise - - for tarinfo in tar: - if tarinfo.isdir(): - self.pkgnames.add(tarinfo.name.rsplit("-", 2)[0]) - self.debug_log("Packages: added %s" % - tarinfo.name.rsplit("-", 2)[0]) - tar.close() - - self.deps['global'] = dict() - self.provides['global'] = dict() - for barch in bdeps: - self.deps[barch] = dict() - self.provides[barch] = dict() - for pkgname in self.pkgnames: - pset = set() - for barch in bdeps: - if pkgname not in bdeps[barch]: - bdeps[barch][pkgname] = [] - pset.add(tuple(bdeps[barch][pkgname])) - if len(pset) == 1: - self.deps['global'][pkgname] = pset.pop() - else: - for barch in bdeps: - self.deps[barch][pkgname] = bdeps[barch][pkgname] - provided = set() - for bprovided in list(bprov.values()): - provided.update(set(bprovided)) - for prov in provided: - prset = set() - for barch in bprov: - if prov not in bprov[barch]: - continue - prset.add(tuple(bprov[barch].get(prov, ()))) - if len(prset) == 1: - self.provides['global'][prov] = prset.pop() - else: - for barch in bprov: - self.provides[barch][prov] = bprov[barch].get(prov, ()) - self.save_state() - - def is_package(self, _, pkg): - return (pkg in self.pkgnames and - pkg not in self.blacklist and - (len(self.whitelist) == 0 or pkg in self.whitelist)) diff --git a/src/lib/Server/Plugins/Packages/PackagesConfig.py b/src/lib/Server/Plugins/Packages/PackagesConfig.py deleted file mode 100644 index 7950f15e6..000000000 --- a/src/lib/Server/Plugins/Packages/PackagesConfig.py +++ /dev/null @@ -1,15 +0,0 @@ -import Bcfg2.Server.Plugin - -class PackagesConfig(Bcfg2.Server.Plugin.SimpleConfig): - _required = False - - def Index(self): - """ Build local data structures """ - Bcfg2.Server.Plugin.SimpleConfig.Index(self) - - if hasattr(self.plugin, "sources") and self.plugin.sources.loaded: - # only reload Packages plugin if sources have been loaded. - # otherwise, this is getting called on server startup, and - # we have to wait until all sources have been indexed - # before we can call Packages.Reload() - self.plugin.Reload() diff --git a/src/lib/Server/Plugins/Packages/PackagesSources.py b/src/lib/Server/Plugins/Packages/PackagesSources.py deleted file mode 100644 index da79c00e9..000000000 --- a/src/lib/Server/Plugins/Packages/PackagesSources.py +++ /dev/null @@ -1,97 +0,0 @@ -import os -import sys -import lxml.etree -import Bcfg2.Server.Plugin -from Bcfg2.Server.Plugins.Packages.Source import SourceInitError - -class PackagesSources(Bcfg2.Server.Plugin.SingleXMLFileBacked, - Bcfg2.Server.Plugin.StructFile, - Bcfg2.Server.Plugin.Debuggable): - __identifier__ = None - - def __init__(self, filename, cachepath, fam, packages, config): - Bcfg2.Server.Plugin.Debuggable.__init__(self) - try: - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, - filename, - fam) - except OSError: - err = sys.exc_info()[1] - msg = "Packages: Failed to read configuration file: %s" % err - if not os.path.exists(self.name): - msg += " Have you created it?" - self.logger.error(msg) - raise Bcfg2.Server.Plugin.PluginInitError(msg) - Bcfg2.Server.Plugin.StructFile.__init__(self, filename) - self.cachepath = cachepath - self.config = config - if not os.path.exists(self.cachepath): - # create cache directory if needed - try: - os.makedirs(self.cachepath) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Could not create Packages cache at %s: %s" % - (self.cachepath, err)) - self.pkg_obj = packages - self.parsed = set() - self.loaded = False - - def toggle_debug(self): - Bcfg2.Server.Plugin.Debuggable.toggle_debug(self) - for source in self.entries: - source.toggle_debug() - - def HandleEvent(self, event=None): - Bcfg2.Server.Plugin.SingleXMLFileBacked.HandleEvent(self, event=event) - if event.filename != self.name: - self.parsed.add(os.path.basename(event.filename)) - - if sorted(list(self.parsed)) == sorted(self.extras): - self.logger.info("Reloading Packages plugin") - self.pkg_obj.Reload() - self.loaded = True - - def Index(self): - Bcfg2.Server.Plugin.SingleXMLFileBacked.Index(self) - self.entries = [] - for xsource in self.xdata.findall('.//Source'): - source = self.source_from_xml(xsource) - if source is not None: - self.entries.append(source) - - def source_from_xml(self, xsource): - """ create a *Source object from its XML representation in - sources.xml """ - stype = xsource.get("type") - if stype is None: - self.logger.error("Packages: No type specified for source, " - "skipping") - return None - - try: - module = getattr(__import__("Bcfg2.Server.Plugins.Packages.%s" % - stype.title()).Server.Plugins.Packages, - stype.title()) - cls = getattr(module, "%sSource" % stype.title()) - except (ImportError, AttributeError): - self.logger.error("Packages: Unknown source type %s" % stype) - return None - - try: - source = cls(self.cachepath, xsource, self.config) - except SourceInitError: - err = sys.exc_info()[1] - self.logger.error("Packages: %s" % err) - source = None - - return source - - def __getitem__(self, key): - return self.entries[key] - - def __repr__(self): - return "PackagesSources: %s" % repr(self.entries) - - def __str__(self): - return "PackagesSources: %s" % str(self.entries) diff --git a/src/lib/Server/Plugins/Packages/Source.py b/src/lib/Server/Plugins/Packages/Source.py deleted file mode 100644 index 1dfeecc40..000000000 --- a/src/lib/Server/Plugins/Packages/Source.py +++ /dev/null @@ -1,278 +0,0 @@ -import os -import re -import sys -import base64 -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import HTTPError, HTTPBasicAuthHandler, \ - HTTPPasswordMgrWithDefaultRealm, install_opener, build_opener, \ - urlopen, file, cPickle - -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - -def fetch_url(url): - if '@' in url: - mobj = re.match('(\w+://)([^:]+):([^@]+)@(.*)$', url) - if not mobj: - raise ValueError - user = mobj.group(2) - passwd = mobj.group(3) - url = mobj.group(1) + mobj.group(4) - auth = HTTPBasicAuthHandler(HTTPPasswordMgrWithDefaultRealm()) - auth.add_password(None, url, user, passwd) - install_opener(build_opener(auth)) - return urlopen(url).read() - - -class SourceInitError(Exception): - pass - - -class Source(Bcfg2.Server.Plugin.Debuggable): - mrepo_re = re.compile(r'/RPMS\.([^/]+)') - pulprepo_re = re.compile(r'pulp/repos/([^/]+)') - genericrepo_re = re.compile(r'https?://[^/]+/(.+?)/?$') - basegroups = [] - - def __init__(self, basepath, xsource, config): - Bcfg2.Server.Plugin.Debuggable.__init__(self) - self.basepath = basepath - self.xsource = xsource - self.config = config - - try: - self.version = xsource.find('Version').text - except AttributeError: - pass - - for key, tag in [('components', 'Component'), ('arches', 'Arch'), - ('blacklist', 'Blacklist'), - ('whitelist', 'Whitelist')]: - self.__dict__[key] = [item.text for item in xsource.findall(tag)] - - self.gpgkeys = [el.text for el in xsource.findall("GPGKey")] - - self.recommended = xsource.get('recommended', 'false').lower() == 'true' - - self.rawurl = xsource.get('rawurl', '') - if self.rawurl and not self.rawurl.endswith("/"): - self.rawurl += "/" - self.url = xsource.get('url', '') - if self.url and not self.url.endswith("/"): - self.url += "/" - self.version = xsource.get('version', '') - - # build the set of conditions to see if this source applies to - # a given set of metadata - self.conditions = [] - self.groups = [] # provided for some limited backwards compat - for el in xsource.iterancestors(): - if el.tag == "Group": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") not in m.groups) - else: - self.groups.append(el.get("name")) - self.conditions.append(lambda m, el=el: - el.get("name") in m.groups) - elif el.tag == "Client": - if el.get("negate", "false").lower() == "true": - self.conditions.append(lambda m, el=el: - el.get("name") != m.hostname) - else: - self.conditions.append(lambda m, el=el: - el.get("name") == m.hostname) - - self.deps = dict() - self.provides = dict() - - self.cachefile = os.path.join(self.basepath, - "cache-%s" % self.cachekey) - self.url_map = [] - - @property - def cachekey(self): - return md5(cPickle.dumps([self.version, self.components, self.url, - self.rawurl, self.arches])).hexdigest() - - def get_relevant_groups(self, metadata): - return sorted(list(set([g for g in metadata.groups - if (g in self.basegroups or - g in self.groups or - g in self.arches)]))) - - def load_state(self): - pass - - def setup_data(self, force_update=False): - should_read = True - should_download = False - if os.path.exists(self.cachefile): - try: - self.load_state() - should_read = False - except: - self.logger.error("Packages: Cachefile %s load failed; " - "falling back to file read" % self.cachefile) - if should_read: - try: - self.read_files() - except: - self.logger.error("Packages: File read failed; " - "falling back to file download") - should_download = True - - if should_download or force_update: - try: - self.update() - self.read_files() - except: - self.logger.error("Packages: Failed to load data for Source " - "of %s. Some Packages will be missing." % - self.urls) - - def get_repo_name(self, url_map): - # try to find a sensible name for a repo - if url_map['component']: - rname = url_map['component'] - else: - name = None - for repo_re in (self.mrepo_re, - self.pulprepo_re, - self.genericrepo_re): - match = repo_re.search(url_map['url']) - if match: - break - if name is None: - # couldn't figure out the name from the URL or URL map - # (which probably means its a screwy URL), so we just - # generate a random one - name = base64.b64encode(os.urandom(16))[:-2] - rname = "%s-%s" % (self.groups[0], name) - # see yum/__init__.py in the yum source, lines 441-449, for - # the source of this regex. yum doesn't like anything but - # string.ascii_letters, string.digits, and [-_.:]. There - # doesn't seem to be a reason for this, because yum. - return re.sub(r'[^A-Za-z0-9-_.:]', '-', rname) - - def __str__(self): - if self.rawurl: - return "%s at %s" % (self.__class__.__name__, self.rawurl) - elif self.url: - return "%s at %s" % (self.__class__.__name__, self.url) - else: - return self.__class__.__name__ - - def get_urls(self): - return [] - urls = property(get_urls) - - def get_files(self): - return [self.escape_url(url) for url in self.urls] - files = property(get_files) - - def get_vpkgs(self, metadata): - agroups = ['global'] + [a for a in self.arches - if a in metadata.groups] - vdict = dict() - for agrp in agroups: - for key, value in list(self.provides[agrp].items()): - if key not in vdict: - vdict[key] = set(value) - else: - vdict[key].update(value) - return vdict - - def is_virtual_package(self, metadata, package): - """ called to determine if a package is a virtual package. - this is only invoked if the package is not listed in the dict - returned by get_vpkgs """ - return False - - def escape_url(self, url): - return os.path.join(self.basepath, url.replace('/', '@')) - - def file_init(self): - pass - - def read_files(self): - pass - - def filter_unknown(self, unknown): - pass - - def update(self): - for url in self.urls: - self.logger.info("Packages: Updating %s" % url) - fname = self.escape_url(url) - try: - data = fetch_url(url) - file(fname, 'w').write(data) - except ValueError: - self.logger.error("Packages: Bad url string %s" % url) - raise - except HTTPError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to fetch url %s. HTTP " - "response code=%s" % (url, err.code)) - raise - - def applies(self, metadata): - # check base groups - if not self.magic_groups_match(metadata): - return False - - # check Group/Client tags from sources.xml - for condition in self.conditions: - if not condition(metadata): - return False - - return True - - def get_arches(self, metadata): - return ['global'] + [a for a in self.arches if a in metadata.groups] - - def get_deps(self, metadata, pkgname): - for arch in self.get_arches(metadata): - if pkgname in self.deps[arch]: - return self.deps[arch][pkgname] - return [] - - def get_provides(self, metadata, required): - for arch in self.get_arches(metadata): - if required in self.provides[arch]: - return self.provides[arch][required] - return [] - - def is_package(self, metadata, _): - return False - - def get_package(self, metadata, package): - return package - - def get_group(self, metadata, group, ptype=None): - return [] - - def magic_groups_match(self, metadata): - """ check to see if this source applies to the given host - metadata by checking 'magic' (base) groups only, or if magic - groups are off """ - # we always check that arch matches - found_arch = False - for arch in self.arches: - if arch in metadata.groups: - found_arch = True - break - if not found_arch: - return False - - if self.config.getboolean("global", "magic_groups", - default=True) == False: - return True - else: - for group in self.basegroups: - if group in metadata.groups: - return True - return False diff --git a/src/lib/Server/Plugins/Packages/Yum.py b/src/lib/Server/Plugins/Packages/Yum.py deleted file mode 100644 index e13b28251..000000000 --- a/src/lib/Server/Plugins/Packages/Yum.py +++ /dev/null @@ -1,688 +0,0 @@ -import os -import sys -import time -import copy -import glob -import socket -import random -import logging -import threading -import lxml.etree -from UserDict import DictMixin -from subprocess import Popen, PIPE, STDOUT -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import StringIO, cPickle, HTTPError, ConfigParser, file -from Bcfg2.Server.Plugins.Packages.Collection import Collection -from Bcfg2.Server.Plugins.Packages.Source import SourceInitError, Source, \ - fetch_url - -logger = logging.getLogger(__name__) - -try: - from pulp.client.consumer.config import ConsumerConfig - from pulp.client.api.repository import RepositoryAPI - from pulp.client.api.consumer import ConsumerAPI - from pulp.client.api import server - has_pulp = True -except ImportError: - has_pulp = False - -try: - import yum - has_yum = True -except ImportError: - has_yum = False - logger.info("Packages: No yum libraries found; forcing use of internal " - "dependency resolver") - -try: - import json -except ImportError: - import simplejson as json - -XP = '{http://linux.duke.edu/metadata/common}' -RP = '{http://linux.duke.edu/metadata/rpm}' -RPO = '{http://linux.duke.edu/metadata/repo}' -FL = '{http://linux.duke.edu/metadata/filelists}' - -PULPSERVER = None -PULPCONFIG = None - - -def _setup_pulp(config): - global PULPSERVER, PULPCONFIG - if not has_pulp: - logger.error("Packages: Cannot create Pulp collection: Pulp libraries not " - "found") - raise Bcfg2.Server.Plugin.PluginInitError - - if PULPSERVER is None: - try: - username = config.get("pulp", "username") - password = config.get("pulp", "password") - except ConfigParser.NoSectionError: - logger.error("Packages: No [pulp] section found in Packages/packages.conf") - raise Bcfg2.Server.Plugin.PluginInitError - except ConfigParser.NoOptionError: - err = sys.exc_info()[1] - logger.error("Packages: Required option not found in " - "Packages/packages.conf: %s" % err) - raise Bcfg2.Server.Plugin.PluginInitError - - PULPCONFIG = ConsumerConfig() - serveropts = PULPCONFIG.server - - PULPSERVER = server.PulpServer(serveropts['host'], - int(serveropts['port']), - serveropts['scheme'], - serveropts['path']) - PULPSERVER.set_basic_auth_credentials(username, password) - server.set_active_server(PULPSERVER) - return PULPSERVER - - -class YumCollection(Collection): - # options that are included in the [yum] section but that should - # not be included in the temporary yum.conf we write out - option_blacklist = ["use_yum_libraries", "helper"] - - def __init__(self, metadata, sources, basepath, debug=False): - Collection.__init__(self, metadata, sources, basepath, debug=debug) - self.keypath = os.path.join(self.basepath, "keys") - - if len(sources): - config = sources[0].config - self.use_yum = has_yum and config.getboolean("yum", - "use_yum_libraries", - default=False) - else: - self.use_yum = False - - if self.use_yum: - self.cachefile = os.path.join(self.cachepath, - "cache-%s" % self.cachekey) - if not os.path.exists(self.cachefile): - os.mkdir(self.cachefile) - - self.configdir = os.path.join(self.basepath, "yum") - if not os.path.exists(self.configdir): - os.mkdir(self.configdir) - self.cfgfile = os.path.join(self.configdir, - "%s-yum.conf" % self.cachekey) - self.write_config() - - self.helper = self.config.get("yum", "helper", - default="/usr/sbin/bcfg2-yum-helper") - if has_pulp: - _setup_pulp(self.config) - - def write_config(self): - if not os.path.exists(self.cfgfile): - yumconf = self.get_config(raw=True) - yumconf.add_section("main") - - mainopts = dict(cachedir=self.cachefile, - keepcache="0", - sslverify="0", - debuglevel="0", - reposdir="/dev/null") - try: - for opt in self.config.options("yum"): - if opt not in self.option_blacklist: - mainopts[opt] = self.config.get("yum", opt) - except ConfigParser.NoSectionError: - pass - - for opt, val in list(mainopts.items()): - yumconf.set("main", opt, val) - - yumconf.write(open(self.cfgfile, 'w')) - - def get_config(self, raw=False): - config = ConfigParser.SafeConfigParser() - for source in self.sources: - # get_urls() loads url_map as a side-effect - source.get_urls() - for url_map in source.url_map: - if url_map['arch'] in self.metadata.groups: - basereponame = source.get_repo_name(url_map) - reponame = basereponame - - added = False - while not added: - try: - config.add_section(reponame) - added = True - except ConfigParser.DuplicateSectionError: - match = re.match("-(\d)", reponame) - if match: - rid = int(match.group(1)) + 1 - else: - rid = 1 - reponame = "%s-%d" % (basereponame, rid) - - config.set(reponame, "name", reponame) - config.set(reponame, "baseurl", url_map['url']) - config.set(reponame, "enabled", "1") - if len(source.gpgkeys): - config.set(reponame, "gpgcheck", "1") - config.set(reponame, "gpgkey", - " ".join(source.gpgkeys)) - else: - config.set(reponame, "gpgcheck", "0") - - if len(source.blacklist): - config.set(reponame, "exclude", - " ".join(source.blacklist)) - if len(source.whitelist): - config.set(reponame, "includepkgs", - " ".join(source.whitelist)) - - if raw: - return config - else: - # configparser only writes to file, so we have to use a - # StringIO object to get the data out as a string - buf = StringIO() - config.write(buf) - return "# This config was generated automatically by the Bcfg2 " \ - "Packages plugin\n\n" + buf.getvalue() - - def build_extra_structures(self, independent): - """ build list of gpg keys to be added to the specification by - validate_structures() """ - needkeys = set() - for source in self.sources: - for key in source.gpgkeys: - needkeys.add(key) - - if len(needkeys): - if has_yum: - # this must be be has_yum, not use_yum, because - # regardless of whether the user wants to use the yum - # resolver we want to include gpg key data - keypkg = lxml.etree.Element('BoundPackage', name="gpg-pubkey", - type=self.ptype, origin='Packages') - else: - self.logger.warning("GPGKeys were specified for yum sources in " - "sources.xml, but no yum libraries were " - "found") - self.logger.warning("GPG key version/release data cannot be " - "determined automatically") - self.logger.warning("Install yum libraries, or manage GPG keys " - "manually") - keypkg = None - - for key in needkeys: - # figure out the path of the key on the client - keydir = self.config.get("global", "gpg_keypath", - default="/etc/pki/rpm-gpg") - remotekey = os.path.join(keydir, os.path.basename(key)) - localkey = os.path.join(self.keypath, os.path.basename(key)) - kdata = open(localkey).read() - - # copy the key to the client - keypath = lxml.etree.Element("BoundPath", name=remotekey, - encoding='ascii', - owner='root', group='root', - type='file', perms='0644', - important='true') - keypath.text = kdata - - # hook to add version/release info if possible - self._add_gpg_instances(keypkg, kdata, localkey, remotekey) - independent.append(keypath) - if keypkg is not None: - independent.append(keypkg) - - # see if there are any pulp sources to handle - has_pulp_sources = False - for source in self.sources: - if source.pulp_id: - has_pulp_sources = True - break - - if has_pulp_sources: - consumerapi = ConsumerAPI() - consumer = self._get_pulp_consumer(consumerapi=consumerapi) - if consumer is None: - consumer = consumerapi.create(self.metadata.hostname, - self.metadata.hostname) - lxml.etree.SubElement(independent, "BoundAction", - name="pulp-update", timing="pre", - when="always", status="check", - command="pulp-consumer consumer update") - - for source in self.sources: - # each pulp source can only have one arch, so we don't - # have to check the arch in url_map - if (source.pulp_id and - source.pulp_id not in consumer['repoids']): - consumerapi.bind(self.metadata.hostname, source.pulp_id) - - crt = lxml.etree.SubElement(independent, "BoundPath", - name="/etc/pki/consumer/cert.pem", - type="file", owner="root", - group="root", perms="0644") - crt.text = consumerapi.certificate(self.metadata.hostname) - - def _get_pulp_consumer(self, consumerapi=None): - if consumerapi is None: - consumerapi = ConsumerAPI() - consumer = None - try: - consumer = consumerapi.consumer(self.metadata.hostname) - except server.ServerRequestError: - # consumer does not exist - pass - except socket.error: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not contact Pulp server: %s" % - err) - except: - err = sys.exc_info()[1] - self.logger.error("Packages: Unknown error querying Pulp server: %s" - % err) - return consumer - - def _add_gpg_instances(self, keyentry, keydata, localkey, remotekey): - """ add gpg keys to the specification to ensure they get - installed """ - # this must be be has_yum, not use_yum, because regardless of - # whether the user wants to use the yum resolver we want to - # include gpg key data - if not has_yum: - return - - try: - kinfo = yum.misc.getgpgkeyinfo(keydata) - version = yum.misc.keyIdToRPMVer(kinfo['keyid']) - release = yum.misc.keyIdToRPMVer(kinfo['timestamp']) - - lxml.etree.SubElement(keyentry, 'Instance', - version=version, - release=release, - simplefile=remotekey) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not read GPG key %s: %s" % - (localkey, err)) - - def is_package(self, package): - if not self.use_yum: - return Collection.is_package(self, package) - elif isinstance(package, tuple): - if package[1] is None and package[2] == (None, None, None): - package = package[0] - else: - return None - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("is_package", package) - - def is_virtual_package(self, package): - if not self.use_yum: - return Collection.is_virtual_package(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("is_virtual_package", package) - - def get_deps(self, package): - if not self.use_yum: - return Collection.get_deps(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("get_deps", package) - - def get_provides(self, required, all=False, silent=False): - if not self.use_yum: - return Collection.get_provides(self, package) - else: - # this should really never get called; it's just provided - # for API completeness - return self.call_helper("get_provides", package) - - def get_group(self, group, ptype="default"): - if not self.use_yum: - self.logger.warning("Packages: Package groups are not supported by " - "Bcfg2's internal Yum dependency generator") - return [] - - if group.startswith("@"): - group = group[1:] - - pkgs = self.call_helper("get_group", dict(group=group, type=ptype)) - return pkgs - - def complete(self, packagelist): - if not self.use_yum: - return Collection.complete(self, packagelist) - - packages = set() - unknown = set(packagelist) - - if unknown: - result = \ - self.call_helper("complete", - dict(packages=list(unknown), - groups=list(self.get_relevant_groups()))) - if result and "packages" in result and "unknown" in result: - # we stringify every package because it gets returned - # in unicode; set.update() doesn't work if some - # elements are unicode and other are strings. (I.e., - # u'foo' and 'foo' get treated as unique elements.) - packages.update([str(p) for p in result['packages']]) - unknown = set([str(p) for p in result['unknown']]) - - self.filter_unknown(unknown) - - return packages, unknown - - def call_helper(self, command, input=None): - """ Make a call to bcfg2-yum-helper. The yum libs have - horrific memory leaks, so apparently the right way to get - around that in long-running processes it to have a short-lived - helper. No, seriously -- check out the yum-updatesd code. - It's pure madness. """ - # it'd be nice if we could change this to be more verbose if - # -v was given to bcfg2-server, but Collection objects don't - # get the 'setup' variable, so we don't know how verbose - # bcfg2-server is. It'd also be nice if we could tell yum to - # log to syslog. So would a unicorn. - cmd = [self.helper, "-c", self.cfgfile] - if self.debug_flag: - cmd.append("-v") - cmd.append(command) - self.debug_log("Packages: running %s" % " ".join(cmd)) - try: - helper = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to execute %s: %s" % - (" ".join(cmd), err)) - return None - - if input: - idata = json.dumps(input) - (stdout, stderr) = helper.communicate(idata) - else: - (stdout, stderr) = helper.communicate() - rv = helper.wait() - if rv: - self.logger.error("Packages: error running bcfg2-yum-helper " - "(returned %d): %s" % (rv, stderr)) - elif self.debug_flag: - self.debug_log("Packages: debug info from bcfg2-yum-helper: %s" % - stderr) - try: - return json.loads(stdout) - except ValueError: - err = sys.exc_info()[1] - self.logger.error("Packages: error reading bcfg2-yum-helper " - "output: %s" % err) - return None - - def setup_data(self, force_update=False): - if not self.use_yum: - return Collection.setup_data(self, force_update) - - if force_update: - # we call this twice: one to clean up data from the old - # config, and once to clean up data from the new config - self.call_helper("clean") - - os.unlink(self.cfgfile) - self.write_config() - - if force_update: - self.call_helper("clean") - - -class YumSource(Source): - basegroups = ['yum', 'redhat', 'centos', 'fedora'] - ptype = 'yum' - - def __init__(self, basepath, xsource, config): - Source.__init__(self, basepath, xsource, config) - self.pulp_id = None - if has_pulp and xsource.get("pulp_id"): - self.pulp_id = xsource.get("pulp_id") - - _setup_pulp(self.config) - repoapi = RepositoryAPI() - try: - self.repo = repoapi.repository(self.pulp_id) - self.gpgkeys = [os.path.join(PULPCONFIG.cds['keyurl'], key) - for key in repoapi.listkeys(self.pulp_id)] - except server.ServerRequestError: - err = sys.exc_info()[1] - if err[0] == 401: - msg = "Packages: Error authenticating to Pulp: %s" % err[1] - elif err[0] == 404: - msg = "Packages: Pulp repo id %s not found: %s" % \ - (self.pulp_id, err[1]) - else: - msg = "Packages: Error %d fetching pulp repo %s: %s" % \ - (err[0], self.pulp_id, err[1]) - raise SourceInitError(msg) - except socket.error: - err = sys.exc_info()[1] - raise SourceInitError("Could not contact Pulp server: %s" % err) - except: - err = sys.exc_info()[1] - raise SourceInitError("Unknown error querying Pulp server: %s" % - err) - self.rawurl = "%s/%s" % (PULPCONFIG.cds['baseurl'], - self.repo['relative_path']) - self.arches = [self.repo['arch']] - - if not self.rawurl: - self.baseurl = self.url + "%(version)s/%(component)s/%(arch)s/" - else: - self.baseurl = self.rawurl - self.packages = dict() - self.deps = dict([('global', dict())]) - self.provides = dict([('global', dict())]) - self.filemap = dict([(x, dict()) - for x in ['global'] + self.arches]) - self.needed_paths = set() - self.file_to_arch = dict() - - self.use_yum = has_yum and config.getboolean("yum", "use_yum_libraries", - default=False) - - def save_state(self): - if not self.use_yum: - cache = file(self.cachefile, 'wb') - cPickle.dump((self.packages, self.deps, self.provides, - self.filemap, self.url_map), cache, 2) - cache.close() - - - def load_state(self): - if not self.use_yum: - data = file(self.cachefile) - (self.packages, self.deps, self.provides, - self.filemap, self.url_map) = cPickle.load(data) - - def get_urls(self): - surls = list() - self.url_map = [] - for arch in self.arches: - if self.url: - usettings = [{'version':self.version, 'component':comp, - 'arch':arch} - for comp in self.components] - else: # rawurl given - usettings = [{'version':self.version, 'component':None, - 'arch':arch}] - - for setting in usettings: - setting['url'] = self.baseurl % setting - self.url_map.append(copy.deepcopy(setting)) - surls.append((arch, [setting['url'] for setting in usettings])) - urls = [] - for (sarch, surl_list) in surls: - for surl in surl_list: - urls.extend(self._get_urls_from_repodata(surl, sarch)) - return urls - urls = property(get_urls) - - def _get_urls_from_repodata(self, url, arch): - if self.use_yum: - return [url] - - rmdurl = '%srepodata/repomd.xml' % url - try: - repomd = fetch_url(rmdurl) - xdata = lxml.etree.XML(repomd) - except ValueError: - self.logger.error("Packages: Bad url string %s" % rmdurl) - return [] - except HTTPError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to fetch url %s. code=%s" % - (rmdurl, err.code)) - return [] - except lxml.etree.XMLSyntaxError: - err = sys.exc_info()[1] - self.logger.error("Packages: Failed to process metadata at %s: %s" % - (rmdurl, err)) - return [] - - urls = [] - for elt in xdata.findall(RPO + 'data'): - if elt.get('type') in ['filelists', 'primary']: - floc = elt.find(RPO + 'location') - fullurl = url + floc.get('href') - urls.append(fullurl) - self.file_to_arch[self.escape_url(fullurl)] = arch - return urls - - def read_files(self): - # we have to read primary.xml first, and filelists.xml afterwards; - primaries = list() - filelists = list() - for fname in self.files: - if fname.endswith('primary.xml.gz'): - primaries.append(fname) - elif fname.endswith('filelists.xml.gz'): - filelists.append(fname) - - for fname in primaries: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_primary(fdata, farch) - for fname in filelists: - farch = self.file_to_arch[fname] - fdata = lxml.etree.parse(fname).getroot() - self.parse_filelist(fdata, farch) - - # merge data - sdata = list(self.packages.values()) - try: - self.packages['global'] = copy.deepcopy(sdata.pop()) - except IndexError: - logger.error("Packages: No packages in repo") - while sdata: - self.packages['global'] = \ - self.packages['global'].intersection(sdata.pop()) - - for key in self.packages: - if key == 'global': - continue - self.packages[key] = \ - self.packages[key].difference(self.packages['global']) - self.save_state() - - def parse_filelist(self, data, arch): - if arch not in self.filemap: - self.filemap[arch] = dict() - for pkg in data.findall(FL + 'package'): - for fentry in pkg.findall(FL + 'file'): - if fentry.text in self.needed_paths: - if fentry.text in self.filemap[arch]: - self.filemap[arch][fentry.text].add(pkg.get('name')) - else: - self.filemap[arch][fentry.text] = \ - set([pkg.get('name')]) - - def parse_primary(self, data, arch): - if arch not in self.packages: - self.packages[arch] = set() - if arch not in self.deps: - self.deps[arch] = dict() - if arch not in self.provides: - self.provides[arch] = dict() - for pkg in data.getchildren(): - if not pkg.tag.endswith('package'): - continue - pkgname = pkg.find(XP + 'name').text - self.packages[arch].add(pkgname) - - pdata = pkg.find(XP + 'format') - self.deps[arch][pkgname] = set() - pre = pdata.find(RP + 'requires') - if pre is not None: - for entry in pre.getchildren(): - self.deps[arch][pkgname].add(entry.get('name')) - if entry.get('name').startswith('/'): - self.needed_paths.add(entry.get('name')) - pro = pdata.find(RP + 'provides') - if pro != None: - for entry in pro.getchildren(): - prov = entry.get('name') - if prov not in self.provides[arch]: - self.provides[arch][prov] = list() - self.provides[arch][prov].append(pkgname) - - def is_package(self, metadata, item): - arch = [a for a in self.arches if a in metadata.groups] - if not arch: - return False - return ((item in self.packages['global'] or - item in self.packages[arch[0]]) and - item not in self.blacklist and - (len(self.whitelist) == 0 or item in self.whitelist)) - - def get_vpkgs(self, metadata): - if self.use_yum: - return dict() - - rv = Source.get_vpkgs(self, metadata) - for arch, fmdata in list(self.filemap.items()): - if arch not in metadata.groups and arch != 'global': - continue - for filename, pkgs in list(fmdata.items()): - rv[filename] = pkgs - return rv - - def filter_unknown(self, unknown): - if self.use_yum: - filtered = set() - for unk in unknown: - try: - if unk.startswith('rpmlib'): - filtered.update(unk) - except AttributeError: - try: - if unk[0].startswith('rpmlib'): - filtered.update(unk) - except (IndexError, AttributeError): - pass - else: - filtered = set([u for u in unknown if u.startswith('rpmlib')]) - unknown.difference_update(filtered) - - def setup_data(self, force_update=False): - if not self.use_yum: - Source.setup_data(self, force_update=force_update) - - def get_repo_name(self, url_map): - if self.pulp_id: - return self.pulp_id - else: - return Source.get_repo_name(self, url_map) diff --git a/src/lib/Server/Plugins/Packages/__init__.py b/src/lib/Server/Plugins/Packages/__init__.py deleted file mode 100644 index da5832e90..000000000 --- a/src/lib/Server/Plugins/Packages/__init__.py +++ /dev/null @@ -1,263 +0,0 @@ -import os -import sys -import time -import copy -import glob -import shutil -import lxml.etree -import Bcfg2.Logger -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import ConfigParser, urlopen -from Bcfg2.Server.Plugins.Packages import Collection -from Bcfg2.Server.Plugins.Packages.PackagesSources import PackagesSources -from Bcfg2.Server.Plugins.Packages.PackagesConfig import PackagesConfig - -class Packages(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.StructureValidator, - Bcfg2.Server.Plugin.Generator, - Bcfg2.Server.Plugin.Connector): - name = 'Packages' - conflicts = ['Pkgmgr'] - experimental = True - __rmi__ = Bcfg2.Server.Plugin.Plugin.__rmi__ + ['Refresh', 'Reload'] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.StructureValidator.__init__(self) - Bcfg2.Server.Plugin.Generator.__init__(self) - Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.Probing.__init__(self) - - self.sentinels = set() - self.cachepath = os.path.join(self.data, 'cache') - self.keypath = os.path.join(self.data, 'keys') - if not os.path.exists(self.keypath): - # create key directory if needed - os.makedirs(self.keypath) - - # set up config files - self.config = PackagesConfig(self) - self.sources = PackagesSources(os.path.join(self.data, "sources.xml"), - self.cachepath, core.fam, self, - self.config) - - def toggle_debug(self): - Bcfg2.Server.Plugin.Plugin.toggle_debug(self) - self.sources.toggle_debug() - - @property - def disableResolver(self): - try: - return not self.config.getboolean("global", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled", which are not handled according to the - # Python docs but appear to be handled properly by - # ConfigParser in at least some versions - return self.config.get("global", "resolver", - default="enabled").lower() == "disabled" - - @property - def disableMetaData(self): - try: - return not self.config.getboolean("global", "resolver") - except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): - return False - except ValueError: - # for historical reasons we also accept "enabled" and - # "disabled" - return self.config.get("global", "metadata", - default="enabled").lower() == "disabled" - - def create_config(self, entry, metadata): - """ create yum/apt config for the specified host """ - attrib = {'encoding': 'ascii', - 'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0644'} - - collection = self._get_collection(metadata) - entry.text = collection.get_config() - for (key, value) in list(attrib.items()): - entry.attrib.__setitem__(key, value) - - def HandleEntry(self, entry, metadata): - if entry.tag == 'Package': - collection = self._get_collection(metadata) - entry.set('version', 'auto') - entry.set('type', collection.ptype) - elif entry.tag == 'Path': - if (entry.get("name") == self.config.get("global", "yum_config", - default="") or - entry.get("name") == self.config.get("global", "apt_config", - default="")): - self.create_config(entry, metadata) - - def HandlesEntry(self, entry, metadata): - if entry.tag == 'Package': - if self.config.getboolean("global", "magic_groups", - default=True) == True: - collection = self._get_collection(metadata) - if collection.magic_groups_match(): - return True - else: - return True - elif entry.tag == 'Path': - # managed entries for yum/apt configs - if (entry.get("name") == self.config.get("global", "yum_config", - default="") or - entry.get("name") == self.config.get("global", "apt_config", - default="")): - return True - return False - - def validate_structures(self, metadata, structures): - '''Ensure client configurations include all needed prerequisites - - Arguments: - metadata - client metadata instance - structures - a list of structure-stage entry combinations - ''' - collection = self._get_collection(metadata) - indep = lxml.etree.Element('Independent') - self._build_packages(metadata, indep, structures, - collection=collection) - collection.build_extra_structures(indep) - structures.append(indep) - - def _build_packages(self, metadata, independent, structures, - collection=None): - """ build list of packages that need to be included in the - specification by validate_structures() """ - if self.disableResolver: - # Config requests no resolver - return - - if collection is None: - collection = self._get_collection(metadata) - # initial is the set of packages that are explicitly specified - # in the configuration - initial = set() - # base is the set of initial packages with groups expanded - base = set() - to_remove = [] - for struct in structures: - for pkg in struct.xpath('//Package | //BoundPackage'): - if pkg.get("name"): - initial.add(pkg.get("name")) - elif pkg.get("group"): - try: - if pkg.get("type"): - gpkgs = collection.get_group(pkg.get("group"), - ptype=pkg.get("type")) - else: - gpkgs = collection.get_group(pkg.get("group")) - base.update(gpkgs) - except TypeError: - raise - self.logger.error("Could not resolve group %s" % - pkg.get("group")) - to_remove.append(pkg) - else: - self.logger.error("Packages: Malformed Package: %s" % - lxml.etree.tostring(pkg)) - base.update(initial) - for el in to_remove: - el.getparent().remove(el) - - packages, unknown = collection.complete(base) - if unknown: - self.logger.info("Packages: Got %d unknown entries" % len(unknown)) - self.logger.info("Packages: %s" % list(unknown)) - newpkgs = list(packages.difference(initial)) - self.debug_log("Packages: %d initial, %d complete, %d new" % - (len(initial), len(packages), len(newpkgs))) - newpkgs.sort() - for pkg in newpkgs: - lxml.etree.SubElement(independent, 'BoundPackage', name=pkg, - version='auto', type=collection.ptype, - origin='Packages') - - def Refresh(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and download sources\n''' - self._load_config(force_update=True) - return True - - def Reload(self): - '''Packages.Refresh() => True|False\nReload configuration - specification and sources\n''' - self._load_config() - return True - - def _load_config(self, force_update=False): - ''' - Load the configuration data and setup sources - - Keyword args: - force_update Force downloading repo data - ''' - self._load_sources(force_update) - self._load_gpg_keys(force_update) - - def _load_sources(self, force_update): - """ Load sources from the config """ - self.sentinels = set() - cachefiles = set() - - for collection in list(Collection.collections.values()): - cachefiles.update(collection.cachefiles) - if not self.disableMetaData: - collection.setup_data(force_update) - self.sentinels.update(collection.basegroups) - - Collection.clear_cache() - - for source in self.sources: - cachefiles.add(source.cachefile) - if not self.disableMetaData: - source.setup_data(force_update) - - for cfile in glob.glob(os.path.join(self.cachepath, "cache-*")): - if cfile not in cachefiles: - try: - if os.path.isdir(cfile): - shutil.rmtree(cfile) - else: - os.unlink(cfile) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Packages: Could not remove cache file " - "%s: %s" % (cfile, err)) - - def _load_gpg_keys(self, force_update): - """ Load gpg keys from the config """ - keyfiles = [] - keys = [] - for source in self.sources: - for key in source.gpgkeys: - localfile = os.path.join(self.keypath, - os.path.basename(key.rstrip("/"))) - if localfile not in keyfiles: - keyfiles.append(localfile) - if ((force_update and key not in keys) or - not os.path.exists(localfile)): - self.logger.info("Packages: Downloading and parsing %s" % key) - response = urlopen(key) - open(localfile, 'w').write(response.read()) - keys.append(key) - - for kfile in glob.glob(os.path.join(self.keypath, "*")): - if kfile not in keyfiles: - os.unlink(kfile) - - def _get_collection(self, metadata): - return Collection.factory(metadata, self.sources, self.data, - debug=self.debug_flag) - - def get_additional_data(self, metadata): - collection = self._get_collection(metadata) - return dict(sources=collection.get_additional_data()) diff --git a/src/lib/Server/Plugins/Pkgmgr.py b/src/lib/Server/Plugins/Pkgmgr.py deleted file mode 100644 index e9254cdcc..000000000 --- a/src/lib/Server/Plugins/Pkgmgr.py +++ /dev/null @@ -1,169 +0,0 @@ -'''This module implements a package management scheme for all images''' - -import logging -import re -import Bcfg2.Server.Plugin -import lxml -try: - set -except NameError: - from sets import Set as set - -logger = logging.getLogger('Bcfg2.Plugins.Pkgmgr') - - -class FuzzyDict(dict): - fuzzy = re.compile('(?P.*):(?P\S+(,\S+)*)') - - def __getitem__(self, key): - if isinstance(key, str): - mdata = self.fuzzy.match(key) - if mdata: - return dict.__getitem__(self, mdata.groupdict()['name']) - else: - print("got non-string key %s" % str(key)) - return dict.__getitem__(self, key) - - def has_key(self, key): - if isinstance(key, str): - mdata = self.fuzzy.match(key) - if self.fuzzy.match(key): - return dict.has_key(self, mdata.groupdict()['name']) - return dict.has_key(self, key) - - def get(self, key, default=None): - try: - return self.__getitem__(key) - except: - if default: - return default - raise - - -class PNode(Bcfg2.Server.Plugin.INode): - """PNode has a list of packages available at a - particular group intersection. - """ - splitters = {'rpm': re.compile('^(.*/)?(?P[\w\+\d\.]+(-[\w\+\d\.]+)*)-' + \ - '(?P[\w\d\.]+-([\w\d\.]+))\.(?P\S+)\.rpm$'), - 'encap': re.compile('^(?P[\w-]+)-(?P[\w\d\.+-]+).encap.*$')} - ignore = ['Package'] - - def Match(self, metadata, data, entry=lxml.etree.Element("None")): - """Return a dictionary of package mappings.""" - if self.predicate(metadata, entry): - for key in self.contents: - try: - data[key].update(self.contents[key]) - except: - data[key] = FuzzyDict() - data[key].update(self.contents[key]) - for child in self.children: - child.Match(metadata, data) - - def __init__(self, data, pdict, parent=None): - # copy local attributes to all child nodes if no local attribute exists - if 'Package' not in pdict: - pdict['Package'] = set() - for child in data.getchildren(): - attrs = set(data.attrib.keys()).difference(child.attrib.keys() + ['name']) - for attr in attrs: - try: - child.set(attr, data.get(attr)) - except: - # don't fail on things like comments and other immutable elements - pass - Bcfg2.Server.Plugin.INode.__init__(self, data, pdict, parent) - if 'Package' not in self.contents: - self.contents['Package'] = FuzzyDict() - for pkg in data.findall('./Package'): - if 'name' in pkg.attrib and pkg.get('name') not in pdict['Package']: - pdict['Package'].add(pkg.get('name')) - if pkg.get('name') != None: - self.contents['Package'][pkg.get('name')] = {} - if pkg.getchildren(): - self.contents['Package'][pkg.get('name')]['__children__'] \ - = pkg.getchildren() - if 'simplefile' in pkg.attrib: - pkg.set('url', "%s/%s" % (pkg.get('uri'), pkg.get('simplefile'))) - self.contents['Package'][pkg.get('name')].update(pkg.attrib) - else: - if 'file' in pkg.attrib: - if 'multiarch' in pkg.attrib: - archs = pkg.get('multiarch').split() - srcs = pkg.get('srcs', pkg.get('multiarch')).split() - url = ' '.join(["%s/%s" % (pkg.get('uri'), - pkg.get('file') % {'src':srcs[idx], - 'arch':archs[idx]}) - for idx in range(len(archs))]) - pkg.set('url', url) - else: - pkg.set('url', '%s/%s' % (pkg.get('uri'), - pkg.get('file'))) - if pkg.get('type') in self.splitters and pkg.get('file') != None: - mdata = self.splitters[pkg.get('type')].match(pkg.get('file')) - if not mdata: - logger.error("Failed to match pkg %s" % pkg.get('file')) - continue - pkgname = mdata.group('name') - self.contents['Package'][pkgname] = mdata.groupdict() - self.contents['Package'][pkgname].update(pkg.attrib) - if pkg.attrib.get('file'): - self.contents['Package'][pkgname]['url'] = pkg.get('url') - self.contents['Package'][pkgname]['type'] = pkg.get('type') - if pkg.get('verify'): - self.contents['Package'][pkgname]['verify'] = pkg.get('verify') - if pkg.get('multiarch'): - self.contents['Package'][pkgname]['multiarch'] = pkg.get('multiarch') - if pkgname not in pdict['Package']: - pdict['Package'].add(pkgname) - if pkg.getchildren(): - self.contents['Package'][pkgname]['__children__'] = pkg.getchildren() - else: - self.contents['Package'][pkg.get('name')].update(pkg.attrib) - - -class PkgSrc(Bcfg2.Server.Plugin.XMLSrc): - """PkgSrc files contain a PNode hierarchy that - returns matching package entries. - """ - __node__ = PNode - __cacheobj__ = FuzzyDict - - -class Pkgmgr(Bcfg2.Server.Plugin.PrioDir): - """This is a generator that handles package assignments.""" - name = 'Pkgmgr' - __author__ = 'bcfg-dev@mcs.anl.gov' - __child__ = PkgSrc - __element__ = 'Package' - - def HandleEvent(self, event): - '''Handle events and update dispatch table''' - Bcfg2.Server.Plugin.XMLDirectoryBacked.HandleEvent(self, event) - for src in list(self.entries.values()): - for itype, children in list(src.items.items()): - for child in children: - try: - self.Entries[itype][child] = self.BindEntry - except KeyError: - self.Entries[itype] = FuzzyDict([(child, - self.BindEntry)]) - - def BindEntry(self, entry, metadata): - """Bind data for entry, and remove instances that are not requested.""" - pname = entry.get('name') - Bcfg2.Server.Plugin.PrioDir.BindEntry(self, entry, metadata) - if entry.findall('Instance'): - mdata = FuzzyDict.fuzzy.match(pname) - if mdata: - arches = mdata.group('alist').split(',') - [entry.remove(inst) for inst in \ - entry.findall('Instance') \ - if inst.get('arch') not in arches] - - def HandlesEntry(self, entry, metadata): - return entry.tag == 'Package' and entry.get('name').split(':')[0] in list(self.Entries['Package'].keys()) - - def HandleEntry(self, entry, metadata): - self.BindEntry(entry, metadata) diff --git a/src/lib/Server/Plugins/Probes.py b/src/lib/Server/Plugins/Probes.py deleted file mode 100644 index 2af3635e9..000000000 --- a/src/lib/Server/Plugins/Probes.py +++ /dev/null @@ -1,285 +0,0 @@ -import time -import lxml.etree -import operator -import re - -try: - import json - has_json = True -except ImportError: - try: - import simplejson as json - has_json = True - except ImportError: - has_json = False - -try: - import syck - has_syck = True -except ImportError: - has_syck = False - try: - import yaml - has_yaml = True - except ImportError: - has_yaml = False - -import Bcfg2.Server.Plugin - -specific_probe_matcher = re.compile("(.*/)?(?P\S+)(.(?P[GH](\d\d)?)_\S+)") -probe_matcher = re.compile("(.*/)?(?P\S+)") - -class ClientProbeDataSet(dict): - """ dict of probe => [probe data] that records a for each host """ - def __init__(self, *args, **kwargs): - if "timestamp" in kwargs and kwargs['timestamp'] is not None: - self.timestamp = kwargs.pop("timestamp") - else: - self.timestamp = time.time() - dict.__init__(self, *args, **kwargs) - - -class ProbeData(object): - """ a ProbeData object emulates a str object, but also has .xdata - and .json properties to provide convenient ways to use ProbeData - objects as XML or JSON data """ - def __init__(self, data): - self.data = data - self._xdata = None - self._json = None - self._yaml = None - - def __str__(self): - return str(self.data) - - def __repr__(self): - return repr(self.data) - - def __getattr__(self, name): - """ make ProbeData act like a str object """ - return getattr(self.data, name) - - def __complex__(self): - return complex(self.data) - - def __int__(self): - return int(self.data) - - def __long__(self): - return long(self.data) - - def __float__(self): - return float(self.data) - - def __eq__(self, other): - return str(self) == str(other) - - def __ne__(self, other): - return str(self) != str(other) - - def __gt__(self, other): - return str(self) > str(other) - - def __lt__(self, other): - return str(self) < str(other) - - def __ge__(self, other): - return self > other or self == other - - def __le__(self, other): - return self < other or self == other - - @property - def xdata(self): - if self._xdata is None: - try: - self._xdata = lxml.etree.XML(self.data) - except lxml.etree.XMLSyntaxError: - pass - return self._xdata - - @property - def json(self): - if self._json is None and has_json: - try: - self._json = json.loads(self.data) - except ValueError: - pass - return self._json - - @property - def yaml(self): - if self._yaml is None: - if has_yaml: - try: - self._yaml = yaml.load(self.data) - except yaml.YAMLError: - pass - elif has_syck: - try: - self._yaml = syck.load(self.data) - except syck.error: - pass - return self._yaml - - -class ProbeSet(Bcfg2.Server.Plugin.EntrySet): - ignore = re.compile("^(\.#.*|.*~|\\..*\\.(tmp|sw[px])|probed\\.xml)$") - - def __init__(self, path, fam, encoding, plugin_name): - fpattern = '[0-9A-Za-z_\-]+' - self.plugin_name = plugin_name - Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path, - Bcfg2.Server.Plugin.SpecificData, - encoding) - fam.AddMonitor(path, self) - self.bangline = re.compile('^#!(?P.*)$') - - def HandleEvent(self, event): - if event.filename != self.path: - if (event.code2str == 'changed' and - event.filename.endswith("probed.xml") and - event.filename not in self.entries): - # for some reason, probed.xml is particularly prone to - # getting changed events before created events, - # because gamin is the worst ever. anyhow, we - # specifically handle it here to avoid a warning on - # every single server startup. - self.entry_init(event) - return - return self.handle_event(event) - - def get_probe_data(self, metadata): - ret = [] - build = dict() - candidates = self.get_matching(metadata) - candidates.sort(key=operator.attrgetter('specific')) - for entry in candidates: - rem = specific_probe_matcher.match(entry.name) - if not rem: - rem = probe_matcher.match(entry.name) - pname = rem.group('basename') - if pname not in build: - build[pname] = entry - - for (name, entry) in list(build.items()): - probe = lxml.etree.Element('probe') - probe.set('name', name.split('/')[-1]) - probe.set('source', self.plugin_name) - probe.text = entry.data - match = self.bangline.match(entry.data.split('\n')[0]) - if match: - probe.set('interpreter', match.group('interpreter')) - else: - probe.set('interpreter', '/bin/sh') - ret.append(probe) - return ret - - -class Probes(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Probing, - Bcfg2.Server.Plugin.Connector): - """A plugin to gather information from a client machine.""" - name = 'Probes' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - Bcfg2.Server.Plugin.Probing.__init__(self) - - try: - self.probes = ProbeSet(self.data, core.fam, core.encoding, - self.name) - except: - raise Bcfg2.Server.Plugin.PluginInitError - - self.probedata = dict() - self.cgroups = dict() - self.load_data() - - def write_data(self): - """Write probe data out for use with bcfg2-info.""" - top = lxml.etree.Element("Probed") - for client, probed in sorted(self.probedata.items()): - cx = lxml.etree.SubElement(top, 'Client', name=client, - timestamp=str(int(probed.timestamp))) - for probe in sorted(probed): - lxml.etree.SubElement(cx, 'Probe', name=probe, - value=str(self.probedata[client][probe])) - for group in sorted(self.cgroups[client]): - lxml.etree.SubElement(cx, "Group", name=group) - data = lxml.etree.tostring(top, encoding='UTF-8', - xml_declaration=True, - pretty_print='true') - try: - datafile = open("%s/%s" % (self.data, 'probed.xml'), 'w') - except IOError: - self.logger.error("Failed to write probed.xml") - datafile.write(data.decode('utf-8')) - - def load_data(self): - try: - data = lxml.etree.parse(self.data + '/probed.xml').getroot() - except: - self.logger.error("Failed to read file probed.xml") - return - self.probedata = {} - self.cgroups = {} - for client in data.getchildren(): - self.probedata[client.get('name')] = \ - ClientProbeDataSet(timestamp=client.get("timestamp")) - self.cgroups[client.get('name')] = [] - for pdata in client: - if (pdata.tag == 'Probe'): - self.probedata[client.get('name')][pdata.get('name')] = \ - ProbeData(pdata.get('value')) - elif (pdata.tag == 'Group'): - self.cgroups[client.get('name')].append(pdata.get('name')) - - def GetProbes(self, meta, force=False): - """Return a set of probes for execution on client.""" - return self.probes.get_probe_data(meta) - - def ReceiveData(self, client, datalist): - self.cgroups[client.hostname] = [] - self.probedata[client.hostname] = ClientProbeDataSet() - for data in datalist: - self.ReceiveDataItem(client, data) - self.write_data() - - def ReceiveDataItem(self, client, data): - """Receive probe results pertaining to client.""" - if client.hostname not in self.cgroups: - self.cgroups[client.hostname] = [] - if data.text == None: - self.logger.error("Got null response to probe %s from %s" % \ - (data.get('name'), client.hostname)) - try: - self.probedata[client.hostname].update({data.get('name'): - ProbeData('')}) - except KeyError: - self.probedata[client.hostname] = \ - ClientProbeDataSet([(data.get('name'), ProbeData(''))]) - return - dlines = data.text.split('\n') - self.logger.debug("%s:probe:%s:%s" % (client.hostname, - data.get('name'), [line.strip() for line in dlines])) - for line in dlines[:]: - if line.split(':')[0] == 'group': - newgroup = line.split(':')[1].strip() - if newgroup not in self.cgroups[client.hostname]: - self.cgroups[client.hostname].append(newgroup) - dlines.remove(line) - dobj = ProbeData("\n".join(dlines)) - try: - self.probedata[client.hostname].update({data.get('name'): dobj}) - except KeyError: - self.probedata[client.hostname] = \ - ClientProbeDataSet([(data.get('name'), dobj)]) - - def get_additional_groups(self, meta): - return self.cgroups.get(meta.hostname, list()) - - def get_additional_data(self, meta): - return self.probedata.get(meta.hostname, ClientProbeDataSet()) diff --git a/src/lib/Server/Plugins/Properties.py b/src/lib/Server/Plugins/Properties.py deleted file mode 100644 index 680881858..000000000 --- a/src/lib/Server/Plugins/Properties.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import re -import sys -import copy -import logging -import lxml.etree -import Bcfg2.Server.Plugin - -logger = logging.getLogger('Bcfg2.Plugins.Properties') - -class PropertyFile(Bcfg2.Server.Plugin.StructFile): - """Class for properties files.""" - def write(self): - """ Write the data in this data structure back to the property - file """ - if self.validate_data(): - try: - open(self.name, - "wb").write(lxml.etree.tostring(self.xdata, - pretty_print=True)) - return True - except IOError: - err = sys.exc_info()[1] - logger.error("Failed to write %s: %s" % (self.name, err)) - return False - else: - return False - - def validate_data(self): - """ ensure that the data in this object validates against the - XML schema for this property file (if a schema exists) """ - schemafile = self.name.replace(".xml", ".xsd") - if os.path.exists(schemafile): - try: - schema = lxml.etree.XMLSchema(file=schemafile) - except: - logger.error("Failed to process schema for %s" % self.name) - return False - else: - # no schema exists - return True - - if not schema.validate(self.xdata): - logger.error("Data for %s fails to validate; run bcfg2-lint for " - "more details" % self.name) - return False - else: - return True - - -class PropDirectoryBacked(Bcfg2.Server.Plugin.DirectoryBacked): - __child__ = PropertyFile - patterns = re.compile(r'.*\.xml$') - - -class Properties(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Connector): - """ - The properties plugin maps property - files into client metadata instances. - """ - name = 'Properties' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Connector.__init__(self) - try: - self.store = PropDirectoryBacked(self.data, core.fam) - except OSError: - e = sys.exc_info()[1] - self.logger.error("Error while creating Properties store: %s %s" % - (e.strerror, e.filename)) - raise Bcfg2.Server.Plugin.PluginInitError - - def get_additional_data(self, _): - return copy.copy(self.store.entries) diff --git a/src/lib/Server/Plugins/Rules.py b/src/lib/Server/Plugins/Rules.py deleted file mode 100644 index 684f0c290..000000000 --- a/src/lib/Server/Plugins/Rules.py +++ /dev/null @@ -1,55 +0,0 @@ -"""This generator provides rule-based entry mappings.""" - -import re -import Bcfg2.Server.Plugin - -class RulesConfig(Bcfg2.Server.Plugin.SimpleConfig): - _required = False - -class Rules(Bcfg2.Server.Plugin.PrioDir): - """This is a generator that handles service assignments.""" - name = 'Rules' - __author__ = 'bcfg-dev@mcs.anl.gov' - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.PrioDir.__init__(self, core, datastore) - self.config = RulesConfig(self) - self._regex_cache = dict() - - def HandlesEntry(self, entry, metadata): - if entry.tag in self.Entries: - return self._matches(entry, metadata, - self.Entries[entry.tag].keys()) - return False - - def HandleEntry(self, entry, metadata): - return self.BindEntry(entry, metadata) - - def BindEntry(self, entry, metadata): - attrs = self.get_attrs(entry, metadata) - for key, val in list(attrs.items()): - if key not in entry.attrib: - entry.attrib[key] = val - - def _matches(self, entry, metadata, rules): - if Bcfg2.Server.Plugin.PrioDir._matches(self, entry, metadata, rules): - return True - elif (entry.tag == "Path" and - ((entry.get('name').endswith("/") and - entry.get('name').rstrip("/") in rules) or - (not entry.get('name').endswith("/") and - entry.get('name') + '/' in rules))): - # special case for Path tags: - # http://trac.mcs.anl.gov/projects/bcfg2/ticket/967 - return True - elif self._regex_enabled: - # attempt regular expression matching - for rule in rules: - if rule not in self._regex_cache: - self._regex_cache[rule] = re.compile("%s$" % rule) - if self._regex_cache[rule].match(entry.get('name')): - return True - return False - - def _regex_enabled(self): - return self.config.getboolean("rules", "regex", default=False) diff --git a/src/lib/Server/Plugins/SGenshi.py b/src/lib/Server/Plugins/SGenshi.py deleted file mode 100644 index 0ba08125e..000000000 --- a/src/lib/Server/Plugins/SGenshi.py +++ /dev/null @@ -1,97 +0,0 @@ -'''This module implements a templating generator based on Genshi''' - -import genshi.input -import genshi.template -import lxml.etree -import logging -import copy -import sys -import os.path - -import Bcfg2.Server.Plugin -import Bcfg2.Server.Plugins.TGenshi - -logger = logging.getLogger('Bcfg2.Plugins.SGenshi') - - -class SGenshiTemplateFile(Bcfg2.Server.Plugins.TGenshi.TemplateFile, - Bcfg2.Server.Plugin.StructFile): - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugins.TGenshi.TemplateFile.__init__(self, name, - specific, encoding) - Bcfg2.Server.Plugin.StructFile.__init__(self, name) - - def get_xml_value(self, metadata): - if not hasattr(self, 'template'): - logger.error("No parsed template information for %s" % (self.name)) - raise Bcfg2.Server.Plugin.PluginExecutionError - try: - stream = self.template.generate(metadata=metadata).filter( \ - Bcfg2.Server.Plugins.TGenshi.removecomment) - data = lxml.etree.XML(stream.render('xml', strip_whitespace=False)) - bundlename = os.path.splitext(os.path.basename(self.name))[0] - bundle = lxml.etree.Element('Bundle', name=bundlename) - for item in self.Match(metadata, data): - bundle.append(copy.deepcopy(item)) - return bundle - except LookupError: - lerror = sys.exc_info()[1] - logger.error('Genshi lookup error: %s' % lerror) - except genshi.template.TemplateError: - terror = sys.exc_info()[1] - logger.error('Genshi template error: %s' % terror) - raise - except genshi.input.ParseError: - perror = sys.exc_info()[1] - logger.error('Genshi parse error: %s' % perror) - raise - - def Match(self, metadata, xdata): - """Return matching fragments of parsed template.""" - rv = [] - for child in xdata.getchildren(): - rv.extend(self._match(child, metadata)) - logger.debug("File %s got %d match(es)" % (self.name, len(rv))) - return rv - -class SGenshiEntrySet(Bcfg2.Server.Plugin.EntrySet): - - def __init__(self, path, fam, encoding): - fpattern = '\S+\.xml' - Bcfg2.Server.Plugin.EntrySet.__init__(self, fpattern, path, - SGenshiTemplateFile, encoding) - fam.AddMonitor(path, self) - - def HandleEvent(self, event): - '''passthrough event handler for old calling convention''' - if event.filename != self.path: - return self.handle_event(event) - - def BuildStructures(self, metadata): - """Build SGenshi structures.""" - ret = [] - for entry in self.get_matching(metadata): - try: - ret.append(entry.get_xml_value(metadata)) - except: - logger.error("SGenshi: Failed to template file %s" % entry.name) - return ret - - -class SGenshi(SGenshiEntrySet, - Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Structure): - """The SGenshi plugin provides templated structures.""" - name = 'SGenshi' - __author__ = 'bcfg-dev@mcs.anl.gov' - deprecated = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Structure.__init__(self) - try: - SGenshiEntrySet.__init__(self, self.data, self.core.fam, core.encoding) - except: - logger.error("Failed to load %s repository; disabling %s" \ - % (self.name, self.name)) - raise Bcfg2.Server.Plugin.PluginInitError diff --git a/src/lib/Server/Plugins/SSHbase.py b/src/lib/Server/Plugins/SSHbase.py deleted file mode 100644 index 2e247caa7..000000000 --- a/src/lib/Server/Plugins/SSHbase.py +++ /dev/null @@ -1,413 +0,0 @@ -'''This module manages ssh key files for bcfg2''' - -import binascii -import re -import os -import socket -import shutil -import sys -import tempfile -from subprocess import Popen, PIPE -import Bcfg2.Server.Plugin -from Bcfg2.Bcfg2Py3k import u_str - -if sys.hexversion >= 0x03000000: - from functools import reduce - -import logging -logger = logging.getLogger(__name__) - -class KeyData(Bcfg2.Server.Plugin.SpecificData): - def __init__(self, name, specific, encoding): - Bcfg2.Server.Plugin.SpecificData.__init__(self, name, specific, - encoding) - self.encoding = encoding - - def bind_entry(self, entry, metadata): - entry.set('type', 'file') - if entry.get('encoding') == 'base64': - entry.text = binascii.b2a_base64(self.data) - else: - try: - entry.text = u_str(self.data, self.encoding) - except UnicodeDecodeError: - e = sys.exc_info()[1] - logger.error("Failed to decode %s: %s" % (entry.get('name'), e)) - logger.error("Please verify you are using the proper encoding.") - raise Bcfg2.Server.Plugin.PluginExecutionError - except ValueError: - e = sys.exc_info()[1] - logger.error("Error in specification for %s" % - entry.get('name')) - logger.error(str(e)) - logger.error("You need to specify base64 encoding for %s." % - entry.get('name')) - raise Bcfg2.Server.Plugin.PluginExecutionError - if entry.text in ['', None]: - entry.set('empty', 'true') - -class HostKeyEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, basename, path): - if basename.startswith("ssh_host_key"): - encoding = "base64" - else: - encoding = None - Bcfg2.Server.Plugin.EntrySet.__init__(self, basename, path, KeyData, - encoding) - self.metadata = {'owner': 'root', - 'group': 'root', - 'type': 'file'} - if encoding is not None: - self.metadata['encoding'] = encoding - if basename.endswith('.pub'): - self.metadata['perms'] = '0644' - else: - self.metadata['perms'] = '0600' - - -class KnownHostsEntrySet(Bcfg2.Server.Plugin.EntrySet): - def __init__(self, path): - Bcfg2.Server.Plugin.EntrySet.__init__(self, "ssh_known_hosts", path, - KeyData, None) - self.metadata = {'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '0644'} - - -class SSHbase(Bcfg2.Server.Plugin.Plugin, - Bcfg2.Server.Plugin.Generator, - Bcfg2.Server.Plugin.PullTarget): - """ - The sshbase generator manages ssh host keys (both v1 and v2) - for hosts. It also manages the ssh_known_hosts file. It can - integrate host keys from other management domains and similarly - export its keys. The repository contains files in the following - formats: - - ssh_host_key.H_(hostname) -> the v1 host private key for - (hostname) - ssh_host_key.pub.H_(hostname) -> the v1 host public key - for (hostname) - ssh_host_(ec)(dr)sa_key.H_(hostname) -> the v2 ssh host - private key for (hostname) - ssh_host_(ec)(dr)sa_key.pub.H_(hostname) -> the v2 ssh host - public key for (hostname) - ssh_known_hosts -> the current known hosts file. this - is regenerated each time a new key is generated. - - """ - name = 'SSHbase' - __author__ = 'bcfg-dev@mcs.anl.gov' - - keypatterns = ["ssh_host_dsa_key", - "ssh_host_ecdsa_key", - "ssh_host_rsa_key", - "ssh_host_key", - "ssh_host_dsa_key.pub", - "ssh_host_ecdsa_key.pub", - "ssh_host_rsa_key.pub", - "ssh_host_key.pub"] - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Generator.__init__(self) - Bcfg2.Server.Plugin.PullTarget.__init__(self) - self.ipcache = {} - self.namecache = {} - self.__skn = False - - # keep track of which bogus keys we've warned about, and only - # do so once - self.badnames = dict() - - core.fam.AddMonitor(self.data, self) - - self.static = dict() - self.entries = dict() - self.Entries['Path'] = dict() - - self.entries['/etc/ssh/ssh_known_hosts'] = KnownHostsEntrySet(self.data) - self.Entries['Path']['/etc/ssh/ssh_known_hosts'] = self.build_skn - for keypattern in self.keypatterns: - self.entries["/etc/ssh/" + keypattern] = HostKeyEntrySet(keypattern, - self.data) - self.Entries['Path']["/etc/ssh/" + keypattern] = self.build_hk - - def get_skn(self): - """Build memory cache of the ssh known hosts file.""" - if not self.__skn: - # if no metadata is registered yet, defer - if len(self.core.metadata.query.all()) == 0: - self.__skn = False - return self.__skn - - skn = [s.data.decode().rstrip() - for s in list(self.static.values())] - - mquery = self.core.metadata.query - - # build hostname cache - names = dict() - for cmeta in mquery.all(): - names[cmeta.hostname] = set([cmeta.hostname]) - names[cmeta.hostname].update(cmeta.aliases) - newnames = set() - newips = set() - for name in names[cmeta.hostname]: - newnames.add(name.split('.')[0]) - try: - newips.add(self.get_ipcache_entry(name)[0]) - except: - continue - names[cmeta.hostname].update(newnames) - names[cmeta.hostname].update(cmeta.addresses) - names[cmeta.hostname].update(newips) - # TODO: Only perform reverse lookups on IPs if an option is set. - if True: - for ip in newips: - try: - names[cmeta.hostname].update(self.get_namecache_entry(ip)) - except: - continue - names[cmeta.hostname] = sorted(names[cmeta.hostname]) - - pubkeys = [pubk for pubk in list(self.entries.keys()) - if pubk.endswith('.pub')] - pubkeys.sort() - for pubkey in pubkeys: - for entry in sorted(self.entries[pubkey].entries.values(), - key=lambda e: e.specific.hostname or e.specific.group): - specific = entry.specific - hostnames = [] - if specific.hostname and specific.hostname in names: - hostnames = names[specific.hostname] - elif specific.group: - hostnames = \ - reduce(lambda x, y: x + y, - [names[cmeta.hostname] - for cmeta in \ - mquery.by_groups([specific.group])], []) - elif specific.all: - # a generic key for all hosts? really? - hostnames = reduce(lambda x, y: x + y, - list(names.values()), []) - if not hostnames: - if specific.hostname: - key = specific.hostname - ktype = "host" - elif specific.group: - key = specific.group - ktype = "group" - else: - # user has added a global SSH key, but - # have no clients yet. don't warn about - # this. - continue - - if key not in self.badnames: - self.badnames[key] = True - self.logger.info("Ignoring key for unknown %s %s" % - (ktype, key)) - continue - - skn.append("%s %s" % (','.join(hostnames), - entry.data.decode().rstrip())) - - self.__skn = "\n".join(skn) + "\n" - return self.__skn - - def set_skn(self, value): - """Set backing data for skn.""" - self.__skn = value - skn = property(get_skn, set_skn) - - def HandleEvent(self, event=None): - """Local event handler that does skn regen on pubkey change.""" - # skip events we don't care about - action = event.code2str() - if action == "endExist" or event.filename == self.data: - return - - for entry in list(self.entries.values()): - if entry.specific.match(event.filename): - entry.handle_event(event) - if event.filename.endswith(".pub"): - self.logger.info("New public key %s; invalidating " - "ssh_known_hosts cache" % event.filename) - self.skn = False - return - - if event.filename in ['info', 'info.xml', ':info']: - for entry in list(self.entries.values()): - entry.handle_event(event) - return - - if event.filename.endswith('.static'): - self.logger.info("Static key %s %s; invalidating ssh_known_hosts " - "cache" % (event.filename, action)) - if action == "deleted" and event.filename in self.static: - del self.static[event.filename] - self.skn = False - else: - self.static[event.filename] = \ - Bcfg2.Server.Plugin.FileBacked(os.path.join(self.data, - event.filename)) - self.static[event.filename].HandleEvent(event) - self.skn = False - return - - self.logger.warn("SSHbase: Got unknown event %s %s" % - (event.filename, action)) - - def get_ipcache_entry(self, client): - """Build a cache of dns results.""" - if client in self.ipcache: - if self.ipcache[client]: - return self.ipcache[client] - else: - raise socket.gaierror - else: - # need to add entry - try: - ipaddr = socket.gethostbyname(client) - self.ipcache[client] = (ipaddr, client) - return (ipaddr, client) - except socket.gaierror: - ipaddr = Popen(["getent", "hosts", client], - stdout=PIPE).stdout.read().strip().split() - if ipaddr: - self.ipcache[client] = (ipaddr, client) - return (ipaddr, client) - self.ipcache[client] = False - self.logger.error("Failed to find IP address for %s" % client) - raise socket.gaierror - - def get_namecache_entry(self, cip): - """Build a cache of name lookups from client IP addresses.""" - if cip in self.namecache: - # lookup cached name from IP - if self.namecache[cip]: - return self.namecache[cip] - else: - raise socket.gaierror - else: - # add an entry that has not been cached - try: - rvlookup = socket.gethostbyaddr(cip) - if rvlookup[0]: - self.namecache[cip] = [rvlookup[0]] - else: - self.namecache[cip] = [] - self.namecache[cip].extend(rvlookup[1]) - return self.namecache[cip] - except socket.gaierror: - self.namecache[cip] = False - self.logger.error("Failed to find any names associated with IP address %s" % cip) - raise - - def build_skn(self, entry, metadata): - """This function builds builds a host specific known_hosts file.""" - try: - rv = self.entries[entry.get('name')].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - client = metadata.hostname - entry.text = self.skn - hostkeys = [] - for k in self.keypatterns: - if k.endswith(".pub"): - try: - hostkeys.append(self.entries["/etc/ssh/" + - k].best_matching(metadata)) - except Bcfg2.Server.Plugin.PluginExecutionError: - pass - hostkeys.sort() - for hostkey in hostkeys: - entry.text += "localhost,localhost.localdomain,127.0.0.1 %s" % ( - hostkey.data.decode()) - self.entries[entry.get('name')].bind_info_to_entry(entry, metadata) - - def build_hk(self, entry, metadata): - """This binds host key data into entries.""" - try: - self.entries[entry.get('name')].bind_entry(entry, metadata) - except Bcfg2.Server.Plugin.PluginExecutionError: - filename = entry.get('name').split('/')[-1] - self.GenerateHostKeyPair(metadata.hostname, filename) - # Service the FAM events queued up by the key generation - # so the data structure entries will be available for - # binding. - # - # NOTE: We wait for up to ten seconds. There is some - # potential for race condition, because if the file - # monitor doesn't get notified about the new key files in - # time, those entries won't be available for binding. In - # practice, this seems "good enough". - tries = 0 - is_bound = False - while not is_bound: - if tries >= 10: - self.logger.error("%s still not registered" % filename) - raise Bcfg2.Server.Plugin.PluginExecutionError - self.core.fam.handle_events_in_interval(1) - tries += 1 - try: - self.entries[entry.get('name')].bind_entry(entry, metadata) - is_bound = True - except Bcfg2.Server.Plugin.PluginExecutionError: - pass - - def GenerateHostKeyPair(self, client, filename): - """Generate new host key pair for client.""" - match = re.search(r'(ssh_host_(?:((?:ecd|d|r)sa)_)?key)', filename) - if match: - hostkey = "%s.H_%s" % (match.group(1), client) - if match.group(2): - keytype = match.group(2) - else: - keytype = 'rsa1' - else: - self.logger.error("Unknown key filename: %s" % filename) - return - - fileloc = "%s/%s" % (self.data, hostkey) - publoc = self.data + '/' + ".".join([hostkey.split('.')[0], 'pub', - "H_%s" % client]) - tempdir = tempfile.mkdtemp() - temploc = "%s/%s" % (tempdir, hostkey) - cmd = ["ssh-keygen", "-q", "-f", temploc, "-N", "", - "-t", keytype, "-C", "root@%s" % client] - proc = Popen(cmd, stdout=PIPE, stdin=PIPE) - proc.communicate() - proc.wait() - - try: - shutil.copy(temploc, fileloc) - shutil.copy("%s.pub" % temploc, publoc) - except IOError: - err = sys.exc_info()[1] - self.logger.error("Temporary SSH keys not found: %s" % err) - - try: - os.unlink(temploc) - os.unlink("%s.pub" % temploc) - os.rmdir(tempdir) - except OSError: - err = sys.exc_info()[1] - self.logger.error("Failed to unlink temporary ssh keys: %s" % err) - - def AcceptChoices(self, _, metadata): - return [Bcfg2.Server.Plugin.Specificity(hostname=metadata.hostname)] - - def AcceptPullData(self, specific, entry, log): - """Per-plugin bcfg2-admin pull support.""" - # specific will always be host specific - filename = "%s/%s.H_%s" % (self.data, entry['name'].split('/')[-1], - specific.hostname) - try: - open(filename, 'w').write(entry['text']) - if log: - print("Wrote file %s" % filename) - except KeyError: - self.logger.error("Failed to pull %s. This file does not currently " - "exist on the client" % entry.get('name')) diff --git a/src/lib/Server/Plugins/SSLCA.py b/src/lib/Server/Plugins/SSLCA.py deleted file mode 100644 index 0072dc62d..000000000 --- a/src/lib/Server/Plugins/SSLCA.py +++ /dev/null @@ -1,274 +0,0 @@ -import Bcfg2.Server.Plugin -import Bcfg2.Options -import lxml.etree -import posixpath -import tempfile -import pipes -import os -from subprocess import Popen, PIPE, STDOUT -# Compatibility import -from Bcfg2.Bcfg2Py3k import ConfigParser - - -class SSLCA(Bcfg2.Server.Plugin.GroupSpool): - """ - The SSLCA generator handles the creation and - management of ssl certificates and their keys. - """ - name = 'SSLCA' - __author__ = 'g.hagger@gmail.com' - __child__ = Bcfg2.Server.Plugin.FileBacked - key_specs = {} - cert_specs = {} - CAs = {} - - def HandleEvent(self, event=None): - """ - Updates which files this plugin handles based upon filesystem events. - Allows configuration items to be added/removed without server restarts. - """ - action = event.code2str() - if event.filename[0] == '/': - return - epath = "".join([self.data, self.handles[event.requestID], - event.filename]) - if posixpath.isdir(epath): - ident = self.handles[event.requestID] + event.filename - else: - ident = self.handles[event.requestID][:-1] - - fname = "".join([ident, '/', event.filename]) - - if event.filename.endswith('.xml'): - if action in ['exists', 'created', 'changed']: - if event.filename.endswith('key.xml'): - key_spec = dict(list(lxml.etree.parse(epath).find('Key').items())) - self.key_specs[ident] = { - 'bits': key_spec.get('bits', 2048), - 'type': key_spec.get('type', 'rsa') - } - self.Entries['Path'][ident] = self.get_key - elif event.filename.endswith('cert.xml'): - cert_spec = dict(list(lxml.etree.parse(epath).find('Cert').items())) - ca = cert_spec.get('ca', 'default') - self.cert_specs[ident] = { - 'ca': ca, - 'format': cert_spec.get('format', 'pem'), - 'key': cert_spec.get('key'), - 'days': cert_spec.get('days', 365), - 'C': cert_spec.get('c'), - 'L': cert_spec.get('l'), - 'ST': cert_spec.get('st'), - 'OU': cert_spec.get('ou'), - 'O': cert_spec.get('o'), - 'emailAddress': cert_spec.get('emailaddress') - } - cp = ConfigParser.ConfigParser() - cp.read(self.core.cfile) - self.CAs[ca] = dict(cp.items('sslca_' + ca)) - self.Entries['Path'][ident] = self.get_cert - if action == 'deleted': - if ident in self.Entries['Path']: - del self.Entries['Path'][ident] - else: - if action in ['exists', 'created']: - if posixpath.isdir(epath): - self.AddDirectoryMonitor(epath[len(self.data):]) - if ident not in self.entries and posixpath.isfile(epath): - self.entries[fname] = self.__child__(epath) - self.entries[fname].HandleEvent(event) - if action == 'changed': - self.entries[fname].HandleEvent(event) - elif action == 'deleted': - if fname in self.entries: - del self.entries[fname] - else: - self.entries[fname].HandleEvent(event) - - def get_key(self, entry, metadata): - """ - either grabs a prexisting key hostfile, or triggers the generation - of a new key if one doesn't exist. - """ - # set path type and permissions, otherwise bcfg2 won't bind the file - permdata = {'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - - # check if we already have a hostfile, or need to generate a new key - # TODO: verify key fits the specs - path = entry.get('name') - filename = "".join([path, '/', path.rsplit('/', 1)[1], - '.H_', metadata.hostname]) - if filename not in list(self.entries.keys()): - key = self.build_key(filename, entry, metadata) - open(self.data + filename, 'w').write(key) - entry.text = key - self.entries[filename] = self.__child__("%s%s" % (self.data, - filename)) - self.entries[filename].HandleEvent() - else: - entry.text = self.entries[filename].data - - def build_key(self, filename, entry, metadata): - """ - generates a new key according the the specification - """ - type = self.key_specs[entry.get('name')]['type'] - bits = self.key_specs[entry.get('name')]['bits'] - if type == 'rsa': - cmd = ["openssl", "genrsa", bits] - elif type == 'dsa': - cmd = ["openssl", "dsaparam", "-noout", "-genkey", bits] - key = Popen(cmd, stdout=PIPE).stdout.read() - return key - - def get_cert(self, entry, metadata): - """ - either grabs a prexisting cert hostfile, or triggers the generation - of a new cert if one doesn't exist. - """ - # set path type and permissions, otherwise bcfg2 won't bind the file - permdata = {'owner': 'root', - 'group': 'root', - 'type': 'file', - 'perms': '644'} - [entry.attrib.__setitem__(key, permdata[key]) for key in permdata] - - path = entry.get('name') - filename = "".join([path, '/', path.rsplit('/', 1)[1], - '.H_', metadata.hostname]) - - # first - ensure we have a key to work with - key = self.cert_specs[entry.get('name')].get('key') - key_filename = "".join([key, '/', key.rsplit('/', 1)[1], - '.H_', metadata.hostname]) - if key_filename not in self.entries: - e = lxml.etree.Element('Path') - e.attrib['name'] = key - self.core.Bind(e, metadata) - - # check if we have a valid hostfile - if filename in list(self.entries.keys()) and self.verify_cert(filename, - key_filename, - entry): - entry.text = self.entries[filename].data - else: - cert = self.build_cert(key_filename, entry, metadata) - open(self.data + filename, 'w').write(cert) - self.entries[filename] = self.__child__("%s%s" % (self.data, - filename)) - self.entries[filename].HandleEvent() - entry.text = cert - - def verify_cert(self, filename, key_filename, entry): - if self.verify_cert_against_ca(filename, entry): - if self.verify_cert_against_key(filename, key_filename): - return True - return False - - def verify_cert_against_ca(self, filename, entry): - """ - check that a certificate validates against the ca cert, - and that it has not expired. - """ - chaincert = self.CAs[self.cert_specs[entry.get('name')]['ca']].get('chaincert') - cert = self.data + filename - res = Popen(["openssl", "verify", "-CAfile", chaincert, cert], - stdout=PIPE, stderr=STDOUT).stdout.read() - if res == cert + ": OK\n": - return True - return False - - def verify_cert_against_key(self, filename, key_filename): - """ - check that a certificate validates against its private key. - """ - cert = self.data + filename - key = self.data + key_filename - cmd = ("openssl x509 -noout -modulus -in %s | openssl md5" % - pipes.quote(cert)) - cert_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read() - cmd = ("openssl rsa -noout -modulus -in %s | openssl md5" % - pipes.quote(key)) - key_md5 = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT).stdout.read() - if cert_md5 == key_md5: - return True - return False - - def build_cert(self, key_filename, entry, metadata): - """ - creates a new certificate according to the specification - """ - req_config = self.build_req_config(entry, metadata) - req = self.build_request(key_filename, req_config, entry) - ca = self.cert_specs[entry.get('name')]['ca'] - ca_config = self.CAs[ca]['config'] - days = self.cert_specs[entry.get('name')]['days'] - passphrase = self.CAs[ca].get('passphrase') - cmd = ["openssl", "ca", "-config", ca_config, "-in", req, - "-days", days, "-batch"] - if passphrase: - cmd.extend(["-passin", "pass:%s" % passphrase]) - cert = Popen(cmd, stdout=PIPE).stdout.read() - try: - os.unlink(req_config) - os.unlink(req) - except OSError: - self.logger.error("Failed to unlink temporary files") - return cert - - def build_req_config(self, entry, metadata): - """ - generates a temporary openssl configuration file that is - used to generate the required certificate request - """ - # create temp request config file - conffile = open(tempfile.mkstemp()[1], 'w') - cp = ConfigParser.ConfigParser({}) - cp.optionxform = str - defaults = { - 'req': { - 'default_md': 'sha1', - 'distinguished_name': 'req_distinguished_name', - 'req_extensions': 'v3_req', - 'x509_extensions': 'v3_req', - 'prompt': 'no' - }, - 'req_distinguished_name': {}, - 'v3_req': { - 'subjectAltName': '@alt_names' - }, - 'alt_names': {} - } - for section in list(defaults.keys()): - cp.add_section(section) - for key in defaults[section]: - cp.set(section, key, defaults[section][key]) - x = 1 - altnames = list(metadata.aliases) - altnames.append(metadata.hostname) - for altname in altnames: - cp.set('alt_names', 'DNS.' + str(x), altname) - x += 1 - for item in ['C', 'L', 'ST', 'O', 'OU', 'emailAddress']: - if self.cert_specs[entry.get('name')][item]: - cp.set('req_distinguished_name', item, self.cert_specs[entry.get('name')][item]) - cp.set('req_distinguished_name', 'CN', metadata.hostname) - cp.write(conffile) - conffile.close() - return conffile.name - - def build_request(self, key_filename, req_config, entry): - """ - creates the certificate request - """ - req = tempfile.mkstemp()[1] - days = self.cert_specs[entry.get('name')]['days'] - key = self.data + key_filename - cmd = ["openssl", "req", "-new", "-config", req_config, - "-days", days, "-key", key, "-text", "-out", req] - res = Popen(cmd, stdout=PIPE).stdout.read() - return req diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py deleted file mode 100644 index aeb3b9f74..000000000 --- a/src/lib/Server/Plugins/Snapshots.py +++ /dev/null @@ -1,142 +0,0 @@ -#import lxml.etree -import logging -import binascii -import difflib -#import sqlalchemy -#import sqlalchemy.orm -import Bcfg2.Server.Plugin -import Bcfg2.Server.Snapshots -import Bcfg2.Logger -from Bcfg2.Server.Snapshots.model import Snapshot -import sys -import time -import threading - -# Compatibility import -from Bcfg2.Bcfg2Py3k import Queue - -logger = logging.getLogger('Snapshots') - -ftypes = ['ConfigFile', 'SymLink', 'Directory'] -datafields = { - 'Package': ['version'], - 'Path': ['type'], - 'Service': ['status'], - 'ConfigFile': ['owner', 'group', 'perms'], - 'Directory': ['owner', 'group', 'perms'], - 'SymLink': ['to'], - } - - -# py3k compatibility -def u_str(string): - if sys.hexversion >= 0x03000000: - return string - else: - return unicode(string) - - -def build_snap_ent(entry): - basefields = [] - if entry.tag in ['Package', 'Service']: - basefields += ['type'] - desired = dict([(key, u_str(entry.get(key))) for key in basefields]) - state = dict([(key, u_str(entry.get(key))) for key in basefields]) - desired.update([(key, u_str(entry.get(key))) for key in \ - datafields[entry.tag]]) - if entry.tag == 'ConfigFile' or \ - ((entry.tag == 'Path') and (entry.get('type') == 'file')): - if entry.text == None: - desired['contents'] = None - else: - if entry.get('encoding', 'ascii') == 'ascii': - desired['contents'] = u_str(entry.text) - else: - desired['contents'] = u_str(binascii.a2b_base64(entry.text)) - - if 'current_bfile' in entry.attrib: - state['contents'] = u_str(binascii.a2b_base64( \ - entry.get('current_bfile'))) - elif 'current_bdiff' in entry.attrib: - diff = binascii.a2b_base64(entry.get('current_bdiff')) - state['contents'] = u_str( \ - '\n'.join(difflib.restore(diff.split('\n'), 1))) - - state.update([(key, u_str(entry.get('current_' + key, entry.get(key)))) \ - for key in datafields[entry.tag]]) - if entry.tag in ['ConfigFile', 'Path'] and entry.get('exists', 'true') == 'false': - state = None - return [desired, state] - - -class Snapshots(Bcfg2.Server.Plugin.Statistics, - Bcfg2.Server.Plugin.Plugin): - name = 'Snapshots' - experimental = True - - def __init__(self, core, datastore): - Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) - Bcfg2.Server.Plugin.Statistics.__init__(self) - self.session = Bcfg2.Server.Snapshots.setup_session(core.cfile) - self.work_queue = Queue() - self.loader = threading.Thread(target=self.load_snapshot) - self.loader.start() - - def load_snapshot(self): - while self.running: - try: - (metadata, data) = self.work_queue.get(block=True, timeout=5) - except: - continue - self.statistics_from_old_stats(metadata, data) - - def process_statistics(self, metadata, data): - return self.work_queue.put((metadata, data)) - - def statistics_from_old_stats(self, metadata, xdata): - # entries are name -> (modified, correct, start, desired, end) - # not sure we can get all of this from old format stats - t1 = time.time() - entries = dict([('Package', dict()), - ('Service', dict()), ('Path', dict())]) - extra = dict([('Package', dict()), ('Service', dict()), - ('Path', dict())]) - bad = [] - state = xdata.find('.//Statistics') - correct = state.get('state') == 'clean' - revision = u_str(state.get('revision', '-1')) - for entry in state.find('.//Bad'): - data = [False, False, u_str(entry.get('name'))] \ - + build_snap_ent(entry) - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - entries[etag][entry.get('name')] = data - for entry in state.find('.//Modified'): - if entry.tag in ftypes: - etag = 'Path' - else: - etag = entry.tag - if entry.get('name') in entries[etag]: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - else: - data = [True, False, u_str(entry.get('name'))] + \ - build_snap_ent(entry) - for entry in state.find('.//Extra'): - if entry.tag in datafields: - data = build_snap_ent(entry)[1] - ename = u_str(entry.get('name')) - data['name'] = ename - extra[entry.tag][ename] = data - else: - print("extra", entry.tag, entry.get('name')) - t2 = time.time() - snap = Snapshot.from_data(self.session, correct, revision, - metadata, entries, extra) - self.session.add(snap) - self.session.commit() - t3 = time.time() - logger.info("Snapshot storage took %fs" % (t3 - t2)) - return True diff --git a/src/lib/Server/Plugins/Statistics.py b/src/lib/Server/Plugins/Statistics.py deleted file mode 100644 index 265ef95a8..000000000 --- a/src/lib/Server/Plugins/Statistics.py +++ /dev/null @@ -1,161 +0,0 @@ -'''This file manages the statistics collected by the BCFG2 Server''' - -import binascii -import copy -import difflib -import logging -from lxml.etree import XML, SubElement, Element, XMLSyntaxError -import lxml.etree -import os -from time import asctime, localtime, time, strptime, mktime -import threading - -import Bcfg2.Server.Plugin - - -class StatisticsStore(object): - """Manages the memory and file copy of statistics collected about client runs.""" - __min_write_delay__ = 0 - - def __init__(self, filename): - self.filename = filename - self.element = Element('Dummy') - self.dirty = 0 - self.lastwrite = 0 - self.logger = logging.getLogger('Bcfg2.Server.Statistics') - self.ReadFromFile() - - def WriteBack(self, force=0): - """Write statistics changes back to persistent store.""" - if (self.dirty and (self.lastwrite + self.__min_write_delay__ <= time())) \ - or force: - try: - fout = open(self.filename + '.new', 'w') - except IOError: - ioerr = sys.exc_info()[1] - self.logger.error("Failed to open %s for writing: %s" % (self.filename + '.new', ioerr)) - else: - fout.write(lxml.etree.tostring(self.element, encoding='UTF-8', xml_declaration=True)) - fout.close() - os.rename(self.filename + '.new', self.filename) - self.dirty = 0 - self.lastwrite = time() - - def ReadFromFile(self): - """Reads current state regarding statistics.""" - try: - fin = open(self.filename, 'r') - data = fin.read() - fin.close() - self.element = XML(data) - self.dirty = 0 - except (IOError, XMLSyntaxError): - self.logger.error("Creating new statistics file %s"%(self.filename)) - self.element = Element('ConfigStatistics') - self.WriteBack() - self.dirty = 0 - - def updateStats(self, xml, client): - """Updates the statistics of a current node with new data.""" - - # Current policy: - # - Keep anything less than 24 hours old - # - Keep latest clean run for clean nodes - # - Keep latest clean and dirty run for dirty nodes - newstat = xml.find('Statistics') - - if newstat.get('state') == 'clean': - node_dirty = 0 - else: - node_dirty = 1 - - # Find correct node entry in stats data - # The following list comprehension should be guarenteed to return at - # most one result - nodes = [elem for elem in self.element.findall('Node') \ - if elem.get('name') == client] - nummatch = len(nodes) - if nummatch == 0: - # Create an entry for this node - node = SubElement(self.element, 'Node', name=client) - elif nummatch == 1 and not node_dirty: - # Delete old instance - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if self.isOlderThan24h(elem.get('time'))] - elif nummatch == 1 and node_dirty: - # Delete old dirty statistics entry - node = nodes[0] - [node.remove(elem) for elem in node.findall('Statistics') \ - if (elem.get('state') == 'dirty' \ - and self.isOlderThan24h(elem.get('time')))] - else: - # Shouldn't be reached - self.logger.error("Duplicate node entry for %s"%(client)) - - # Set current time for stats - newstat.set('time', asctime(localtime())) - - # Add statistic - node.append(copy.copy(newstat)) - - # Set dirty - self.dirty = 1 - self.WriteBack(force=1) - - def isOlderThan24h(self, testTime): - """Helper function to determine if