From 1e3f8604797459a1cb45db617958256557cde9d8 Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Mon, 2 Mar 2009 04:05:06 +0000 Subject: Snapshots update - storage of packages works from clients (old-style statistics) git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5092 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Server/Admin/Snapshots.py | 29 +++++++++++++++++++ src/lib/Server/Admin/__init__.py | 2 +- src/lib/Server/Plugins/Snapshots.py | 56 ++++++++++++++++++++++++++++++++++++ src/lib/Server/Snapshots/__init__.py | 22 +++++++++++++- src/lib/Server/Snapshots/model.py | 42 +++++++++++++++++++++------ 5 files changed, 140 insertions(+), 11 deletions(-) create mode 100644 src/lib/Server/Admin/Snapshots.py create mode 100644 src/lib/Server/Plugins/Snapshots.py diff --git a/src/lib/Server/Admin/Snapshots.py b/src/lib/Server/Admin/Snapshots.py new file mode 100644 index 000000000..17db0af62 --- /dev/null +++ b/src/lib/Server/Admin/Snapshots.py @@ -0,0 +1,29 @@ +import Bcfg2.Server.Admin +import sqlalchemy, sqlalchemy.orm +import Bcfg2.Server.Snapshots +import Bcfg2.Server.Snapshots.model + +class Snapshots(Bcfg2.Server.Admin.Mode): + __shorthelp__ = "Interact with the Snapshots system" + __longhelp__ = (__shorthelp__) + __usage__ = ("bcfg2-admin snapshots [init|query qtype] ") + + q_dispatch = {'client':Bcfg2.Server.Snapshots.model.Client, + 'group':Bcfg2.Server.Snapshots.model.Group, + 'snapshot':Bcfg2.Server.Snapshots.model.Snapshot, + } + + def __init__(self, configfile): + Bcfg2.Server.Admin.Mode.__init__(self, configfile) + self.session = Bcfg2.Server.Snapshots.setup_session(debug=True) + + def __call__(self, args): + if args[0] == 'query': + if args[1] in self.q_dispatch: + q_obj = self.q_dispatch[args[1]] + results = self.session.query(q_obj).all() + else: + print 'error' + raise SystemExit, 1 + for result in results: + print result.name diff --git a/src/lib/Server/Admin/__init__.py b/src/lib/Server/Admin/__init__.py index 075822a56..e92020e5d 100644 --- a/src/lib/Server/Admin/__init__.py +++ b/src/lib/Server/Admin/__init__.py @@ -1,7 +1,7 @@ __revision__ = '$Revision$' __all__ = ['Mode', 'Client', 'Compare', 'Fingerprint', 'Init', 'Minestruct', - 'Pull', 'Query', 'Tidy', 'Viz'] + 'Pull', 'Query', 'Snapshots', 'Tidy', 'Viz'] import ConfigParser, lxml.etree, logging, sys import Bcfg2.Server.Core diff --git a/src/lib/Server/Plugins/Snapshots.py b/src/lib/Server/Plugins/Snapshots.py new file mode 100644 index 000000000..f5b69d18f --- /dev/null +++ b/src/lib/Server/Plugins/Snapshots.py @@ -0,0 +1,56 @@ +import lxml.etree +import sqlalchemy +import sqlalchemy.orm +import Bcfg2.Server.Plugin +import Bcfg2.Server.Snapshots +from Bcfg2.Server.Snapshots.model import Snapshot +import time + +class Snapshots(Bcfg2.Server.Plugin.Statistics, + Bcfg2.Server.Plugin.Plugin): + name = 'Snapshots' + experimental = True + def __init__(self, core, datastore): + Bcfg2.Server.Plugin.Plugin.__init__(self, core, datastore) + Bcfg2.Server.Plugin.Statistics.__init__(self) + self.session = Bcfg2.Server.Snapshots.db_from_config() + + def process_statistics(self, metadata, data): + return self.statistics_from_old_stats(metadata, data) + + def statistics_from_old_stats(self, metadata, xdata): + # entries are name -> (modified, correct, start, desired, end) + # not sure we can get all of this from old format stats + t1 = time.time() + entries = dict([('Package', dict()), + ('Service', dict()), ('Path', dict())]) + extra = dict([('Package', list()), ('Service', list()), + ('Path', list())]) + for entry in xdata.find('.//Bad'): + print entry.tag, entry.get('name') + if entry.tag == 'Package': + data = [False, False, unicode(entry.get('type')), + unicode(entry.get('current_version')), + unicode(entry.get('current_version'))] + entries['Package'][entry.get('name')] = data + for entry in xdata.find('.//Modified'): + print entry.tag, entry.get('name') + if entry.tag == 'Package': + if entry.get('name') in entries['Package']: + entries['Package'][entry.get('name')][0] = True + else: + data = [True, True, unicode(entry.get('type')), + unicode(entry.get('current_version')), + unicode(entry.get('version'))] + for entry in xdata.find('.//Extra'): + if entry.tag == 'Package': + edata = dict([('name', unicode(entry.get('name'))), + ('type', unicode(entry.get('type'))), + ('version', unicode(entry.get('version')))]) + extra['Package'].append(edata) + t2 = time.time() + snap = Snapshot.from_data(self.session, metadata, entries, extra) + self.session.save(snap) + self.session.commit() + t3 = time.time() + return True diff --git a/src/lib/Server/Snapshots/__init__.py b/src/lib/Server/Snapshots/__init__.py index 74a3894d7..b94511ebd 100644 --- a/src/lib/Server/Snapshots/__init__.py +++ b/src/lib/Server/Snapshots/__init__.py @@ -1 +1,21 @@ -__all__ = ['models'] +__all__ = ['models', 'db_from_config', 'setup_session'] + +import sqlalchemy, sqlalchemy.orm, ConfigParser + +def db_from_config(fname='/etc/bcfg2.conf'): + cp = ConfigParser.ConfigParser() + cp.read([fname]) + driver = cp.get('snapshots', 'driver') + if driver == 'sqlite': + path = cp.get('snapshots', 'database') + return 'sqlite:///%s' % path + else: + raise Exception, "not done yet" + + +def setup_session(debug=False): + engine = sqlalchemy.create_engine(db_from_config(), + echo=debug) + Session = sqlalchemy.orm.sessionmaker() + Session.configure(bind=engine) + return Session() diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 5489edff5..97331ffbf 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -1,5 +1,5 @@ from sqlalchemy import Table, Column, Integer, Unicode, MetaData, ForeignKey, Boolean, DateTime, create_engine, UnicodeText - +import datetime import sqlalchemy.exceptions from sqlalchemy.orm import relation, backref, sessionmaker from sqlalchemy.ext.declarative import declarative_base @@ -63,11 +63,11 @@ class Metadata(Base): timestamp = Column(DateTime) @classmethod - def from_metadata(cls, session, metadata): - client = Client.by_value(session, name=metadata.hostname) + def from_metadata(cls, mysession, mymetadata): + client = Client.by_value(mysession, name=unicode(mymetadata.hostname)) m = cls(client=client) for group in metadata.groups: - m.groups.append(Group.by_value(session, name=unicode(group))) + m.groups.append(Group.by_value(mysession, name=unicode(group))) for connector in metadata.connectors: data = getattr(metadata, connector) if not isinstance(data, dict): @@ -75,13 +75,13 @@ class Metadata(Base): for key, value in data.iteritems(): if not isinstance(value, str): continue - m.keyvals.append(ConnectorKeyVal.by_value(session, + m.keyvals.append(ConnectorKeyVal.by_value(mysession, connector=unicode(connector), key=unicode(key), value=unicode(value))) return m -class Package(Base): +class Package(Base, Uniquer): __tablename__ = 'package' id = Column(Integer, primary_key=True) name = Column(Unicode(24)) @@ -99,6 +99,18 @@ class PackageCorrespondence(Base): modified = Column(Boolean) correct = Column(Boolean) + @classmethod + def from_record(cls, mysession, name, record): + (mod, corr, ptype, s_vers, e_vers) = record + start = Package.by_value(mysession, name=unicode(name), type=ptype, + version=s_vers) + if s_vers != e_vers: + start = Package.by_value(mysession, name=unicode(name), type=ptype, + version=e_vers) + else: + end = start + return cls(start=start, end=end, modified=mod, correct=corr) + package_snap = Table('package_snap', Base.metadata, Column('ppair_id', Integer, ForeignKey('package_pair.id')), Column('snapshot_id', Integer, ForeignKey('snapshot.id'))) @@ -142,7 +154,7 @@ class FileCorrespondence(Base): end_id = Column(Integer, ForeignKey('file.id'), nullable=True) end = relation(File, primaryjoin=end_id == File.id) modified = Column(Boolean) - correct = Column(Boolean) + correct = Column(Boolean) file_snap = Table('file_snap', Base.metadata, Column('fpair_id', Integer, ForeignKey('file_pair.id')), @@ -176,7 +188,7 @@ class Snapshot(Base): id = Column(Integer, primary_key=True) metadata_id = Column(Integer, ForeignKey('metadata.id')) client_metadata = relation(Metadata, primaryjoin=metadata_id==Metadata.id) - timestamp = Column(DateTime) + timestamp = Column(DateTime, default=datetime.datetime.now) client_id = Column(Integer, ForeignKey('client.id')) client = relation(Client, backref=backref('snapshots')) packages = relation(PackageCorrespondence, secondary=package_snap) @@ -187,8 +199,20 @@ class Snapshot(Base): extra_services = relation(Service, secondary=extra_service_snap) extra_files = relation(File, secondary=extra_file_snap) + @classmethod + def from_data(cls, session, metadata, entries, extra): + dbm = Metadata.from_metadata(session, metadata) + snap = cls(client_metadata=dbm, timestamp=datetime.datetime.now()) + for pkg, pdata in entries['Package'].iteritems(): + snap.packages.append(\ + PackageCorrespondence.from_record(session, pkg, pdata)) + for data in extra['Package']: + extra_pkg = Package.by_value(session, **data) + snap.extra_packages.append(extra_pkg) + return snap + if __name__ == '__main__': - engine = create_engine('sqlite:///:memory:', echo=True) + engine = create_engine('sqlite:////tmp/snapshots.db', echo=True) metadata = Base.metadata metadata.create_all(engine) Session = sessionmaker() -- cgit v1.2.3-1-g7c22