From 65355e11b840687c6e1076fe5a7ce6d35d5de2fc Mon Sep 17 00:00:00 2001 From: Narayan Desai Date: Mon, 16 Feb 2009 22:21:18 +0000 Subject: Snapshots: Implement metadata loads and basic row de-duplication git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5079 ce84e21b-d406-0410-9b95-82705330c041 --- src/lib/Server/Snapshots/model.py | 73 +++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py index 29bf059fe..4076bed9f 100644 --- a/src/lib/Server/Snapshots/model.py +++ b/src/lib/Server/Snapshots/model.py @@ -1,40 +1,51 @@ from sqlalchemy import Table, Column, Integer, Unicode, MetaData, ForeignKey, Boolean, DateTime, create_engine, UnicodeText -from sqlalchemy.orm import relation, backref +import sqlalchemy.exceptions +from sqlalchemy.orm import relation, backref, sessionmaker from sqlalchemy.ext.declarative import declarative_base # TODO pingtime? -# TODO modified entries? +# backlinks Client -> Snapshot +# entry missing fields +# extra entries + +class Uniquer(object): + @classmethod + def by_value(cls, session, **kwargs): + try: + return session.query(cls).filter_by(**kwargs).one() + except sqlalchemy.exceptions.InvalidRequestError: + return cls(**kwargs) Base = declarative_base() -class Administrator(Base): +class Administrator(Uniquer, Base): __tablename__ = 'administrator' id = Column(Integer, primary_key=True) - name = Column(Unicode(20)) + name = Column(Unicode(20), unique=True) email = Column(Unicode(64)) admin_client = Table('admin_client', Base.metadata, Column('admin_id', Integer, ForeignKey('administrator.id')), Column('client_id', Integer, ForeignKey('client.id'))) -class Client(Base): +class Client(Uniquer, Base): __tablename__ = 'client' id = Column(Integer, primary_key=True) - name = Column(Unicode(64)) + name = Column(Unicode(64), unique=True) admins = relation("Administrator", secondary=admin_client) - active = Column(Boolean) + active = Column(Boolean, default=True) -class Group(Base): +class Group(Uniquer, Base): __tablename__ = 'group' - id = Column(Integer, primary_key=True) - name = Column(Unicode(32)) + id = Column(Integer, primary_key=True) + name = Column(Unicode(32), unique=True) -class ConnectorKeyVal(Base): +class ConnectorKeyVal(Uniquer, Base): __tablename__ = 'connkeyval' id = Column(Integer, primary_key=True) connector = Column(Unicode(16)) - key = Column(UnicodeText) + key = Column(Unicode(32)) value = Column(UnicodeText) meta_group = Table('meta_group', Base.metadata, @@ -47,11 +58,31 @@ meta_conn = Table('meta_conn', Base.metadata, class Metadata(Base): __tablename__ = 'metadata' - id = Column(Integer, primary_key=True) - name = Column(Unicode(64)) + id = Column(Integer, primary_key=True) + client_id = Column(Integer, ForeignKey('client.id')) + client = relation(Client) groups = relation("Group", secondary=meta_group) keyvals = relation(ConnectorKeyVal, secondary=meta_conn) + @classmethod + def from_metadata(cls, session, metadata): + client = Client.by_value(session, name=metadata.hostname) + m = cls(client=client) + for group in metadata.groups: + m.groups.append(Group.by_value(session, name=unicode(group))) + for connector in metadata.connectors: + data = getattr(metadata, connector) + if not isinstance(data, dict): + continue + for key, value in data.iteritems(): + if not isinstance(value, str): + continue + m.keyvals.append(ConnectorKeyVal.by_value(session, + connector=unicode(connector), + key=unicode(key), + value=unicode(value))) + return m + class Package(Base): __tablename__ = 'package' id = Column(Integer, primary_key=True) @@ -67,6 +98,7 @@ class PackageCorrespondence(Base): desired = relation(Package, primaryjoin=desired_id == Package.id) incorrect_id = Column(Integer, ForeignKey('package.id'), nullable=True) incorrect = relation(Package, primaryjoin=incorrect_id == Package.id) + modified = Column(Boolean) package_snap = Table('package_snap', Base.metadata, Column('ppair_id', Integer, ForeignKey('package_pair.id')), @@ -86,6 +118,7 @@ class ServiceCorrespondence(Base): desired = relation(Service, primaryjoin=desired_id == Service.id) incorrect_id = Column(Integer, ForeignKey('service.id'), nullable=True) incorrect = relation(Service, primaryjoin=incorrect_id == Service.id) + modified = Column(Boolean) service_snap = Table('service_snap', Base.metadata, Column('spair_id', Integer, ForeignKey('service_pair.id')), @@ -108,6 +141,7 @@ class FileCorrespondence(Base): desired = relation(File, primaryjoin=desired_id == File.id) incorrect_id = Column(Integer, ForeignKey('file.id'), nullable=True) incorrect = relation(File, primaryjoin=incorrect_id == File.id) + modified = Column(Boolean) file_snap = Table('file_snap', Base.metadata, Column('fpair_id', Integer, ForeignKey('file_pair.id')), @@ -126,12 +160,15 @@ class Snapshot(Base): metadata_id = Column(Integer, ForeignKey('metadata.id')) client_metadata = relation(Metadata, primaryjoin=metadata_id==Metadata.id) timestamp = Column(DateTime) + client_id = Column(Integer, ForeignKey('client.id')) client = relation(Client, backref=backref('snapshots', order_by=timestamp)) packages = relation(PackageCorrespondence, secondary=package_snap) services = relation(ServiceCorrespondence, secondary=service_snap) files = relation(FileCorrespondence, secondary=file_snap) -if __name__ == '__main__': - engine = create_engine('sqlite:///:memory:', echo=True) - metadata = Base.metadata - metadata.create_all(engine) +engine = create_engine('sqlite:///:memory:', echo=True) +metadata = Base.metadata +metadata.create_all(engine) +Session = sessionmaker() +Session.configure(bind=engine) +session = Session() -- cgit v1.2.3-1-g7c22