Snapshots: Implement metadata loads and basic row de-duplication

git-svn-id: https://svn.mcs.anl.gov/repos/bcfg/trunk/bcfg2@5079 ce84e21b-d406-0410-9b95-82705330c041

1 files changed, 55 insertions, 18 deletions

diff --git a/src/lib/Server/Snapshots/model.py b/src/lib/Server/Snapshots/model.py
index 29bf059fe..4076bed9f 100644
--- a/src/lib/Server/Snapshots/model.py
+++ b/src/lib/Server/Snapshots/model.py
@@ -1,40 +1,51 @@
 from sqlalchemy import Table, Column, Integer, Unicode, MetaData, ForeignKey, Boolean, DateTime, create_engine, UnicodeText
 
-from sqlalchemy.orm import relation, backref
+import sqlalchemy.exceptions
+from sqlalchemy.orm import relation, backref, sessionmaker
 from sqlalchemy.ext.declarative import declarative_base
 
 # TODO pingtime?
-# TODO modified entries?
+# backlinks Client -> Snapshot
+# entry missing fields
+# extra entries
+
+class Uniquer(object):
+    @classmethod
+    def by_value(cls, session, **kwargs):
+        try:
+            return session.query(cls).filter_by(**kwargs).one()
+        except sqlalchemy.exceptions.InvalidRequestError:
+            return cls(**kwargs)
 
 Base = declarative_base()
 
-class Administrator(Base):
+class Administrator(Uniquer, Base):
     __tablename__ = 'administrator'
     id = Column(Integer, primary_key=True)
-    name = Column(Unicode(20))
+    name = Column(Unicode(20), unique=True)
     email = Column(Unicode(64))
 
 admin_client = Table('admin_client', Base.metadata,
                      Column('admin_id', Integer, ForeignKey('administrator.id')),
                      Column('client_id', Integer, ForeignKey('client.id')))
 
-class Client(Base):
+class Client(Uniquer, Base):
     __tablename__ = 'client'
     id = Column(Integer, primary_key=True)        
-    name = Column(Unicode(64))
+    name = Column(Unicode(64), unique=True)
     admins = relation("Administrator", secondary=admin_client)
-    active = Column(Boolean)
+    active = Column(Boolean, default=True)
 
-class Group(Base):
+class Group(Uniquer, Base):
     __tablename__ = 'group'
-    id = Column(Integer, primary_key=True)        
-    name = Column(Unicode(32))
+    id = Column(Integer, primary_key=True)
+    name = Column(Unicode(32), unique=True)
 
-class ConnectorKeyVal(Base):
+class ConnectorKeyVal(Uniquer, Base):
     __tablename__ = 'connkeyval'
     id = Column(Integer, primary_key=True)    
     connector = Column(Unicode(16))
-    key = Column(UnicodeText)
+    key = Column(Unicode(32))
     value = Column(UnicodeText)
 
 meta_group = Table('meta_group', Base.metadata,
@@ -47,11 +58,31 @@ meta_conn = Table('meta_conn', Base.metadata,
 
 class Metadata(Base):
     __tablename__ = 'metadata'
-    id = Column(Integer, primary_key=True)        
-    name = Column(Unicode(64))
+    id = Column(Integer, primary_key=True)
+    client_id = Column(Integer, ForeignKey('client.id'))
+    client = relation(Client)
     groups = relation("Group", secondary=meta_group)
     keyvals = relation(ConnectorKeyVal, secondary=meta_conn)
 
+    @classmethod
+    def from_metadata(cls, session, metadata):
+        client = Client.by_value(session, name=metadata.hostname)
+        m = cls(client=client)
+        for group in metadata.groups:
+            m.groups.append(Group.by_value(session, name=unicode(group)))
+        for connector in metadata.connectors:
+            data = getattr(metadata, connector)
+            if not isinstance(data, dict):
+                continue
+            for key, value in data.iteritems():
+                if not isinstance(value, str):
+                    continue
+                m.keyvals.append(ConnectorKeyVal.by_value(session,
+                                                          connector=unicode(connector),
+                                                          key=unicode(key),
+                                                          value=unicode(value)))
+        return m
+
 class Package(Base):
     __tablename__ = 'package'
     id = Column(Integer, primary_key=True)        
@@ -67,6 +98,7 @@ class PackageCorrespondence(Base):
     desired = relation(Package, primaryjoin=desired_id == Package.id)
     incorrect_id = Column(Integer, ForeignKey('package.id'), nullable=True)
     incorrect = relation(Package, primaryjoin=incorrect_id == Package.id)
+    modified = Column(Boolean)
 
 package_snap = Table('package_snap', Base.metadata,
                      Column('ppair_id', Integer, ForeignKey('package_pair.id')),
@@ -86,6 +118,7 @@ class ServiceCorrespondence(Base):
     desired = relation(Service, primaryjoin=desired_id == Service.id)
     incorrect_id = Column(Integer, ForeignKey('service.id'), nullable=True)
     incorrect = relation(Service, primaryjoin=incorrect_id == Service.id)
+    modified = Column(Boolean)
 
 service_snap = Table('service_snap', Base.metadata,
                      Column('spair_id', Integer, ForeignKey('service_pair.id')),
@@ -108,6 +141,7 @@ class FileCorrespondence(Base):
     desired = relation(File, primaryjoin=desired_id == File.id)
     incorrect_id = Column(Integer, ForeignKey('file.id'), nullable=True)
     incorrect = relation(File, primaryjoin=incorrect_id == File.id)
+    modified = Column(Boolean)
 
 file_snap = Table('file_snap', Base.metadata,
                   Column('fpair_id', Integer, ForeignKey('file_pair.id')),
@@ -126,12 +160,15 @@ class Snapshot(Base):
     metadata_id = Column(Integer, ForeignKey('metadata.id'))
     client_metadata = relation(Metadata, primaryjoin=metadata_id==Metadata.id)
     timestamp = Column(DateTime)
+    client_id = Column(Integer, ForeignKey('client.id'))
     client = relation(Client, backref=backref('snapshots', order_by=timestamp))
     packages = relation(PackageCorrespondence, secondary=package_snap)
     services = relation(ServiceCorrespondence, secondary=service_snap)
     files = relation(FileCorrespondence, secondary=file_snap)
 
-if __name__ == '__main__':
-    engine = create_engine('sqlite:///:memory:', echo=True)
-    metadata = Base.metadata
-    metadata.create_all(engine) 
+engine = create_engine('sqlite:///:memory:', echo=True)
+metadata = Base.metadata
+metadata.create_all(engine) 
+Session = sessionmaker()
+Session.configure(bind=engine)
+session = Session()