Merge pull request #2442 from mattermost/PLT-2115

PLT-2115 adding compliance feature

5 files changed, 474 insertions, 2 deletions

diff --git a/store/sql_audit_store.go b/store/sql_audit_store.go
index dbcb9a616..7609ebc25 100644
--- a/store/sql_audit_store.go
+++ b/store/sql_audit_store.go
@@ -18,8 +18,8 @@ func NewSqlAuditStore(sqlStore *SqlStore) AuditStore {
 		table := db.AddTableWithName(model.Audit{}, "Audits").SetKeys(false, "Id")
 		table.ColMap("Id").SetMaxSize(26)
 		table.ColMap("UserId").SetMaxSize(26)
-		table.ColMap("Action").SetMaxSize(64)
-		table.ColMap("ExtraInfo").SetMaxSize(128)
+		table.ColMap("Action").SetMaxSize(512)
+		table.ColMap("ExtraInfo").SetMaxSize(1024)
 		table.ColMap("IpAddress").SetMaxSize(64)
 		table.ColMap("SessionId").SetMaxSize(26)
 	}
@@ -28,6 +28,17 @@ func NewSqlAuditStore(sqlStore *SqlStore) AuditStore {
 }
 
 func (s SqlAuditStore) UpgradeSchemaIfNeeded() {
+	// ADDED for 2.2 REMOVE for 2.6
+	extraLength := s.GetMaxLengthOfColumnIfExists("Audits", "ExtraInfo")
+	if len(extraLength) > 0 && extraLength != "1024" {
+		s.AlterColumnTypeIfExists("Audits", "ExtraInfo", "VARCHAR(1024)", "VARCHAR(1024)")
+	}
+
+	// ADDED for 2.2 REMOVE for 2.6
+	actionLength := s.GetMaxLengthOfColumnIfExists("Audits", "Action")
+	if len(actionLength) > 0 && actionLength != "512" {
+		s.AlterColumnTypeIfExists("Audits", "Action", "VARCHAR(512)", "VARCHAR(512)")
+	}
 }
 
 func (s SqlAuditStore) CreateIndexesIfNotExists() {
diff --git a/store/sql_compliance_store.go b/store/sql_compliance_store.go
new file mode 100644
index 000000000..57872aef4
--- /dev/null
+++ b/store/sql_compliance_store.go
@@ -0,0 +1,234 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package store
+
+import (
+	"github.com/mattermost/platform/model"
+	"strconv"
+	"strings"
+)
+
+type SqlComplianceStore struct {
+	*SqlStore
+}
+
+func NewSqlComplianceStore(sqlStore *SqlStore) ComplianceStore {
+	s := &SqlComplianceStore{sqlStore}
+
+	for _, db := range sqlStore.GetAllConns() {
+		table := db.AddTableWithName(model.Compliance{}, "Compliances").SetKeys(false, "Id")
+		table.ColMap("Id").SetMaxSize(26)
+		table.ColMap("UserId").SetMaxSize(26)
+		table.ColMap("Status").SetMaxSize(64)
+		table.ColMap("Desc").SetMaxSize(512)
+		table.ColMap("Type").SetMaxSize(64)
+		table.ColMap("Keywords").SetMaxSize(512)
+		table.ColMap("Emails").SetMaxSize(1024)
+	}
+
+	return s
+}
+
+func (s SqlComplianceStore) UpgradeSchemaIfNeeded() {
+}
+
+func (s SqlComplianceStore) CreateIndexesIfNotExists() {
+}
+
+func (s SqlComplianceStore) Save(compliance *model.Compliance) StoreChannel {
+
+	storeChannel := make(StoreChannel)
+
+	go func() {
+		result := StoreResult{}
+
+		compliance.PreSave()
+		if result.Err = compliance.IsValid(); result.Err != nil {
+			storeChannel <- result
+			close(storeChannel)
+			return
+		}
+
+		if err := s.GetMaster().Insert(compliance); err != nil {
+			result.Err = model.NewLocAppError("SqlComplianceStore.Save", "store.sql_compliance.save.saving.app_error", nil, err.Error())
+		} else {
+			result.Data = compliance
+		}
+
+		storeChannel <- result
+		close(storeChannel)
+	}()
+
+	return storeChannel
+}
+
+func (us SqlComplianceStore) Update(compliance *model.Compliance) StoreChannel {
+
+	storeChannel := make(StoreChannel)
+
+	go func() {
+		result := StoreResult{}
+
+		if result.Err = compliance.IsValid(); result.Err != nil {
+			storeChannel <- result
+			close(storeChannel)
+			return
+		}
+
+		if _, err := us.GetMaster().Update(compliance); err != nil {
+			result.Err = model.NewLocAppError("SqlComplianceStore.Update", "store.sql_compliance.save.saving.app_error", nil, err.Error())
+		} else {
+			result.Data = compliance
+		}
+
+		storeChannel <- result
+		close(storeChannel)
+	}()
+
+	return storeChannel
+}
+
+func (s SqlComplianceStore) GetAll() StoreChannel {
+
+	storeChannel := make(StoreChannel)
+
+	go func() {
+		result := StoreResult{}
+
+		query := "SELECT * FROM Compliances ORDER BY CreateAt DESC"
+
+		var compliances model.Compliances
+		if _, err := s.GetReplica().Select(&compliances, query); err != nil {
+			result.Err = model.NewLocAppError("SqlComplianceStore.Get", "store.sql_compliance.get.finding.app_error", nil, err.Error())
+		} else {
+			result.Data = compliances
+		}
+
+		storeChannel <- result
+		close(storeChannel)
+	}()
+
+	return storeChannel
+}
+
+func (us SqlComplianceStore) Get(id string) StoreChannel {
+
+	storeChannel := make(StoreChannel)
+
+	go func() {
+		result := StoreResult{}
+
+		if obj, err := us.GetReplica().Get(model.Compliance{}, id); err != nil {
+			result.Err = model.NewLocAppError("SqlComplianceStore.Get", "store.sql_compliance.get.finding.app_error", nil, err.Error())
+		} else if obj == nil {
+			result.Err = model.NewLocAppError("SqlComplianceStore.Get", "store.sql_compliance.get.finding.app_error", nil, err.Error())
+		} else {
+			result.Data = obj.(*model.Compliance)
+		}
+
+		storeChannel <- result
+		close(storeChannel)
+
+	}()
+
+	return storeChannel
+}
+
+func (s SqlComplianceStore) ComplianceExport(job *model.Compliance) StoreChannel {
+	storeChannel := make(StoreChannel)
+
+	go func() {
+		result := StoreResult{}
+
+		props := map[string]interface{}{"StartTime": job.StartAt, "EndTime": job.EndAt}
+
+		keywordQuery := ""
+		keywords := strings.Fields(strings.TrimSpace(strings.ToLower(strings.Replace(job.Keywords, ",", " ", -1))))
+		if len(keywords) > 0 {
+
+			keywordQuery = "AND ("
+
+			for index, keyword := range keywords {
+				if index >= 1 {
+					keywordQuery += " OR LOWER(Posts.Message) LIKE :Keyword" + strconv.Itoa(index)
+				} else {
+					keywordQuery += "LOWER(Posts.Message) LIKE :Keyword" + strconv.Itoa(index)
+				}
+
+				props["Keyword"+strconv.Itoa(index)] = "%" + keyword + "%"
+			}
+
+			keywordQuery += ")"
+		}
+
+		emailQuery := ""
+		emails := strings.Fields(strings.TrimSpace(strings.ToLower(strings.Replace(job.Emails, ",", " ", -1))))
+		if len(emails) > 0 {
+
+			emailQuery = "AND ("
+
+			for index, email := range emails {
+				if index >= 1 {
+					emailQuery += " OR Users.Email = :Email" + strconv.Itoa(index)
+				} else {
+					emailQuery += "Users.Email = :Email" + strconv.Itoa(index)
+				}
+
+				props["Email"+strconv.Itoa(index)] = email
+			}
+
+			emailQuery += ")"
+		}
+
+		query :=
+			`SELECT
+			    Teams.Name AS TeamName,
+			    Teams.DisplayName AS TeamDisplayName,
+			    Channels.Name AS ChannelName,
+			    Channels.DisplayName AS ChannelDisplayName,
+			    Users.Username AS UserUsername,
+			    Users.Email AS UserEmail,
+			    Users.Nickname AS UserNickname,
+			    Posts.Id AS PostId,
+			    Posts.CreateAt AS PostCreateAt,
+			    Posts.UpdateAt AS PostUpdateAt,
+			    Posts.DeleteAt AS PostDeleteAt,
+			    Posts.RootId AS PostRootId,
+			    Posts.ParentId AS PostParentId,
+			    Posts.OriginalId AS PostOriginalId,
+			    Posts.Message AS PostMessage,
+			    Posts.Type AS PostType,
+			    Posts.Props AS PostProps,
+			    Posts.Hashtags AS PostHashtags,
+			    Posts.Filenames AS PostFilenames
+			FROM
+			    Teams,
+			    Channels,
+			    Users,
+			    Posts
+			WHERE
+			    Teams.Id = Channels.TeamId
+			        AND Posts.ChannelId = Channels.Id
+			        AND Posts.UserId = Users.Id
+			        AND Posts.CreateAt > :StartTime
+			        AND Posts.CreateAt <= :EndTime
+			        ` + emailQuery + `
+			        ` + keywordQuery + `
+			ORDER BY Posts.CreateAt
+			LIMIT 30000`
+
+		var cposts []*model.CompliancePost
+
+		if _, err := s.GetReplica().Select(&cposts, query, props); err != nil {
+			result.Err = model.NewLocAppError("SqlPostStore.ComplianceExport", "store.sql_post.compliance_export.app_error", nil, err.Error())
+		} else {
+			result.Data = cposts
+		}
+
+		storeChannel <- result
+		close(storeChannel)
+	}()
+
+	return storeChannel
+}
diff --git a/store/sql_compliance_store_test.go b/store/sql_compliance_store_test.go
new file mode 100644
index 000000000..1a41fa389
--- /dev/null
+++ b/store/sql_compliance_store_test.go
@@ -0,0 +1,210 @@
+// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package store
+
+import (
+	"github.com/mattermost/platform/model"
+	"testing"
+	"time"
+)
+
+func TestSqlComplianceStore(t *testing.T) {
+	Setup()
+
+	compliance1 := &model.Compliance{Desc: "Audit for federal subpoena case #22443", UserId: model.NewId(), Status: model.COMPLIANCE_STATUS_FAILED, StartAt: model.GetMillis() - 1, EndAt: model.GetMillis() + 1, Type: model.COMPLIANCE_TYPE_ADHOC}
+	Must(store.Compliance().Save(compliance1))
+	time.Sleep(100 * time.Millisecond)
+
+	compliance2 := &model.Compliance{Desc: "Audit for federal subpoena case #11458", UserId: model.NewId(), Status: model.COMPLIANCE_STATUS_RUNNING, StartAt: model.GetMillis() - 1, EndAt: model.GetMillis() + 1, Type: model.COMPLIANCE_TYPE_ADHOC}
+	Must(store.Compliance().Save(compliance2))
+	time.Sleep(100 * time.Millisecond)
+
+	c := store.Compliance().GetAll()
+	result := <-c
+	compliances := result.Data.(model.Compliances)
+
+	if compliances[0].Status != model.COMPLIANCE_STATUS_RUNNING && compliance2.Id != compliances[0].Id {
+		t.Fatal()
+	}
+
+	compliance2.Status = model.COMPLIANCE_STATUS_FAILED
+	Must(store.Compliance().Update(compliance2))
+
+	c = store.Compliance().GetAll()
+	result = <-c
+	compliances = result.Data.(model.Compliances)
+
+	if compliances[0].Status != model.COMPLIANCE_STATUS_FAILED && compliance2.Id != compliances[0].Id {
+		t.Fatal()
+	}
+
+	rc2 := (<-store.Compliance().Get(compliance2.Id)).Data.(*model.Compliance)
+	if rc2.Status != compliance2.Status {
+		t.Fatal()
+	}
+}
+
+func TestComplianceExport(t *testing.T) {
+	Setup()
+
+	time.Sleep(100 * time.Millisecond)
+
+	t1 := &model.Team{}
+	t1.DisplayName = "DisplayName"
+	t1.Name = "a" + model.NewId() + "b"
+	t1.Email = model.NewId() + "@nowhere.com"
+	t1.Type = model.TEAM_OPEN
+	t1 = Must(store.Team().Save(t1)).(*model.Team)
+
+	u1 := &model.User{}
+	u1.TeamId = t1.Id
+	u1.Email = model.NewId()
+	u1.Username = model.NewId()
+	u1 = Must(store.User().Save(u1)).(*model.User)
+
+	u2 := &model.User{}
+	u2.TeamId = t1.Id
+	u2.Email = model.NewId()
+	u2.Username = model.NewId()
+	u2 = Must(store.User().Save(u2)).(*model.User)
+
+	c1 := &model.Channel{}
+	c1.TeamId = t1.Id
+	c1.DisplayName = "Channel2"
+	c1.Name = "a" + model.NewId() + "b"
+	c1.Type = model.CHANNEL_OPEN
+	c1 = Must(store.Channel().Save(c1)).(*model.Channel)
+
+	o1 := &model.Post{}
+	o1.ChannelId = c1.Id
+	o1.UserId = u1.Id
+	o1.CreateAt = model.GetMillis()
+	o1.Message = "a" + model.NewId() + "b"
+	o1 = Must(store.Post().Save(o1)).(*model.Post)
+
+	o1a := &model.Post{}
+	o1a.ChannelId = c1.Id
+	o1a.UserId = u1.Id
+	o1a.CreateAt = o1.CreateAt + 10
+	o1a.Message = "a" + model.NewId() + "b"
+	o1a = Must(store.Post().Save(o1a)).(*model.Post)
+
+	o2 := &model.Post{}
+	o2.ChannelId = c1.Id
+	o2.UserId = u1.Id
+	o2.CreateAt = o1.CreateAt + 20
+	o2.Message = "a" + model.NewId() + "b"
+	o2 = Must(store.Post().Save(o2)).(*model.Post)
+
+	o2a := &model.Post{}
+	o2a.ChannelId = c1.Id
+	o2a.UserId = u2.Id
+	o2a.CreateAt = o1.CreateAt + 30
+	o2a.Message = "a" + model.NewId() + "b"
+	o2a = Must(store.Post().Save(o2a)).(*model.Post)
+
+	time.Sleep(100 * time.Millisecond)
+
+	cr1 := &model.Compliance{Desc: "test" + model.NewId(), StartAt: o1.CreateAt - 1, EndAt: o2a.CreateAt + 1}
+	if r1 := <-store.Compliance().ComplianceExport(cr1); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		cposts := r1.Data.([]*model.CompliancePost)
+
+		if len(cposts) != 4 {
+			t.Fatal("return wrong results length")
+		}
+
+		if cposts[0].PostId != o1.Id {
+			t.Fatal("Wrong sort")
+		}
+
+		if cposts[3].PostId != o2a.Id {
+			t.Fatal("Wrong sort")
+		}
+	}
+
+	cr2 := &model.Compliance{Desc: "test" + model.NewId(), StartAt: o1.CreateAt - 1, EndAt: o2a.CreateAt + 1, Emails: u2.Email}
+	if r1 := <-store.Compliance().ComplianceExport(cr2); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		cposts := r1.Data.([]*model.CompliancePost)
+
+		if len(cposts) != 1 {
+			t.Fatal("return wrong results length")
+		}
+
+		if cposts[0].PostId != o2a.Id {
+			t.Fatal("Wrong sort")
+		}
+	}
+
+	cr3 := &model.Compliance{Desc: "test" + model.NewId(), StartAt: o1.CreateAt - 1, EndAt: o2a.CreateAt + 1, Emails: u2.Email + ", " + u1.Email}
+	if r1 := <-store.Compliance().ComplianceExport(cr3); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		cposts := r1.Data.([]*model.CompliancePost)
+
+		if len(cposts) != 4 {
+			t.Fatal("return wrong results length")
+		}
+
+		if cposts[0].PostId != o1.Id {
+			t.Fatal("Wrong sort")
+		}
+
+		if cposts[3].PostId != o2a.Id {
+			t.Fatal("Wrong sort")
+		}
+	}
+
+	cr4 := &model.Compliance{Desc: "test" + model.NewId(), StartAt: o1.CreateAt - 1, EndAt: o2a.CreateAt + 1, Keywords: o2a.Message}
+	if r1 := <-store.Compliance().ComplianceExport(cr4); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		cposts := r1.Data.([]*model.CompliancePost)
+
+		if len(cposts) != 1 {
+			t.Fatal("return wrong results length")
+		}
+
+		if cposts[0].PostId != o2a.Id {
+			t.Fatal("Wrong sort")
+		}
+	}
+
+	cr5 := &model.Compliance{Desc: "test" + model.NewId(), StartAt: o1.CreateAt - 1, EndAt: o2a.CreateAt + 1, Keywords: o2a.Message + " " + o1.Message}
+	if r1 := <-store.Compliance().ComplianceExport(cr5); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		cposts := r1.Data.([]*model.CompliancePost)
+
+		if len(cposts) != 2 {
+			t.Fatal("return wrong results length")
+		}
+
+		if cposts[0].PostId != o1.Id {
+			t.Fatal("Wrong sort")
+		}
+	}
+
+	cr6 := &model.Compliance{Desc: "test" + model.NewId(), StartAt: o1.CreateAt - 1, EndAt: o2a.CreateAt + 1, Emails: u2.Email + ", " + u1.Email, Keywords: o2a.Message + " " + o1.Message}
+	if r1 := <-store.Compliance().ComplianceExport(cr6); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		cposts := r1.Data.([]*model.CompliancePost)
+
+		if len(cposts) != 2 {
+			t.Fatal("return wrong results length")
+		}
+
+		if cposts[0].PostId != o1.Id {
+			t.Fatal("Wrong sort")
+		}
+
+		if cposts[1].PostId != o2a.Id {
+			t.Fatal("Wrong sort")
+		}
+	}
+}
diff --git a/store/sql_store.go b/store/sql_store.go
index de23f4db3..8ff5da6f7 100644
--- a/store/sql_store.go
+++ b/store/sql_store.go
@@ -43,6 +43,7 @@ type SqlStore struct {
 	post       PostStore
 	user       UserStore
 	audit      AuditStore
+	compliance ComplianceStore
 	session    SessionStore
 	oauth      OAuthStore
 	system     SystemStore
@@ -98,6 +99,7 @@ func NewSqlStore() Store {
 	sqlStore.post = NewSqlPostStore(sqlStore)
 	sqlStore.user = NewSqlUserStore(sqlStore)
 	sqlStore.audit = NewSqlAuditStore(sqlStore)
+	sqlStore.compliance = NewSqlComplianceStore(sqlStore)
 	sqlStore.session = NewSqlSessionStore(sqlStore)
 	sqlStore.oauth = NewSqlOAuthStore(sqlStore)
 	sqlStore.system = NewSqlSystemStore(sqlStore)
@@ -116,6 +118,7 @@ func NewSqlStore() Store {
 	sqlStore.post.(*SqlPostStore).UpgradeSchemaIfNeeded()
 	sqlStore.user.(*SqlUserStore).UpgradeSchemaIfNeeded()
 	sqlStore.audit.(*SqlAuditStore).UpgradeSchemaIfNeeded()
+	sqlStore.compliance.(*SqlComplianceStore).UpgradeSchemaIfNeeded()
 	sqlStore.session.(*SqlSessionStore).UpgradeSchemaIfNeeded()
 	sqlStore.oauth.(*SqlOAuthStore).UpgradeSchemaIfNeeded()
 	sqlStore.system.(*SqlSystemStore).UpgradeSchemaIfNeeded()
@@ -129,6 +132,7 @@ func NewSqlStore() Store {
 	sqlStore.post.(*SqlPostStore).CreateIndexesIfNotExists()
 	sqlStore.user.(*SqlUserStore).CreateIndexesIfNotExists()
 	sqlStore.audit.(*SqlAuditStore).CreateIndexesIfNotExists()
+	sqlStore.compliance.(*SqlComplianceStore).CreateIndexesIfNotExists()
 	sqlStore.session.(*SqlSessionStore).CreateIndexesIfNotExists()
 	sqlStore.oauth.(*SqlOAuthStore).CreateIndexesIfNotExists()
 	sqlStore.system.(*SqlSystemStore).CreateIndexesIfNotExists()
@@ -591,6 +595,10 @@ func (ss SqlStore) Audit() AuditStore {
 	return ss.audit
 }
 
+func (ss SqlStore) Compliance() ComplianceStore {
+	return ss.compliance
+}
+
 func (ss SqlStore) OAuth() OAuthStore {
 	return ss.oauth
 }
diff --git a/store/store.go b/store/store.go
index 1738ba84e..7ec5ac3a5 100644
--- a/store/store.go
+++ b/store/store.go
@@ -33,6 +33,7 @@ type Store interface {
 	Post() PostStore
 	User() UserStore
 	Audit() AuditStore
+	Compliance() ComplianceStore
 	Session() SessionStore
 	OAuth() OAuthStore
 	System() SystemStore
@@ -153,6 +154,14 @@ type AuditStore interface {
 	PermanentDeleteByUser(userId string) StoreChannel
 }
 
+type ComplianceStore interface {
+	Save(compliance *model.Compliance) StoreChannel
+	Update(compliance *model.Compliance) StoreChannel
+	Get(id string) StoreChannel
+	GetAll() StoreChannel
+	ComplianceExport(compliance *model.Compliance) StoreChannel
+}
+
 type OAuthStore interface {
 	SaveApp(app *model.OAuthApp) StoreChannel
 	UpdateApp(app *model.OAuthApp) StoreChannel