From f24ea30a75ad52b695f5f275139af1c0495624ac Mon Sep 17 00:00:00 2001 From: JoramWilander Date: Fri, 16 Oct 2015 11:17:24 -0400 Subject: Refactor to hit database less often. --- api/post.go | 146 ++++++++++++++++++---------------------- config/config.json | 3 +- model/outgoing_webhook.go | 14 ++++ store/sql_store.go | 9 +-- store/sql_webhook_store.go | 51 +------------- store/sql_webhook_store_test.go | 25 ++----- store/store.go | 2 +- 7 files changed, 89 insertions(+), 161 deletions(-) diff --git a/api/post.go b/api/post.go index 98c394727..e4e67dc66 100644 --- a/api/post.go +++ b/api/post.go @@ -137,55 +137,23 @@ func CreatePost(c *Context, post *model.Post, triggerWebhooks bool) (*model.Post } else { rpost = result.Data.(*model.Post) - fireAndForgetNotifications(rpost, c.Session.TeamId, c.GetSiteURL()) - - if triggerWebhooks { - fireAndForgetWebhookEvent(c, rpost) - } + handlePostEventsAndForget(c, rpost, triggerWebhooks) } return rpost, nil } -func fireAndForgetWebhookEvent(c *Context, post *model.Post) { - +func handlePostEventsAndForget(c *Context, post *model.Post, triggerWebhooks bool) { go func() { - - chchan := Srv.Store.Webhook().GetOutgoingByChannel(post.ChannelId) - - firstWord := strings.Split(post.Message, " ")[0] - - var thchan store.StoreChannel - if len(firstWord) != 0 { - thchan = Srv.Store.Webhook().GetOutgoingByTriggerWord(c.Session.TeamId, post.ChannelId, firstWord) - } - - hooks := []*model.OutgoingWebhook{} - - if result := <-chchan; result.Err != nil { - l4g.Error("Encountered error getting webhook by channel, err=%v", result.Err) - return - } else { - hooks = append(hooks, result.Data.([]*model.OutgoingWebhook)...) - } - - if thchan != nil { - if result := <-thchan; result.Err != nil { - l4g.Error("Encountered error getting webhook by trigger word, err=%v", result.Err) - return - } else { - hooks = append(hooks, result.Data.([]*model.OutgoingWebhook)...) - } - } - + tchan := Srv.Store.Team().Get(c.Session.TeamId) cchan := Srv.Store.Channel().Get(post.ChannelId) uchan := Srv.Store.User().Get(post.UserId) - tchan := Srv.Store.Team().Get(c.Session.TeamId) var team *model.Team if result := <-tchan; result.Err != nil { l4g.Error("Encountered error getting team, team_id=%s, err=%v", c.Session.TeamId, result.Err) + return } else { team = result.Data.(*model.Team) } @@ -193,43 +161,76 @@ func fireAndForgetWebhookEvent(c *Context, post *model.Post) { var channel *model.Channel if result := <-cchan; result.Err != nil { l4g.Error("Encountered error getting channel, channel_id=%s, err=%v", post.ChannelId, result.Err) + return } else { channel = result.Data.(*model.Channel) } + fireAndForgetNotifications(c, post, team, channel) + var user *model.User if result := <-uchan; result.Err != nil { l4g.Error("Encountered error getting user, user_id=%s, err=%v", post.UserId, result.Err) + return } else { user = result.Data.(*model.User) } + if triggerWebhooks { + handleWebhookEventsAndForget(c, post, team, channel, user) + } + }() +} + +func handleWebhookEventsAndForget(c *Context, post *model.Post, team *model.Team, channel *model.Channel, user *model.User) { + go func() { + hchan := Srv.Store.Webhook().GetOutgoingByTeam(c.Session.TeamId) + + hooks := []*model.OutgoingWebhook{} + + if result := <-hchan; result.Err != nil { + l4g.Error("Encountered error getting webhooks by team, err=%v", result.Err) + return + } else { + hooks = result.Data.([]*model.OutgoingWebhook) + } + + if len(hooks) == 0 { + return + } + + firstWord := strings.Split(post.Message, " ")[0] + + relevantHooks := []*model.OutgoingWebhook{} + for _, hook := range hooks { + if hook.ChannelId == post.ChannelId { + if len(hook.TriggerWords) == 0 || hook.HasTriggerWord(firstWord) { + relevantHooks = append(relevantHooks, hook) + } + } else if len(hook.ChannelId) == 0 && hook.HasTriggerWord(firstWord) { + relevantHooks = append(relevantHooks, hook) + } + } + + for _, hook := range relevantHooks { go func() { p := url.Values{} p.Set("token", hook.Token) - p.Set("team_id", hook.TeamId) - if team != nil { - p.Set("team_domain", team.Name) - } + p.Set("team_id", hook.TeamId) + p.Set("team_domain", team.Name) p.Set("channel_id", post.ChannelId) - if channel != nil { - p.Set("channel_name", channel.Name) - } + p.Set("channel_name", channel.Name) p.Set("timestamp", strconv.FormatInt(post.CreateAt/1000, 10)) p.Set("user_id", post.UserId) - if user != nil { - p.Set("user_name", user.Username) - } + p.Set("user_name", user.Username) p.Set("text", post.Message) - if len(hook.TriggerWords) > 0 { - p.Set("trigger_word", firstWord) - } + p.Set("trigger_word", firstWord) client := &http.Client{} @@ -260,38 +261,29 @@ func fireAndForgetWebhookEvent(c *Context, post *model.Post) { } -func fireAndForgetNotifications(post *model.Post, teamId, siteURL string) { +func fireAndForgetNotifications(c *Context, post *model.Post, team *model.Team, channel *model.Channel) { go func() { // Get a list of user names (to be used as keywords) and ids for the given team - uchan := Srv.Store.User().GetProfiles(teamId) + uchan := Srv.Store.User().GetProfiles(c.Session.TeamId) echan := Srv.Store.Channel().GetMembers(post.ChannelId) - cchan := Srv.Store.Channel().Get(post.ChannelId) - tchan := Srv.Store.Team().Get(teamId) - var channel *model.Channel var channelName string var bodyText string var subjectText string - if result := <-cchan; result.Err != nil { - l4g.Error("Failed to retrieve channel channel_id=%v, err=%v", post.ChannelId, result.Err) - return + if channel.Type == model.CHANNEL_DIRECT { + bodyText = "You have one new message." + subjectText = "New Direct Message" } else { - channel = result.Data.(*model.Channel) - if channel.Type == model.CHANNEL_DIRECT { - bodyText = "You have one new message." - subjectText = "New Direct Message" - } else { - bodyText = "You have one new mention." - subjectText = "New Mention" - channelName = channel.DisplayName - } + bodyText = "You have one new mention." + subjectText = "New Mention" + channelName = channel.DisplayName } var mentionedUsers []string if result := <-uchan; result.Err != nil { - l4g.Error("Failed to retrieve user profiles team_id=%v, err=%v", teamId, result.Err) + l4g.Error("Failed to retrieve user profiles team_id=%v, err=%v", c.Session.TeamId, result.Err) return } else { profileMap := result.Data.(map[string]*model.User) @@ -414,23 +406,15 @@ func fireAndForgetNotifications(post *model.Post, teamId, siteURL string) { mentionedUsers = append(mentionedUsers, k) } - var teamDisplayName string - var teamURL string - if result := <-tchan; result.Err != nil { - l4g.Error("Failed to retrieve team team_id=%v, err=%v", teamId, result.Err) - return - } else { - teamDisplayName = result.Data.(*model.Team).DisplayName - teamURL = siteURL + "/" + result.Data.(*model.Team).Name - } + teamURL := c.GetSiteURL() + "/" + team.Name // Build and send the emails location, _ := time.LoadLocation("UTC") tm := time.Unix(post.CreateAt/1000, 0).In(location) subjectPage := NewServerTemplatePage("post_subject") - subjectPage.Props["SiteURL"] = siteURL - subjectPage.Props["TeamDisplayName"] = teamDisplayName + subjectPage.Props["SiteURL"] = c.GetSiteURL() + subjectPage.Props["TeamDisplayName"] = team.DisplayName subjectPage.Props["SubjectText"] = subjectText subjectPage.Props["Month"] = tm.Month().String()[:3] subjectPage.Props["Day"] = fmt.Sprintf("%d", tm.Day()) @@ -448,9 +432,9 @@ func fireAndForgetNotifications(post *model.Post, teamId, siteURL string) { } bodyPage := NewServerTemplatePage("post_body") - bodyPage.Props["SiteURL"] = siteURL + bodyPage.Props["SiteURL"] = c.GetSiteURL() bodyPage.Props["Nickname"] = profileMap[id].FirstName - bodyPage.Props["TeamDisplayName"] = teamDisplayName + bodyPage.Props["TeamDisplayName"] = team.DisplayName bodyPage.Props["ChannelName"] = channelName bodyPage.Props["BodyText"] = bodyText bodyPage.Props["SenderName"] = senderName @@ -517,7 +501,7 @@ func fireAndForgetNotifications(post *model.Post, teamId, siteURL string) { } } - message := model.NewMessage(teamId, post.ChannelId, post.UserId, model.ACTION_POSTED) + message := model.NewMessage(c.Session.TeamId, post.ChannelId, post.UserId, model.ACTION_POSTED) message.Add("post", post.ToJson()) if len(post.Filenames) != 0 { diff --git a/config/config.json b/config/config.json index 3980d4b98..37109428d 100644 --- a/config/config.json +++ b/config/config.json @@ -11,7 +11,6 @@ "EnablePostIconOverride": false, "EnableTesting": false, "EnableSecurityFixAlert": true - "EnableTesting": false }, "TeamSettings": { "SiteName": "Mattermost", @@ -91,4 +90,4 @@ "TokenEndpoint": "", "UserApiEndpoint": "" } -} +} \ No newline at end of file diff --git a/model/outgoing_webhook.go b/model/outgoing_webhook.go index 64d58ac9d..8958dd5b0 100644 --- a/model/outgoing_webhook.go +++ b/model/outgoing_webhook.go @@ -119,3 +119,17 @@ func (o *OutgoingWebhook) PreSave() { func (o *OutgoingWebhook) PreUpdate() { o.UpdateAt = GetMillis() } + +func (o *OutgoingWebhook) HasTriggerWord(word string) bool { + if len(o.TriggerWords) == 0 || len(word) == 0 { + return false + } + + for _, trigger := range o.TriggerWords { + if trigger == word { + return true + } + } + + return false +} diff --git a/store/sql_store.go b/store/sql_store.go index a1a542691..c5bf840a1 100644 --- a/store/sql_store.go +++ b/store/sql_store.go @@ -32,7 +32,6 @@ import ( const ( INDEX_TYPE_FULL_TEXT = "full_text" - INDEX_TYPE_PATTERN = "pattern" INDEX_TYPE_DEFAULT = "default" ) @@ -376,10 +375,6 @@ func (ss SqlStore) CreateFullTextIndexIfNotExists(indexName string, tableName st ss.createIndexIfNotExists(indexName, tableName, columnName, INDEX_TYPE_FULL_TEXT) } -func (ss SqlStore) CreatePatternIndexIfNotExists(indexName string, tableName string, columnName string) { - ss.createIndexIfNotExists(indexName, tableName, columnName, INDEX_TYPE_PATTERN) -} - func (ss SqlStore) createIndexIfNotExists(indexName string, tableName string, columnName string, indexType string) { if utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES { @@ -392,8 +387,6 @@ func (ss SqlStore) createIndexIfNotExists(indexName string, tableName string, co query := "" if indexType == INDEX_TYPE_FULL_TEXT { query = "CREATE INDEX " + indexName + " ON " + tableName + " USING gin(to_tsvector('english', " + columnName + "))" - } else if indexType == INDEX_TYPE_PATTERN { - query = "CREATE INDEX " + indexName + " ON " + tableName + " (" + columnName + " text_pattern_ops)" } else { query = "CREATE INDEX " + indexName + " ON " + tableName + " (" + columnName + ")" } @@ -418,7 +411,7 @@ func (ss SqlStore) createIndexIfNotExists(indexName string, tableName string, co } fullTextIndex := "" - if indexType == INDEX_TYPE_FULL_TEXT || indexType == INDEX_TYPE_PATTERN { + if indexType == INDEX_TYPE_FULL_TEXT { fullTextIndex = " FULLTEXT " } diff --git a/store/sql_webhook_store.go b/store/sql_webhook_store.go index 68a3236eb..14469e2e7 100644 --- a/store/sql_webhook_store.go +++ b/store/sql_webhook_store.go @@ -5,7 +5,6 @@ package store import ( "github.com/mattermost/platform/model" - "github.com/mattermost/platform/utils" ) type SqlWebhookStore struct { @@ -42,8 +41,6 @@ func (s SqlWebhookStore) CreateIndexesIfNotExists() { s.CreateIndexIfNotExists("idx_incoming_webhook_user_id", "IncomingWebhooks", "UserId") s.CreateIndexIfNotExists("idx_incoming_webhook_team_id", "IncomingWebhooks", "TeamId") s.CreateIndexIfNotExists("idx_outgoing_webhook_channel_id", "OutgoingWebhooks", "ChannelId") - - s.CreatePatternIndexIfNotExists("idx_outgoing_webhook_trigger_txt", "OutgoingWebhooks", "TriggerWords") } func (s SqlWebhookStore) SaveIncoming(webhook *model.IncomingWebhook) StoreChannel { @@ -237,7 +234,7 @@ func (s SqlWebhookStore) GetOutgoingByChannel(channelId string) StoreChannel { return storeChannel } -func (s SqlWebhookStore) GetOutgoingByTriggerWord(teamId, channelId, triggerWord string) StoreChannel { +func (s SqlWebhookStore) GetOutgoingByTeam(teamId string) StoreChannel { storeChannel := make(StoreChannel) go func() { @@ -245,50 +242,8 @@ func (s SqlWebhookStore) GetOutgoingByTriggerWord(teamId, channelId, triggerWord var webhooks []*model.OutgoingWebhook - var err error - - if utils.Cfg.SqlSettings.DriverName == "postgres" { - - searchQuery := `SELECT - * - FROM - OutgoingWebhooks - WHERE - DeleteAt = 0 - AND TeamId = $1 - AND TriggerWords LIKE '%' || $2 || '%'` - - if len(channelId) != 0 { - searchQuery += " AND (ChannelId = $3 OR ChannelId = '')" - _, err = s.GetReplica().Select(&webhooks, searchQuery, teamId, triggerWord, channelId) - } else { - searchQuery += " AND ChannelId = ''" - _, err = s.GetReplica().Select(&webhooks, searchQuery, teamId, triggerWord) - } - - } else if utils.Cfg.SqlSettings.DriverName == "mysql" { - searchQuery := `SELECT - * - FROM - OutgoingWebhooks - WHERE - DeleteAt = 0 - AND TeamId = ? - AND MATCH (TriggerWords) AGAINST (? IN BOOLEAN MODE)` - - triggerWord = "+" + triggerWord - - if len(channelId) != 0 { - searchQuery += " AND (ChannelId = ? OR ChannelId = '')" - _, err = s.GetReplica().Select(&webhooks, searchQuery, teamId, triggerWord, channelId) - } else { - searchQuery += " AND ChannelId = ''" - _, err = s.GetReplica().Select(&webhooks, searchQuery, teamId, triggerWord) - } - } - - if err != nil { - result.Err = model.NewAppError("SqlPostStore.GetOutgoingByTriggerWord", "We encounted an error while getting the outgoing webhooks by trigger word", "teamId="+teamId+", channelId="+channelId+", triggerWord="+triggerWord+", err="+err.Error()) + if _, err := s.GetReplica().Select(&webhooks, "SELECT * FROM OutgoingWebhooks WHERE TeamId = :TeamId AND DeleteAt = 0", map[string]interface{}{"TeamId": teamId}); err != nil { + result.Err = model.NewAppError("SqlWebhookStore.GetOutgoingByTeam", "We couldn't get the webhooks", "teamId="+teamId+", err="+err.Error()) } result.Data = webhooks diff --git a/store/sql_webhook_store_test.go b/store/sql_webhook_store_test.go index 6098b2131..1fb990f3e 100644 --- a/store/sql_webhook_store_test.go +++ b/store/sql_webhook_store_test.go @@ -201,27 +201,18 @@ func TestWebhookStoreGetOutgoingByCreator(t *testing.T) { } } -func TestWebhookStoreGetOutgoingByTriggerWord(t *testing.T) { +func TestWebhookStoreGetOutgoingByTeam(t *testing.T) { Setup() o1 := &model.OutgoingWebhook{} + o1.ChannelId = model.NewId() o1.CreatorId = model.NewId() o1.TeamId = model.NewId() - o1.TriggerWords = []string{"trigger"} o1.CallbackURLs = []string{"http://nowhere.com/"} o1 = (<-store.Webhook().SaveOutgoing(o1)).Data.(*model.OutgoingWebhook) - o2 := &model.OutgoingWebhook{} - o2.CreatorId = model.NewId() - o2.TeamId = o1.TeamId - o2.ChannelId = model.NewId() - o2.TriggerWords = []string{"trigger"} - o2.CallbackURLs = []string{"http://nowhere.com/"} - - o2 = (<-store.Webhook().SaveOutgoing(o2)).Data.(*model.OutgoingWebhook) - - if r1 := <-store.Webhook().GetOutgoingByTriggerWord(o1.TeamId, "", "trigger"); r1.Err != nil { + if r1 := <-store.Webhook().GetOutgoingByTeam(o1.TeamId); r1.Err != nil { t.Fatal(r1.Err) } else { if r1.Data.([]*model.OutgoingWebhook)[0].CreateAt != o1.CreateAt { @@ -229,15 +220,7 @@ func TestWebhookStoreGetOutgoingByTriggerWord(t *testing.T) { } } - if r1 := <-store.Webhook().GetOutgoingByTriggerWord(o2.TeamId, o2.ChannelId, "trigger"); r1.Err != nil { - t.Fatal(r1.Err) - } else { - if len(r1.Data.([]*model.OutgoingWebhook)) != 2 { - t.Fatal("wrong number of webhooks returned") - } - } - - if result := <-store.Webhook().GetOutgoingByTriggerWord(o1.TeamId, "", "blargh"); result.Err != nil { + if result := <-store.Webhook().GetOutgoingByTeam("123"); result.Err != nil { t.Fatal(result.Err) } else { if len(result.Data.([]*model.OutgoingWebhook)) != 0 { diff --git a/store/store.go b/store/store.go index 2a099975c..70980a15c 100644 --- a/store/store.go +++ b/store/store.go @@ -154,7 +154,7 @@ type WebhookStore interface { GetOutgoing(id string) StoreChannel GetOutgoingByCreator(userId string) StoreChannel GetOutgoingByChannel(channelId string) StoreChannel - GetOutgoingByTriggerWord(teamId, channelId, triggerWord string) StoreChannel + GetOutgoingByTeam(teamId string) StoreChannel DeleteOutgoing(webhookId string, time int64) StoreChannel UpdateOutgoing(hook *model.OutgoingWebhook) StoreChannel } -- cgit v1.2.3-1-g7c22