From 97558f6a6ec4c53fa69035fb430ead209d9c222d Mon Sep 17 00:00:00 2001 From: Joram Wilander Date: Fri, 13 Jan 2017 13:53:37 -0500 Subject: PLT-4938 Add app package and move logic over from api package (#4931) * Add app package and move logic over from api package * Change app package functions to return errors * Move non-api tests into app package * Fix merge --- app/apptestlib.go | 191 ++++++++++++ app/channel.go | 216 +++++++++++++ app/command.go | 31 ++ app/email_batching.go | 252 ++++++++++++++++ app/email_batching_test.go | 193 ++++++++++++ app/notification.go | 732 +++++++++++++++++++++++++++++++++++++++++++++ app/notification_test.go | 312 +++++++++++++++++++ app/post.go | 196 ++++++++++++ app/server.go | 217 ++++++++++++++ app/session.go | 94 ++++++ app/session_test.go | 31 ++ app/status.go | 255 ++++++++++++++++ app/team.go | 82 +++++ app/user.go | 60 ++++ app/web_conn.go | 254 ++++++++++++++++ app/web_hub.go | 241 +++++++++++++++ app/webhook.go | 155 ++++++++++ app/websocket_router.go | 96 ++++++ 18 files changed, 3608 insertions(+) create mode 100644 app/apptestlib.go create mode 100644 app/channel.go create mode 100644 app/command.go create mode 100644 app/email_batching.go create mode 100644 app/email_batching_test.go create mode 100644 app/notification.go create mode 100644 app/notification_test.go create mode 100644 app/post.go create mode 100644 app/server.go create mode 100644 app/session.go create mode 100644 app/session_test.go create mode 100644 app/status.go create mode 100644 app/team.go create mode 100644 app/user.go create mode 100644 app/web_conn.go create mode 100644 app/web_hub.go create mode 100644 app/webhook.go create mode 100644 app/websocket_router.go (limited to 'app') diff --git a/app/apptestlib.go b/app/apptestlib.go new file mode 100644 index 000000000..dee6f0fd9 --- /dev/null +++ b/app/apptestlib.go @@ -0,0 +1,191 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "time" + + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" + + l4g "github.com/alecthomas/log4go" +) + +type TestHelper struct { + BasicTeam *model.Team + BasicUser *model.User + BasicUser2 *model.User + BasicChannel *model.Channel + BasicPost *model.Post +} + +func SetupEnterprise() *TestHelper { + if Srv == nil { + utils.TranslationsPreInit() + utils.LoadConfig("config.json") + utils.InitTranslations(utils.Cfg.LocalizationSettings) + utils.Cfg.TeamSettings.MaxUsersPerTeam = 50 + *utils.Cfg.RateLimitSettings.Enable = false + utils.DisableDebugLogForTest() + utils.License.Features.SetDefaults() + NewServer() + InitStores() + StartServer() + utils.InitHTML() + utils.EnableDebugLogForTest() + Srv.Store.MarkSystemRanUnitTests() + + *utils.Cfg.TeamSettings.EnableOpenServer = true + } + + return &TestHelper{} +} + +func Setup() *TestHelper { + if Srv == nil { + utils.TranslationsPreInit() + utils.LoadConfig("config.json") + utils.InitTranslations(utils.Cfg.LocalizationSettings) + utils.Cfg.TeamSettings.MaxUsersPerTeam = 50 + *utils.Cfg.RateLimitSettings.Enable = false + utils.DisableDebugLogForTest() + NewServer() + InitStores() + StartServer() + utils.EnableDebugLogForTest() + Srv.Store.MarkSystemRanUnitTests() + + *utils.Cfg.TeamSettings.EnableOpenServer = true + } + + return &TestHelper{} +} + +func (me *TestHelper) InitBasic() *TestHelper { + me.BasicTeam = me.CreateTeam() + me.BasicUser = me.CreateUser() + LinkUserToTeam(me.BasicUser, me.BasicTeam) + me.BasicUser2 = me.CreateUser() + LinkUserToTeam(me.BasicUser2, me.BasicTeam) + me.BasicChannel = me.CreateChannel(me.BasicTeam) + me.BasicPost = me.CreatePost(me.BasicChannel) + + return me +} + +func (me *TestHelper) CreateTeam() *model.Team { + id := model.NewId() + team := &model.Team{ + DisplayName: "dn_" + id, + Name: "name" + id, + Email: "success+" + id + "@simulator.amazonses.com", + Type: model.TEAM_OPEN, + } + + utils.DisableDebugLogForTest() + var err *model.AppError + if team, err = CreateTeam(team); err != nil { + l4g.Error(err.Error()) + l4g.Close() + time.Sleep(time.Second) + panic(err) + } + utils.EnableDebugLogForTest() + return team +} + +func (me *TestHelper) CreateUser() *model.User { + id := model.NewId() + + user := &model.User{ + Email: "success+" + id + "@simulator.amazonses.com", + Username: "un_" + id, + Nickname: "nn_" + id, + Password: "Password1", + EmailVerified: true, + } + + utils.DisableDebugLogForTest() + var err *model.AppError + if user, err = CreateUser(user); err != nil { + l4g.Error(err.Error()) + l4g.Close() + time.Sleep(time.Second) + panic(err) + } + utils.EnableDebugLogForTest() + return user +} + +func (me *TestHelper) CreateChannel(team *model.Team) *model.Channel { + return me.createChannel(team, model.CHANNEL_OPEN) +} + +func (me *TestHelper) CreatePrivateChannel(team *model.Team) *model.Channel { + return me.createChannel(team, model.CHANNEL_PRIVATE) +} + +func (me *TestHelper) createChannel(team *model.Team, channelType string) *model.Channel { + id := model.NewId() + + channel := &model.Channel{ + DisplayName: "dn_" + id, + Name: "name_" + id, + Type: channelType, + TeamId: team.Id, + CreatorId: me.BasicUser.Id, + } + + utils.DisableDebugLogForTest() + var err *model.AppError + if channel, err = CreateChannel(channel, true); err != nil { + l4g.Error(err.Error()) + l4g.Close() + time.Sleep(time.Second) + panic(err) + } + utils.EnableDebugLogForTest() + return channel +} + +func (me *TestHelper) CreatePost(channel *model.Channel) *model.Post { + id := model.NewId() + + post := &model.Post{ + UserId: me.BasicUser.Id, + ChannelId: channel.Id, + Message: "message_" + id, + } + + utils.DisableDebugLogForTest() + var err *model.AppError + if post, err = CreatePost(post, channel.TeamId, false); err != nil { + l4g.Error(err.Error()) + l4g.Close() + time.Sleep(time.Second) + panic(err) + } + utils.EnableDebugLogForTest() + return post +} + +func LinkUserToTeam(user *model.User, team *model.Team) { + utils.DisableDebugLogForTest() + + err := JoinUserToTeam(team, user) + if err != nil { + l4g.Error(err.Error()) + l4g.Close() + time.Sleep(time.Second) + panic(err) + } + + utils.EnableDebugLogForTest() +} + +func TearDown() { + if Srv != nil { + StopServer() + } +} diff --git a/app/channel.go b/app/channel.go new file mode 100644 index 000000000..1771c856b --- /dev/null +++ b/app/channel.go @@ -0,0 +1,216 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "fmt" + + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/store" + "github.com/mattermost/platform/utils" +) + +func MakeDirectChannelVisible(channelId string) *model.AppError { + var members []model.ChannelMember + if result := <-Srv.Store.Channel().GetMembers(channelId); result.Err != nil { + return result.Err + } else { + members = result.Data.([]model.ChannelMember) + } + + if len(members) != 2 { + return model.NewLocAppError("MakeDirectChannelVisible", "api.post.make_direct_channel_visible.get_2_members.error", map[string]interface{}{"ChannelId": channelId}, "") + } + + // make sure the channel is visible to both members + for i, member := range members { + otherUserId := members[1-i].UserId + + if result := <-Srv.Store.Preference().Get(member.UserId, model.PREFERENCE_CATEGORY_DIRECT_CHANNEL_SHOW, otherUserId); result.Err != nil { + // create a new preference since one doesn't exist yet + preference := &model.Preference{ + UserId: member.UserId, + Category: model.PREFERENCE_CATEGORY_DIRECT_CHANNEL_SHOW, + Name: otherUserId, + Value: "true", + } + + if saveResult := <-Srv.Store.Preference().Save(&model.Preferences{*preference}); saveResult.Err != nil { + return saveResult.Err + } else { + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_PREFERENCE_CHANGED, "", "", member.UserId, nil) + message.Add("preference", preference.ToJson()) + + go Publish(message) + } + } else { + preference := result.Data.(model.Preference) + + if preference.Value != "true" { + // update the existing preference to make the channel visible + preference.Value = "true" + + if updateResult := <-Srv.Store.Preference().Save(&model.Preferences{preference}); updateResult.Err != nil { + return updateResult.Err + } else { + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_PREFERENCE_CHANGED, "", "", member.UserId, nil) + message.Add("preference", preference.ToJson()) + + go Publish(message) + } + } + } + } + + return nil +} + +func CreateDefaultChannels(teamId string) ([]*model.Channel, *model.AppError) { + townSquare := &model.Channel{DisplayName: utils.T("api.channel.create_default_channels.town_square"), Name: "town-square", Type: model.CHANNEL_OPEN, TeamId: teamId} + + if _, err := CreateChannel(townSquare, false); err != nil { + return nil, err + } + + offTopic := &model.Channel{DisplayName: utils.T("api.channel.create_default_channels.off_topic"), Name: "off-topic", Type: model.CHANNEL_OPEN, TeamId: teamId} + + if _, err := CreateChannel(offTopic, false); err != nil { + return nil, err + } + + channels := []*model.Channel{townSquare, offTopic} + return channels, nil +} + +func JoinDefaultChannels(teamId string, user *model.User, channelRole string) *model.AppError { + var err *model.AppError = nil + + if result := <-Srv.Store.Channel().GetByName(teamId, "town-square"); result.Err != nil { + err = result.Err + } else { + cm := &model.ChannelMember{ChannelId: result.Data.(*model.Channel).Id, UserId: user.Id, + Roles: channelRole, NotifyProps: model.GetDefaultChannelNotifyProps()} + + if cmResult := <-Srv.Store.Channel().SaveMember(cm); cmResult.Err != nil { + err = cmResult.Err + } + + post := &model.Post{ + ChannelId: result.Data.(*model.Channel).Id, + Message: fmt.Sprintf(utils.T("api.channel.join_channel.post_and_forget"), user.Username), + Type: model.POST_JOIN_LEAVE, + UserId: user.Id, + } + + InvalidateCacheForChannel(result.Data.(*model.Channel).Id) + + if _, err := CreatePost(post, teamId, false); err != nil { + l4g.Error(utils.T("api.channel.post_user_add_remove_message_and_forget.error"), err) + } + } + + if result := <-Srv.Store.Channel().GetByName(teamId, "off-topic"); result.Err != nil { + err = result.Err + } else { + cm := &model.ChannelMember{ChannelId: result.Data.(*model.Channel).Id, UserId: user.Id, + Roles: channelRole, NotifyProps: model.GetDefaultChannelNotifyProps()} + + if cmResult := <-Srv.Store.Channel().SaveMember(cm); cmResult.Err != nil { + err = cmResult.Err + } + + post := &model.Post{ + ChannelId: result.Data.(*model.Channel).Id, + Message: fmt.Sprintf(utils.T("api.channel.join_channel.post_and_forget"), user.Username), + Type: model.POST_JOIN_LEAVE, + UserId: user.Id, + } + + InvalidateCacheForChannel(result.Data.(*model.Channel).Id) + + if _, err := CreatePost(post, teamId, false); err != nil { + l4g.Error(utils.T("api.channel.post_user_add_remove_message_and_forget.error"), err) + } + } + + return err +} + +func CreateChannel(channel *model.Channel, addMember bool) (*model.Channel, *model.AppError) { + if result := <-Srv.Store.Channel().Save(channel); result.Err != nil { + return nil, result.Err + } else { + sc := result.Data.(*model.Channel) + + if addMember { + cm := &model.ChannelMember{ + ChannelId: sc.Id, + UserId: channel.CreatorId, + Roles: model.ROLE_CHANNEL_USER.Id + " " + model.ROLE_CHANNEL_ADMIN.Id, + NotifyProps: model.GetDefaultChannelNotifyProps(), + } + + if cmresult := <-Srv.Store.Channel().SaveMember(cm); cmresult.Err != nil { + return nil, cmresult.Err + } + + InvalidateCacheForUser(channel.CreatorId) + } + + return sc, nil + } +} + +func AddUserToChannel(user *model.User, channel *model.Channel) (*model.ChannelMember, *model.AppError) { + if channel.DeleteAt > 0 { + return nil, model.NewLocAppError("AddUserToChannel", "api.channel.add_user_to_channel.deleted.app_error", nil, "") + } + + if channel.Type != model.CHANNEL_OPEN && channel.Type != model.CHANNEL_PRIVATE { + return nil, model.NewLocAppError("AddUserToChannel", "api.channel.add_user_to_channel.type.app_error", nil, "") + } + + tmchan := Srv.Store.Team().GetMember(channel.TeamId, user.Id) + cmchan := Srv.Store.Channel().GetMember(channel.Id, user.Id) + + if result := <-tmchan; result.Err != nil { + return nil, result.Err + } else { + teamMember := result.Data.(model.TeamMember) + if teamMember.DeleteAt > 0 { + return nil, model.NewLocAppError("AddUserToChannel", "api.channel.add_user.to.channel.failed.deleted.app_error", nil, "") + } + } + + if result := <-cmchan; result.Err != nil { + if result.Err.Id != store.MISSING_CHANNEL_MEMBER_ERROR { + return nil, result.Err + } + } else { + channelMember := result.Data.(model.ChannelMember) + return &channelMember, nil + } + + newMember := &model.ChannelMember{ + ChannelId: channel.Id, + UserId: user.Id, + NotifyProps: model.GetDefaultChannelNotifyProps(), + Roles: model.ROLE_CHANNEL_USER.Id, + } + if result := <-Srv.Store.Channel().SaveMember(newMember); result.Err != nil { + l4g.Error("Failed to add member user_id=%v channel_id=%v err=%v", user.Id, channel.Id, result.Err) + return nil, model.NewLocAppError("AddUserToChannel", "api.channel.add_user.to.channel.failed.app_error", nil, "") + } + + InvalidateCacheForUser(user.Id) + InvalidateCacheForChannel(channel.Id) + + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_USER_ADDED, "", channel.Id, "", nil) + message.Add("user_id", user.Id) + message.Add("team_id", channel.TeamId) + go Publish(message) + + return newMember, nil +} diff --git a/app/command.go b/app/command.go new file mode 100644 index 000000000..2d5861206 --- /dev/null +++ b/app/command.go @@ -0,0 +1,31 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "github.com/mattermost/platform/model" +) + +func CreateCommandPost(post *model.Post, teamId string, response *model.CommandResponse) (*model.Post, *model.AppError) { + post.Message = parseSlackLinksToMarkdown(response.Text) + post.CreateAt = model.GetMillis() + + if response.Attachments != nil { + parseSlackAttachment(post, response.Attachments) + } + + switch response.ResponseType { + case model.COMMAND_RESPONSE_TYPE_IN_CHANNEL: + return CreatePost(post, teamId, true) + case model.COMMAND_RESPONSE_TYPE_EPHEMERAL: + if response.Text == "" { + return post, nil + } + + post.ParentId = "" + SendEphemeralPost(teamId, post.UserId, post) + } + + return post, nil +} diff --git a/app/email_batching.go b/app/email_batching.go new file mode 100644 index 000000000..fc2fb1cea --- /dev/null +++ b/app/email_batching.go @@ -0,0 +1,252 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "database/sql" + "fmt" + "html/template" + "strconv" + "time" + + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" + + l4g "github.com/alecthomas/log4go" + "github.com/nicksnyder/go-i18n/i18n" +) + +const ( + EMAIL_BATCHING_TASK_NAME = "Email Batching" +) + +var emailBatchingJob *EmailBatchingJob + +func InitEmailBatching() { + if *utils.Cfg.EmailSettings.EnableEmailBatching { + if emailBatchingJob == nil { + emailBatchingJob = MakeEmailBatchingJob(*utils.Cfg.EmailSettings.EmailBatchingBufferSize) + } + + // note that we don't support changing EmailBatchingBufferSize without restarting the server + + emailBatchingJob.Start() + } +} + +func AddNotificationEmailToBatch(user *model.User, post *model.Post, team *model.Team) *model.AppError { + if !*utils.Cfg.EmailSettings.EnableEmailBatching { + return model.NewLocAppError("AddNotificationEmailToBatch", "api.email_batching.add_notification_email_to_batch.disabled.app_error", nil, "") + } + + if !emailBatchingJob.Add(user, post, team) { + l4g.Error(utils.T("api.email_batching.add_notification_email_to_batch.channel_full.app_error")) + return model.NewLocAppError("AddNotificationEmailToBatch", "api.email_batching.add_notification_email_to_batch.channel_full.app_error", nil, "") + } + + return nil +} + +type batchedNotification struct { + userId string + post *model.Post + teamName string +} + +type EmailBatchingJob struct { + newNotifications chan *batchedNotification + pendingNotifications map[string][]*batchedNotification +} + +func MakeEmailBatchingJob(bufferSize int) *EmailBatchingJob { + return &EmailBatchingJob{ + newNotifications: make(chan *batchedNotification, bufferSize), + pendingNotifications: make(map[string][]*batchedNotification), + } +} + +func (job *EmailBatchingJob) Start() { + if task := model.GetTaskByName(EMAIL_BATCHING_TASK_NAME); task != nil { + task.Cancel() + } + + l4g.Debug(utils.T("api.email_batching.start.starting"), *utils.Cfg.EmailSettings.EmailBatchingInterval) + model.CreateRecurringTask(EMAIL_BATCHING_TASK_NAME, job.CheckPendingEmails, time.Duration(*utils.Cfg.EmailSettings.EmailBatchingInterval)*time.Second) +} + +func (job *EmailBatchingJob) Add(user *model.User, post *model.Post, team *model.Team) bool { + notification := &batchedNotification{ + userId: user.Id, + post: post, + teamName: team.Name, + } + + select { + case job.newNotifications <- notification: + return true + default: + // return false if we couldn't queue the email notification so that we can send an immediate email + return false + } +} + +func (job *EmailBatchingJob) CheckPendingEmails() { + job.handleNewNotifications() + + // it's a bit weird to pass the send email function through here, but it makes it so that we can test + // without actually sending emails + job.checkPendingNotifications(time.Now(), sendBatchedEmailNotification) + + l4g.Debug(utils.T("api.email_batching.check_pending_emails.finished_running"), len(job.pendingNotifications)) +} + +func (job *EmailBatchingJob) handleNewNotifications() { + receiving := true + + // read in new notifications to send + for receiving { + select { + case notification := <-job.newNotifications: + userId := notification.userId + + if _, ok := job.pendingNotifications[userId]; !ok { + job.pendingNotifications[userId] = []*batchedNotification{notification} + } else { + job.pendingNotifications[userId] = append(job.pendingNotifications[userId], notification) + } + default: + receiving = false + } + } +} + +func (job *EmailBatchingJob) checkPendingNotifications(now time.Time, handler func(string, []*batchedNotification)) { + // look for users who've acted since pending posts were received + for userId, notifications := range job.pendingNotifications { + schan := Srv.Store.Status().Get(userId) + pchan := Srv.Store.Preference().Get(userId, model.PREFERENCE_CATEGORY_NOTIFICATIONS, model.PREFERENCE_NAME_EMAIL_INTERVAL) + batchStartTime := notifications[0].post.CreateAt + + // check if the user has been active and would've seen any new posts + if result := <-schan; result.Err != nil { + l4g.Error(utils.T("api.email_batching.check_pending_emails.status.app_error"), result.Err) + delete(job.pendingNotifications, userId) + continue + } else if status := result.Data.(*model.Status); status.LastActivityAt >= batchStartTime { + delete(job.pendingNotifications, userId) + continue + } + + // get how long we need to wait to send notifications to the user + var interval int64 + if result := <-pchan; result.Err != nil { + // default to 30 seconds to match the send "immediate" setting + interval, _ = strconv.ParseInt(model.PREFERENCE_DEFAULT_EMAIL_INTERVAL, 10, 64) + } else { + preference := result.Data.(model.Preference) + + if value, err := strconv.ParseInt(preference.Value, 10, 64); err != nil { + interval, _ = strconv.ParseInt(model.PREFERENCE_DEFAULT_EMAIL_INTERVAL, 10, 64) + } else { + interval = value + } + } + + // send the email notification if it's been long enough + if now.Sub(time.Unix(batchStartTime/1000, 0)) > time.Duration(interval)*time.Second { + go handler(userId, notifications) + delete(job.pendingNotifications, userId) + } + } +} + +func sendBatchedEmailNotification(userId string, notifications []*batchedNotification) { + uchan := Srv.Store.User().Get(userId) + pchan := Srv.Store.Preference().Get(userId, model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, model.PREFERENCE_NAME_DISPLAY_NAME_FORMAT) + + var user *model.User + if result := <-uchan; result.Err != nil { + l4g.Warn("api.email_batching.send_batched_email_notification.user.app_error") + return + } else { + user = result.Data.(*model.User) + } + + translateFunc := utils.GetUserTranslations(user.Locale) + + var displayNameFormat string + if result := <-pchan; result.Err != nil && result.Err.DetailedError != sql.ErrNoRows.Error() { + l4g.Warn("api.email_batching.send_batched_email_notification.preferences.app_error") + return + } else if result.Err != nil { + // no display name format saved, so fall back to default + displayNameFormat = model.PREFERENCE_DEFAULT_DISPLAY_NAME_FORMAT + } else { + displayNameFormat = result.Data.(model.Preference).Value + } + + var contents string + for _, notification := range notifications { + template := utils.NewHTMLTemplate("post_batched_post", user.Locale) + + contents += renderBatchedPost(template, notification.post, notification.teamName, displayNameFormat, translateFunc) + } + + tm := time.Unix(notifications[0].post.CreateAt/1000, 0) + + subject := translateFunc("api.email_batching.send_batched_email_notification.subject", len(notifications), map[string]interface{}{ + "SiteName": utils.Cfg.TeamSettings.SiteName, + "Year": tm.Year(), + "Month": translateFunc(tm.Month().String()), + "Day": tm.Day(), + }) + + body := utils.NewHTMLTemplate("post_batched_body", user.Locale) + body.Props["SiteURL"] = *utils.Cfg.ServiceSettings.SiteURL + body.Props["Posts"] = template.HTML(contents) + body.Props["BodyText"] = translateFunc("api.email_batching.send_batched_email_notification.body_text", len(notifications)) + + if err := utils.SendMail(user.Email, subject, body.Render()); err != nil { + l4g.Warn(utils.T("api.email_batchings.send_batched_email_notification.send.app_error"), user.Email, err) + } +} + +func renderBatchedPost(template *utils.HTMLTemplate, post *model.Post, teamName string, displayNameFormat string, translateFunc i18n.TranslateFunc) string { + schan := Srv.Store.User().Get(post.UserId) + cchan := Srv.Store.Channel().Get(post.ChannelId, true) + + template.Props["Button"] = translateFunc("api.email_batching.render_batched_post.go_to_post") + template.Props["PostMessage"] = GetMessageForNotification(post, translateFunc) + template.Props["PostLink"] = *utils.Cfg.ServiceSettings.SiteURL + "/" + teamName + "/pl/" + post.Id + + tm := time.Unix(post.CreateAt/1000, 0) + timezone, _ := tm.Zone() + + template.Props["Date"] = translateFunc("api.email_batching.render_batched_post.date", map[string]interface{}{ + "Year": tm.Year(), + "Month": translateFunc(tm.Month().String()), + "Day": tm.Day(), + "Hour": tm.Hour(), + "Minute": fmt.Sprintf("%02d", tm.Minute()), + "Timezone": timezone, + }) + + if result := <-schan; result.Err != nil { + l4g.Warn(utils.T("api.email_batching.render_batched_post.sender.app_error")) + return "" + } else { + template.Props["SenderName"] = result.Data.(*model.User).GetDisplayNameForPreference(displayNameFormat) + } + + if result := <-cchan; result.Err != nil { + l4g.Warn(utils.T("api.email_batching.render_batched_post.channel.app_error")) + return "" + } else if channel := result.Data.(*model.Channel); channel.Type == model.CHANNEL_DIRECT { + template.Props["ChannelName"] = translateFunc("api.email_batching.render_batched_post.direct_message") + } else { + template.Props["ChannelName"] = channel.DisplayName + } + + return template.Render() +} diff --git a/app/email_batching_test.go b/app/email_batching_test.go new file mode 100644 index 000000000..23722facd --- /dev/null +++ b/app/email_batching_test.go @@ -0,0 +1,193 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "testing" + "time" + + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/store" +) + +func TestHandleNewNotifications(t *testing.T) { + Setup() + + id1 := model.NewId() + id2 := model.NewId() + id3 := model.NewId() + + // test queueing of received posts by user + job := MakeEmailBatchingJob(128) + + job.handleNewNotifications() + + if len(job.pendingNotifications) != 0 { + t.Fatal("shouldn't have added any pending notifications") + } + + job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test"}, &model.Team{Name: "team"}) + if len(job.pendingNotifications) != 0 { + t.Fatal("shouldn't have added any pending notifications") + } + + job.handleNewNotifications() + if len(job.pendingNotifications) != 1 { + t.Fatal("should have received posts for 1 user") + } else if len(job.pendingNotifications[id1]) != 1 { + t.Fatal("should have received 1 post for user") + } + + job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test"}, &model.Team{Name: "team"}) + job.handleNewNotifications() + if len(job.pendingNotifications) != 1 { + t.Fatal("should have received posts for 1 user") + } else if len(job.pendingNotifications[id1]) != 2 { + t.Fatal("should have received 2 posts for user1", job.pendingNotifications[id1]) + } + + job.Add(&model.User{Id: id2}, &model.Post{UserId: id1, Message: "test"}, &model.Team{Name: "team"}) + job.handleNewNotifications() + if len(job.pendingNotifications) != 2 { + t.Fatal("should have received posts for 2 users") + } else if len(job.pendingNotifications[id1]) != 2 { + t.Fatal("should have received 2 posts for user1") + } else if len(job.pendingNotifications[id2]) != 1 { + t.Fatal("should have received 1 post for user2") + } + + job.Add(&model.User{Id: id2}, &model.Post{UserId: id2, Message: "test"}, &model.Team{Name: "team"}) + job.Add(&model.User{Id: id1}, &model.Post{UserId: id3, Message: "test"}, &model.Team{Name: "team"}) + job.Add(&model.User{Id: id3}, &model.Post{UserId: id3, Message: "test"}, &model.Team{Name: "team"}) + job.Add(&model.User{Id: id2}, &model.Post{UserId: id2, Message: "test"}, &model.Team{Name: "team"}) + job.handleNewNotifications() + if len(job.pendingNotifications) != 3 { + t.Fatal("should have received posts for 3 users") + } else if len(job.pendingNotifications[id1]) != 3 { + t.Fatal("should have received 3 posts for user1") + } else if len(job.pendingNotifications[id2]) != 3 { + t.Fatal("should have received 3 posts for user2") + } else if len(job.pendingNotifications[id3]) != 1 { + t.Fatal("should have received 1 post for user3") + } + + // test ordering of received posts + job = MakeEmailBatchingJob(128) + + job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test1"}, &model.Team{Name: "team"}) + job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test2"}, &model.Team{Name: "team"}) + job.Add(&model.User{Id: id2}, &model.Post{UserId: id1, Message: "test3"}, &model.Team{Name: "team"}) + job.Add(&model.User{Id: id1}, &model.Post{UserId: id1, Message: "test4"}, &model.Team{Name: "team"}) + job.Add(&model.User{Id: id2}, &model.Post{UserId: id1, Message: "test5"}, &model.Team{Name: "team"}) + job.handleNewNotifications() + if job.pendingNotifications[id1][0].post.Message != "test1" || + job.pendingNotifications[id1][1].post.Message != "test2" || + job.pendingNotifications[id1][2].post.Message != "test4" { + t.Fatal("incorrect order of received posts for user1") + } else if job.pendingNotifications[id2][0].post.Message != "test3" || + job.pendingNotifications[id2][1].post.Message != "test5" { + t.Fatal("incorrect order of received posts for user2") + } +} + +func TestCheckPendingNotifications(t *testing.T) { + Setup() + + id1 := model.NewId() + + job := MakeEmailBatchingJob(128) + job.pendingNotifications[id1] = []*batchedNotification{ + { + post: &model.Post{ + UserId: id1, + CreateAt: 10000000, + }, + }, + } + + store.Must(Srv.Store.Status().SaveOrUpdate(&model.Status{ + UserId: id1, + LastActivityAt: 9999000, + })) + store.Must(Srv.Store.Preference().Save(&model.Preferences{{ + UserId: id1, + Category: model.PREFERENCE_CATEGORY_NOTIFICATIONS, + Name: model.PREFERENCE_NAME_EMAIL_INTERVAL, + Value: "60", + }})) + + // test that notifications aren't sent before interval + job.checkPendingNotifications(time.Unix(10001, 0), func(string, []*batchedNotification) {}) + + if job.pendingNotifications[id1] == nil || len(job.pendingNotifications[id1]) != 1 { + t.Fatal("should'nt have sent queued post") + } + + // test that notifications are cleared if the user has acted + store.Must(Srv.Store.Status().SaveOrUpdate(&model.Status{ + UserId: id1, + LastActivityAt: 10001000, + })) + + job.checkPendingNotifications(time.Unix(10002, 0), func(string, []*batchedNotification) {}) + + if job.pendingNotifications[id1] != nil && len(job.pendingNotifications[id1]) != 0 { + t.Fatal("should've remove queued post since user acted") + } + + // test that notifications are sent if enough time passes since the first message + job.pendingNotifications[id1] = []*batchedNotification{ + { + post: &model.Post{ + UserId: id1, + CreateAt: 10060000, + Message: "post1", + }, + }, + { + post: &model.Post{ + UserId: id1, + CreateAt: 10090000, + Message: "post2", + }, + }, + } + + received := make(chan *model.Post, 2) + timeout := make(chan bool) + + job.checkPendingNotifications(time.Unix(10130, 0), func(s string, notifications []*batchedNotification) { + for _, notification := range notifications { + received <- notification.post + } + }) + + go func() { + // start a timeout to make sure that we don't get stuck here on a failed test + time.Sleep(5 * time.Second) + timeout <- true + }() + + if job.pendingNotifications[id1] != nil && len(job.pendingNotifications[id1]) != 0 { + t.Fatal("should've remove queued posts when sending messages") + } + + select { + case post := <-received: + if post.Message != "post1" { + t.Fatal("should've received post1 first") + } + case _ = <-timeout: + t.Fatal("timed out waiting for first post notification") + } + + select { + case post := <-received: + if post.Message != "post2" { + t.Fatal("should've received post2 second") + } + case _ = <-timeout: + t.Fatal("timed out waiting for second post notification") + } +} diff --git a/app/notification.go b/app/notification.go new file mode 100644 index 000000000..d5e3c7b13 --- /dev/null +++ b/app/notification.go @@ -0,0 +1,732 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "crypto/tls" + "fmt" + "html" + "html/template" + "io/ioutil" + "net/http" + "net/url" + "path/filepath" + "sort" + "strings" + "time" + + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/einterfaces" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/store" + "github.com/mattermost/platform/utils" + "github.com/nicksnyder/go-i18n/i18n" +) + +func SendNotifications(post *model.Post, team *model.Team, channel *model.Channel) ([]string, *model.AppError) { + mentionedUsersList := make([]string, 0) + var fchan store.StoreChannel + var senderUsername string + + if post.IsSystemMessage() { + senderUsername = utils.T("system.message.name") + } else { + pchan := Srv.Store.User().GetProfilesInChannel(channel.Id, -1, -1, true) + fchan = Srv.Store.FileInfo().GetForPost(post.Id) + + var profileMap map[string]*model.User + if result := <-pchan; result.Err != nil { + return nil, result.Err + } else { + profileMap = result.Data.(map[string]*model.User) + } + + // If the user who made the post isn't in the channel don't send a notification + if _, ok := profileMap[post.UserId]; !ok { + l4g.Debug(utils.T("api.post.send_notifications.user_id.debug"), post.Id, channel.Id, post.UserId) + return []string{}, nil + } + + mentionedUserIds := make(map[string]bool) + allActivityPushUserIds := []string{} + hereNotification := false + channelNotification := false + allNotification := false + updateMentionChans := []store.StoreChannel{} + + if channel.Type == model.CHANNEL_DIRECT { + var otherUserId string + if userIds := strings.Split(channel.Name, "__"); userIds[0] == post.UserId { + otherUserId = userIds[1] + } else { + otherUserId = userIds[0] + } + + mentionedUserIds[otherUserId] = true + if post.Props["from_webhook"] == "true" { + mentionedUserIds[post.UserId] = true + } + } else { + keywords := GetMentionKeywordsInChannel(profileMap) + + var potentialOtherMentions []string + mentionedUserIds, potentialOtherMentions, hereNotification, channelNotification, allNotification = GetExplicitMentions(post.Message, keywords) + + // get users that have comment thread mentions enabled + if len(post.RootId) > 0 { + if result := <-Srv.Store.Post().Get(post.RootId); result.Err != nil { + return nil, result.Err + } else { + list := result.Data.(*model.PostList) + + for _, threadPost := range list.Posts { + if profile, ok := profileMap[threadPost.UserId]; ok { + if profile.NotifyProps["comments"] == "any" || (profile.NotifyProps["comments"] == "root" && threadPost.Id == list.Order[0]) { + mentionedUserIds[threadPost.UserId] = true + } + } + } + } + } + + // prevent the user from mentioning themselves + if post.Props["from_webhook"] != "true" { + delete(mentionedUserIds, post.UserId) + } + + if len(potentialOtherMentions) > 0 { + if result := <-Srv.Store.User().GetProfilesByUsernames(potentialOtherMentions, team.Id); result.Err == nil { + outOfChannelMentions := result.Data.(map[string]*model.User) + go sendOutOfChannelMentions(post, team.Id, outOfChannelMentions) + } + } + + // find which users in the channel are set up to always receive mobile notifications + for _, profile := range profileMap { + if profile.NotifyProps["push"] == model.USER_NOTIFY_ALL && + (post.UserId != profile.Id || post.Props["from_webhook"] == "true") { + allActivityPushUserIds = append(allActivityPushUserIds, profile.Id) + } + } + } + + mentionedUsersList = make([]string, 0, len(mentionedUserIds)) + for id := range mentionedUserIds { + mentionedUsersList = append(mentionedUsersList, id) + updateMentionChans = append(updateMentionChans, Srv.Store.Channel().IncrementMentionCount(post.ChannelId, id)) + } + + var sender *model.User + senderName := make(map[string]string) + for _, id := range mentionedUsersList { + senderName[id] = "" + if profile, ok := profileMap[post.UserId]; ok { + if value, ok := post.Props["override_username"]; ok && post.Props["from_webhook"] == "true" { + senderName[id] = value.(string) + } else { + //Get the Display name preference from the receiver + if result := <-Srv.Store.Preference().Get(id, model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, "name_format"); result.Err != nil { + // Show default sender's name if user doesn't set display settings. + senderName[id] = profile.Username + } else { + senderName[id] = profile.GetDisplayNameForPreference(result.Data.(model.Preference).Value) + } + } + sender = profile + } + } + + if value, ok := post.Props["override_username"]; ok && post.Props["from_webhook"] == "true" { + senderUsername = value.(string) + } else { + senderUsername = profileMap[post.UserId].Username + } + + if utils.Cfg.EmailSettings.SendEmailNotifications { + for _, id := range mentionedUsersList { + userAllowsEmails := profileMap[id].NotifyProps["email"] != "false" + + var status *model.Status + var err *model.AppError + if status, err = GetStatus(id); err != nil { + status = &model.Status{ + UserId: id, + Status: model.STATUS_OFFLINE, + Manual: false, + LastActivityAt: 0, + ActiveChannel: "", + } + } + + if userAllowsEmails && status.Status != model.STATUS_ONLINE && profileMap[id].DeleteAt == 0 { + if err := sendNotificationEmail(post, profileMap[id], channel, team, senderName[id], sender); err != nil { + l4g.Error(err.Error()) + } + } + } + } + + T := utils.GetUserTranslations(profileMap[post.UserId].Locale) + + // If the channel has more than 1K users then @here is disabled + if hereNotification && int64(len(profileMap)) > *utils.Cfg.TeamSettings.MaxNotificationsPerChannel { + hereNotification = false + SendEphemeralPost( + team.Id, + post.UserId, + &model.Post{ + ChannelId: post.ChannelId, + Message: T("api.post.disabled_here", map[string]interface{}{"Users": *utils.Cfg.TeamSettings.MaxNotificationsPerChannel}), + CreateAt: post.CreateAt + 1, + }, + ) + } + + // If the channel has more than 1K users then @channel is disabled + if channelNotification && int64(len(profileMap)) > *utils.Cfg.TeamSettings.MaxNotificationsPerChannel { + SendEphemeralPost( + team.Id, + post.UserId, + &model.Post{ + ChannelId: post.ChannelId, + Message: T("api.post.disabled_channel", map[string]interface{}{"Users": *utils.Cfg.TeamSettings.MaxNotificationsPerChannel}), + CreateAt: post.CreateAt + 1, + }, + ) + } + + // If the channel has more than 1K users then @all is disabled + if allNotification && int64(len(profileMap)) > *utils.Cfg.TeamSettings.MaxNotificationsPerChannel { + SendEphemeralPost( + team.Id, + post.UserId, + &model.Post{ + ChannelId: post.ChannelId, + Message: T("api.post.disabled_all", map[string]interface{}{"Users": *utils.Cfg.TeamSettings.MaxNotificationsPerChannel}), + CreateAt: post.CreateAt + 1, + }, + ) + } + + if hereNotification { + statuses := GetAllStatuses() + for _, status := range statuses { + if status.UserId == post.UserId { + continue + } + + _, profileFound := profileMap[status.UserId] + _, alreadyMentioned := mentionedUserIds[status.UserId] + + if status.Status == model.STATUS_ONLINE && profileFound && !alreadyMentioned { + mentionedUsersList = append(mentionedUsersList, status.UserId) + updateMentionChans = append(updateMentionChans, Srv.Store.Channel().IncrementMentionCount(post.ChannelId, status.UserId)) + } + } + } + + // Make sure all mention updates are complete to prevent race + // Probably better to batch these DB updates in the future + // MUST be completed before push notifications send + for _, uchan := range updateMentionChans { + if result := <-uchan; result.Err != nil { + l4g.Warn(utils.T("api.post.update_mention_count_and_forget.update_error"), post.Id, post.ChannelId, result.Err) + } + } + + sendPushNotifications := false + if *utils.Cfg.EmailSettings.SendPushNotifications { + pushServer := *utils.Cfg.EmailSettings.PushNotificationServer + if pushServer == model.MHPNS && (!utils.IsLicensed || !*utils.License.Features.MHPNS) { + l4g.Warn(utils.T("api.post.send_notifications_and_forget.push_notification.mhpnsWarn")) + sendPushNotifications = false + } else { + sendPushNotifications = true + } + } + + if sendPushNotifications { + for _, id := range mentionedUsersList { + var status *model.Status + var err *model.AppError + if status, err = GetStatus(id); err != nil { + status = &model.Status{id, model.STATUS_OFFLINE, false, 0, ""} + } + + if DoesStatusAllowPushNotification(profileMap[id], status, post.ChannelId) { + if err := sendPushNotification(post, profileMap[id], channel, senderName[id], true); err != nil { + l4g.Error(err.Error()) + } + } + } + + for _, id := range allActivityPushUserIds { + if _, ok := mentionedUserIds[id]; !ok { + var status *model.Status + var err *model.AppError + if status, err = GetStatus(id); err != nil { + status = &model.Status{id, model.STATUS_OFFLINE, false, 0, ""} + } + + if DoesStatusAllowPushNotification(profileMap[id], status, post.ChannelId) { + if err := sendPushNotification(post, profileMap[id], channel, senderName[id], false); err != nil { + l4g.Error(err.Error()) + } + } + } + } + } + } + + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POSTED, "", post.ChannelId, "", nil) + message.Add("post", post.ToJson()) + message.Add("channel_type", channel.Type) + message.Add("channel_display_name", channel.DisplayName) + message.Add("channel_name", channel.Name) + message.Add("sender_name", senderUsername) + message.Add("team_id", team.Id) + + if len(post.FileIds) != 0 && fchan != nil { + message.Add("otherFile", "true") + + var infos []*model.FileInfo + if result := <-fchan; result.Err != nil { + l4g.Warn(utils.T("api.post.send_notifications.files.error"), post.Id, result.Err) + } else { + infos = result.Data.([]*model.FileInfo) + } + + for _, info := range infos { + if info.IsImage() { + message.Add("image", "true") + break + } + } + } + + if len(mentionedUsersList) != 0 { + message.Add("mentions", model.ArrayToJson(mentionedUsersList)) + } + + Publish(message) + return mentionedUsersList, nil +} + +func sendNotificationEmail(post *model.Post, user *model.User, channel *model.Channel, team *model.Team, senderName string, sender *model.User) *model.AppError { + + if channel.Type == model.CHANNEL_DIRECT && channel.TeamId != team.Id { + // this message is a cross-team DM so it we need to find a team that the recipient is on to use in the link + if result := <-Srv.Store.Team().GetTeamsByUserId(user.Id); result.Err != nil { + return result.Err + } else { + // if the recipient isn't in the current user's team, just pick one + teams := result.Data.([]*model.Team) + found := false + + for i := range teams { + if teams[i].Id == team.Id { + found = true + team = teams[i] + break + } + } + + if !found { + if len(teams) > 0 { + team = teams[0] + } else { + // in case the user hasn't joined any teams we send them to the select_team page + team = &model.Team{Name: "select_team", DisplayName: utils.Cfg.TeamSettings.SiteName} + } + } + } + } + if *utils.Cfg.EmailSettings.EnableEmailBatching { + var sendBatched bool + + if result := <-Srv.Store.Preference().Get(user.Id, model.PREFERENCE_CATEGORY_NOTIFICATIONS, model.PREFERENCE_NAME_EMAIL_INTERVAL); result.Err != nil { + // if the call fails, assume it hasn't been set and use the default + sendBatched = false + } else { + // default to not using batching if the setting is set to immediate + sendBatched = result.Data.(model.Preference).Value != model.PREFERENCE_DEFAULT_EMAIL_INTERVAL + } + + if sendBatched { + if err := AddNotificationEmailToBatch(user, post, team); err == nil { + return nil + } + } + + // fall back to sending a single email if we can't batch it for some reason + } + + var channelName string + var bodyText string + var subjectText string + var mailTemplate string + var mailParameters map[string]interface{} + + teamURL := utils.GetSiteURL() + "/" + team.Name + tm := time.Unix(post.CreateAt/1000, 0) + + userLocale := utils.GetUserTranslations(user.Locale) + month := userLocale(tm.Month().String()) + day := fmt.Sprintf("%d", tm.Day()) + year := fmt.Sprintf("%d", tm.Year()) + zone, _ := tm.Zone() + + if channel.Type == model.CHANNEL_DIRECT { + bodyText = userLocale("api.post.send_notifications_and_forget.message_body") + subjectText = userLocale("api.post.send_notifications_and_forget.message_subject") + + senderDisplayName := senderName + + mailTemplate = "api.templates.post_subject_in_direct_message" + mailParameters = map[string]interface{}{"SubjectText": subjectText, "TeamDisplayName": team.DisplayName, + "SenderDisplayName": senderDisplayName, "Month": month, "Day": day, "Year": year} + } else { + bodyText = userLocale("api.post.send_notifications_and_forget.mention_body") + subjectText = userLocale("api.post.send_notifications_and_forget.mention_subject") + channelName = channel.DisplayName + mailTemplate = "api.templates.post_subject_in_channel" + mailParameters = map[string]interface{}{"SubjectText": subjectText, "TeamDisplayName": team.DisplayName, + "ChannelName": channelName, "Month": month, "Day": day, "Year": year} + } + + subject := fmt.Sprintf("[%v] %v", utils.Cfg.TeamSettings.SiteName, userLocale(mailTemplate, mailParameters)) + + bodyPage := utils.NewHTMLTemplate("post_body", user.Locale) + bodyPage.Props["SiteURL"] = utils.GetSiteURL() + bodyPage.Props["PostMessage"] = GetMessageForNotification(post, userLocale) + if team.Name != "select_team" { + bodyPage.Props["TeamLink"] = teamURL + "/pl/" + post.Id + } else { + bodyPage.Props["TeamLink"] = teamURL + } + + bodyPage.Props["BodyText"] = bodyText + bodyPage.Props["Button"] = userLocale("api.templates.post_body.button") + bodyPage.Html["Info"] = template.HTML(userLocale("api.templates.post_body.info", + map[string]interface{}{"ChannelName": channelName, "SenderName": senderName, + "Hour": fmt.Sprintf("%02d", tm.Hour()), "Minute": fmt.Sprintf("%02d", tm.Minute()), + "TimeZone": zone, "Month": month, "Day": day})) + + if err := utils.SendMail(user.Email, html.UnescapeString(subject), bodyPage.Render()); err != nil { + return err + } + + if einterfaces.GetMetricsInterface() != nil { + einterfaces.GetMetricsInterface().IncrementPostSentEmail() + } + + return nil +} + +func GetMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string { + if len(strings.TrimSpace(post.Message)) != 0 || len(post.FileIds) == 0 { + return post.Message + } + + // extract the filenames from their paths and determine what type of files are attached + var infos []*model.FileInfo + if result := <-Srv.Store.FileInfo().GetForPost(post.Id); result.Err != nil { + l4g.Warn(utils.T("api.post.get_message_for_notification.get_files.error"), post.Id, result.Err) + } else { + infos = result.Data.([]*model.FileInfo) + } + + filenames := make([]string, len(infos)) + onlyImages := true + for i, info := range infos { + if escaped, err := url.QueryUnescape(filepath.Base(info.Name)); err != nil { + // this should never error since filepath was escaped using url.QueryEscape + filenames[i] = escaped + } else { + filenames[i] = info.Name + } + + onlyImages = onlyImages && info.IsImage() + } + + props := map[string]interface{}{"Filenames": strings.Join(filenames, ", ")} + + if onlyImages { + return translateFunc("api.post.get_message_for_notification.images_sent", len(filenames), props) + } else { + return translateFunc("api.post.get_message_for_notification.files_sent", len(filenames), props) + } +} + +func sendPushNotification(post *model.Post, user *model.User, channel *model.Channel, senderName string, wasMentioned bool) *model.AppError { + sessions, err := getMobileAppSessions(user.Id) + if err != nil { + return err + } + + var channelName string + + if channel.Type == model.CHANNEL_DIRECT { + channelName = senderName + } else { + channelName = channel.DisplayName + } + + userLocale := utils.GetUserTranslations(user.Locale) + + msg := model.PushNotification{} + if badge := <-Srv.Store.User().GetUnreadCount(user.Id); badge.Err != nil { + msg.Badge = 1 + l4g.Error(utils.T("store.sql_user.get_unread_count.app_error"), user.Id, badge.Err) + } else { + msg.Badge = int(badge.Data.(int64)) + } + msg.Type = model.PUSH_TYPE_MESSAGE + msg.TeamId = channel.TeamId + msg.ChannelId = channel.Id + msg.ChannelName = channel.Name + + if *utils.Cfg.EmailSettings.PushNotificationContents == model.FULL_NOTIFICATION { + if channel.Type == model.CHANNEL_DIRECT { + msg.Category = model.CATEGORY_DM + msg.Message = "@" + senderName + ": " + model.ClearMentionTags(post.Message) + } else { + msg.Message = senderName + userLocale("api.post.send_notifications_and_forget.push_in") + channelName + ": " + model.ClearMentionTags(post.Message) + } + } else { + if channel.Type == model.CHANNEL_DIRECT { + msg.Category = model.CATEGORY_DM + msg.Message = senderName + userLocale("api.post.send_notifications_and_forget.push_message") + } else if wasMentioned { + msg.Message = senderName + userLocale("api.post.send_notifications_and_forget.push_mention") + channelName + } else { + msg.Message = senderName + userLocale("api.post.send_notifications_and_forget.push_non_mention") + channelName + } + } + + l4g.Debug(utils.T("api.post.send_notifications_and_forget.push_notification.debug"), msg.DeviceId, msg.Message) + + for _, session := range sessions { + tmpMessage := *model.PushNotificationFromJson(strings.NewReader(msg.ToJson())) + tmpMessage.SetDeviceIdAndPlatform(session.DeviceId) + if err := sendToPushProxy(tmpMessage); err != nil { + l4g.Error(err.Error) + } + if einterfaces.GetMetricsInterface() != nil { + einterfaces.GetMetricsInterface().IncrementPostSentPush() + } + } + + return nil +} + +func ClearPushNotification(userId string, channelId string) *model.AppError { + sessions, err := getMobileAppSessions(userId) + if err != nil { + return err + } + + msg := model.PushNotification{} + msg.Type = model.PUSH_TYPE_CLEAR + msg.ChannelId = channelId + msg.ContentAvailable = 0 + if badge := <-Srv.Store.User().GetUnreadCount(userId); badge.Err != nil { + msg.Badge = 0 + l4g.Error(utils.T("store.sql_user.get_unread_count.app_error"), userId, badge.Err) + } else { + msg.Badge = int(badge.Data.(int64)) + } + + l4g.Debug(utils.T("api.post.send_notifications_and_forget.clear_push_notification.debug"), msg.DeviceId, msg.ChannelId) + for _, session := range sessions { + tmpMessage := *model.PushNotificationFromJson(strings.NewReader(msg.ToJson())) + tmpMessage.SetDeviceIdAndPlatform(session.DeviceId) + if err := sendToPushProxy(tmpMessage); err != nil { + l4g.Error(err.Error) + } + } + + return nil +} + +func sendToPushProxy(msg model.PushNotification) *model.AppError { + msg.ServerId = utils.CfgDiagnosticId + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: *utils.Cfg.ServiceSettings.EnableInsecureOutgoingConnections}, + } + httpClient := &http.Client{Transport: tr} + request, _ := http.NewRequest("POST", *utils.Cfg.EmailSettings.PushNotificationServer+model.API_URL_SUFFIX_V1+"/send_push", strings.NewReader(msg.ToJson())) + + if resp, err := httpClient.Do(request); err != nil { + return model.NewLocAppError("sendToPushProxy", "api.post.send_notifications_and_forget.push_notification.error", map[string]interface{}{"DeviceId": msg.DeviceId, "Error": err.Error()}, "") + } else { + ioutil.ReadAll(resp.Body) + resp.Body.Close() + } + + return nil +} + +func getMobileAppSessions(userId string) ([]*model.Session, *model.AppError) { + if result := <-Srv.Store.Session().GetSessionsWithActiveDeviceIds(userId); result.Err != nil { + return nil, result.Err + } else { + return result.Data.([]*model.Session), nil + } +} + +func sendOutOfChannelMentions(post *model.Post, teamId string, profiles map[string]*model.User) *model.AppError { + if len(profiles) == 0 { + return nil + } + + var usernames []string + for _, user := range profiles { + usernames = append(usernames, user.Username) + } + sort.Strings(usernames) + + T := utils.GetUserTranslations(profiles[post.UserId].Locale) + + var message string + if len(usernames) == 1 { + message = T("api.post.check_for_out_of_channel_mentions.message.one", map[string]interface{}{ + "Username": usernames[0], + }) + } else { + message = T("api.post.check_for_out_of_channel_mentions.message.multiple", map[string]interface{}{ + "Usernames": strings.Join(usernames[:len(usernames)-1], ", "), + "LastUsername": usernames[len(usernames)-1], + }) + } + + SendEphemeralPost( + teamId, + post.UserId, + &model.Post{ + ChannelId: post.ChannelId, + Message: message, + CreateAt: post.CreateAt + 1, + }, + ) + + return nil +} + +// Given a message and a map mapping mention keywords to the users who use them, returns a map of mentioned +// users and a slice of potential mention users not in the channel and whether or not @here was mentioned. +func GetExplicitMentions(message string, keywords map[string][]string) (map[string]bool, []string, bool, bool, bool) { + mentioned := make(map[string]bool) + potentialOthersMentioned := make([]string, 0) + systemMentions := map[string]bool{"@here": true, "@channel": true, "@all": true} + hereMentioned := false + allMentioned := false + channelMentioned := false + + addMentionedUsers := func(ids []string) { + for _, id := range ids { + mentioned[id] = true + } + } + + for _, word := range strings.Fields(message) { + isMention := false + + if word == "@here" { + hereMentioned = true + } + + if word == "@channel" { + channelMentioned = true + } + + if word == "@all" { + allMentioned = true + } + + // Non-case-sensitive check for regular keys + if ids, match := keywords[strings.ToLower(word)]; match { + addMentionedUsers(ids) + isMention = true + } + + // Case-sensitive check for first name + if ids, match := keywords[word]; match { + addMentionedUsers(ids) + isMention = true + } + + if !isMention { + // No matches were found with the string split just on whitespace so try further splitting + // the message on punctuation + splitWords := strings.FieldsFunc(word, func(c rune) bool { + return model.SplitRunes[c] + }) + + for _, splitWord := range splitWords { + if splitWord == "@here" { + hereMentioned = true + } + + if splitWord == "@all" { + allMentioned = true + } + + if splitWord == "@channel" { + channelMentioned = true + } + + // Non-case-sensitive check for regular keys + if ids, match := keywords[strings.ToLower(splitWord)]; match { + addMentionedUsers(ids) + } + + // Case-sensitive check for first name + if ids, match := keywords[splitWord]; match { + addMentionedUsers(ids) + } else if _, ok := systemMentions[word]; !ok && strings.HasPrefix(word, "@") { + username := word[1:len(splitWord)] + potentialOthersMentioned = append(potentialOthersMentioned, username) + } + } + } + } + + return mentioned, potentialOthersMentioned, hereMentioned, channelMentioned, allMentioned +} + +// Given a map of user IDs to profiles, returns a list of mention +// keywords for all users in the channel. +func GetMentionKeywordsInChannel(profiles map[string]*model.User) map[string][]string { + keywords := make(map[string][]string) + + for id, profile := range profiles { + userMention := "@" + strings.ToLower(profile.Username) + keywords[userMention] = append(keywords[userMention], id) + + if len(profile.NotifyProps["mention_keys"]) > 0 { + // Add all the user's mention keys + splitKeys := strings.Split(profile.NotifyProps["mention_keys"], ",") + for _, k := range splitKeys { + // note that these are made lower case so that we can do a case insensitive check for them + key := strings.ToLower(k) + keywords[key] = append(keywords[key], id) + } + } + + // If turned on, add the user's case sensitive first name + if profile.NotifyProps["first_name"] == "true" { + keywords[profile.FirstName] = append(keywords[profile.FirstName], profile.Id) + } + + // Add @channel and @all to keywords if user has them turned on + if int64(len(profiles)) < *utils.Cfg.TeamSettings.MaxNotificationsPerChannel && profile.NotifyProps["channel"] == "true" { + keywords["@channel"] = append(keywords["@channel"], profile.Id) + keywords["@all"] = append(keywords["@all"], profile.Id) + } + } + + return keywords +} diff --git a/app/notification_test.go b/app/notification_test.go new file mode 100644 index 000000000..d3aea214c --- /dev/null +++ b/app/notification_test.go @@ -0,0 +1,312 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "testing" + + "github.com/mattermost/platform/model" +) + +func TestSendNotifications(t *testing.T) { + th := Setup().InitBasic() + + AddUserToChannel(th.BasicUser2, th.BasicChannel) + + post1, postErr := CreatePost(&model.Post{ + UserId: th.BasicUser.Id, + ChannelId: th.BasicChannel.Id, + Message: "@" + th.BasicUser2.Username, + }, th.BasicTeam.Id, true) + + if postErr != nil { + t.Fatal(postErr) + } + + mentions, err := SendNotifications(post1, th.BasicTeam, th.BasicChannel) + if err != nil { + t.Fatal(err) + } else if mentions == nil { + t.Log(mentions) + t.Fatal("user should have been mentioned") + } else if mentions[0] != th.BasicUser2.Id { + t.Log(mentions) + t.Fatal("user should have been mentioned") + } +} + +func TestGetExplicitMentions(t *testing.T) { + id1 := model.NewId() + id2 := model.NewId() + + // not mentioning anybody + message := "this is a message" + keywords := map[string][]string{} + if mentions, potential, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 0 || len(potential) != 0 { + t.Fatal("shouldn't have mentioned anybody or have any potencial mentions") + } + + // mentioning a user that doesn't exist + message = "this is a message for @user" + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 0 { + t.Fatal("shouldn't have mentioned user that doesn't exist") + } + + // mentioning one person + keywords = map[string][]string{"@user": {id1}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 1 || !mentions[id1] { + t.Fatal("should've mentioned @user") + } + + // mentioning one person without an @mention + message = "this is a message for @user" + keywords = map[string][]string{"this": {id1}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 1 || !mentions[id1] { + t.Fatal("should've mentioned this") + } + + // mentioning multiple people with one word + message = "this is a message for @user" + keywords = map[string][]string{"@user": {id1, id2}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 2 || !mentions[id1] || !mentions[id2] { + t.Fatal("should've mentioned two users with @user") + } + + // mentioning only one of multiple people + keywords = map[string][]string{"@user": {id1}, "@mention": {id2}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 1 || !mentions[id1] || mentions[id2] { + t.Fatal("should've mentioned @user and not @mention") + } + + // mentioning multiple people with multiple words + message = "this is an @mention for @user" + keywords = map[string][]string{"@user": {id1}, "@mention": {id2}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 2 || !mentions[id1] || !mentions[id2] { + t.Fatal("should've mentioned two users with @user and @mention") + } + + // mentioning @channel (not a special case, but it's good to double check) + message = "this is an message for @channel" + keywords = map[string][]string{"@channel": {id1, id2}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 2 || !mentions[id1] || !mentions[id2] { + t.Fatal("should've mentioned two users with @channel") + } + + // mentioning @all (not a special case, but it's good to double check) + message = "this is an message for @all" + keywords = map[string][]string{"@all": {id1, id2}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 2 || !mentions[id1] || !mentions[id2] { + t.Fatal("should've mentioned two users with @all") + } + + // mentioning user.period without mentioning user (PLT-3222) + message = "user.period doesn't complicate things at all by including periods in their username" + keywords = map[string][]string{"user.period": {id1}, "user": {id2}} + if mentions, _, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 1 || !mentions[id1] || mentions[id2] { + t.Fatal("should've mentioned user.period and not user") + } + + // mentioning a potential out of channel user + message = "this is an message for @potential and @user" + keywords = map[string][]string{"@user": {id1}} + if mentions, potential, _, _, _ := GetExplicitMentions(message, keywords); len(mentions) != 1 || !mentions[id1] || len(potential) != 1 { + t.Fatal("should've mentioned user and have a potential not in channel") + } +} + +func TestGetExplicitMentionsAtHere(t *testing.T) { + // test all the boundary cases that we know can break up terms (and those that we know won't) + cases := map[string]bool{ + "": false, + "here": false, + "@here": true, + " @here ": true, + "\t@here\t": true, + "\n@here\n": true, + // "!@here!": true, + // "@@here@": true, + // "#@here#": true, + // "$@here$": true, + // "%@here%": true, + // "^@here^": true, + // "&@here&": true, + // "*@here*": true, + "(@here(": true, + ")@here)": true, + // "-@here-": true, + // "_@here_": true, + // "=@here=": true, + "+@here+": true, + "[@here[": true, + "{@here{": true, + "]@here]": true, + "}@here}": true, + "\\@here\\": true, + // "|@here|": true, + ";@here;": true, + ":@here:": true, + // "'@here'": true, + // "\"@here\"": true, + ",@here,": true, + "<@here<": true, + ".@here.": true, + ">@here>": true, + "/@here/": true, + "?@here?": true, + // "`@here`": true, + // "~@here~": true, + } + + for message, shouldMention := range cases { + if _, _, hereMentioned, _, _ := GetExplicitMentions(message, nil); hereMentioned && !shouldMention { + t.Fatalf("shouldn't have mentioned @here with \"%v\"", message) + } else if !hereMentioned && shouldMention { + t.Fatalf("should've have mentioned @here with \"%v\"", message) + } + } + + // mentioning @here and someone + id := model.NewId() + if mentions, potential, hereMentioned, _, _ := GetExplicitMentions("@here @user @potential", map[string][]string{"@user": {id}}); !hereMentioned { + t.Fatal("should've mentioned @here with \"@here @user\"") + } else if len(mentions) != 1 || !mentions[id] { + t.Fatal("should've mentioned @user with \"@here @user\"") + } else if len(potential) > 1 { + t.Fatal("should've potential mentions for @potential") + } +} + +func TestGetMentionKeywords(t *testing.T) { + // user with username or custom mentions enabled + user1 := &model.User{ + Id: model.NewId(), + FirstName: "First", + Username: "User", + NotifyProps: map[string]string{ + "mention_keys": "User,@User,MENTION", + }, + } + + profiles := map[string]*model.User{user1.Id: user1} + mentions := GetMentionKeywordsInChannel(profiles) + if len(mentions) != 3 { + t.Fatal("should've returned three mention keywords") + } else if ids, ok := mentions["user"]; !ok || ids[0] != user1.Id { + t.Fatal("should've returned mention key of user") + } else if ids, ok := mentions["@user"]; !ok || ids[0] != user1.Id { + t.Fatal("should've returned mention key of @user") + } else if ids, ok := mentions["mention"]; !ok || ids[0] != user1.Id { + t.Fatal("should've returned mention key of mention") + } + + // user with first name mention enabled + user2 := &model.User{ + Id: model.NewId(), + FirstName: "First", + Username: "User", + NotifyProps: map[string]string{ + "first_name": "true", + }, + } + + profiles = map[string]*model.User{user2.Id: user2} + mentions = GetMentionKeywordsInChannel(profiles) + if len(mentions) != 2 { + t.Fatal("should've returned two mention keyword") + } else if ids, ok := mentions["First"]; !ok || ids[0] != user2.Id { + t.Fatal("should've returned mention key of First") + } + + // user with @channel/@all mentions enabled + user3 := &model.User{ + Id: model.NewId(), + FirstName: "First", + Username: "User", + NotifyProps: map[string]string{ + "channel": "true", + }, + } + + profiles = map[string]*model.User{user3.Id: user3} + mentions = GetMentionKeywordsInChannel(profiles) + if len(mentions) != 3 { + t.Fatal("should've returned three mention keywords") + } else if ids, ok := mentions["@channel"]; !ok || ids[0] != user3.Id { + t.Fatal("should've returned mention key of @channel") + } else if ids, ok := mentions["@all"]; !ok || ids[0] != user3.Id { + t.Fatal("should've returned mention key of @all") + } + + // user with all types of mentions enabled + user4 := &model.User{ + Id: model.NewId(), + FirstName: "First", + Username: "User", + NotifyProps: map[string]string{ + "mention_keys": "User,@User,MENTION", + "first_name": "true", + "channel": "true", + }, + } + + profiles = map[string]*model.User{user4.Id: user4} + mentions = GetMentionKeywordsInChannel(profiles) + if len(mentions) != 6 { + t.Fatal("should've returned six mention keywords") + } else if ids, ok := mentions["user"]; !ok || ids[0] != user4.Id { + t.Fatal("should've returned mention key of user") + } else if ids, ok := mentions["@user"]; !ok || ids[0] != user4.Id { + t.Fatal("should've returned mention key of @user") + } else if ids, ok := mentions["mention"]; !ok || ids[0] != user4.Id { + t.Fatal("should've returned mention key of mention") + } else if ids, ok := mentions["First"]; !ok || ids[0] != user4.Id { + t.Fatal("should've returned mention key of First") + } else if ids, ok := mentions["@channel"]; !ok || ids[0] != user4.Id { + t.Fatal("should've returned mention key of @channel") + } else if ids, ok := mentions["@all"]; !ok || ids[0] != user4.Id { + t.Fatal("should've returned mention key of @all") + } + + dup_count := func(list []string) map[string]int { + + duplicate_frequency := make(map[string]int) + + for _, item := range list { + // check if the item/element exist in the duplicate_frequency map + + _, exist := duplicate_frequency[item] + + if exist { + duplicate_frequency[item] += 1 // increase counter by 1 if already in the map + } else { + duplicate_frequency[item] = 1 // else start counting from 1 + } + } + return duplicate_frequency + } + + // multiple users + profiles = map[string]*model.User{ + user1.Id: user1, + user2.Id: user2, + user3.Id: user3, + user4.Id: user4, + } + mentions = GetMentionKeywordsInChannel(profiles) + if len(mentions) != 6 { + t.Fatal("should've returned six mention keywords") + } else if ids, ok := mentions["user"]; !ok || len(ids) != 2 || (ids[0] != user1.Id && ids[1] != user1.Id) || (ids[0] != user4.Id && ids[1] != user4.Id) { + t.Fatal("should've mentioned user1 and user4 with user") + } else if ids := dup_count(mentions["@user"]); len(ids) != 4 || (ids[user1.Id] != 2) || (ids[user4.Id] != 2) { + t.Fatal("should've mentioned user1 and user4 with @user") + } else if ids, ok := mentions["mention"]; !ok || len(ids) != 2 || (ids[0] != user1.Id && ids[1] != user1.Id) || (ids[0] != user4.Id && ids[1] != user4.Id) { + t.Fatal("should've mentioned user1 and user4 with mention") + } else if ids, ok := mentions["First"]; !ok || len(ids) != 2 || (ids[0] != user2.Id && ids[1] != user2.Id) || (ids[0] != user4.Id && ids[1] != user4.Id) { + t.Fatal("should've mentioned user2 and user4 with mention") + } else if ids, ok := mentions["@channel"]; !ok || len(ids) != 2 || (ids[0] != user3.Id && ids[1] != user3.Id) || (ids[0] != user4.Id && ids[1] != user4.Id) { + t.Fatal("should've mentioned user3 and user4 with @channel") + } else if ids, ok := mentions["@all"]; !ok || len(ids) != 2 || (ids[0] != user3.Id && ids[1] != user3.Id) || (ids[0] != user4.Id && ids[1] != user4.Id) { + t.Fatal("should've mentioned user3 and user4 with @all") + } +} diff --git a/app/post.go b/app/post.go new file mode 100644 index 000000000..7eebe905f --- /dev/null +++ b/app/post.go @@ -0,0 +1,196 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "regexp" + + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/einterfaces" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/store" + "github.com/mattermost/platform/utils" +) + +func CreatePost(post *model.Post, teamId string, triggerWebhooks bool) (*model.Post, *model.AppError) { + var pchan store.StoreChannel + if len(post.RootId) > 0 { + pchan = Srv.Store.Post().Get(post.RootId) + } + + // Verify the parent/child relationships are correct + if pchan != nil { + if presult := <-pchan; presult.Err != nil { + return nil, model.NewLocAppError("createPost", "api.post.create_post.root_id.app_error", nil, "") + } else { + list := presult.Data.(*model.PostList) + if len(list.Posts) == 0 || !list.IsChannelId(post.ChannelId) { + return nil, model.NewLocAppError("createPost", "api.post.create_post.channel_root_id.app_error", nil, "") + } + + if post.ParentId == "" { + post.ParentId = post.RootId + } + + if post.RootId != post.ParentId { + parent := list.Posts[post.ParentId] + if parent == nil { + return nil, model.NewLocAppError("createPost", "api.post.create_post.parent_id.app_error", nil, "") + } + } + } + } + + post.Hashtags, _ = model.ParseHashtags(post.Message) + + var rpost *model.Post + if result := <-Srv.Store.Post().Save(post); result.Err != nil { + return nil, result.Err + } else { + rpost = result.Data.(*model.Post) + } + + if einterfaces.GetMetricsInterface() != nil { + einterfaces.GetMetricsInterface().IncrementPostCreate() + } + + if len(post.FileIds) > 0 { + // There's a rare bug where the client sends up duplicate FileIds so protect against that + post.FileIds = utils.RemoveDuplicatesFromStringArray(post.FileIds) + + for _, fileId := range post.FileIds { + if result := <-Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil { + l4g.Error(utils.T("api.post.create_post.attach_files.error"), post.Id, post.FileIds, post.UserId, result.Err) + } + } + + if einterfaces.GetMetricsInterface() != nil { + einterfaces.GetMetricsInterface().IncrementPostFileAttachment(len(post.FileIds)) + } + } + + InvalidateCacheForChannel(rpost.ChannelId) + InvalidateCacheForChannelPosts(rpost.ChannelId) + + if err := handlePostEvents(rpost, teamId, triggerWebhooks); err != nil { + return nil, err + } + + return rpost, nil +} + +func handlePostEvents(post *model.Post, teamId string, triggerWebhooks bool) *model.AppError { + tchan := Srv.Store.Team().Get(teamId) + cchan := Srv.Store.Channel().Get(post.ChannelId, true) + uchan := Srv.Store.User().Get(post.UserId) + + var team *model.Team + if result := <-tchan; result.Err != nil { + return result.Err + } else { + team = result.Data.(*model.Team) + } + + var channel *model.Channel + if result := <-cchan; result.Err != nil { + return result.Err + } else { + channel = result.Data.(*model.Channel) + } + + if _, err := SendNotifications(post, team, channel); err != nil { + return err + } + + var user *model.User + if result := <-uchan; result.Err != nil { + return result.Err + } else { + user = result.Data.(*model.User) + } + + if triggerWebhooks { + go func() { + if err := handleWebhookEvents(post, team, channel, user); err != nil { + l4g.Error(err.Error()) + } + }() + } + + if channel.Type == model.CHANNEL_DIRECT { + go func() { + if err := MakeDirectChannelVisible(post.ChannelId); err != nil { + l4g.Error(err.Error()) + } + }() + } + + return nil +} + +var linkWithTextRegex *regexp.Regexp = regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) + +// This method only parses and processes the attachments, +// all else should be set in the post which is passed +func parseSlackAttachment(post *model.Post, attachments interface{}) { + post.Type = model.POST_SLACK_ATTACHMENT + + if list, success := attachments.([]interface{}); success { + for i, aInt := range list { + attachment := aInt.(map[string]interface{}) + if aText, ok := attachment["text"].(string); ok { + aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})") + attachment["text"] = aText + list[i] = attachment + } + if aText, ok := attachment["pretext"].(string); ok { + aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})") + attachment["pretext"] = aText + list[i] = attachment + } + if fVal, ok := attachment["fields"]; ok { + if fields, ok := fVal.([]interface{}); ok { + // parse attachment field links into Markdown format + for j, fInt := range fields { + field := fInt.(map[string]interface{}) + if fValue, ok := field["value"].(string); ok { + fValue = linkWithTextRegex.ReplaceAllString(fValue, "[${2}](${1})") + field["value"] = fValue + fields[j] = field + } + } + attachment["fields"] = fields + list[i] = attachment + } + } + } + post.AddProp("attachments", list) + } +} + +func parseSlackLinksToMarkdown(text string) string { + return linkWithTextRegex.ReplaceAllString(text, "[${2}](${1})") +} + +func SendEphemeralPost(teamId, userId string, post *model.Post) *model.Post { + post.Type = model.POST_EPHEMERAL + + // fill in fields which haven't been specified which have sensible defaults + if post.Id == "" { + post.Id = model.NewId() + } + if post.CreateAt == 0 { + post.CreateAt = model.GetMillis() + } + if post.Props == nil { + post.Props = model.StringInterface{} + } + + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil) + message.Add("post", post.ToJson()) + + go Publish(message) + + return post +} diff --git a/app/server.go b/app/server.go new file mode 100644 index 000000000..972c91ea3 --- /dev/null +++ b/app/server.go @@ -0,0 +1,217 @@ +// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "crypto/tls" + "net" + "net/http" + "strings" + "time" + + l4g "github.com/alecthomas/log4go" + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/store" + "github.com/mattermost/platform/utils" + "github.com/rsc/letsencrypt" + "github.com/tylerb/graceful" + "gopkg.in/throttled/throttled.v2" + "gopkg.in/throttled/throttled.v2/store/memstore" +) + +type Server struct { + Store store.Store + WebSocketRouter *WebSocketRouter + Router *mux.Router + GracefulServer *graceful.Server +} + +var allowedMethods []string = []string{ + "POST", + "GET", + "OPTIONS", + "PUT", + "PATCH", + "DELETE", +} + +type CorsWrapper struct { + router *mux.Router +} + +func (cw *CorsWrapper) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if len(*utils.Cfg.ServiceSettings.AllowCorsFrom) > 0 { + origin := r.Header.Get("Origin") + if *utils.Cfg.ServiceSettings.AllowCorsFrom == "*" || strings.Contains(*utils.Cfg.ServiceSettings.AllowCorsFrom, origin) { + w.Header().Set("Access-Control-Allow-Origin", origin) + + if r.Method == "OPTIONS" { + w.Header().Set( + "Access-Control-Allow-Methods", + strings.Join(allowedMethods, ", ")) + + w.Header().Set( + "Access-Control-Allow-Headers", + r.Header.Get("Access-Control-Request-Headers")) + } + } + } + + if r.Method == "OPTIONS" { + return + } + + cw.router.ServeHTTP(w, r) +} + +const TIME_TO_WAIT_FOR_CONNECTIONS_TO_CLOSE_ON_SERVER_SHUTDOWN = time.Second + +var Srv *Server + +func NewServer() { + l4g.Info(utils.T("api.server.new_server.init.info")) + + Srv = &Server{} +} + +func InitStores() { + Srv.Store = store.NewSqlStore() +} + +type VaryBy struct{} + +func (m *VaryBy) Key(r *http.Request) string { + return utils.GetIpAddress(r) +} + +func initalizeThrottledVaryBy() *throttled.VaryBy { + vary := throttled.VaryBy{} + + if utils.Cfg.RateLimitSettings.VaryByRemoteAddr { + vary.RemoteAddr = true + } + + if len(utils.Cfg.RateLimitSettings.VaryByHeader) > 0 { + vary.Headers = strings.Fields(utils.Cfg.RateLimitSettings.VaryByHeader) + + if utils.Cfg.RateLimitSettings.VaryByRemoteAddr { + l4g.Warn(utils.T("api.server.start_server.rate.warn")) + vary.RemoteAddr = false + } + } + + return &vary +} + +func redirectHTTPToHTTPS(w http.ResponseWriter, r *http.Request) { + if r.Host == "" { + http.Error(w, "Not Found", http.StatusNotFound) + } + + url := r.URL + url.Host = r.Host + url.Scheme = "https" + http.Redirect(w, r, url.String(), http.StatusFound) +} + +func StartServer() { + l4g.Info(utils.T("api.server.start_server.starting.info")) + + var handler http.Handler = &CorsWrapper{Srv.Router} + + if *utils.Cfg.RateLimitSettings.Enable { + l4g.Info(utils.T("api.server.start_server.rate.info")) + + store, err := memstore.New(utils.Cfg.RateLimitSettings.MemoryStoreSize) + if err != nil { + l4g.Critical(utils.T("api.server.start_server.rate_limiting_memory_store")) + return + } + + quota := throttled.RateQuota{ + MaxRate: throttled.PerSec(utils.Cfg.RateLimitSettings.PerSec), + MaxBurst: *utils.Cfg.RateLimitSettings.MaxBurst, + } + + rateLimiter, err := throttled.NewGCRARateLimiter(store, quota) + if err != nil { + l4g.Critical(utils.T("api.server.start_server.rate_limiting_rate_limiter")) + return + } + + httpRateLimiter := throttled.HTTPRateLimiter{ + RateLimiter: rateLimiter, + VaryBy: &VaryBy{}, + DeniedHandler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + l4g.Error("%v: Denied due to throttling settings code=429 ip=%v", r.URL.Path, utils.GetIpAddress(r)) + throttled.DefaultDeniedHandler.ServeHTTP(w, r) + }), + } + + handler = httpRateLimiter.RateLimit(handler) + } + + Srv.GracefulServer = &graceful.Server{ + Timeout: TIME_TO_WAIT_FOR_CONNECTIONS_TO_CLOSE_ON_SERVER_SHUTDOWN, + Server: &http.Server{ + Addr: utils.Cfg.ServiceSettings.ListenAddress, + Handler: handlers.RecoveryHandler(handlers.PrintRecoveryStack(true))(handler), + ReadTimeout: time.Duration(*utils.Cfg.ServiceSettings.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(*utils.Cfg.ServiceSettings.WriteTimeout) * time.Second, + }, + } + l4g.Info(utils.T("api.server.start_server.listening.info"), utils.Cfg.ServiceSettings.ListenAddress) + + if *utils.Cfg.ServiceSettings.Forward80To443 { + go func() { + listener, err := net.Listen("tcp", ":80") + if err != nil { + l4g.Error("Unable to setup forwarding") + return + } + defer listener.Close() + + http.Serve(listener, http.HandlerFunc(redirectHTTPToHTTPS)) + }() + } + + go func() { + var err error + if *utils.Cfg.ServiceSettings.ConnectionSecurity == model.CONN_SECURITY_TLS { + if *utils.Cfg.ServiceSettings.UseLetsEncrypt { + var m letsencrypt.Manager + m.CacheFile(*utils.Cfg.ServiceSettings.LetsEncryptCertificateCacheFile) + + tlsConfig := &tls.Config{ + GetCertificate: m.GetCertificate, + } + + tlsConfig.NextProtos = append(tlsConfig.NextProtos, "h2") + + err = Srv.GracefulServer.ListenAndServeTLSConfig(tlsConfig) + } else { + err = Srv.GracefulServer.ListenAndServeTLS(*utils.Cfg.ServiceSettings.TLSCertFile, *utils.Cfg.ServiceSettings.TLSKeyFile) + } + } else { + err = Srv.GracefulServer.ListenAndServe() + } + if err != nil { + l4g.Critical(utils.T("api.server.start_server.starting.critical"), err) + time.Sleep(time.Second) + } + }() +} + +func StopServer() { + + l4g.Info(utils.T("api.server.stop_server.stopping.info")) + + Srv.GracefulServer.Stop(TIME_TO_WAIT_FOR_CONNECTIONS_TO_CLOSE_ON_SERVER_SHUTDOWN) + Srv.Store.Close() + HubStop() + + l4g.Info(utils.T("api.server.stop_server.stopped.info")) +} diff --git a/app/session.go b/app/session.go new file mode 100644 index 000000000..29c961e81 --- /dev/null +++ b/app/session.go @@ -0,0 +1,94 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "github.com/mattermost/platform/einterfaces" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/store" + "github.com/mattermost/platform/utils" + + l4g "github.com/alecthomas/log4go" +) + +var sessionCache *utils.Cache = utils.NewLru(model.SESSION_CACHE_SIZE) + +func GetSession(token string) (*model.Session, *model.AppError) { + metrics := einterfaces.GetMetricsInterface() + + var session *model.Session + if ts, ok := sessionCache.Get(token); ok { + session = ts.(*model.Session) + if metrics != nil { + metrics.IncrementMemCacheHitCounter("Session") + } + } else { + if metrics != nil { + metrics.IncrementMemCacheMissCounter("Session") + } + } + + if session == nil { + if sessionResult := <-Srv.Store.Session().Get(token); sessionResult.Err != nil { + return nil, model.NewLocAppError("GetSession", "api.context.invalid_token.error", map[string]interface{}{"Token": token, "Error": sessionResult.Err.DetailedError}, "") + } else { + session = sessionResult.Data.(*model.Session) + + if session.IsExpired() || session.Token != token { + return nil, model.NewLocAppError("GetSession", "api.context.invalid_token.error", map[string]interface{}{"Token": token, "Error": sessionResult.Err.DetailedError}, "") + } else { + AddSessionToCache(session) + return session, nil + } + } + } + + if session == nil || session.IsExpired() { + return nil, model.NewLocAppError("GetSession", "api.context.invalid_token.error", map[string]interface{}{"Token": token}, "") + } + + return session, nil +} + +func RemoveAllSessionsForUserId(userId string) { + + RemoveAllSessionsForUserIdSkipClusterSend(userId) + + if einterfaces.GetClusterInterface() != nil { + einterfaces.GetClusterInterface().RemoveAllSessionsForUserId(userId) + } +} + +func RemoveAllSessionsForUserIdSkipClusterSend(userId string) { + keys := sessionCache.Keys() + + for _, key := range keys { + if ts, ok := sessionCache.Get(key); ok { + session := ts.(*model.Session) + if session.UserId == userId { + sessionCache.Remove(key) + } + } + } + + InvalidateWebConnSessionCacheForUser(userId) + +} + +func AddSessionToCache(session *model.Session) { + sessionCache.AddWithExpiresInSecs(session.Token, session, int64(*utils.Cfg.ServiceSettings.SessionCacheInMinutes*60)) +} + +func InvalidateAllCaches() { + l4g.Info(utils.T("api.context.invalidate_all_caches")) + sessionCache.Purge() + ClearStatusCache() + store.ClearChannelCaches() + store.ClearUserCaches() + store.ClearPostCaches() +} + +func SessionCacheLength() int { + return sessionCache.Len() +} diff --git a/app/session_test.go b/app/session_test.go new file mode 100644 index 000000000..352395c76 --- /dev/null +++ b/app/session_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "github.com/mattermost/platform/model" + "testing" +) + +func TestCache(t *testing.T) { + session := &model.Session{ + Id: model.NewId(), + Token: model.NewId(), + UserId: model.NewId(), + } + + sessionCache.AddWithExpiresInSecs(session.Token, session, 5*60) + + keys := sessionCache.Keys() + if len(keys) <= 0 { + t.Fatal("should have items") + } + + RemoveAllSessionsForUserId(session.UserId) + + rkeys := sessionCache.Keys() + if len(rkeys) != len(keys)-1 { + t.Fatal("should have one less") + } +} diff --git a/app/status.go b/app/status.go new file mode 100644 index 000000000..98cdb0dc0 --- /dev/null +++ b/app/status.go @@ -0,0 +1,255 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + l4g "github.com/alecthomas/log4go" + + "github.com/mattermost/platform/einterfaces" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/store" + "github.com/mattermost/platform/utils" +) + +var statusCache *utils.Cache = utils.NewLru(model.STATUS_CACHE_SIZE) + +func ClearStatusCache() { + statusCache.Purge() +} + +func AddStatusCacheSkipClusterSend(status *model.Status) { + statusCache.Add(status.UserId, status) +} + +func AddStatusCache(status *model.Status) { + AddStatusCacheSkipClusterSend(status) + + if einterfaces.GetClusterInterface() != nil { + einterfaces.GetClusterInterface().UpdateStatus(status) + } +} + +func GetAllStatuses() map[string]*model.Status { + userIds := statusCache.Keys() + statusMap := map[string]*model.Status{} + + for _, userId := range userIds { + if id, ok := userId.(string); !ok { + continue + } else { + status := GetStatusFromCache(id) + if status != nil { + statusMap[id] = status + } + } + } + + return statusMap +} + +func GetStatusesByIds(userIds []string) (map[string]interface{}, *model.AppError) { + statusMap := map[string]interface{}{} + metrics := einterfaces.GetMetricsInterface() + + missingUserIds := []string{} + for _, userId := range userIds { + if result, ok := statusCache.Get(userId); ok { + statusMap[userId] = result.(*model.Status).Status + if metrics != nil { + metrics.IncrementMemCacheHitCounter("Status") + } + } else { + missingUserIds = append(missingUserIds, userId) + if metrics != nil { + metrics.IncrementMemCacheMissCounter("Status") + } + } + } + + if len(missingUserIds) > 0 { + if result := <-Srv.Store.Status().GetByIds(missingUserIds); result.Err != nil { + return nil, result.Err + } else { + statuses := result.Data.([]*model.Status) + + for _, s := range statuses { + AddStatusCache(s) + statusMap[s.UserId] = s.Status + } + } + } + + // For the case where the user does not have a row in the Status table and cache + for _, userId := range missingUserIds { + if _, ok := statusMap[userId]; !ok { + statusMap[userId] = model.STATUS_OFFLINE + } + } + + return statusMap, nil +} + +func SetStatusOnline(userId string, sessionId string, manual bool) { + broadcast := false + + var oldStatus string = model.STATUS_OFFLINE + var oldTime int64 = 0 + var oldManual bool = false + var status *model.Status + var err *model.AppError + + if status, err = GetStatus(userId); err != nil { + status = &model.Status{userId, model.STATUS_ONLINE, false, model.GetMillis(), ""} + broadcast = true + } else { + if status.Manual && !manual { + return // manually set status always overrides non-manual one + } + + if status.Status != model.STATUS_ONLINE { + broadcast = true + } + + oldStatus = status.Status + oldTime = status.LastActivityAt + oldManual = status.Manual + + status.Status = model.STATUS_ONLINE + status.Manual = false // for "online" there's no manual setting + status.LastActivityAt = model.GetMillis() + } + + AddStatusCache(status) + + // Only update the database if the status has changed, the status has been manually set, + // or enough time has passed since the previous action + if status.Status != oldStatus || status.Manual != oldManual || status.LastActivityAt-oldTime > model.STATUS_MIN_UPDATE_TIME { + achan := Srv.Store.Session().UpdateLastActivityAt(sessionId, status.LastActivityAt) + + var schan store.StoreChannel + if broadcast { + schan = Srv.Store.Status().SaveOrUpdate(status) + } else { + schan = Srv.Store.Status().UpdateLastActivityAt(status.UserId, status.LastActivityAt) + } + + if result := <-achan; result.Err != nil { + l4g.Error(utils.T("api.status.last_activity.error"), userId, sessionId, result.Err) + } + + if result := <-schan; result.Err != nil { + l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err) + } + } + + if broadcast { + event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil) + event.Add("status", model.STATUS_ONLINE) + event.Add("user_id", status.UserId) + go Publish(event) + } +} + +func SetStatusOffline(userId string, manual bool) { + status, err := GetStatus(userId) + if err == nil && status.Manual && !manual { + return // manually set status always overrides non-manual one + } + + status = &model.Status{userId, model.STATUS_OFFLINE, manual, model.GetMillis(), ""} + + AddStatusCache(status) + + if result := <-Srv.Store.Status().SaveOrUpdate(status); result.Err != nil { + l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err) + } + + event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil) + event.Add("status", model.STATUS_OFFLINE) + event.Add("user_id", status.UserId) + go Publish(event) +} + +func SetStatusAwayIfNeeded(userId string, manual bool) { + status, err := GetStatus(userId) + + if err != nil { + status = &model.Status{userId, model.STATUS_OFFLINE, manual, 0, ""} + } + + if !manual && status.Manual { + return // manually set status always overrides non-manual one + } + + if !manual { + if status.Status == model.STATUS_AWAY { + return + } + + if !IsUserAway(status.LastActivityAt) { + return + } + } + + status.Status = model.STATUS_AWAY + status.Manual = manual + status.ActiveChannel = "" + + AddStatusCache(status) + + if result := <-Srv.Store.Status().SaveOrUpdate(status); result.Err != nil { + l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err) + } + + event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil) + event.Add("status", model.STATUS_AWAY) + event.Add("user_id", status.UserId) + go Publish(event) +} + +func GetStatusFromCache(userId string) *model.Status { + if result, ok := statusCache.Get(userId); ok { + status := result.(*model.Status) + statusCopy := &model.Status{} + *statusCopy = *status + return statusCopy + } + + return nil +} + +func GetStatus(userId string) (*model.Status, *model.AppError) { + status := GetStatusFromCache(userId) + if status != nil { + return status, nil + } + + if result := <-Srv.Store.Status().Get(userId); result.Err != nil { + return nil, result.Err + } else { + return result.Data.(*model.Status), nil + } +} + +func IsUserAway(lastActivityAt int64) bool { + return model.GetMillis()-lastActivityAt >= *utils.Cfg.TeamSettings.UserStatusAwayTimeout*1000 +} + +func DoesStatusAllowPushNotification(user *model.User, status *model.Status, channelId string) bool { + props := user.NotifyProps + + if props["push"] == "none" { + return false + } + + if pushStatus, ok := props["push_status"]; (pushStatus == model.STATUS_ONLINE || !ok) && (status.ActiveChannel != channelId || model.GetMillis()-status.LastActivityAt > model.STATUS_CHANNEL_TIMEOUT) { + return true + } else if pushStatus == model.STATUS_AWAY && (status.Status == model.STATUS_AWAY || status.Status == model.STATUS_OFFLINE) { + return true + } else if pushStatus == model.STATUS_OFFLINE && status.Status == model.STATUS_OFFLINE { + return true + } + + return false +} diff --git a/app/team.go b/app/team.go new file mode 100644 index 000000000..98b6894a5 --- /dev/null +++ b/app/team.go @@ -0,0 +1,82 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + l4g "github.com/alecthomas/log4go" + + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +func CreateTeam(team *model.Team) (*model.Team, *model.AppError) { + if result := <-Srv.Store.Team().Save(team); result.Err != nil { + return nil, result.Err + } else { + rteam := result.Data.(*model.Team) + + if _, err := CreateDefaultChannels(rteam.Id); err != nil { + return nil, err + } + + return rteam, nil + } +} + +func JoinUserToTeamById(teamId string, user *model.User) *model.AppError { + if result := <-Srv.Store.Team().Get(teamId); result.Err != nil { + return result.Err + } else { + return JoinUserToTeam(result.Data.(*model.Team), user) + } +} + +func JoinUserToTeam(team *model.Team, user *model.User) *model.AppError { + + tm := &model.TeamMember{ + TeamId: team.Id, + UserId: user.Id, + Roles: model.ROLE_TEAM_USER.Id, + } + + channelRole := model.ROLE_CHANNEL_USER.Id + + if team.Email == user.Email { + tm.Roles = model.ROLE_TEAM_USER.Id + " " + model.ROLE_TEAM_ADMIN.Id + channelRole = model.ROLE_CHANNEL_USER.Id + " " + model.ROLE_CHANNEL_ADMIN.Id + } + + if etmr := <-Srv.Store.Team().GetMember(team.Id, user.Id); etmr.Err == nil { + // Membership alredy exists. Check if deleted and and update, otherwise do nothing + rtm := etmr.Data.(model.TeamMember) + + // Do nothing if already added + if rtm.DeleteAt == 0 { + return nil + } + + if tmr := <-Srv.Store.Team().UpdateMember(tm); tmr.Err != nil { + return tmr.Err + } + } else { + // Membership appears to be missing. Lets try to add. + if tmr := <-Srv.Store.Team().SaveMember(tm); tmr.Err != nil { + return tmr.Err + } + } + + if uua := <-Srv.Store.User().UpdateUpdateAt(user.Id); uua.Err != nil { + return uua.Err + } + + // Soft error if there is an issue joining the default channels + if err := JoinDefaultChannels(team.Id, user, channelRole); err != nil { + l4g.Error(utils.T("api.user.create_user.joining.error"), user.Id, team.Id, err) + } + + RemoveAllSessionsForUserId(user.Id) + InvalidateCacheForUser(user.Id) + + return nil +} diff --git a/app/user.go b/app/user.go new file mode 100644 index 000000000..5acd9dcaa --- /dev/null +++ b/app/user.go @@ -0,0 +1,60 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +func CreateUser(user *model.User) (*model.User, *model.AppError) { + + user.Roles = model.ROLE_SYSTEM_USER.Id + + // Below is a special case where the first user in the entire + // system is granted the system_admin role + if result := <-Srv.Store.User().GetTotalUsersCount(); result.Err != nil { + return nil, result.Err + } else { + count := result.Data.(int64) + if count <= 0 { + user.Roles = model.ROLE_SYSTEM_ADMIN.Id + " " + model.ROLE_SYSTEM_USER.Id + } + } + + user.MakeNonNil() + user.Locale = *utils.Cfg.LocalizationSettings.DefaultClientLocale + + if err := utils.IsPasswordValid(user.Password); user.AuthService == "" && err != nil { + return nil, err + } + + if result := <-Srv.Store.User().Save(user); result.Err != nil { + l4g.Error(utils.T("api.user.create_user.save.error"), result.Err) + return nil, result.Err + } else { + ruser := result.Data.(*model.User) + + if user.EmailVerified { + if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil { + l4g.Error(utils.T("api.user.create_user.verified.error"), cresult.Err) + } + } + + pref := model.Preference{UserId: ruser.Id, Category: model.PREFERENCE_CATEGORY_TUTORIAL_STEPS, Name: ruser.Id, Value: "0"} + if presult := <-Srv.Store.Preference().Save(&model.Preferences{pref}); presult.Err != nil { + l4g.Error(utils.T("api.user.create_user.tutorial.error"), presult.Err.Message) + } + + ruser.Sanitize(map[string]bool{}) + + // This message goes to everyone, so the teamId, channelId and userId are irrelevant + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_NEW_USER, "", "", "", nil) + message.Add("user_id", ruser.Id) + go Publish(message) + + return ruser, nil + } +} diff --git a/app/web_conn.go b/app/web_conn.go new file mode 100644 index 000000000..02c3b2642 --- /dev/null +++ b/app/web_conn.go @@ -0,0 +1,254 @@ +// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "fmt" + "time" + + "github.com/mattermost/platform/einterfaces" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" + + l4g "github.com/alecthomas/log4go" + "github.com/gorilla/websocket" + goi18n "github.com/nicksnyder/go-i18n/i18n" +) + +const ( + WRITE_WAIT = 30 * time.Second + PONG_WAIT = 100 * time.Second + PING_PERIOD = (PONG_WAIT * 6) / 10 + AUTH_TIMEOUT = 5 * time.Second +) + +type WebConn struct { + WebSocket *websocket.Conn + Send chan model.WebSocketMessage + SessionToken string + SessionExpiresAt int64 + UserId string + T goi18n.TranslateFunc + Locale string + AllChannelMembers map[string]string + LastAllChannelMembersTime int64 +} + +func NewWebConn(ws *websocket.Conn, session model.Session, t goi18n.TranslateFunc, locale string) *WebConn { + if len(session.UserId) > 0 { + go SetStatusOnline(session.UserId, session.Id, false) + } + + return &WebConn{ + Send: make(chan model.WebSocketMessage, 256), + WebSocket: ws, + UserId: session.UserId, + SessionToken: session.Token, + SessionExpiresAt: session.ExpiresAt, + T: t, + Locale: locale, + } +} + +func (c *WebConn) ReadPump() { + defer func() { + HubUnregister(c) + c.WebSocket.Close() + }() + c.WebSocket.SetReadLimit(model.SOCKET_MAX_MESSAGE_SIZE_KB) + c.WebSocket.SetReadDeadline(time.Now().Add(PONG_WAIT)) + c.WebSocket.SetPongHandler(func(string) error { + c.WebSocket.SetReadDeadline(time.Now().Add(PONG_WAIT)) + if c.IsAuthenticated() { + go SetStatusAwayIfNeeded(c.UserId, false) + } + return nil + }) + + for { + var req model.WebSocketRequest + if err := c.WebSocket.ReadJSON(&req); err != nil { + // browsers will appear as CloseNoStatusReceived + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseNoStatusReceived) { + l4g.Debug(fmt.Sprintf("websocket.read: client side closed socket userId=%v", c.UserId)) + } else { + l4g.Debug(fmt.Sprintf("websocket.read: closing websocket for userId=%v error=%v", c.UserId, err.Error())) + } + + return + } else { + Srv.WebSocketRouter.ServeWebSocket(c, &req) + } + } +} + +func (c *WebConn) WritePump() { + ticker := time.NewTicker(PING_PERIOD) + authTicker := time.NewTicker(AUTH_TIMEOUT) + + defer func() { + ticker.Stop() + authTicker.Stop() + c.WebSocket.Close() + }() + + for { + select { + case msg, ok := <-c.Send: + if !ok { + c.WebSocket.SetWriteDeadline(time.Now().Add(WRITE_WAIT)) + c.WebSocket.WriteMessage(websocket.CloseMessage, []byte{}) + return + } + + c.WebSocket.SetWriteDeadline(time.Now().Add(WRITE_WAIT)) + if err := c.WebSocket.WriteMessage(websocket.TextMessage, msg.GetPreComputeJson()); err != nil { + // browsers will appear as CloseNoStatusReceived + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseNoStatusReceived) { + l4g.Debug(fmt.Sprintf("websocket.send: client side closed socket userId=%v", c.UserId)) + } else { + l4g.Debug(fmt.Sprintf("websocket.send: closing websocket for userId=%v, error=%v", c.UserId, err.Error())) + } + + return + } + + if msg.EventType() == model.WEBSOCKET_EVENT_POSTED { + if einterfaces.GetMetricsInterface() != nil { + einterfaces.GetMetricsInterface().IncrementPostBroadcast() + } + } + + case <-ticker.C: + c.WebSocket.SetWriteDeadline(time.Now().Add(WRITE_WAIT)) + if err := c.WebSocket.WriteMessage(websocket.PingMessage, []byte{}); err != nil { + // browsers will appear as CloseNoStatusReceived + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseNoStatusReceived) { + l4g.Debug(fmt.Sprintf("websocket.ticker: client side closed socket userId=%v", c.UserId)) + } else { + l4g.Debug(fmt.Sprintf("websocket.ticker: closing websocket for userId=%v error=%v", c.UserId, err.Error())) + } + + return + } + + case <-authTicker.C: + if c.SessionToken == "" { + l4g.Debug(fmt.Sprintf("websocket.authTicker: did not authenticate ip=%v", c.WebSocket.RemoteAddr())) + return + } + authTicker.Stop() + } + } +} + +func (webCon *WebConn) InvalidateCache() { + webCon.AllChannelMembers = nil + webCon.LastAllChannelMembersTime = 0 + webCon.SessionExpiresAt = 0 +} + +func (webCon *WebConn) IsAuthenticated() bool { + // Check the expiry to see if we need to check for a new session + if webCon.SessionExpiresAt < model.GetMillis() { + if webCon.SessionToken == "" { + return false + } + + session, err := GetSession(webCon.SessionToken) + if err != nil { + l4g.Error(utils.T("api.websocket.invalid_session.error"), err.Error()) + webCon.SessionToken = "" + webCon.SessionExpiresAt = 0 + return false + } + + webCon.SessionToken = session.Token + webCon.SessionExpiresAt = session.ExpiresAt + } + + return true +} + +func (webCon *WebConn) SendHello() { + msg := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_HELLO, "", "", webCon.UserId, nil) + msg.Add("server_version", fmt.Sprintf("%v.%v.%v", model.CurrentVersion, model.BuildNumber, utils.CfgHash)) + msg.DoPreComputeJson() + webCon.Send <- msg +} + +func (webCon *WebConn) ShouldSendEvent(msg *model.WebSocketEvent) bool { + // IMPORTANT: Do not send event if WebConn does not have a session + if !webCon.IsAuthenticated() { + return false + } + + // If the event is destined to a specific user + if len(msg.Broadcast.UserId) > 0 && webCon.UserId != msg.Broadcast.UserId { + return false + } + + // if the user is omitted don't send the message + if len(msg.Broadcast.OmitUsers) > 0 { + if _, ok := msg.Broadcast.OmitUsers[webCon.UserId]; ok { + return false + } + } + + // Only report events to users who are in the channel for the event + if len(msg.Broadcast.ChannelId) > 0 { + + // Only broadcast typing messages if less than 1K people in channel + if msg.Event == model.WEBSOCKET_EVENT_TYPING { + if Srv.Store.Channel().GetMemberCountFromCache(msg.Broadcast.ChannelId) > *utils.Cfg.TeamSettings.MaxNotificationsPerChannel { + return false + } + } + + if model.GetMillis()-webCon.LastAllChannelMembersTime > 1000*60*15 { // 15 minutes + webCon.AllChannelMembers = nil + webCon.LastAllChannelMembersTime = 0 + } + + if webCon.AllChannelMembers == nil { + if result := <-Srv.Store.Channel().GetAllChannelMembersForUser(webCon.UserId, true); result.Err != nil { + l4g.Error("webhub.shouldSendEvent: " + result.Err.Error()) + return false + } else { + webCon.AllChannelMembers = result.Data.(map[string]string) + webCon.LastAllChannelMembersTime = model.GetMillis() + } + } + + if _, ok := webCon.AllChannelMembers[msg.Broadcast.ChannelId]; ok { + return true + } else { + return false + } + } + + // Only report events to users who are in the team for the event + if len(msg.Broadcast.TeamId) > 0 { + return webCon.IsMemberOfTeam(msg.Broadcast.TeamId) + + } + + return true +} + +func (webCon *WebConn) IsMemberOfTeam(teamId string) bool { + session, err := GetSession(webCon.SessionToken) + if err != nil { + l4g.Error(utils.T("api.websocket.invalid_session.error"), err.Error()) + return false + } else { + member := session.GetTeamByTeamId(teamId) + + if member != nil { + return true + } else { + return false + } + } +} diff --git a/app/web_hub.go b/app/web_hub.go new file mode 100644 index 000000000..28d2c0095 --- /dev/null +++ b/app/web_hub.go @@ -0,0 +1,241 @@ +// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "fmt" + "hash/fnv" + "runtime" + + l4g "github.com/alecthomas/log4go" + + "github.com/mattermost/platform/einterfaces" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +type Hub struct { + connections map[*WebConn]bool + register chan *WebConn + unregister chan *WebConn + broadcast chan *model.WebSocketEvent + stop chan string + invalidateUser chan string +} + +var hubs []*Hub = make([]*Hub, 0) + +func NewWebHub() *Hub { + return &Hub{ + register: make(chan *WebConn), + unregister: make(chan *WebConn), + connections: make(map[*WebConn]bool, model.SESSION_CACHE_SIZE), + broadcast: make(chan *model.WebSocketEvent, 4096), + stop: make(chan string), + invalidateUser: make(chan string), + } +} + +func TotalWebsocketConnections() int { + // This is racy, but it's only used for reporting information + // so it's probably OK + count := 0 + for _, hub := range hubs { + count = count + len(hub.connections) + } + + return count +} + +func HubStart() { + l4g.Info(utils.T("api.web_hub.start.starting.debug"), runtime.NumCPU()*2) + + // Total number of hubs is twice the number of CPUs. + hubs = make([]*Hub, runtime.NumCPU()*2) + + for i := 0; i < len(hubs); i++ { + hubs[i] = NewWebHub() + hubs[i].Start() + } +} + +func HubStop() { + l4g.Info(utils.T("api.web_hub.start.stopping.debug")) + + for _, hub := range hubs { + hub.Stop() + } + + hubs = make([]*Hub, 0) +} + +func GetHubForUserId(userId string) *Hub { + hash := fnv.New32a() + hash.Write([]byte(userId)) + index := hash.Sum32() % uint32(len(hubs)) + return hubs[index] +} + +func HubRegister(webConn *WebConn) { + GetHubForUserId(webConn.UserId).Register(webConn) +} + +func HubUnregister(webConn *WebConn) { + GetHubForUserId(webConn.UserId).Unregister(webConn) +} + +func Publish(message *model.WebSocketEvent) { + message.DoPreComputeJson() + for _, hub := range hubs { + hub.Broadcast(message) + } + + if einterfaces.GetClusterInterface() != nil { + einterfaces.GetClusterInterface().Publish(message) + } +} + +func PublishSkipClusterSend(message *model.WebSocketEvent) { + message.DoPreComputeJson() + for _, hub := range hubs { + hub.Broadcast(message) + } +} + +func InvalidateCacheForChannel(channelId string) { + InvalidateCacheForChannelSkipClusterSend(channelId) + + if cluster := einterfaces.GetClusterInterface(); cluster != nil { + cluster.InvalidateCacheForChannel(channelId) + } +} + +func InvalidateCacheForChannelSkipClusterSend(channelId string) { + Srv.Store.User().InvalidateProfilesInChannelCache(channelId) + Srv.Store.Channel().InvalidateMemberCount(channelId) + Srv.Store.Channel().InvalidateChannel(channelId) +} + +func InvalidateCacheForChannelPosts(channelId string) { + InvalidateCacheForChannelPostsSkipClusterSend(channelId) + + if cluster := einterfaces.GetClusterInterface(); cluster != nil { + cluster.InvalidateCacheForChannelPosts(channelId) + } +} + +func InvalidateCacheForChannelPostsSkipClusterSend(channelId string) { + Srv.Store.Post().InvalidateLastPostTimeCache(channelId) +} + +func InvalidateCacheForUser(userId string) { + InvalidateCacheForUserSkipClusterSend(userId) + + if einterfaces.GetClusterInterface() != nil { + einterfaces.GetClusterInterface().InvalidateCacheForUser(userId) + } +} + +func InvalidateCacheForUserSkipClusterSend(userId string) { + Srv.Store.Channel().InvalidateAllChannelMembersForUser(userId) + Srv.Store.User().InvalidateProfilesInChannelCacheByUser(userId) + Srv.Store.User().InvalidatProfileCacheForUser(userId) + + if len(hubs) != 0 { + GetHubForUserId(userId).InvalidateUser(userId) + } +} + +func InvalidateWebConnSessionCacheForUser(userId string) { + if len(hubs) != 0 { + GetHubForUserId(userId).InvalidateUser(userId) + } +} + +func (h *Hub) Register(webConn *WebConn) { + h.register <- webConn + + if webConn.IsAuthenticated() { + webConn.SendHello() + } +} + +func (h *Hub) Unregister(webConn *WebConn) { + h.unregister <- webConn +} + +func (h *Hub) Broadcast(message *model.WebSocketEvent) { + if message != nil { + h.broadcast <- message + } +} + +func (h *Hub) InvalidateUser(userId string) { + h.invalidateUser <- userId +} + +func (h *Hub) Stop() { + h.stop <- "all" +} + +func (h *Hub) Start() { + go func() { + for { + select { + case webCon := <-h.register: + h.connections[webCon] = true + + case webCon := <-h.unregister: + userId := webCon.UserId + if _, ok := h.connections[webCon]; ok { + delete(h.connections, webCon) + close(webCon.Send) + } + + if len(userId) == 0 { + continue + } + + found := false + for webCon := range h.connections { + if userId == webCon.UserId { + found = true + break + } + } + + if !found { + go SetStatusOffline(userId, false) + } + + case userId := <-h.invalidateUser: + for webCon := range h.connections { + if webCon.UserId == userId { + webCon.InvalidateCache() + } + } + + case msg := <-h.broadcast: + for webCon := range h.connections { + if webCon.ShouldSendEvent(msg) { + select { + case webCon.Send <- msg: + default: + l4g.Error(fmt.Sprintf("webhub.broadcast: cannot send, closing websocket for userId=%v", webCon.UserId)) + close(webCon.Send) + delete(h.connections, webCon) + } + } + } + + case <-h.stop: + for webCon := range h.connections { + webCon.WebSocket.Close() + } + + return + } + } + }() +} diff --git a/app/webhook.go b/app/webhook.go new file mode 100644 index 000000000..dfd59349f --- /dev/null +++ b/app/webhook.go @@ -0,0 +1,155 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "strings" + + l4g "github.com/alecthomas/log4go" + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +const ( + TRIGGERWORDS_FULL = 0 + TRIGGERWORDS_STARTSWITH = 1 +) + +func handleWebhookEvents(post *model.Post, team *model.Team, channel *model.Channel, user *model.User) *model.AppError { + if !utils.Cfg.ServiceSettings.EnableOutgoingWebhooks { + return nil + } + + if channel.Type != model.CHANNEL_OPEN { + return nil + } + + hchan := Srv.Store.Webhook().GetOutgoingByTeam(team.Id) + result := <-hchan + if result.Err != nil { + return result.Err + } + + hooks := result.Data.([]*model.OutgoingWebhook) + if len(hooks) == 0 { + return nil + } + + splitWords := strings.Fields(post.Message) + if len(splitWords) == 0 { + return nil + } + firstWord := splitWords[0] + + relevantHooks := []*model.OutgoingWebhook{} + for _, hook := range hooks { + if hook.ChannelId == post.ChannelId || len(hook.ChannelId) == 0 { + if hook.ChannelId == post.ChannelId && len(hook.TriggerWords) == 0 { + relevantHooks = append(relevantHooks, hook) + } else if hook.TriggerWhen == TRIGGERWORDS_FULL && hook.HasTriggerWord(firstWord) { + relevantHooks = append(relevantHooks, hook) + } else if hook.TriggerWhen == TRIGGERWORDS_STARTSWITH && hook.TriggerWordStartsWith(firstWord) { + relevantHooks = append(relevantHooks, hook) + } + } + } + + for _, hook := range relevantHooks { + go func(hook *model.OutgoingWebhook) { + payload := &model.OutgoingWebhookPayload{ + Token: hook.Token, + TeamId: hook.TeamId, + TeamDomain: team.Name, + ChannelId: post.ChannelId, + ChannelName: channel.Name, + Timestamp: post.CreateAt, + UserId: post.UserId, + UserName: user.Username, + PostId: post.Id, + Text: post.Message, + TriggerWord: firstWord, + } + var body io.Reader + var contentType string + if hook.ContentType == "application/json" { + body = strings.NewReader(payload.ToJSON()) + contentType = "application/json" + } else { + body = strings.NewReader(payload.ToFormValues()) + contentType = "application/x-www-form-urlencoded" + } + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: *utils.Cfg.ServiceSettings.EnableInsecureOutgoingConnections}, + } + client := &http.Client{Transport: tr} + + for _, url := range hook.CallbackURLs { + go func(url string) { + req, _ := http.NewRequest("POST", url, body) + req.Header.Set("Content-Type", contentType) + req.Header.Set("Accept", "application/json") + if resp, err := client.Do(req); err != nil { + l4g.Error(utils.T("api.post.handle_webhook_events_and_forget.event_post.error"), err.Error()) + } else { + defer func() { + ioutil.ReadAll(resp.Body) + resp.Body.Close() + }() + respProps := model.MapFromJson(resp.Body) + + if text, ok := respProps["text"]; ok { + if _, err := CreateWebhookPost(hook.CreatorId, hook.TeamId, post.ChannelId, text, respProps["username"], respProps["icon_url"], post.Props, post.Type); err != nil { + l4g.Error(utils.T("api.post.handle_webhook_events_and_forget.create_post.error"), err) + } + } + } + }(url) + } + + }(hook) + } + + return nil +} + +func CreateWebhookPost(userId, teamId, channelId, text, overrideUsername, overrideIconUrl string, props model.StringInterface, postType string) (*model.Post, *model.AppError) { + post := &model.Post{UserId: userId, ChannelId: channelId, Message: text, Type: postType} + post.AddProp("from_webhook", "true") + + if utils.Cfg.ServiceSettings.EnablePostUsernameOverride { + if len(overrideUsername) != 0 { + post.AddProp("override_username", overrideUsername) + } else { + post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME) + } + } + + if utils.Cfg.ServiceSettings.EnablePostIconOverride { + if len(overrideIconUrl) != 0 { + post.AddProp("override_icon_url", overrideIconUrl) + } + } + + post.Message = parseSlackLinksToMarkdown(post.Message) + + if len(props) > 0 { + for key, val := range props { + if key == "attachments" { + parseSlackAttachment(post, val) + } else if key != "override_icon_url" && key != "override_username" && key != "from_webhook" { + post.AddProp(key, val) + } + } + } + + if _, err := CreatePost(post, teamId, false); err != nil { + return nil, model.NewLocAppError("CreateWebhookPost", "api.post.create_webhook_post.creating.app_error", nil, "err="+err.Message) + } + + return post, nil +} diff --git a/app/websocket_router.go b/app/websocket_router.go new file mode 100644 index 000000000..984b9d17e --- /dev/null +++ b/app/websocket_router.go @@ -0,0 +1,96 @@ +// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + l4g "github.com/alecthomas/log4go" + + "github.com/mattermost/platform/model" + "github.com/mattermost/platform/utils" +) + +type webSocketHandler interface { + ServeWebSocket(*WebConn, *model.WebSocketRequest) +} + +type WebSocketRouter struct { + handlers map[string]webSocketHandler +} + +func NewWebSocketRouter() *WebSocketRouter { + router := &WebSocketRouter{} + router.handlers = make(map[string]webSocketHandler) + return router +} + +func (wr *WebSocketRouter) Handle(action string, handler webSocketHandler) { + wr.handlers[action] = handler +} + +func (wr *WebSocketRouter) ServeWebSocket(conn *WebConn, r *model.WebSocketRequest) { + if r.Action == "" { + err := model.NewLocAppError("ServeWebSocket", "api.web_socket_router.no_action.app_error", nil, "") + ReturnWebSocketError(conn, r, err) + return + } + + if r.Seq <= 0 { + err := model.NewLocAppError("ServeWebSocket", "api.web_socket_router.bad_seq.app_error", nil, "") + ReturnWebSocketError(conn, r, err) + return + } + + if r.Action == model.WEBSOCKET_AUTHENTICATION_CHALLENGE { + token, ok := r.Data["token"].(string) + if !ok { + conn.WebSocket.Close() + return + } + + session, err := GetSession(token) + + if err != nil { + conn.WebSocket.Close() + } else { + go SetStatusOnline(session.UserId, session.Id, false) + + conn.SessionToken = session.Token + conn.UserId = session.UserId + + resp := model.NewWebSocketResponse(model.STATUS_OK, r.Seq, nil) + resp.DoPreComputeJson() + conn.Send <- resp + conn.SendHello() + } + + return + } + + if !conn.IsAuthenticated() { + err := model.NewLocAppError("ServeWebSocket", "api.web_socket_router.not_authenticated.app_error", nil, "") + ReturnWebSocketError(conn, r, err) + return + } + + var handler webSocketHandler + if h, ok := wr.handlers[r.Action]; !ok { + err := model.NewLocAppError("ServeWebSocket", "api.web_socket_router.bad_action.app_error", nil, "") + ReturnWebSocketError(conn, r, err) + return + } else { + handler = h + } + + handler.ServeWebSocket(conn, r) +} + +func ReturnWebSocketError(conn *WebConn, r *model.WebSocketRequest, err *model.AppError) { + l4g.Error(utils.T("api.web_socket_router.log.error"), r.Seq, conn.UserId, err.SystemMessage(utils.T), err.DetailedError) + + err.DetailedError = "" + errorResp := model.NewWebSocketError(r.Seq, err) + errorResp.DoPreComputeJson() + + conn.Send <- errorResp +} -- cgit v1.2.3-1-g7c22