From 365b8b465e8a53ebb2da2bf3aef659ac81a2bc6a Mon Sep 17 00:00:00 2001 From: Joram Wilander Date: Wed, 19 Oct 2016 14:49:25 -0400 Subject: Merging performance branch into master (#4268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improve performance on sendNotifications * Fix SQL queries * Remove get direct profiles, not needed anymore * Add raw data to error details if AppError fails to decode * men * Fix decode (#4052) * Fixing json decode * Adding unit test * Initial work for client scaling (#4051) * Begin adding paging to profiles API * Added more paging functionality * Finish hooking up admin console user lists * Add API for searching users and add searching to all user lists * Add lazy loading of profiles * Revert config.json * Fix unit tests and some style issues * Add GetProfilesFromList to Go driver and fix web unit test * Update etag for GetProfiles * Updating ui for filters and pagination (#4044) * Updating UI for pagination * Adjusting margins for filter row * Adjusting margin for specific modals * Adding relative padding to system console * Adjusting responsive view * Update client user tests * Minor fixes for direct messages modal (#4056) * Remove some unneeded initial load calls (#4057) * UX updates to user lists, added smart counts and bug fixes (#4059) * Improved getExplicitMentions and unit tests (#4064) * Refactor getting posts to lazy load profiles correctly (#4062) * Comment out SetActiveChannel test (#4066) * Profiler cpu, block, and memory profiler. (#4081) * Fix TestSetActiveChannel unit test (#4071) * Fixing build failure caused by dependancies updating (#4076) * Adding profiler * Fix admin_team_member_dropdown eslint errors * Bumping session cache size (#4077) * Bumping session cache size * Bumping status cache * Refactor how the client handles channel members to be large team friendly (#4106) * Refactor how the client handles channel members to be large team friendly * Change Id to ChannelId in ChannelStats model * Updated getChannelMember and getProfilesByIds routes to match proposal * Performance improvements (#4100) * Performance improvements * Fixing re-connect issue * Fixing error message * Some other minor perf tweaks * Some other minor perf tweaks * Fixing config file * Fixing buffer size * Fixing web socket send message * adding some error logging * fix getMe to be user required * Fix websocket event for new user * Fixing shutting down * Reverting web socket changes * Fixing logging lvl * Adding caching to GetMember * Adding some logging * Fixing caching * Fixing caching invalidate * Fixing direct message caching * Fixing caching * Fixing caching * Remove GetDirectProfiles from initial load * Adding logging and fixing websocket client * Adding back caching from bad merge. * Explicitly close go driver requests (#4162) * Refactored how the client handles team members to be more large team friendly (#4159) * Refactor getProfilesForDirectMessageList API into getAllProfiles API * Refactored how the client handles team members to be more large team friendly * Fix js error when receiving a notification * Fix JS error caused by current user being overwritten with sanitized version (#4165) * Adding error message to status failure (#4167) * Fix a few bugs caused by client scaling refactoring (#4170) * When there is no read replica, don't open a second set of connections to the master database (#4173) * Adding connection tacking to stats (#4174) * Reduce DB writes for statuses and other status related changes (#4175) * Fix bug preventing opening of DM channels from more modal (#4181) * Fixing socket timing error (#4183) * Fixing ping/pong handler * Fixing socket timing error * Commenting out status broadcasting * Removing user status changes * Removing user status changes * Removing user status changes * Removing user status changes * Adding DoPreComputeJson() * Performance improvements (#4194) * * Fix System Console Analytics queries * Add db.SetConnMaxLifetime to 15 minutes * Add "net/http/pprof" for profiling * Add FreeOSMemory() to manually release memory on reload config * Add flag to enable http profiler * Fix memory leak (#4197) * Fix memory leak * removed unneeded nil assignment * Fixing go routine leak (#4208) * Merge fixes * Merge fix * Refactored statuses to be queried by the client rather than broadcast by the server (#4212) * Refactored server code to reduce status broadcasts and to allow getting statuses by IDs * Refactor client code to periodically fetch statuses * Add store unit test for getting statuses by ids * Fix status unit test * Add getStatusesByIds REST API and move the client over to use that instead of the WebSocket * Adding multiple threads to websocket hub (#4230) * Adding multiple threads to websocket hub * Fixing unit tests * Fixing so websocket connections from the same user end up in the same… (#4240) * Fixing so websocket connections from the same user end up in the same list * Removing old comment * Refactor user autocomplete to query the server (#4239) * Add API for autocompleting users * Converted at mention autocomplete to query server * Converted user search autocomplete to query server * Switch autocomplete API naming to use term instead of username * Split autocomplete API into two, one for channels and for teams * Fix copy/paste error * Some final client scaling fixes (#4246) * Add lazy loading of profiles to integration pages * Add lazy loading of profiles to emoji page * Fix JS error when receiving post in select team menu and also clean up channel store --- api/status.go | 131 +++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 111 insertions(+), 20 deletions(-) (limited to 'api/status.go') diff --git a/api/status.go b/api/status.go index a1a5ac496..897102a4b 100644 --- a/api/status.go +++ b/api/status.go @@ -31,9 +31,11 @@ func AddStatusCache(status *model.Status) { func InitStatus() { l4g.Debug(utils.T("api.status.init.debug")) - BaseRoutes.Users.Handle("/status", ApiUserRequiredActivity(getStatusesHttp, false)).Methods("GET") - BaseRoutes.Users.Handle("/status/set_active_channel", ApiUserRequiredActivity(setActiveChannel, false)).Methods("POST") + BaseRoutes.Users.Handle("/status", ApiUserRequired(getStatusesHttp)).Methods("GET") + BaseRoutes.Users.Handle("/status/ids", ApiUserRequired(getStatusesByIdsHttp)).Methods("POST") + BaseRoutes.Users.Handle("/status/set_active_channel", ApiUserRequired(setActiveChannel)).Methods("POST") BaseRoutes.WebSocket.Handle("get_statuses", ApiWebSocketHandler(getStatusesWebSocket)) + BaseRoutes.WebSocket.Handle("get_statuses_by_ids", ApiWebSocketHandler(getStatusesByIdsWebSocket)) } func getStatusesHttp(c *Context, w http.ResponseWriter, r *http.Request) { @@ -55,6 +57,7 @@ func getStatusesWebSocket(req *model.WebSocketRequest) (map[string]interface{}, return statusMap, nil } +// Only returns 300 statuses max func GetAllStatuses() (map[string]interface{}, *model.AppError) { if result := <-Srv.Store.Status().GetOnlineAway(); result.Err != nil { return nil, result.Err @@ -70,11 +73,82 @@ func GetAllStatuses() (map[string]interface{}, *model.AppError) { } } +func getStatusesByIdsHttp(c *Context, w http.ResponseWriter, r *http.Request) { + userIds := model.ArrayFromJson(r.Body) + + if len(userIds) == 0 { + c.SetInvalidParam("getStatusesByIdsHttp", "user_ids") + return + } + + statusMap, err := GetStatusesByIds(userIds) + if err != nil { + c.Err = err + return + } + + w.Write([]byte(model.StringInterfaceToJson(statusMap))) +} + +func getStatusesByIdsWebSocket(req *model.WebSocketRequest) (map[string]interface{}, *model.AppError) { + var userIds []string + if userIds = model.ArrayFromInterface(req.Data["user_ids"]); len(userIds) == 0 { + l4g.Error(model.StringInterfaceToJson(req.Data)) + return nil, NewInvalidWebSocketParamError(req.Action, "user_ids") + } + + statusMap, err := GetStatusesByIds(userIds) + if err != nil { + return nil, err + } + + return statusMap, nil +} + +func GetStatusesByIds(userIds []string) (map[string]interface{}, *model.AppError) { + statusMap := map[string]interface{}{} + + missingUserIds := []string{} + for _, userId := range userIds { + if result, ok := statusCache.Get(userId); ok { + statusMap[userId] = result.(*model.Status).Status + } else { + missingUserIds = append(missingUserIds, userId) + } + } + + if len(missingUserIds) > 0 { + if result := <-Srv.Store.Status().GetByIds(missingUserIds); result.Err != nil { + return nil, result.Err + } else { + statuses := result.Data.([]*model.Status) + + for _, s := range statuses { + AddStatusCache(s) + statusMap[s.UserId] = s.Status + } + } + } + + // For the case where the user does not have a row in the Status table and cache + for _, userId := range missingUserIds { + if _, ok := statusMap[userId]; !ok { + statusMap[userId] = model.STATUS_OFFLINE + } + } + + return statusMap, nil +} + func SetStatusOnline(userId string, sessionId string, manual bool) { broadcast := false + var oldStatus string = model.STATUS_OFFLINE + var oldTime int64 = 0 + var oldManual bool = false var status *model.Status var err *model.AppError + if status, err = GetStatus(userId); err != nil { status = &model.Status{userId, model.STATUS_ONLINE, false, model.GetMillis(), ""} broadcast = true @@ -82,35 +156,45 @@ func SetStatusOnline(userId string, sessionId string, manual bool) { if status.Manual && !manual { return // manually set status always overrides non-manual one } + if status.Status != model.STATUS_ONLINE { broadcast = true } + + oldStatus = status.Status + oldTime = status.LastActivityAt + oldManual = status.Manual + status.Status = model.STATUS_ONLINE - status.Manual = false // for "online" there's no manually or auto set + status.Manual = false // for "online" there's no manual setting status.LastActivityAt = model.GetMillis() } AddStatusCache(status) - achan := Srv.Store.Session().UpdateLastActivityAt(sessionId, model.GetMillis()) + // Only update the database if the status has changed, the status has been manually set, + // or enough time has passed since the previous action + if status.Status != oldStatus || status.Manual != oldManual || status.LastActivityAt-oldTime > model.STATUS_MIN_UPDATE_TIME { + achan := Srv.Store.Session().UpdateLastActivityAt(sessionId, status.LastActivityAt) - var schan store.StoreChannel - if broadcast { - schan = Srv.Store.Status().SaveOrUpdate(status) - } else { - schan = Srv.Store.Status().UpdateLastActivityAt(status.UserId, status.LastActivityAt) - } + var schan store.StoreChannel + if broadcast { + schan = Srv.Store.Status().SaveOrUpdate(status) + } else { + schan = Srv.Store.Status().UpdateLastActivityAt(status.UserId, status.LastActivityAt) + } - if result := <-achan; result.Err != nil { - l4g.Error(utils.T("api.status.last_activity.error"), userId, sessionId, result.Err) - } + if result := <-achan; result.Err != nil { + l4g.Error(utils.T("api.status.last_activity.error"), userId, sessionId, result.Err) + } - if result := <-schan; result.Err != nil { - l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err) + if result := <-schan; result.Err != nil { + l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err) + } } if broadcast { - event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", "", nil) + event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil) event.Add("status", model.STATUS_ONLINE) event.Add("user_id", status.UserId) go Publish(event) @@ -131,7 +215,7 @@ func SetStatusOffline(userId string, manual bool) { l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err) } - event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", "", nil) + event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil) event.Add("status", model.STATUS_OFFLINE) event.Add("user_id", status.UserId) go Publish(event) @@ -168,15 +252,18 @@ func SetStatusAwayIfNeeded(userId string, manual bool) { l4g.Error(utils.T("api.status.save_status.error"), userId, result.Err) } - event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", "", nil) + event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil) event.Add("status", model.STATUS_AWAY) event.Add("user_id", status.UserId) go Publish(event) } func GetStatus(userId string) (*model.Status, *model.AppError) { - if status, ok := statusCache.Get(userId); ok { - return status.(*model.Status), nil + if result, ok := statusCache.Get(userId); ok { + status := result.(*model.Status) + statusCopy := &model.Status{} + *statusCopy = *status + return statusCopy, nil } if result := <-Srv.Store.Status().Get(userId); result.Err != nil { @@ -232,6 +319,10 @@ func SetActiveChannel(userId string, channelId string) *model.AppError { status = &model.Status{userId, model.STATUS_ONLINE, false, model.GetMillis(), channelId} } else { status.ActiveChannel = channelId + if !status.Manual { + status.Status = model.STATUS_ONLINE + } + status.LastActivityAt = model.GetMillis() } AddStatusCache(status) -- cgit v1.2.3-1-g7c22