From 365b8b465e8a53ebb2da2bf3aef659ac81a2bc6a Mon Sep 17 00:00:00 2001 From: Joram Wilander Date: Wed, 19 Oct 2016 14:49:25 -0400 Subject: Merging performance branch into master (#4268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improve performance on sendNotifications * Fix SQL queries * Remove get direct profiles, not needed anymore * Add raw data to error details if AppError fails to decode * men * Fix decode (#4052) * Fixing json decode * Adding unit test * Initial work for client scaling (#4051) * Begin adding paging to profiles API * Added more paging functionality * Finish hooking up admin console user lists * Add API for searching users and add searching to all user lists * Add lazy loading of profiles * Revert config.json * Fix unit tests and some style issues * Add GetProfilesFromList to Go driver and fix web unit test * Update etag for GetProfiles * Updating ui for filters and pagination (#4044) * Updating UI for pagination * Adjusting margins for filter row * Adjusting margin for specific modals * Adding relative padding to system console * Adjusting responsive view * Update client user tests * Minor fixes for direct messages modal (#4056) * Remove some unneeded initial load calls (#4057) * UX updates to user lists, added smart counts and bug fixes (#4059) * Improved getExplicitMentions and unit tests (#4064) * Refactor getting posts to lazy load profiles correctly (#4062) * Comment out SetActiveChannel test (#4066) * Profiler cpu, block, and memory profiler. (#4081) * Fix TestSetActiveChannel unit test (#4071) * Fixing build failure caused by dependancies updating (#4076) * Adding profiler * Fix admin_team_member_dropdown eslint errors * Bumping session cache size (#4077) * Bumping session cache size * Bumping status cache * Refactor how the client handles channel members to be large team friendly (#4106) * Refactor how the client handles channel members to be large team friendly * Change Id to ChannelId in ChannelStats model * Updated getChannelMember and getProfilesByIds routes to match proposal * Performance improvements (#4100) * Performance improvements * Fixing re-connect issue * Fixing error message * Some other minor perf tweaks * Some other minor perf tweaks * Fixing config file * Fixing buffer size * Fixing web socket send message * adding some error logging * fix getMe to be user required * Fix websocket event for new user * Fixing shutting down * Reverting web socket changes * Fixing logging lvl * Adding caching to GetMember * Adding some logging * Fixing caching * Fixing caching invalidate * Fixing direct message caching * Fixing caching * Fixing caching * Remove GetDirectProfiles from initial load * Adding logging and fixing websocket client * Adding back caching from bad merge. * Explicitly close go driver requests (#4162) * Refactored how the client handles team members to be more large team friendly (#4159) * Refactor getProfilesForDirectMessageList API into getAllProfiles API * Refactored how the client handles team members to be more large team friendly * Fix js error when receiving a notification * Fix JS error caused by current user being overwritten with sanitized version (#4165) * Adding error message to status failure (#4167) * Fix a few bugs caused by client scaling refactoring (#4170) * When there is no read replica, don't open a second set of connections to the master database (#4173) * Adding connection tacking to stats (#4174) * Reduce DB writes for statuses and other status related changes (#4175) * Fix bug preventing opening of DM channels from more modal (#4181) * Fixing socket timing error (#4183) * Fixing ping/pong handler * Fixing socket timing error * Commenting out status broadcasting * Removing user status changes * Removing user status changes * Removing user status changes * Removing user status changes * Adding DoPreComputeJson() * Performance improvements (#4194) * * Fix System Console Analytics queries * Add db.SetConnMaxLifetime to 15 minutes * Add "net/http/pprof" for profiling * Add FreeOSMemory() to manually release memory on reload config * Add flag to enable http profiler * Fix memory leak (#4197) * Fix memory leak * removed unneeded nil assignment * Fixing go routine leak (#4208) * Merge fixes * Merge fix * Refactored statuses to be queried by the client rather than broadcast by the server (#4212) * Refactored server code to reduce status broadcasts and to allow getting statuses by IDs * Refactor client code to periodically fetch statuses * Add store unit test for getting statuses by ids * Fix status unit test * Add getStatusesByIds REST API and move the client over to use that instead of the WebSocket * Adding multiple threads to websocket hub (#4230) * Adding multiple threads to websocket hub * Fixing unit tests * Fixing so websocket connections from the same user end up in the same… (#4240) * Fixing so websocket connections from the same user end up in the same list * Removing old comment * Refactor user autocomplete to query the server (#4239) * Add API for autocompleting users * Converted at mention autocomplete to query server * Converted user search autocomplete to query server * Switch autocomplete API naming to use term instead of username * Split autocomplete API into two, one for channels and for teams * Fix copy/paste error * Some final client scaling fixes (#4246) * Add lazy loading of profiles to integration pages * Add lazy loading of profiles to emoji page * Fix JS error when receiving post in select team menu and also clean up channel store --- store/sql_user_store_test.go | 428 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 419 insertions(+), 9 deletions(-) (limited to 'store/sql_user_store_test.go') diff --git a/store/sql_user_store_test.go b/store/sql_user_store_test.go index 076be1a81..7ffb68a47 100644 --- a/store/sql_user_store_test.go +++ b/store/sql_user_store_test.go @@ -205,7 +205,7 @@ func TestUserStoreGetAllProfiles(t *testing.T) { Must(store.User().Save(u2)) Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id})) - if r1 := <-store.User().GetAllProfiles(); r1.Err != nil { + if r1 := <-store.User().GetAllProfiles(0, 100); r1.Err != nil { t.Fatal(r1.Err) } else { users := r1.Data.(map[string]*model.User) @@ -213,6 +213,15 @@ func TestUserStoreGetAllProfiles(t *testing.T) { t.Fatal("invalid returned users") } } + + if r2 := <-store.User().GetAllProfiles(0, 1); r2.Err != nil { + t.Fatal(r2.Err) + } else { + users := r2.Data.(map[string]*model.User) + if len(users) != 1 { + t.Fatal("invalid returned users, limit did not work") + } + } } func TestUserStoreGetProfiles(t *testing.T) { @@ -230,7 +239,7 @@ func TestUserStoreGetProfiles(t *testing.T) { Must(store.User().Save(u2)) Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id})) - if r1 := <-store.User().GetProfiles(teamId); r1.Err != nil { + if r1 := <-store.User().GetProfiles(teamId, 0, 100); r1.Err != nil { t.Fatal(r1.Err) } else { users := r1.Data.(map[string]*model.User) @@ -243,7 +252,7 @@ func TestUserStoreGetProfiles(t *testing.T) { } } - if r2 := <-store.User().GetProfiles("123"); r2.Err != nil { + if r2 := <-store.User().GetProfiles("123", 0, 100); r2.Err != nil { t.Fatal(r2.Err) } else { if len(r2.Data.(map[string]*model.User)) != 0 { @@ -252,7 +261,7 @@ func TestUserStoreGetProfiles(t *testing.T) { } } -func TestUserStoreGetDirectProfiles(t *testing.T) { +func TestUserStoreGetProfilesInChannel(t *testing.T) { Setup() teamId := model.NewId() @@ -267,22 +276,166 @@ func TestUserStoreGetDirectProfiles(t *testing.T) { Must(store.User().Save(u2)) Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id})) - if r1 := <-store.User().GetDirectProfiles(u1.Id); r1.Err != nil { + c1 := model.Channel{} + c1.TeamId = teamId + c1.DisplayName = "Profiles in channel" + c1.Name = "profiles-" + model.NewId() + c1.Type = model.CHANNEL_OPEN + + c2 := model.Channel{} + c2.TeamId = teamId + c2.DisplayName = "Profiles in private" + c2.Name = "profiles-" + model.NewId() + c2.Type = model.CHANNEL_PRIVATE + + Must(store.Channel().Save(&c1)) + Must(store.Channel().Save(&c2)) + + m1 := model.ChannelMember{} + m1.ChannelId = c1.Id + m1.UserId = u1.Id + m1.NotifyProps = model.GetDefaultChannelNotifyProps() + + m2 := model.ChannelMember{} + m2.ChannelId = c1.Id + m2.UserId = u2.Id + m2.NotifyProps = model.GetDefaultChannelNotifyProps() + + m3 := model.ChannelMember{} + m3.ChannelId = c2.Id + m3.UserId = u1.Id + m3.NotifyProps = model.GetDefaultChannelNotifyProps() + + Must(store.Channel().SaveMember(&m1)) + Must(store.Channel().SaveMember(&m2)) + Must(store.Channel().SaveMember(&m3)) + + if r1 := <-store.User().GetProfilesInChannel(c1.Id, -1, -1, false); r1.Err != nil { t.Fatal(r1.Err) } else { users := r1.Data.(map[string]*model.User) - if len(users) != 0 { + if len(users) != 2 { t.Fatal("invalid returned users") } + + if users[u1.Id].Id != u1.Id { + t.Fatal("invalid returned user") + } } - if r2 := <-store.User().GetDirectProfiles("123"); r2.Err != nil { + if r2 := <-store.User().GetProfilesInChannel(c2.Id, -1, -1, false); r2.Err != nil { t.Fatal(r2.Err) } else { - if len(r2.Data.(map[string]*model.User)) != 0 { + if len(r2.Data.(map[string]*model.User)) != 1 { + t.Fatal("should have returned empty map") + } + } + + if r2 := <-store.User().GetProfilesInChannel(c2.Id, -1, -1, true); r2.Err != nil { + t.Fatal(r2.Err) + } else { + if len(r2.Data.(map[string]*model.User)) != 1 { + t.Fatal("should have returned empty map") + } + } + + if r2 := <-store.User().GetProfilesInChannel(c2.Id, -1, -1, true); r2.Err != nil { + t.Fatal(r2.Err) + } else { + if len(r2.Data.(map[string]*model.User)) != 1 { t.Fatal("should have returned empty map") } } + + store.User().InvalidateProfilesInChannelCache(c2.Id) +} + +func TestUserStoreGetProfilesNotInChannel(t *testing.T) { + Setup() + + teamId := model.NewId() + + u1 := &model.User{} + u1.Email = model.NewId() + Must(store.User().Save(u1)) + Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id})) + + u2 := &model.User{} + u2.Email = model.NewId() + Must(store.User().Save(u2)) + Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id})) + + c1 := model.Channel{} + c1.TeamId = teamId + c1.DisplayName = "Profiles in channel" + c1.Name = "profiles-" + model.NewId() + c1.Type = model.CHANNEL_OPEN + + c2 := model.Channel{} + c2.TeamId = teamId + c2.DisplayName = "Profiles in private" + c2.Name = "profiles-" + model.NewId() + c2.Type = model.CHANNEL_PRIVATE + + Must(store.Channel().Save(&c1)) + Must(store.Channel().Save(&c2)) + + if r1 := <-store.User().GetProfilesNotInChannel(teamId, c1.Id, 0, 100); r1.Err != nil { + t.Fatal(r1.Err) + } else { + users := r1.Data.(map[string]*model.User) + if len(users) != 2 { + t.Fatal("invalid returned users") + } + + if users[u1.Id].Id != u1.Id { + t.Fatal("invalid returned user") + } + } + + if r2 := <-store.User().GetProfilesNotInChannel(teamId, c2.Id, 0, 100); r2.Err != nil { + t.Fatal(r2.Err) + } else { + if len(r2.Data.(map[string]*model.User)) != 2 { + t.Fatal("invalid returned users") + } + } + + m1 := model.ChannelMember{} + m1.ChannelId = c1.Id + m1.UserId = u1.Id + m1.NotifyProps = model.GetDefaultChannelNotifyProps() + + m2 := model.ChannelMember{} + m2.ChannelId = c1.Id + m2.UserId = u2.Id + m2.NotifyProps = model.GetDefaultChannelNotifyProps() + + m3 := model.ChannelMember{} + m3.ChannelId = c2.Id + m3.UserId = u1.Id + m3.NotifyProps = model.GetDefaultChannelNotifyProps() + + Must(store.Channel().SaveMember(&m1)) + Must(store.Channel().SaveMember(&m2)) + Must(store.Channel().SaveMember(&m3)) + + if r1 := <-store.User().GetProfilesNotInChannel(teamId, c1.Id, 0, 100); r1.Err != nil { + t.Fatal(r1.Err) + } else { + users := r1.Data.(map[string]*model.User) + if len(users) != 0 { + t.Fatal("invalid returned users") + } + } + + if r2 := <-store.User().GetProfilesNotInChannel(teamId, c2.Id, 0, 100); r2.Err != nil { + t.Fatal(r2.Err) + } else { + if len(r2.Data.(map[string]*model.User)) != 1 { + t.Fatal("should have had 1 user not in channel") + } + } } func TestUserStoreGetProfilesByIds(t *testing.T) { @@ -326,7 +479,7 @@ func TestUserStoreGetProfilesByIds(t *testing.T) { } } - if r2 := <-store.User().GetProfiles("123"); r2.Err != nil { + if r2 := <-store.User().GetProfiles("123", 0, 100); r2.Err != nil { t.Fatal(r2.Err) } else { if len(r2.Data.(map[string]*model.User)) != 0 { @@ -335,6 +488,50 @@ func TestUserStoreGetProfilesByIds(t *testing.T) { } } +func TestUserStoreGetProfilesByUsernames(t *testing.T) { + Setup() + + teamId := model.NewId() + + u1 := &model.User{} + u1.Email = model.NewId() + u1.Username = "username1" + model.NewId() + Must(store.User().Save(u1)) + Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id})) + + u2 := &model.User{} + u2.Email = model.NewId() + u2.Username = "username2" + model.NewId() + Must(store.User().Save(u2)) + Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id})) + + if r1 := <-store.User().GetProfilesByUsernames([]string{u1.Username, u2.Username}, teamId); r1.Err != nil { + t.Fatal(r1.Err) + } else { + users := r1.Data.(map[string]*model.User) + if len(users) != 2 { + t.Fatal("invalid returned users") + } + + if users[u1.Id].Id != u1.Id { + t.Fatal("invalid returned user") + } + } + + if r1 := <-store.User().GetProfilesByUsernames([]string{u1.Username}, teamId); r1.Err != nil { + t.Fatal(r1.Err) + } else { + users := r1.Data.(map[string]*model.User) + if len(users) != 1 { + t.Fatal("invalid returned users") + } + + if users[u1.Id].Id != u1.Id { + t.Fatal("invalid returned user") + } + } +} + func TestUserStoreGetSystemAdminProfiles(t *testing.T) { Setup() @@ -713,3 +910,216 @@ func TestUserStoreUpdateMfaActive(t *testing.T) { t.Fatal(err) } } + +func TestUserStoreGetRecentlyActiveUsersForTeam(t *testing.T) { + Setup() + + u1 := &model.User{} + u1.Email = model.NewId() + Must(store.User().Save(u1)) + Must(store.Status().SaveOrUpdate(&model.Status{u1.Id, model.STATUS_ONLINE, false, model.GetMillis(), ""})) + tid := model.NewId() + Must(store.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id})) + + if r1 := <-store.User().GetRecentlyActiveUsersForTeam(tid); r1.Err != nil { + t.Fatal(r1.Err) + } +} + +func TestUserStoreSearch(t *testing.T) { + Setup() + + u1 := &model.User{} + u1.Username = "jimbo" + model.NewId() + u1.FirstName = "Tim" + u1.LastName = "Bill" + u1.Nickname = "Rob" + u1.Email = model.NewId() + Must(store.User().Save(u1)) + + tid := model.NewId() + Must(store.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id})) + + if r1 := <-store.User().Search(tid, "jimb", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } + + if r1 := <-store.User().Search("", "jimb", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } + + if r1 := <-store.User().Search(tid, "", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } + + c1 := model.Channel{} + c1.TeamId = tid + c1.DisplayName = "NameName" + c1.Name = "a" + model.NewId() + "b" + c1.Type = model.CHANNEL_OPEN + c1 = *Must(store.Channel().Save(&c1)).(*model.Channel) + + if r1 := <-store.User().SearchNotInChannel(tid, c1.Id, "jimb", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } + + if r1 := <-store.User().SearchNotInChannel("", c1.Id, "jimb", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } + + if r1 := <-store.User().SearchNotInChannel("junk", c1.Id, "jimb", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if found { + t.Fatal("should not have found user") + } + } + + if r1 := <-store.User().SearchInChannel(c1.Id, "jimb", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if found { + t.Fatal("should not have found user") + } + } + + Must(store.Channel().SaveMember(&model.ChannelMember{ChannelId: c1.Id, UserId: u1.Id, NotifyProps: model.GetDefaultChannelNotifyProps()})) + + if r1 := <-store.User().SearchInChannel(c1.Id, "jimb", USER_SEARCH_TYPE_USERNAME); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } + + if r1 := <-store.User().Search(tid, "Tim", USER_SEARCH_TYPE_ALL); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } + + if r1 := <-store.User().Search(tid, "Bill", USER_SEARCH_TYPE_ALL); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } + + if r1 := <-store.User().Search(tid, "Rob", USER_SEARCH_TYPE_ALL); r1.Err != nil { + t.Fatal(r1.Err) + } else { + profiles := r1.Data.([]*model.User) + found := false + for _, profile := range profiles { + if profile.Id == u1.Id { + found = true + break + } + } + + if !found { + t.Fatal("should have found user") + } + } +} -- cgit v1.2.3-1-g7c22