Merging performance branch into master (#4268)

* improve performance on sendNotifications

* Fix SQL queries

* Remove get direct profiles, not needed anymore

* Add raw data to error details if AppError fails to decode

* men

* Fix decode (#4052)

* Fixing json decode

* Adding unit test

* Initial work for client scaling (#4051)

* Begin adding paging to profiles API

* Added more paging functionality

* Finish hooking up admin console user lists

* Add API for searching users and add searching to all user lists

* Add lazy loading of profiles

* Revert config.json

* Fix unit tests and some style issues

* Add GetProfilesFromList to Go driver and fix web unit test

* Update etag for GetProfiles

* Updating ui for filters and pagination (#4044)

* Updating UI for pagination

* Adjusting margins for filter row

* Adjusting margin for specific modals

* Adding relative padding to system console

* Adjusting responsive view

* Update client user tests

* Minor fixes for direct messages modal (#4056)

* Remove some unneeded initial load calls (#4057)

* UX updates to user lists, added smart counts and bug fixes (#4059)

* Improved getExplicitMentions and unit tests (#4064)

* Refactor getting posts to lazy load profiles correctly (#4062)

* Comment out SetActiveChannel test (#4066)

* Profiler cpu, block, and memory profiler. (#4081)

* Fix TestSetActiveChannel unit test (#4071)

* Fixing build failure caused by dependancies updating (#4076)

* Adding profiler

* Fix admin_team_member_dropdown eslint errors

* Bumping session cache size (#4077)

* Bumping session cache size

* Bumping status cache

* Refactor how the client handles channel members to be large team friendly (#4106)

* Refactor how the client handles channel members to be large team friendly

* Change Id to ChannelId in ChannelStats model

* Updated getChannelMember and getProfilesByIds routes to match proposal

* Performance improvements (#4100)

* Performance improvements

* Fixing re-connect issue

* Fixing error message

* Some other minor perf tweaks

* Some other minor perf tweaks

* Fixing config file

* Fixing buffer size

* Fixing web socket send message

* adding some error logging

* fix getMe to be user required

* Fix websocket event for new user

* Fixing shutting down

* Reverting web socket changes

* Fixing logging lvl

* Adding caching to GetMember

* Adding some logging

* Fixing caching

* Fixing caching invalidate

* Fixing direct message caching

* Fixing caching

* Fixing caching

* Remove GetDirectProfiles from initial load

* Adding logging and fixing websocket client

* Adding back caching from bad merge.

* Explicitly close go driver requests (#4162)

* Refactored how the client handles team members to be more large team friendly (#4159)

* Refactor getProfilesForDirectMessageList API into getAllProfiles API

* Refactored how the client handles team members to be more large team friendly

* Fix js error when receiving a notification

* Fix JS error caused by current user being overwritten with sanitized version (#4165)

* Adding error message to status failure (#4167)

* Fix a few bugs caused by client scaling refactoring (#4170)

* When there is no read replica, don't open a second set of connections to the master database (#4173)

* Adding connection tacking to stats (#4174)

* Reduce DB writes for statuses and other status related changes (#4175)

* Fix bug preventing opening of DM channels from more modal (#4181)

* 	Fixing socket timing error (#4183)

* Fixing ping/pong handler

* Fixing socket timing error

* Commenting out status broadcasting

* Removing user status changes

* Removing user status changes

* Removing user status changes

* Removing user status changes

* Adding DoPreComputeJson()

* Performance improvements (#4194)

* * Fix System Console Analytics queries
* Add db.SetConnMaxLifetime to 15 minutes
* Add "net/http/pprof" for profiling
* Add FreeOSMemory() to manually release memory on reload config

* Add flag to enable http profiler

* Fix memory leak (#4197)

* Fix memory leak

* removed unneeded nil assignment

* Fixing go routine leak (#4208)

* Merge fixes

* Merge fix

* Refactored statuses to be queried by the client rather than broadcast by the server (#4212)

* Refactored server code to reduce status broadcasts and to allow getting statuses by IDs

* Refactor client code to periodically fetch statuses

* Add store unit test for getting statuses by ids

* Fix status unit test

* Add getStatusesByIds REST API and move the client over to use that instead of the WebSocket

* Adding multiple threads to websocket hub (#4230)

* Adding multiple threads to websocket hub

* Fixing unit tests

* Fixing so websocket connections from the same user end up in the same… (#4240)

* Fixing so websocket connections from the same user end up in the same list

* Removing old comment

* Refactor user autocomplete to query the server (#4239)

* Add API for autocompleting users

* Converted at mention autocomplete to query server

* Converted user search autocomplete to query server

* Switch autocomplete API naming to use term instead of username

* Split autocomplete API into two, one for channels and for teams

* Fix copy/paste error

* Some final client scaling fixes (#4246)

* Add lazy loading of profiles to integration pages

* Add lazy loading of profiles to emoji page

* Fix JS error when receiving post in select team menu and also clean up channel store

1 files changed, 148 insertions, 16 deletions

diff --git a/model/client.go b/model/client.go
index f9a56b86e..f5aeea4db 100644
--- a/model/client.go
+++ b/model/client.go
@@ -131,6 +131,7 @@ func (c *Client) GetFileRoute(fileId string) string {
 func (c *Client) DoPost(url, data, contentType string) (*http.Response, *AppError) {
 	rq, _ := http.NewRequest("POST", c.Url+url, strings.NewReader(data))
 	rq.Header.Set("Content-Type", contentType)
+	rq.Close = true
 
 	if rp, err := c.HttpClient.Do(rq); err != nil {
 		return nil, NewLocAppError(url, "model.client.connecting.app_error", nil, err.Error())
@@ -144,6 +145,7 @@ func (c *Client) DoPost(url, data, contentType string) (*http.Response, *AppErro
 
 func (c *Client) DoApiPost(url string, data string) (*http.Response, *AppError) {
 	rq, _ := http.NewRequest("POST", c.ApiUrl+url, strings.NewReader(data))
+	rq.Close = true
 
 	if len(c.AuthToken) > 0 {
 		rq.Header.Set(HEADER_AUTH, c.AuthType+" "+c.AuthToken)
@@ -161,6 +163,7 @@ func (c *Client) DoApiPost(url string, data string) (*http.Response, *AppError)
 
 func (c *Client) DoApiGet(url string, data string, etag string) (*http.Response, *AppError) {
 	rq, _ := http.NewRequest("GET", c.ApiUrl+url, strings.NewReader(data))
+	rq.Close = true
 
 	if len(etag) > 0 {
 		rq.Header.Set(HEADER_ETAG_CLIENT, etag)
@@ -508,10 +511,9 @@ func (c *Client) GetMe(etag string) (*Result, *AppError) {
 	}
 }
 
-// GetProfilesForDirectMessageList returns a map of users for a team that can be direct
-// messaged, using user id as the key. Must be authenticated.
-func (c *Client) GetProfilesForDirectMessageList(teamId string) (*Result, *AppError) {
-	if r, err := c.DoApiGet("/users/profiles_for_dm_list/"+teamId, "", ""); err != nil {
+// GetProfiles returns a map of users using user id as the key. Must be authenticated.
+func (c *Client) GetProfiles(offset int, limit int, etag string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(fmt.Sprintf("/users/%v/%v", offset, limit), "", etag); err != nil {
 		return nil, err
 	} else {
 		defer closeBody(r)
@@ -520,10 +522,10 @@ func (c *Client) GetProfilesForDirectMessageList(teamId string) (*Result, *AppEr
 	}
 }
 
-// GetProfiles returns a map of users for a team using user id as the key. Must
+// GetProfilesInTeam returns a map of users for a team using user id as the key. Must
 // be authenticated.
-func (c *Client) GetProfiles(teamId string, etag string) (*Result, *AppError) {
-	if r, err := c.DoApiGet("/users/profiles/"+teamId, "", etag); err != nil {
+func (c *Client) GetProfilesInTeam(teamId string, offset int, limit int, etag string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(fmt.Sprintf("/teams/%v/users/%v/%v", teamId, offset, limit), "", etag); err != nil {
 		return nil, err
 	} else {
 		defer closeBody(r)
@@ -532,10 +534,22 @@ func (c *Client) GetProfiles(teamId string, etag string) (*Result, *AppError) {
 	}
 }
 
-// GetDirectProfiles gets a map of users that are currently shown in the sidebar,
-// using user id as the key. Must be authenticated.
-func (c *Client) GetDirectProfiles(etag string) (*Result, *AppError) {
-	if r, err := c.DoApiGet("/users/direct_profiles", "", etag); err != nil {
+// GetProfilesInChannel returns a map of users for a channel using user id as the key. Must
+// be authenticated.
+func (c *Client) GetProfilesInChannel(channelId string, offset int, limit int, etag string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(fmt.Sprintf(c.GetChannelRoute(channelId)+"/users/%v/%v", offset, limit), "", etag); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), UserMapFromJson(r.Body)}, nil
+	}
+}
+
+// GetProfilesNotInChannel returns a map of users not in a channel but on the team using user id as the key. Must
+// be authenticated.
+func (c *Client) GetProfilesNotInChannel(channelId string, offset int, limit int, etag string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(fmt.Sprintf(c.GetChannelRoute(channelId)+"/users/not_in_channel/%v/%v", offset, limit), "", etag); err != nil {
 		return nil, err
 	} else {
 		defer closeBody(r)
@@ -544,6 +558,60 @@ func (c *Client) GetDirectProfiles(etag string) (*Result, *AppError) {
 	}
 }
 
+// GetProfilesByIds returns a map of users based on the user ids provided. Must
+// be authenticated.
+func (c *Client) GetProfilesByIds(userIds []string) (*Result, *AppError) {
+	if r, err := c.DoApiPost("/users/ids", ArrayToJson(userIds)); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), UserMapFromJson(r.Body)}, nil
+	}
+}
+
+// SearchUsers returns a list of users that have a username matching or similar to the search term. Must
+// be authenticated.
+func (c *Client) SearchUsers(term string, teamId string, options map[string]string) (*Result, *AppError) {
+	options["term"] = term
+	options["team_id"] = teamId
+	if r, err := c.DoApiPost("/users/search", MapToJson(options)); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), UserListFromJson(r.Body)}, nil
+	}
+}
+
+// AutocompleteUsersInChannel returns two lists for autocompletion of users in a channel. The first list "in_channel",
+// specifies users in the channel. The second list "out_of_channel" specifies users outside of the
+// channel. Term, the string to search against, is required, channel id is also required. Must be authenticated.
+func (c *Client) AutocompleteUsersInChannel(term string, channelId string) (*Result, *AppError) {
+	url := fmt.Sprintf("%s/users/autocomplete?term=%s", c.GetChannelRoute(channelId), url.QueryEscape(term))
+	if r, err := c.DoApiGet(url, "", ""); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), UserAutocompleteInChannelFromJson(r.Body)}, nil
+	}
+}
+
+// AutocompleteUsersInTeam returns a list for autocompletion of users in a team. The list "in_team" specifies
+// the users in the team that match the provided term, matching against username, full name and
+// nickname. Must be authenticated.
+func (c *Client) AutocompleteUsersInTeam(term string) (*Result, *AppError) {
+	url := fmt.Sprintf("%s/users/autocomplete?term=%s", c.GetTeamRoute(), url.QueryEscape(term))
+	if r, err := c.DoApiGet(url, "", ""); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), UserAutocompleteInTeamFromJson(r.Body)}, nil
+	}
+}
+
 // LoginById authenticates a user by user id and password.
 func (c *Client) LoginById(id string, password string) (*Result, *AppError) {
 	m := make(map[string]string)
@@ -942,6 +1010,7 @@ func (c *Client) SaveComplianceReport(job *Compliance) (*Result, *AppError) {
 func (c *Client) DownloadComplianceReport(id string) (*Result, *AppError) {
 	var rq *http.Request
 	rq, _ = http.NewRequest("GET", c.ApiUrl+"/admin/download_compliance_report/"+id, nil)
+	rq.Close = true
 
 	if len(c.AuthToken) > 0 {
 		rq.Header.Set(HEADER_AUTH, "BEARER "+c.AuthToken)
@@ -1174,13 +1243,23 @@ func (c *Client) UpdateLastViewedAt(channelId string, active bool) (*Result, *Ap
 	}
 }
 
-func (c *Client) GetChannelExtraInfo(id string, memberLimit int, etag string) (*Result, *AppError) {
-	if r, err := c.DoApiGet(c.GetChannelRoute(id)+"/extra_info/"+strconv.FormatInt(int64(memberLimit), 10), "", etag); err != nil {
+func (c *Client) GetChannelStats(id string, etag string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(c.GetChannelRoute(id)+"/stats", "", etag); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), ChannelStatsFromJson(r.Body)}, nil
+	}
+}
+
+func (c *Client) GetChannelMember(channelId string, userId string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(c.GetChannelRoute(channelId)+"/members/"+userId, "", ""); err != nil {
 		return nil, err
 	} else {
 		defer closeBody(r)
 		return &Result{r.Header.Get(HEADER_REQUEST_ID),
-			r.Header.Get(HEADER_ETAG_SERVER), ChannelExtraFromJson(r.Body)}, nil
+			r.Header.Get(HEADER_ETAG_SERVER), ChannelMemberFromJson(r.Body)}, nil
 	}
 }
 
@@ -1325,6 +1404,7 @@ func (c *Client) UploadPostAttachment(data []byte, channelId string, filename st
 func (c *Client) uploadFile(url string, data []byte, contentType string) (*Result, *AppError) {
 	rq, _ := http.NewRequest("POST", url, bytes.NewReader(data))
 	rq.Header.Set("Content-Type", contentType)
+	rq.Close = true
 
 	if len(c.AuthToken) > 0 {
 		rq.Header.Set(HEADER_AUTH, "BEARER "+c.AuthToken)
@@ -1525,6 +1605,18 @@ func (c *Client) GetStatuses() (*Result, *AppError) {
 	}
 }
 
+// GetStatusesByIds returns a map of string statuses using user id as the key,
+// based on the provided user ids
+func (c *Client) GetStatusesByIds(userIds []string) (*Result, *AppError) {
+	if r, err := c.DoApiPost("/users/status/ids", ArrayToJson(userIds)); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), MapFromJson(r.Body)}, nil
+	}
+}
+
 // SetActiveChannel sets the the channel id the user is currently viewing.
 // The channelId key is required but the value can be blank. Returns standard
 // response.
@@ -1550,8 +1642,46 @@ func (c *Client) GetMyTeam(etag string) (*Result, *AppError) {
 	}
 }
 
-func (c *Client) GetTeamMembers(teamId string) (*Result, *AppError) {
-	if r, err := c.DoApiGet("/teams/members/"+teamId, "", ""); err != nil {
+// GetTeamMembers will return a page of team member objects as an array paged based on the
+// team id, offset and limit provided. Must be authenticated.
+func (c *Client) GetTeamMembers(teamId string, offset int, limit int) (*Result, *AppError) {
+	if r, err := c.DoApiGet(fmt.Sprintf("/teams/%v/members/%v/%v", teamId, offset, limit), "", ""); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), TeamMembersFromJson(r.Body)}, nil
+	}
+}
+
+// GetTeamMember will return a team member object based on the team id and user id provided.
+// Must be authenticated.
+func (c *Client) GetTeamMember(teamId string, userId string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(fmt.Sprintf("/teams/%v/members/%v", teamId, userId), "", ""); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), TeamMemberFromJson(r.Body)}, nil
+	}
+}
+
+// GetTeamStats will return a team stats object containing the number of users on the team
+// based on the team id provided. Must be authenticated.
+func (c *Client) GetTeamStats(teamId string) (*Result, *AppError) {
+	if r, err := c.DoApiGet(fmt.Sprintf("/teams/%v/stats", teamId), "", ""); err != nil {
+		return nil, err
+	} else {
+		defer closeBody(r)
+		return &Result{r.Header.Get(HEADER_REQUEST_ID),
+			r.Header.Get(HEADER_ETAG_SERVER), TeamStatsFromJson(r.Body)}, nil
+	}
+}
+
+// GetTeamMembersByIds will return team member objects as an array based on the
+// team id and a list of user ids provided. Must be authenticated.
+func (c *Client) GetTeamMembersByIds(teamId string, userIds []string) (*Result, *AppError) {
+	if r, err := c.DoApiPost(fmt.Sprintf("/teams/%v/members/ids", teamId), ArrayToJson(userIds)); err != nil {
 		return nil, err
 	} else {
 		defer closeBody(r)
@@ -1866,6 +1996,7 @@ func (c *Client) CreateEmoji(emoji *Emoji, image []byte, filename string) (*Emoj
 
 	rq, _ := http.NewRequest("POST", c.ApiUrl+c.GetEmojiRoute()+"/create", body)
 	rq.Header.Set("Content-Type", writer.FormDataContentType())
+	rq.Close = true
 
 	if len(c.AuthToken) > 0 {
 		rq.Header.Set(HEADER_AUTH, "BEARER "+c.AuthToken)
@@ -1908,6 +2039,7 @@ func (c *Client) UploadCertificateFile(data []byte, contentType string) *AppErro
 	url := c.ApiUrl + "/admin/add_certificate"
 	rq, _ := http.NewRequest("POST", url, bytes.NewReader(data))
 	rq.Header.Set("Content-Type", contentType)
+	rq.Close = true
 
 	if len(c.AuthToken) > 0 {
 		rq.Header.Set(HEADER_AUTH, "BEARER "+c.AuthToken)