PLT-4357 adding performance monitoring (#4622)

* WIP

* WIP

* Adding metrics collection

* updating vendor packages

* Adding metrics to config

* Adding admin console page for perf monitoring

* Updating glide

* switching to tylerb/graceful

7 files changed, 62 insertions, 21 deletions

diff --git a/api/admin.go b/api/admin.go
index 0edfb246b..16ec98fcf 100644
--- a/api/admin.go
+++ b/api/admin.go
@@ -176,6 +176,14 @@ func saveConfig(c *Context, w http.ResponseWriter, r *http.Request) {
 	utils.SaveConfig(utils.CfgFileName, cfg)
 	utils.LoadConfig(utils.CfgFileName)
 
+	if einterfaces.GetMetricsInterface() != nil {
+		if *utils.Cfg.MetricsSettings.Enable {
+			einterfaces.GetMetricsInterface().StartServer()
+		} else {
+			einterfaces.GetMetricsInterface().StopServer()
+		}
+	}
+
 	// Future feature is to sync the configuration files
 	// if einterfaces.GetClusterInterface() != nil {
 	// 	err := einterfaces.GetClusterInterface().ConfigChanged(cfg, oldCfg, true)
diff --git a/api/apitestlib.go b/api/apitestlib.go
index 37367b71d..9345d3fc4 100644
--- a/api/apitestlib.go
+++ b/api/apitestlib.go
@@ -36,7 +36,7 @@ func SetupEnterprise() *TestHelper {
 		*utils.Cfg.RateLimitSettings.Enable = false
 		utils.DisableDebugLogForTest()
 		utils.License.Features.SetDefaults()
-		NewServer(false)
+		NewServer()
 		StartServer()
 		utils.InitHTML()
 		InitApi()
@@ -57,7 +57,7 @@ func Setup() *TestHelper {
 		utils.Cfg.TeamSettings.MaxUsersPerTeam = 50
 		*utils.Cfg.RateLimitSettings.Enable = false
 		utils.DisableDebugLogForTest()
-		NewServer(false)
+		NewServer()
 		StartServer()
 		InitApi()
 		utils.EnableDebugLogForTest()
diff --git a/api/context.go b/api/context.go
index 7466d0b05..3a867624b 100644
--- a/api/context.go
+++ b/api/context.go
@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	l4g "github.com/alecthomas/log4go"
 	"github.com/gorilla/mux"
@@ -103,6 +104,7 @@ type handler struct {
 }
 
 func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	now := time.Now()
 	l4g.Debug("%v", r.URL.Path)
 
 	c := &Context{}
@@ -228,6 +230,10 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		if h.isApi {
 			w.WriteHeader(c.Err.StatusCode)
 			w.Write([]byte(c.Err.ToJson()))
+
+			if einterfaces.GetMetricsInterface() != nil {
+				einterfaces.GetMetricsInterface().IncrementHttpError()
+			}
 		} else {
 			if c.Err.StatusCode == http.StatusUnauthorized {
 				http.Redirect(w, r, c.GetTeamURL()+"/?redirect="+url.QueryEscape(r.URL.Path), http.StatusTemporaryRedirect)
@@ -235,6 +241,16 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				RenderWebError(c.Err, w, r)
 			}
 		}
+
+	}
+
+	if h.isApi && einterfaces.GetMetricsInterface() != nil {
+		einterfaces.GetMetricsInterface().IncrementHttpRequest()
+
+		if r.URL.Path != model.API_URL_SUFFIX+"/users/websocket" {
+			elapsed := float64(time.Since(now)) / float64(time.Second)
+			einterfaces.GetMetricsInterface().ObserveHttpRequestDuration(elapsed)
+		}
 	}
 }
 
diff --git a/api/post.go b/api/post.go
index c982b65ca..ad6c2fdbe 100644
--- a/api/post.go
+++ b/api/post.go
@@ -21,6 +21,7 @@ import (
 
 	l4g "github.com/alecthomas/log4go"
 	"github.com/gorilla/mux"
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/store"
 	"github.com/mattermost/platform/utils"
@@ -146,6 +147,10 @@ func CreatePost(c *Context, post *model.Post, triggerWebhooks bool) (*model.Post
 		rpost = result.Data.(*model.Post)
 	}
 
+	if einterfaces.GetMetricsInterface() != nil {
+		einterfaces.GetMetricsInterface().IncrementPostCreate()
+	}
+
 	if len(post.FileIds) > 0 {
 		// There's a rare bug where the client sends up duplicate FileIds so protect against that
 		post.FileIds = utils.RemoveDuplicatesFromStringArray(post.FileIds)
@@ -155,6 +160,10 @@ func CreatePost(c *Context, post *model.Post, triggerWebhooks bool) (*model.Post
 				l4g.Error(utils.T("api.post.create_post.attach_files.error"), post.Id, post.FileIds, c.Session.UserId, result.Err)
 			}
 		}
+
+		if einterfaces.GetMetricsInterface() != nil {
+			einterfaces.GetMetricsInterface().IncrementPostFileAttachment(len(post.FileIds))
+		}
 	}
 
 	handlePostEvents(c, rpost, triggerWebhooks)
@@ -869,6 +878,10 @@ func sendNotificationEmail(c *Context, post *model.Post, user *model.User, chann
 	if err := utils.SendMail(user.Email, html.UnescapeString(subject), bodyPage.Render()); err != nil {
 		l4g.Error(utils.T("api.post.send_notifications_and_forget.send.error"), user.Email, err)
 	}
+
+	if einterfaces.GetMetricsInterface() != nil {
+		einterfaces.GetMetricsInterface().IncrementPostSentEmail()
+	}
 }
 
 func getMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string {
@@ -959,6 +972,9 @@ func sendPushNotification(post *model.Post, user *model.User, channel *model.Cha
 		tmpMessage := *model.PushNotificationFromJson(strings.NewReader(msg.ToJson()))
 		tmpMessage.SetDeviceIdAndPlatform(session.DeviceId)
 		sendToPushProxy(tmpMessage)
+		if einterfaces.GetMetricsInterface() != nil {
+			einterfaces.GetMetricsInterface().IncrementPostSentPush()
+		}
 	}
 }
 
diff --git a/api/server.go b/api/server.go
index fee74e373..f5a374507 100644
--- a/api/server.go
+++ b/api/server.go
@@ -7,7 +7,6 @@ import (
 	"crypto/tls"
 	"net"
 	"net/http"
-	"net/http/pprof"
 	"strings"
 	"time"
 
@@ -37,20 +36,7 @@ const TIME_TO_WAIT_FOR_CONNECTIONS_TO_CLOSE_ON_SERVER_SHUTDOWN = time.Second
 
 var Srv *Server
 
-func AttachProfiler(router *mux.Router) {
-	router.HandleFunc("/debug/pprof/", pprof.Index)
-	router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
-	router.HandleFunc("/debug/pprof/profile", pprof.Profile)
-	router.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-
-	// Manually add support for paths linked to by index page at /debug/pprof/
-	router.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
-	router.Handle("/debug/pprof/heap", pprof.Handler("heap"))
-	router.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
-	router.Handle("/debug/pprof/block", pprof.Handler("block"))
-}
-
-func NewServer(enableProfiler bool) {
+func NewServer() {
 
 	l4g.Info(utils.T("api.server.new_server.init.info"))
 
@@ -58,10 +44,6 @@ func NewServer(enableProfiler bool) {
 	Srv.Store = store.NewSqlStore()
 
 	Srv.Router = mux.NewRouter()
-	if enableProfiler {
-		AttachProfiler(Srv.Router)
-		l4g.Info("Enabled HTTP Profiler")
-	}
 	Srv.Router.NotFoundHandler = http.HandlerFunc(Handle404)
 }
 
diff --git a/api/user.go b/api/user.go
index 2507bd740..3a303bee4 100644
--- a/api/user.go
+++ b/api/user.go
@@ -469,6 +469,9 @@ func login(c *Context, w http.ResponseWriter, r *http.Request) {
 			c.LogAuditWithUserId(user.Id, "failure")
 			c.Err = result.Err
 			c.Err.StatusCode = http.StatusBadRequest
+			if einterfaces.GetMetricsInterface() != nil {
+				einterfaces.GetMetricsInterface().IncrementLoginFail()
+			}
 			return
 		} else {
 			user = result.Data.(*model.User)
@@ -479,6 +482,9 @@ func login(c *Context, w http.ResponseWriter, r *http.Request) {
 		if user, err = getUserForLogin(loginId, ldapOnly); err != nil {
 			c.LogAudit("failure")
 			c.Err = err
+			if einterfaces.GetMetricsInterface() != nil {
+				einterfaces.GetMetricsInterface().IncrementLoginFail()
+			}
 			return
 		}
 
@@ -489,10 +495,16 @@ func login(c *Context, w http.ResponseWriter, r *http.Request) {
 	if user, err = authenticateUser(user, password, mfaToken); err != nil {
 		c.LogAuditWithUserId(user.Id, "failure")
 		c.Err = err
+		if einterfaces.GetMetricsInterface() != nil {
+			einterfaces.GetMetricsInterface().IncrementLoginFail()
+		}
 		return
 	}
 
 	c.LogAuditWithUserId(user.Id, "success")
+	if einterfaces.GetMetricsInterface() != nil {
+		einterfaces.GetMetricsInterface().IncrementLogin()
+	}
 
 	doLogin(c, w, r, user, deviceId)
 	if c.Err != nil {
diff --git a/api/web_conn.go b/api/web_conn.go
index ae2a274d4..f145950b3 100644
--- a/api/web_conn.go
+++ b/api/web_conn.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
 
@@ -111,6 +112,12 @@ func (c *WebConn) writePump() {
 				return
 			}
 
+			if msg.EventType() == model.WEBSOCKET_EVENT_POSTED {
+				if einterfaces.GetMetricsInterface() != nil {
+					einterfaces.GetMetricsInterface().IncrementPostBroadcast()
+				}
+			}
+
 		case <-ticker.C:
 			c.WebSocket.SetWriteDeadline(time.Now().Add(WRITE_WAIT))
 			if err := c.WebSocket.WriteMessage(websocket.PingMessage, []byte{}); err != nil {