[PLT-5465/APIV4] GET /system/health - Improve ping health check to have limits (#6331)

* implement PLT-5465 - Improve ping health check to have limits

* update /ping and delete /health

* remove permission check

6 files changed, 54 insertions, 14 deletions

diff --git a/api4/system.go b/api4/system.go
index 1f4589bf5..3a077283c 100644
--- a/api4/system.go
+++ b/api4/system.go
@@ -5,6 +5,7 @@ package api4
 
 import (
 	"net/http"
+	"runtime"
 	"strconv"
 
 	l4g "github.com/alecthomas/log4go"
@@ -17,6 +18,7 @@ func InitSystem() {
 	l4g.Debug(utils.T("api.system.init.debug"))
 
 	BaseRoutes.System.Handle("/ping", ApiHandler(getSystemPing)).Methods("GET")
+
 	BaseRoutes.ApiRoot.Handle("/config", ApiSessionRequired(getConfig)).Methods("GET")
 	BaseRoutes.ApiRoot.Handle("/config", ApiSessionRequired(updateConfig)).Methods("PUT")
 	BaseRoutes.ApiRoot.Handle("/config/reload", ApiSessionRequired(configReload)).Methods("POST")
@@ -34,7 +36,19 @@ func InitSystem() {
 }
 
 func getSystemPing(c *Context, w http.ResponseWriter, r *http.Request) {
-	ReturnStatusOK(w)
+
+	actualGoroutines := runtime.NumGoroutine()
+	if *utils.Cfg.ServiceSettings.GoroutineHealthThreshold <= 0 || actualGoroutines <= *utils.Cfg.ServiceSettings.GoroutineHealthThreshold {
+		ReturnStatusOK(w)
+	} else {
+		rdata := map[string]string{}
+		rdata["status"] = "unhealthy"
+
+		l4g.Warn(utils.T("api.system.go_routines"), actualGoroutines, *utils.Cfg.ServiceSettings.GoroutineHealthThreshold)
+
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(model.MapToJson(rdata)))
+	}
 }
 
 func testEmail(c *Context, w http.ResponseWriter, r *http.Request) {
diff --git a/api4/system_test.go b/api4/system_test.go
index a6b12d326..a46e14782 100644
--- a/api4/system_test.go
+++ b/api4/system_test.go
@@ -12,13 +12,26 @@ import (
 )
 
 func TestGetPing(t *testing.T) {
-	th := Setup().InitBasic()
+	th := Setup().InitBasic().InitSystemAdmin()
 	defer TearDown()
 	Client := th.Client
 
-	b, _ := Client.GetPing()
-	if b == false {
-		t.Fatal()
+	goRoutineHealthThreshold := *utils.Cfg.ServiceSettings.GoroutineHealthThreshold
+	defer func() {
+		*utils.Cfg.ServiceSettings.GoroutineHealthThreshold = goRoutineHealthThreshold
+	}()
+
+	status, resp := Client.GetPing()
+	CheckNoError(t, resp)
+	if status != "OK" {
+		t.Fatal("should return OK")
+	}
+
+	*utils.Cfg.ServiceSettings.GoroutineHealthThreshold = 10
+	status, resp = th.SystemAdminClient.GetPing()
+	CheckInternalErrorStatus(t, resp)
+	if status != "unhealthy" {
+		t.Fatal("should return unhealthy")
 	}
 }
 
@@ -342,5 +355,4 @@ func TestPostLog(t *testing.T) {
 	if len(logMessage) == 0 {
 		t.Fatal("should return the log message")
 	}
-
 }
diff --git a/config/config.json b/config/config.json
index fe47c20d8..d8384a9e3 100644
--- a/config/config.json
+++ b/config/config.json
@@ -12,6 +12,7 @@
         "ReadTimeout": 300,
         "WriteTimeout": 300,
         "MaximumLoginAttempts": 10,
+        "GoroutineHealthThreshold": -1,
         "GoogleDeveloperKey": "",
         "EnableOAuthServiceProvider": false,
         "EnableIncomingWebhooks": true,
@@ -279,4 +280,4 @@
     "DataRetentionSettings": {
         "Enable": false
     }
-}
+}
\ No newline at end of file
diff --git a/i18n/en.json b/i18n/en.json
index 0f5d8c9de..4c0f1b46e 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -6234,5 +6234,9 @@
   {
     "id": "wsapi.webrtc.init.debug",
     "translation": "Initializing webrtc WebSocket API routes"
+  },
+  {
+    "id": "api.system.go_routines",
+    "translation": "The number of running goroutines is over the health threshold %v of %v"
   }
 ]
diff --git a/model/client4.go b/model/client4.go
index 28434c2e5..fce4c678f 100644
--- a/model/client4.go
+++ b/model/client4.go
@@ -1737,15 +1737,18 @@ func (c *Client4) GetFileInfosForPost(postId string, etag string) ([]*FileInfo,
 	}
 }
 
-// General Section
-
-// GetPing will ping the server and to see if it is up and running.
-func (c *Client4) GetPing() (bool, *Response) {
-	if r, err := c.DoApiGet(c.GetSystemRoute()+"/ping", ""); err != nil {
-		return false, &Response{StatusCode: r.StatusCode, Error: err}
+// General/System Section
+
+// GetPing will return ok if the running goRoutines are below the threshold and unhealthy for above.
+func (c *Client4) GetPing() (string, *Response) {
+	if r, err := c.DoApiGet(c.GetSystemRoute()+"/ping", ""); r.StatusCode == 500 {
+		defer r.Body.Close()
+		return "unhealthy", &Response{StatusCode: r.StatusCode, Error: err}
+	} else if err != nil {
+		return "", &Response{StatusCode: r.StatusCode, Error: err}
 	} else {
 		defer closeBody(r)
-		return CheckStatusOK(r), BuildResponse(r)
+		return MapFromJson(r.Body)["status"], BuildResponse(r)
 	}
 }
 
diff --git a/model/config.go b/model/config.go
index b5ba7845f..f3ce6f2f1 100644
--- a/model/config.go
+++ b/model/config.go
@@ -126,6 +126,7 @@ type ServiceSettings struct {
 	ReadTimeout                              *int
 	WriteTimeout                             *int
 	MaximumLoginAttempts                     int
+	GoroutineHealthThreshold                 *int
 	GoogleDeveloperKey                       string
 	EnableOAuthServiceProvider               bool
 	EnableIncomingWebhooks                   bool
@@ -1170,6 +1171,11 @@ func (o *Config) SetDefaults() {
 		*o.RateLimitSettings.Enable = false
 	}
 
+	if o.ServiceSettings.GoroutineHealthThreshold == nil {
+		o.ServiceSettings.GoroutineHealthThreshold = new(int)
+		*o.ServiceSettings.GoroutineHealthThreshold = -1
+	}
+
 	if o.RateLimitSettings.MaxBurst == nil {
 		o.RateLimitSettings.MaxBurst = new(int)
 		*o.RateLimitSettings.MaxBurst = 100