PLT-6358: Server HTTP client improvements (#6980)

* restrict untrusted, internal http connections by default

* command test fix

* more test fixes

* change setting from toggle to whitelist

* requested ui changes

* add isdefault diagnostic

* fix tests

8 files changed, 66 insertions, 78 deletions

diff --git a/app/command.go b/app/command.go
index bfb97ae0c..7fe11fffc 100644
--- a/app/command.go
+++ b/app/command.go
@@ -209,7 +209,7 @@ func ExecuteCommand(args *model.CommandArgs) (*model.CommandResponse, *model.App
 						req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 					}
 
-					if resp, err := utils.HttpClient().Do(req); err != nil {
+					if resp, err := utils.HttpClient(false).Do(req); err != nil {
 						return nil, model.NewAppError("command", "api.command.execute_command.failed.app_error", map[string]interface{}{"Trigger": trigger}, err.Error(), http.StatusInternalServerError)
 					} else {
 						if resp.StatusCode == http.StatusOK {
diff --git a/app/diagnostics.go b/app/diagnostics.go
index 983ce8077..aff223855 100644
--- a/app/diagnostics.go
+++ b/app/diagnostics.go
@@ -174,49 +174,50 @@ func trackActivity() {
 
 func trackConfig() {
 	SendDiagnostic(TRACK_CONFIG_SERVICE, map[string]interface{}{
-		"web_server_mode":                               *utils.Cfg.ServiceSettings.WebserverMode,
-		"enable_security_fix_alert":                     *utils.Cfg.ServiceSettings.EnableSecurityFixAlert,
-		"enable_insecure_outgoing_connections":          *utils.Cfg.ServiceSettings.EnableInsecureOutgoingConnections,
-		"enable_incoming_webhooks":                      utils.Cfg.ServiceSettings.EnableIncomingWebhooks,
-		"enable_outgoing_webhooks":                      utils.Cfg.ServiceSettings.EnableOutgoingWebhooks,
-		"enable_commands":                               *utils.Cfg.ServiceSettings.EnableCommands,
-		"enable_only_admin_integrations":                *utils.Cfg.ServiceSettings.EnableOnlyAdminIntegrations,
-		"enable_post_username_override":                 utils.Cfg.ServiceSettings.EnablePostUsernameOverride,
-		"enable_post_icon_override":                     utils.Cfg.ServiceSettings.EnablePostIconOverride,
-		"enable_apiv3":                                  *utils.Cfg.ServiceSettings.EnableAPIv3,
-		"enable_user_access_tokens":                     *utils.Cfg.ServiceSettings.EnableUserAccessTokens,
-		"enable_custom_emoji":                           *utils.Cfg.ServiceSettings.EnableCustomEmoji,
-		"enable_emoji_picker":                           *utils.Cfg.ServiceSettings.EnableEmojiPicker,
-		"restrict_custom_emoji_creation":                *utils.Cfg.ServiceSettings.RestrictCustomEmojiCreation,
-		"enable_testing":                                utils.Cfg.ServiceSettings.EnableTesting,
-		"enable_developer":                              *utils.Cfg.ServiceSettings.EnableDeveloper,
-		"enable_multifactor_authentication":             *utils.Cfg.ServiceSettings.EnableMultifactorAuthentication,
-		"enforce_multifactor_authentication":            *utils.Cfg.ServiceSettings.EnforceMultifactorAuthentication,
-		"enable_oauth_service_provider":                 utils.Cfg.ServiceSettings.EnableOAuthServiceProvider,
-		"connection_security":                           *utils.Cfg.ServiceSettings.ConnectionSecurity,
-		"uses_letsencrypt":                              *utils.Cfg.ServiceSettings.UseLetsEncrypt,
-		"forward_80_to_443":                             *utils.Cfg.ServiceSettings.Forward80To443,
-		"maximum_login_attempts":                        utils.Cfg.ServiceSettings.MaximumLoginAttempts,
-		"session_length_web_in_days":                    *utils.Cfg.ServiceSettings.SessionLengthWebInDays,
-		"session_length_mobile_in_days":                 *utils.Cfg.ServiceSettings.SessionLengthMobileInDays,
-		"session_length_sso_in_days":                    *utils.Cfg.ServiceSettings.SessionLengthSSOInDays,
-		"session_cache_in_minutes":                      *utils.Cfg.ServiceSettings.SessionCacheInMinutes,
-		"isdefault_site_url":                            isDefault(*utils.Cfg.ServiceSettings.SiteURL, model.SERVICE_SETTINGS_DEFAULT_SITE_URL),
-		"isdefault_tls_cert_file":                       isDefault(*utils.Cfg.ServiceSettings.TLSCertFile, model.SERVICE_SETTINGS_DEFAULT_TLS_CERT_FILE),
-		"isdefault_tls_key_file":                        isDefault(*utils.Cfg.ServiceSettings.TLSKeyFile, model.SERVICE_SETTINGS_DEFAULT_TLS_KEY_FILE),
-		"isdefault_read_timeout":                        isDefault(*utils.Cfg.ServiceSettings.ReadTimeout, model.SERVICE_SETTINGS_DEFAULT_READ_TIMEOUT),
-		"isdefault_write_timeout":                       isDefault(*utils.Cfg.ServiceSettings.WriteTimeout, model.SERVICE_SETTINGS_DEFAULT_WRITE_TIMEOUT),
-		"isdefault_google_developer_key":                isDefault(utils.Cfg.ServiceSettings.GoogleDeveloperKey, ""),
-		"isdefault_allow_cors_from":                     isDefault(*utils.Cfg.ServiceSettings.AllowCorsFrom, model.SERVICE_SETTINGS_DEFAULT_ALLOW_CORS_FROM),
-		"restrict_post_delete":                          *utils.Cfg.ServiceSettings.RestrictPostDelete,
-		"allow_edit_post":                               *utils.Cfg.ServiceSettings.AllowEditPost,
-		"post_edit_time_limit":                          *utils.Cfg.ServiceSettings.PostEditTimeLimit,
-		"enable_user_typing_messages":                   *utils.Cfg.ServiceSettings.EnableUserTypingMessages,
-		"enable_channel_viewed_messages":                *utils.Cfg.ServiceSettings.EnableChannelViewedMessages,
-		"time_between_user_typing_updates_milliseconds": *utils.Cfg.ServiceSettings.TimeBetweenUserTypingUpdatesMilliseconds,
-		"cluster_log_timeout_milliseconds":              *utils.Cfg.ServiceSettings.ClusterLogTimeoutMilliseconds,
-		"enable_post_search":                            *utils.Cfg.ServiceSettings.EnablePostSearch,
-		"enable_user_statuses":                          *utils.Cfg.ServiceSettings.EnableUserStatuses,
+		"web_server_mode":                                  *utils.Cfg.ServiceSettings.WebserverMode,
+		"enable_security_fix_alert":                        *utils.Cfg.ServiceSettings.EnableSecurityFixAlert,
+		"enable_insecure_outgoing_connections":             *utils.Cfg.ServiceSettings.EnableInsecureOutgoingConnections,
+		"enable_incoming_webhooks":                         utils.Cfg.ServiceSettings.EnableIncomingWebhooks,
+		"enable_outgoing_webhooks":                         utils.Cfg.ServiceSettings.EnableOutgoingWebhooks,
+		"enable_commands":                                  *utils.Cfg.ServiceSettings.EnableCommands,
+		"enable_only_admin_integrations":                   *utils.Cfg.ServiceSettings.EnableOnlyAdminIntegrations,
+		"enable_post_username_override":                    utils.Cfg.ServiceSettings.EnablePostUsernameOverride,
+		"enable_post_icon_override":                        utils.Cfg.ServiceSettings.EnablePostIconOverride,
+		"enable_apiv3":                                     *utils.Cfg.ServiceSettings.EnableAPIv3,
+		"enable_user_access_tokens":                        *utils.Cfg.ServiceSettings.EnableUserAccessTokens,
+		"enable_custom_emoji":                              *utils.Cfg.ServiceSettings.EnableCustomEmoji,
+		"enable_emoji_picker":                              *utils.Cfg.ServiceSettings.EnableEmojiPicker,
+		"restrict_custom_emoji_creation":                   *utils.Cfg.ServiceSettings.RestrictCustomEmojiCreation,
+		"enable_testing":                                   utils.Cfg.ServiceSettings.EnableTesting,
+		"enable_developer":                                 *utils.Cfg.ServiceSettings.EnableDeveloper,
+		"enable_multifactor_authentication":                *utils.Cfg.ServiceSettings.EnableMultifactorAuthentication,
+		"enforce_multifactor_authentication":               *utils.Cfg.ServiceSettings.EnforceMultifactorAuthentication,
+		"enable_oauth_service_provider":                    utils.Cfg.ServiceSettings.EnableOAuthServiceProvider,
+		"connection_security":                              *utils.Cfg.ServiceSettings.ConnectionSecurity,
+		"uses_letsencrypt":                                 *utils.Cfg.ServiceSettings.UseLetsEncrypt,
+		"forward_80_to_443":                                *utils.Cfg.ServiceSettings.Forward80To443,
+		"maximum_login_attempts":                           utils.Cfg.ServiceSettings.MaximumLoginAttempts,
+		"session_length_web_in_days":                       *utils.Cfg.ServiceSettings.SessionLengthWebInDays,
+		"session_length_mobile_in_days":                    *utils.Cfg.ServiceSettings.SessionLengthMobileInDays,
+		"session_length_sso_in_days":                       *utils.Cfg.ServiceSettings.SessionLengthSSOInDays,
+		"session_cache_in_minutes":                         *utils.Cfg.ServiceSettings.SessionCacheInMinutes,
+		"isdefault_site_url":                               isDefault(*utils.Cfg.ServiceSettings.SiteURL, model.SERVICE_SETTINGS_DEFAULT_SITE_URL),
+		"isdefault_tls_cert_file":                          isDefault(*utils.Cfg.ServiceSettings.TLSCertFile, model.SERVICE_SETTINGS_DEFAULT_TLS_CERT_FILE),
+		"isdefault_tls_key_file":                           isDefault(*utils.Cfg.ServiceSettings.TLSKeyFile, model.SERVICE_SETTINGS_DEFAULT_TLS_KEY_FILE),
+		"isdefault_read_timeout":                           isDefault(*utils.Cfg.ServiceSettings.ReadTimeout, model.SERVICE_SETTINGS_DEFAULT_READ_TIMEOUT),
+		"isdefault_write_timeout":                          isDefault(*utils.Cfg.ServiceSettings.WriteTimeout, model.SERVICE_SETTINGS_DEFAULT_WRITE_TIMEOUT),
+		"isdefault_google_developer_key":                   isDefault(utils.Cfg.ServiceSettings.GoogleDeveloperKey, ""),
+		"isdefault_allow_cors_from":                        isDefault(*utils.Cfg.ServiceSettings.AllowCorsFrom, model.SERVICE_SETTINGS_DEFAULT_ALLOW_CORS_FROM),
+		"isdefault_allowed_untrusted_internal_connections": isDefault(*utils.Cfg.ServiceSettings.AllowedUntrustedInternalConnections, ""),
+		"restrict_post_delete":                             *utils.Cfg.ServiceSettings.RestrictPostDelete,
+		"allow_edit_post":                                  *utils.Cfg.ServiceSettings.AllowEditPost,
+		"post_edit_time_limit":                             *utils.Cfg.ServiceSettings.PostEditTimeLimit,
+		"enable_user_typing_messages":                      *utils.Cfg.ServiceSettings.EnableUserTypingMessages,
+		"enable_channel_viewed_messages":                   *utils.Cfg.ServiceSettings.EnableChannelViewedMessages,
+		"time_between_user_typing_updates_milliseconds":    *utils.Cfg.ServiceSettings.TimeBetweenUserTypingUpdatesMilliseconds,
+		"cluster_log_timeout_milliseconds":                 *utils.Cfg.ServiceSettings.ClusterLogTimeoutMilliseconds,
+		"enable_post_search":                               *utils.Cfg.ServiceSettings.EnablePostSearch,
+		"enable_user_statuses":                             *utils.Cfg.ServiceSettings.EnableUserStatuses,
 	})
 
 	SendDiagnostic(TRACK_CONFIG_TEAM, map[string]interface{}{
diff --git a/app/notification.go b/app/notification.go
index b9153037e..f8b0dd7ce 100644
--- a/app/notification.go
+++ b/app/notification.go
@@ -682,7 +682,7 @@ func sendToPushProxy(msg model.PushNotification, session *model.Session) {
 
 	request, _ := http.NewRequest("POST", *utils.Cfg.EmailSettings.PushNotificationServer+model.API_URL_SUFFIX_V1+"/send_push", strings.NewReader(msg.ToJson()))
 
-	if resp, err := utils.HttpClient().Do(request); err != nil {
+	if resp, err := utils.HttpClient(true).Do(request); err != nil {
 		l4g.Error("Device push reported as error for UserId=%v SessionId=%v message=%v", session.UserId, session.Id, err.Error())
 	} else {
 		pushResponse := model.PushResponseFromJson(resp.Body)
diff --git a/app/oauth.go b/app/oauth.go
index 4174f8146..d9129dd6f 100644
--- a/app/oauth.go
+++ b/app/oauth.go
@@ -673,7 +673,7 @@ func AuthorizeOAuthUser(w http.ResponseWriter, r *http.Request, service, code, s
 
 	var ar *model.AccessResponse
 	var bodyBytes []byte
-	if resp, err := utils.HttpClient().Do(req); err != nil {
+	if resp, err := utils.HttpClient(true).Do(req); err != nil {
 		return nil, "", stateProps, model.NewLocAppError("AuthorizeOAuthUser", "api.user.authorize_oauth_user.token_failed.app_error", nil, err.Error())
 	} else {
 		bodyBytes, _ = ioutil.ReadAll(resp.Body)
@@ -702,7 +702,7 @@ func AuthorizeOAuthUser(w http.ResponseWriter, r *http.Request, service, code, s
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("Authorization", "Bearer "+ar.AccessToken)
 
-	if resp, err := utils.HttpClient().Do(req); err != nil {
+	if resp, err := utils.HttpClient(true).Do(req); err != nil {
 		return nil, "", stateProps, model.NewLocAppError("AuthorizeOAuthUser", "api.user.authorize_oauth_user.service.app_error",
 			map[string]interface{}{"Service": service}, err.Error())
 	} else {
diff --git a/app/post.go b/app/post.go
index 055f98373..a4b177859 100644
--- a/app/post.go
+++ b/app/post.go
@@ -619,7 +619,7 @@ func GetFileInfosForPost(postId string, readFromMaster bool) ([]*model.FileInfo,
 func GetOpenGraphMetadata(url string) *opengraph.OpenGraph {
 	og := opengraph.NewOpenGraph()
 
-	res, err := utils.HttpClient().Get(url)
+	res, err := utils.HttpClient(false).Get(url)
 	if err != nil {
 		l4g.Error("GetOpenGraphMetadata request failed for url=%v with err=%v", url, err.Error())
 		return og
diff --git a/app/webhook.go b/app/webhook.go
index 29f642ea8..4606c207f 100644
--- a/app/webhook.go
+++ b/app/webhook.go
@@ -102,7 +102,7 @@ func TriggerWebhook(payload *model.OutgoingWebhookPayload, hook *model.OutgoingW
 			req, _ := http.NewRequest("POST", url, body)
 			req.Header.Set("Content-Type", contentType)
 			req.Header.Set("Accept", "application/json")
-			if resp, err := utils.HttpClient().Do(req); err != nil {
+			if resp, err := utils.HttpClient(false).Do(req); err != nil {
 				l4g.Error(utils.T("api.post.handle_webhook_events_and_forget.event_post.error"), err.Error())
 			} else {
 				defer CloseBody(resp)
diff --git a/app/webrtc.go b/app/webrtc.go
index 4d44234a8..f8361bbb2 100644
--- a/app/webrtc.go
+++ b/app/webrtc.go
@@ -59,7 +59,7 @@ func GetWebrtcToken(sessionId string) (string, *model.AppError) {
 	rq, _ := http.NewRequest("POST", *utils.Cfg.WebrtcSettings.GatewayAdminUrl, strings.NewReader(model.MapToJson(data)))
 	rq.Header.Set("Content-Type", "application/json")
 
-	if rp, err := utils.HttpClient().Do(rq); err != nil {
+	if rp, err := utils.HttpClient(true).Do(rq); err != nil {
 		return "", model.NewAppError("WebRTC.Token", "model.client.connecting.app_error", nil, err.Error(), http.StatusInternalServerError)
 	} else if rp.StatusCode >= 300 {
 		defer CloseBody(rp)
@@ -80,3 +80,18 @@ func GenerateTurnPassword(username string, secret string) string {
 	h.Write([]byte(username))
 	return base64.StdEncoding.EncodeToString(h.Sum(nil))
 }
+
+func RevokeWebrtcToken(sessionId string) {
+	token := base64.StdEncoding.EncodeToString([]byte(sessionId))
+	data := make(map[string]string)
+	data["janus"] = "remove_token"
+	data["token"] = token
+	data["transaction"] = model.NewId()
+	data["admin_secret"] = *utils.Cfg.WebrtcSettings.GatewayAdminSecret
+
+	rq, _ := http.NewRequest("POST", *utils.Cfg.WebrtcSettings.GatewayAdminUrl, strings.NewReader(model.MapToJson(data)))
+	rq.Header.Set("Content-Type", "application/json")
+
+	// we do not care about the response
+	utils.HttpClient(true).Do(rq)
+}
diff --git a/app/webtrc.go b/app/webtrc.go
deleted file mode 100644
index 76e173651..000000000
--- a/app/webtrc.go
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
-// See License.txt for license information.
-
-package app
-
-import (
-	"encoding/base64"
-	"net/http"
-	"strings"
-
-	"github.com/mattermost/platform/model"
-	"github.com/mattermost/platform/utils"
-)
-
-func RevokeWebrtcToken(sessionId string) {
-	token := base64.StdEncoding.EncodeToString([]byte(sessionId))
-	data := make(map[string]string)
-	data["janus"] = "remove_token"
-	data["token"] = token
-	data["transaction"] = model.NewId()
-	data["admin_secret"] = *utils.Cfg.WebrtcSettings.GatewayAdminSecret
-
-	rq, _ := http.NewRequest("POST", *utils.Cfg.WebrtcSettings.GatewayAdminUrl, strings.NewReader(model.MapToJson(data)))
-	rq.Header.Set("Content-Type", "application/json")
-
-	// we do not care about the response
-	utils.HttpClient().Do(rq)
-}