From 4e59a27293394b6d5529efd13ad711daebbc0eb3 Mon Sep 17 00:00:00 2001 From: Harrison Healey Date: Wed, 26 Sep 2018 12:42:51 -0400 Subject: Move HTTPService and ConfigService into services package (#9422) * Move HTTPService and ConfigService into utils package * Re-add StaticConfigService * Move config and http services into their own packages --- services/configservice/configservice.go | 15 ++++ services/httpservice/client.go | 133 ++++++++++++++++++++++++++++++++ services/httpservice/client_test.go | 120 ++++++++++++++++++++++++++++ services/httpservice/httpservice.go | 67 ++++++++++++++++ 4 files changed, 335 insertions(+) create mode 100644 services/configservice/configservice.go create mode 100644 services/httpservice/client.go create mode 100644 services/httpservice/client_test.go create mode 100644 services/httpservice/httpservice.go (limited to 'services') diff --git a/services/configservice/configservice.go b/services/configservice/configservice.go new file mode 100644 index 000000000..bb854d21f --- /dev/null +++ b/services/configservice/configservice.go @@ -0,0 +1,15 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package configservice + +import ( + "github.com/mattermost/mattermost-server/model" +) + +// An interface representing something that contains a Config, such as the app.App struct +type ConfigService interface { + Config() *model.Config + AddConfigListener(func(old, current *model.Config)) string + RemoveConfigListener(string) +} diff --git a/services/httpservice/client.go b/services/httpservice/client.go new file mode 100644 index 000000000..268f63b24 --- /dev/null +++ b/services/httpservice/client.go @@ -0,0 +1,133 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package httpservice + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + "time" +) + +const ( + connectTimeout = 3 * time.Second + requestTimeout = 30 * time.Second +) + +var reservedIPRanges []*net.IPNet + +func IsReservedIP(ip net.IP) bool { + for _, ipRange := range reservedIPRanges { + if ipRange.Contains(ip) { + return true + } + } + return false +} + +func init() { + for _, cidr := range []string{ + // See https://tools.ietf.org/html/rfc6890 + "0.0.0.0/8", // This host on this network + "10.0.0.0/8", // Private-Use + "127.0.0.0/8", // Loopback + "169.254.0.0/16", // Link Local + "172.16.0.0/12", // Private-Use Networks + "192.168.0.0/16", // Private-Use Networks + "::/128", // Unspecified Address + "::1/128", // Loopback Address + "fc00::/7", // Unique-Local + "fe80::/10", // Linked-Scoped Unicast + } { + _, parsed, err := net.ParseCIDR(cidr) + if err != nil { + panic(err) + } + reservedIPRanges = append(reservedIPRanges, parsed) + } +} + +type DialContextFunction func(ctx context.Context, network, addr string) (net.Conn, error) + +var AddressForbidden error = errors.New("address forbidden, you may need to set AllowedUntrustedInternalConnections to allow an integration access to your internal network") + +func dialContextFilter(dial DialContextFunction, allowHost func(host string) bool, allowIP func(ip net.IP) bool) DialContextFunction { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + if allowHost != nil && allowHost(host) { + return dial(ctx, network, addr) + } + + ips, err := net.LookupIP(host) + if err != nil { + return nil, err + } + + var firstErr error + for _, ip := range ips { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + if allowIP == nil || !allowIP(ip) { + continue + } + + conn, err := dial(ctx, network, net.JoinHostPort(ip.String(), port)) + if err == nil { + return conn, nil + } + if firstErr == nil { + firstErr = err + } + } + if firstErr == nil { + return nil, AddressForbidden + } + return nil, firstErr + } +} + +// NewHTTPClient returns a variation the default implementation of Client. +// It uses a Transport with the same settings as the default Transport +// but with the following modifications: +// - shorter timeout for dial and TLS handshake (defined as constant +// "connectTimeout") +// - timeout for the end-to-end request (defined as constant +// "requestTimeout") +func NewHTTPClient(enableInsecureConnections bool, allowHost func(host string) bool, allowIP func(ip net.IP) bool) *http.Client { + dialContext := (&net.Dialer{ + Timeout: connectTimeout, + KeepAlive: 30 * time.Second, + }).DialContext + + if allowHost != nil || allowIP != nil { + dialContext = dialContextFilter(dialContext, allowHost, allowIP) + } + + client := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: dialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: connectTimeout, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: enableInsecureConnections, + }, + }, + Timeout: requestTimeout, + } + + return client +} diff --git a/services/httpservice/client_test.go b/services/httpservice/client_test.go new file mode 100644 index 000000000..ceb133140 --- /dev/null +++ b/services/httpservice/client_test.go @@ -0,0 +1,120 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package httpservice + +import ( + "context" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestHTTPClient(t *testing.T) { + for _, allowInternal := range []bool{true, false} { + c := NewHTTPClient(false, func(_ string) bool { return false }, func(ip net.IP) bool { return allowInternal || !IsReservedIP(ip) }) + for _, tc := range []struct { + URL string + IsInternal bool + }{ + { + URL: "https://google.com", + IsInternal: false, + }, + { + URL: "https://127.0.0.1", + IsInternal: true, + }, + } { + _, err := c.Get(tc.URL) + if !tc.IsInternal { + if err != nil { + t.Fatal("google is down?") + } + } else { + allowed := !tc.IsInternal || allowInternal + success := err == nil + switch e := err.(type) { + case *net.OpError: + success = e.Err != AddressForbidden + case *url.Error: + success = e.Err != AddressForbidden + } + if success != allowed { + t.Fatalf("failed for %v. allowed: %v, success %v", tc.URL, allowed, success) + } + } + } + } +} + +func TestHTTPClientWithProxy(t *testing.T) { + proxy := createProxyServer() + defer proxy.Close() + + c := NewHTTPClient(true, nil, nil) + purl, _ := url.Parse(proxy.URL) + c.Transport.(*http.Transport).Proxy = http.ProxyURL(purl) + + resp, err := c.Get("http://acme.com") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(body) != "proxy" { + t.FailNow() + } +} + +func createProxyServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "text/plain; charset=us-ascii") + fmt.Fprint(w, "proxy") + })) +} + +func TestDialContextFilter(t *testing.T) { + for _, tc := range []struct { + Addr string + IsValid bool + }{ + { + Addr: "google.com:80", + IsValid: true, + }, + { + Addr: "8.8.8.8:53", + IsValid: true, + }, + { + Addr: "127.0.0.1:80", + }, + { + Addr: "10.0.0.1:80", + IsValid: true, + }, + } { + didDial := false + filter := dialContextFilter(func(ctx context.Context, network, addr string) (net.Conn, error) { + didDial = true + return nil, nil + }, func(host string) bool { return host == "10.0.0.1" }, func(ip net.IP) bool { return !IsReservedIP(ip) }) + _, err := filter(context.Background(), "", tc.Addr) + switch { + case tc.IsValid == (err == AddressForbidden) || (err != nil && err != AddressForbidden): + t.Errorf("unexpected err for %v (%v)", tc.Addr, err) + case tc.IsValid != didDial: + t.Errorf("unexpected didDial for %v", tc.Addr) + } + } +} diff --git a/services/httpservice/httpservice.go b/services/httpservice/httpservice.go new file mode 100644 index 000000000..5ed42a12d --- /dev/null +++ b/services/httpservice/httpservice.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package httpservice + +import ( + "net" + "net/http" + "strings" + + "github.com/mattermost/mattermost-server/services/configservice" +) + +// Wraps the functionality for creating a new http.Client to encapsulate that and allow it to be mocked when testing +type HTTPService interface { + MakeClient(trustURLs bool) *http.Client + Close() +} + +type HTTPServiceImpl struct { + configService configservice.ConfigService +} + +func MakeHTTPService(configService configservice.ConfigService) HTTPService { + return &HTTPServiceImpl{configService} +} + +func (h *HTTPServiceImpl) MakeClient(trustURLs bool) *http.Client { + insecure := h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections != nil && *h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections + + if trustURLs { + return NewHTTPClient(insecure, nil, nil) + } + + allowHost := func(host string) bool { + if h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections == nil { + return false + } + for _, allowed := range strings.Fields(*h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections) { + if host == allowed { + return true + } + } + return false + } + + allowIP := func(ip net.IP) bool { + if !IsReservedIP(ip) { + return true + } + if h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections == nil { + return false + } + for _, allowed := range strings.Fields(*h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections) { + if _, ipRange, err := net.ParseCIDR(allowed); err == nil && ipRange.Contains(ip) { + return true + } + } + return false + } + + return NewHTTPClient(insecure, allowHost, allowIP) +} + +func (h *HTTPServiceImpl) Close() { + // Does nothing, but allows this to be overridden when mocking the service +} -- cgit v1.2.3-1-g7c22