MM-9072/MM-10185 Force-convert the encoding of OpenGraph metadata to UTF-8 (#8631)

* Force-convert non-UTF8 HTML to UTF8 before opengraph processing

* Split the force-encoding function

* Add benchmark Test for the forceHTMLEncodingToUTF8()

```
Running tool: /home/comet/go-v1.9.2/bin/go test -benchmem -run=^$ github.com/mattermost/mattermost-server/app -bench ^BenchmarkForceHTMLEncodingToUTF8$

[03:32:58 KST 2018/04/21] [INFO] (github.com/mattermost/mattermost-server/app.TestMain:28) -test.run used, not creating temporary containers
goos: linux
goarch: amd64
pkg: github.com/mattermost/mattermost-server/app
BenchmarkForceHTMLEncodingToUTF8/with_converting-4         	  100000	     11201 ns/op	   18704 B/op	      32 allocs/op
BenchmarkForceHTMLEncodingToUTF8/without_converting-4      	  300000	      3931 ns/op	    4632 B/op	      13 allocs/op
PASS
ok  	github.com/mattermost/mattermost-server/app	2.703s
Success: Benchmarks passed.
```

* Remove an unnecessary constraint

* Add pre-check if content-type header is already utf-8

* Move the checking for utf-8 into forceHTMLEncodingToUTF8() for testing

* Revert df3f347213faa0d023c26d201fa6531f46391086..HEAD, without Gopkg.lock

1 files changed, 82 insertions, 0 deletions

diff --git a/vendor/golang.org/x/text/encoding/unicode/override.go b/vendor/golang.org/x/text/encoding/unicode/override.go
new file mode 100644
index 000000000..35d62fcc9
--- /dev/null
+++ b/vendor/golang.org/x/text/encoding/unicode/override.go
@@ -0,0 +1,82 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package unicode
+
+import (
+	"golang.org/x/text/transform"
+)
+
+// BOMOverride returns a new decoder transformer that is identical to fallback,
+// except that the presence of a Byte Order Mark at the start of the input
+// causes it to switch to the corresponding Unicode decoding. It will only
+// consider BOMs for UTF-8, UTF-16BE, and UTF-16LE.
+//
+// This differs from using ExpectBOM by allowing a BOM to switch to UTF-8, not
+// just UTF-16 variants, and allowing falling back to any encoding scheme.
+//
+// This technique is recommended by the W3C for use in HTML 5: "For
+// compatibility with deployed content, the byte order mark (also known as BOM)
+// is considered more authoritative than anything else."
+// http://www.w3.org/TR/encoding/#specification-hooks
+//
+// Using BOMOverride is mostly intended for use cases where the first characters
+// of a fallback encoding are known to not be a BOM, for example, for valid HTML
+// and most encodings.
+func BOMOverride(fallback transform.Transformer) transform.Transformer {
+	// TODO: possibly allow a variadic argument of unicode encodings to allow
+	// specifying details of which fallbacks are supported as well as
+	// specifying the details of the implementations. This would also allow for
+	// support for UTF-32, which should not be supported by default.
+	return &bomOverride{fallback: fallback}
+}
+
+type bomOverride struct {
+	fallback transform.Transformer
+	current  transform.Transformer
+}
+
+func (d *bomOverride) Reset() {
+	d.current = nil
+	d.fallback.Reset()
+}
+
+var (
+	// TODO: we could use decode functions here, instead of allocating a new
+	// decoder on every NewDecoder as IgnoreBOM decoders can be stateless.
+	utf16le = UTF16(LittleEndian, IgnoreBOM)
+	utf16be = UTF16(BigEndian, IgnoreBOM)
+)
+
+const utf8BOM = "\ufeff"
+
+func (d *bomOverride) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	if d.current != nil {
+		return d.current.Transform(dst, src, atEOF)
+	}
+	if len(src) < 3 && !atEOF {
+		return 0, 0, transform.ErrShortSrc
+	}
+	d.current = d.fallback
+	bomSize := 0
+	if len(src) >= 2 {
+		if src[0] == 0xFF && src[1] == 0xFE {
+			d.current = utf16le.NewDecoder()
+			bomSize = 2
+		} else if src[0] == 0xFE && src[1] == 0xFF {
+			d.current = utf16be.NewDecoder()
+			bomSize = 2
+		} else if len(src) >= 3 &&
+			src[0] == utf8BOM[0] &&
+			src[1] == utf8BOM[1] &&
+			src[2] == utf8BOM[2] {
+			d.current = transform.Nop
+			bomSize = 3
+		}
+	}
+	if bomSize < len(src) {
+		nDst, nSrc, err = d.current.Transform(dst, src[bomSize:], atEOF)
+	}
+	return nDst, nSrc + bomSize, err
+}