From d3a0e561c265e32a305cd5c6ed5e80f461d9f4eb Mon Sep 17 00:00:00 2001 From: Harrison Healey Date: Thu, 8 Feb 2018 10:45:25 -0500 Subject: ICU-669 Ensured all URLs returned from OpenGraph are absolute --- app/post.go | 59 ++++++++++++++++++++++++++++++++++++++--- app/post_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 4 deletions(-) (limited to 'app') diff --git a/app/post.go b/app/post.go index 005624605..1e170d363 100644 --- a/app/post.go +++ b/app/post.go @@ -12,6 +12,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "regexp" "strings" @@ -734,18 +735,68 @@ func (a *App) GetFileInfosForPost(postId string, readFromMaster bool) ([]*model. return infos, nil } -func (a *App) GetOpenGraphMetadata(url string) *opengraph.OpenGraph { +func (a *App) GetOpenGraphMetadata(requestURL string) *opengraph.OpenGraph { og := opengraph.NewOpenGraph() - res, err := a.HTTPClient(false).Get(url) + res, err := a.HTTPClient(false).Get(requestURL) if err != nil { - l4g.Error("GetOpenGraphMetadata request failed for url=%v with err=%v", url, err.Error()) + l4g.Error("GetOpenGraphMetadata request failed for url=%v with err=%v", requestURL, err.Error()) return og } defer consumeAndClose(res) if err := og.ProcessHTML(res.Body); err != nil { - l4g.Error("GetOpenGraphMetadata processing failed for url=%v with err=%v", url, err.Error()) + l4g.Error("GetOpenGraphMetadata processing failed for url=%v with err=%v", requestURL, err.Error()) + } + + og = makeOpenGraphURLsAbsolute(og, requestURL) + + return og +} + +func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) *opengraph.OpenGraph { + parsedRequestURL, err := url.Parse(requestURL) + if err != nil { + l4g.Warn("makeOpenGraphURLsAbsolute failed to parse url=%v", requestURL) + return og + } + + makeURLAbsolute := func(resultURL string) string { + if resultURL == "" { + return resultURL + } + + parsedResultURL, err := url.Parse(resultURL) + if err != nil { + l4g.Warn("makeOpenGraphURLsAbsolute failed to parse result url=%v", resultURL) + return resultURL + } + + if parsedResultURL.IsAbs() { + return resultURL + } + + parsedResultURL.Scheme = parsedRequestURL.Scheme + parsedResultURL.Host = parsedRequestURL.Host + + return parsedResultURL.String() + } + + og.URL = makeURLAbsolute(og.URL) + + for _, image := range og.Images { + image.URL = makeURLAbsolute(image.URL) + image.SecureURL = makeURLAbsolute(image.SecureURL) + } + + for _, audio := range og.Audios { + audio.URL = makeURLAbsolute(audio.URL) + audio.SecureURL = makeURLAbsolute(audio.SecureURL) + } + + for _, video := range og.Videos { + video.URL = makeURLAbsolute(video.URL) + video.SecureURL = makeURLAbsolute(video.SecureURL) } return og diff --git a/app/post_test.go b/app/post_test.go index 3f3783265..987879a72 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -8,9 +8,11 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" "time" + "github.com/dyatlov/go-opengraph/opengraph" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -250,6 +252,84 @@ func TestImageProxy(t *testing.T) { } } +func TestMakeOpenGraphURLsAbsolute(t *testing.T) { + for name, tc := range map[string]struct { + HTML string + RequestURL string + URL string + ImageURL string + }{ + "absolute URLs": { + HTML: ` + + + + + + `, + RequestURL: "https://example.com", + URL: "https://example.com/apps/mattermost", + ImageURL: "https://images.example.com/image.png", + }, + "relative URLs": { + HTML: ` + + + + + + `, + RequestURL: "http://example.com", + URL: "http://example.com/apps/mattermost", + ImageURL: "http://example.com/image.png", + }, + "relative URLs with HTTPS": { + HTML: ` + + + + + + `, + RequestURL: "https://example.com", + URL: "https://example.com/apps/mattermost", + ImageURL: "https://example.com/image.png", + }, + "missing image URL": { + HTML: ` + + + + + `, + RequestURL: "http://example.com", + URL: "http://example.com/apps/mattermost", + ImageURL: "", + }, + } { + t.Run(name, func(t *testing.T) { + og := opengraph.NewOpenGraph() + if err := og.ProcessHTML(strings.NewReader(tc.HTML)); err != nil { + t.Fatal(err) + } + + og = makeOpenGraphURLsAbsolute(og, tc.RequestURL) + + if og.URL != tc.URL { + t.Fatalf("incorrect url, expected %v, got %v", tc.URL, og.URL) + } + + if len(og.Images) > 0 { + if og.Images[0].URL != tc.ImageURL { + t.Fatalf("incorrect image url, expected %v, got %v", tc.ImageURL, og.Images[0].URL) + } + } else if tc.ImageURL != "" { + t.Fatal("missing image url, expected %v, got nothing", tc.ImageURL) + } + }) + } +} + var imageProxyBenchmarkSink *model.Post func BenchmarkPostWithProxyRemovedFromImageURLs(b *testing.B) { -- cgit v1.2.3-1-g7c22 From e2b5f9217f55074e4a64c90f4121803cd68f0b97 Mon Sep 17 00:00:00 2001 From: Harrison Healey Date: Fri, 9 Feb 2018 10:05:23 -0500 Subject: ICU-669 Handle relative links better --- app/post.go | 13 ++++--------- app/post_test.go | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 12 deletions(-) (limited to 'app') diff --git a/app/post.go b/app/post.go index 1e170d363..f8a371fc0 100644 --- a/app/post.go +++ b/app/post.go @@ -749,16 +749,16 @@ func (a *App) GetOpenGraphMetadata(requestURL string) *opengraph.OpenGraph { l4g.Error("GetOpenGraphMetadata processing failed for url=%v with err=%v", requestURL, err.Error()) } - og = makeOpenGraphURLsAbsolute(og, requestURL) + makeOpenGraphURLsAbsolute(og, requestURL) return og } -func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) *opengraph.OpenGraph { +func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) { parsedRequestURL, err := url.Parse(requestURL) if err != nil { l4g.Warn("makeOpenGraphURLsAbsolute failed to parse url=%v", requestURL) - return og + return } makeURLAbsolute := func(resultURL string) string { @@ -776,10 +776,7 @@ func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) *open return resultURL } - parsedResultURL.Scheme = parsedRequestURL.Scheme - parsedResultURL.Host = parsedRequestURL.Host - - return parsedResultURL.String() + return parsedRequestURL.ResolveReference(parsedResultURL).String() } og.URL = makeURLAbsolute(og.URL) @@ -798,8 +795,6 @@ func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) *open video.URL = makeURLAbsolute(video.URL) video.SecureURL = makeURLAbsolute(video.SecureURL) } - - return og } func (a *App) DoPostAction(postId string, actionId string, userId string) *model.AppError { diff --git a/app/post_test.go b/app/post_test.go index 987879a72..62098c865 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -271,7 +271,7 @@ func TestMakeOpenGraphURLsAbsolute(t *testing.T) { URL: "https://example.com/apps/mattermost", ImageURL: "https://images.example.com/image.png", }, - "relative URLs": { + "URLs starting with /": { HTML: ` @@ -283,7 +283,7 @@ func TestMakeOpenGraphURLsAbsolute(t *testing.T) { URL: "http://example.com/apps/mattermost", ImageURL: "http://example.com/image.png", }, - "relative URLs with HTTPS": { + "HTTPS URLs starting with /": { HTML: ` @@ -306,6 +306,18 @@ func TestMakeOpenGraphURLsAbsolute(t *testing.T) { URL: "http://example.com/apps/mattermost", ImageURL: "", }, + "relative URLs": { + HTML: ` + + + + + + `, + RequestURL: "http://example.com/content/index.html", + URL: "http://example.com/content/index.html", + ImageURL: "http://example.com/resources/image.png", + }, } { t.Run(name, func(t *testing.T) { og := opengraph.NewOpenGraph() @@ -313,7 +325,7 @@ func TestMakeOpenGraphURLsAbsolute(t *testing.T) { t.Fatal(err) } - og = makeOpenGraphURLsAbsolute(og, tc.RequestURL) + makeOpenGraphURLsAbsolute(og, tc.RequestURL) if og.URL != tc.URL { t.Fatalf("incorrect url, expected %v, got %v", tc.URL, og.URL) -- cgit v1.2.3-1-g7c22 From 87fb19b8279c86c72ffec623e55b80ce35b7d64f Mon Sep 17 00:00:00 2001 From: Harrison Healey Date: Mon, 12 Feb 2018 08:58:38 -0500 Subject: Fixed typo in unit test --- app/post_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app') diff --git a/app/post_test.go b/app/post_test.go index 62098c865..f5a5a23cb 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -336,7 +336,7 @@ func TestMakeOpenGraphURLsAbsolute(t *testing.T) { t.Fatalf("incorrect image url, expected %v, got %v", tc.ImageURL, og.Images[0].URL) } } else if tc.ImageURL != "" { - t.Fatal("missing image url, expected %v, got nothing", tc.ImageURL) + t.Fatalf("missing image url, expected %v, got nothing", tc.ImageURL) } }) } -- cgit v1.2.3-1-g7c22