diff options
Diffstat (limited to 'model')
-rw-r--r-- | model/config.go | 27 | ||||
-rw-r--r-- | model/post.go | 126 | ||||
-rw-r--r-- | model/post_list.go | 9 | ||||
-rw-r--r-- | model/post_test.go | 127 | ||||
-rw-r--r-- | model/testdata/markdown-sample-with-rewritten-image-urls.md | 245 | ||||
-rw-r--r-- | model/testdata/markdown-sample.md | 245 |
6 files changed, 766 insertions, 13 deletions
diff --git a/model/config.go b/model/config.go index d1bb0a41c..6e3e8859e 100644 --- a/model/config.go +++ b/model/config.go @@ -214,6 +214,9 @@ type ServiceSettings struct { EnablePreviewFeatures *bool EnableTutorial *bool ExperimentalEnableDefaultChannelLeaveJoinMessages *bool + ImageProxyType *string + ImageProxyURL *string + ImageProxyOptions *string } func (s *ServiceSettings) SetDefaults() { @@ -250,7 +253,7 @@ func (s *ServiceSettings) SetDefaults() { } if s.AllowedUntrustedInternalConnections == nil { - s.AllowedUntrustedInternalConnections = new(string) + s.AllowedUntrustedInternalConnections = NewString("") } if s.EnableMultifactorAuthentication == nil { @@ -418,6 +421,18 @@ func (s *ServiceSettings) SetDefaults() { if s.ExperimentalEnableDefaultChannelLeaveJoinMessages == nil { s.ExperimentalEnableDefaultChannelLeaveJoinMessages = NewBool(true) } + + if s.ImageProxyType == nil { + s.ImageProxyType = NewString("") + } + + if s.ImageProxyURL == nil { + s.ImageProxyURL = NewString("") + } + + if s.ImageProxyOptions == nil { + s.ImageProxyOptions = NewString("") + } } type ClusterSettings struct { @@ -2050,6 +2065,16 @@ func (ss *ServiceSettings) isValid() *AppError { return NewAppError("Config.IsValid", "model.config.is_valid.listen_address.app_error", nil, "", http.StatusBadRequest) } + switch *ss.ImageProxyType { + case "", "willnorris/imageproxy": + case "atmos/camo": + if *ss.ImageProxyOptions == "" { + return NewAppError("Config.IsValid", "model.config.is_valid.atmos_camo_image_proxy_options.app_error", nil, "", http.StatusBadRequest) + } + default: + return NewAppError("Config.IsValid", "model.config.is_valid.image_proxy_type.app_error", nil, "", http.StatusBadRequest) + } + return nil } diff --git a/model/post.go b/model/post.go index 2ae8d902d..950bf401c 100644 --- a/model/post.go +++ b/model/post.go @@ -8,8 +8,11 @@ import ( "io" "net/http" "regexp" + "sort" "strings" "unicode/utf8" + + "github.com/mattermost/mattermost-server/utils/markdown" ) const ( @@ -43,18 +46,25 @@ const ( ) type Post struct { - Id string `json:"id"` - CreateAt int64 `json:"create_at"` - UpdateAt int64 `json:"update_at"` - EditAt int64 `json:"edit_at"` - DeleteAt int64 `json:"delete_at"` - IsPinned bool `json:"is_pinned"` - UserId string `json:"user_id"` - ChannelId string `json:"channel_id"` - RootId string `json:"root_id"` - ParentId string `json:"parent_id"` - OriginalId string `json:"original_id"` - Message string `json:"message"` + Id string `json:"id"` + CreateAt int64 `json:"create_at"` + UpdateAt int64 `json:"update_at"` + EditAt int64 `json:"edit_at"` + DeleteAt int64 `json:"delete_at"` + IsPinned bool `json:"is_pinned"` + UserId string `json:"user_id"` + ChannelId string `json:"channel_id"` + RootId string `json:"root_id"` + ParentId string `json:"parent_id"` + OriginalId string `json:"original_id"` + + Message string `json:"message"` + + // MessageSource will contain the message as submitted by the user if Message has been modified + // by Mattermost for presentation (e.g if an image proxy is being used). It should be used to + // populate edit boxes if present. + MessageSource string `json:"message_source,omitempty" db:"-"` + Type string `json:"type"` Props StringInterface `json:"props"` Hashtags string `json:"hashtags"` @@ -72,6 +82,14 @@ type PostPatch struct { HasReactions *bool `json:"has_reactions"` } +func (o *PostPatch) WithRewrittenImageURLs(f func(string) string) *PostPatch { + copy := *o + if copy.Message != nil { + *copy.Message = RewriteImageURLs(*o.Message, f) + } + return © +} + type PostForIndexing struct { Post TeamId string `json:"team_id"` @@ -392,3 +410,87 @@ func (o *Post) GenerateActionIds() { } } } + +var markdownDestinationEscaper = strings.NewReplacer( + `\`, `\\`, + `<`, `\<`, + `>`, `\>`, + `(`, `\(`, + `)`, `\)`, +) + +// WithRewrittenImageURLs returns a new shallow copy of the post where the message has been +// rewritten via RewriteImageURLs. +func (o *Post) WithRewrittenImageURLs(f func(string) string) *Post { + copy := *o + copy.Message = RewriteImageURLs(o.Message, f) + if copy.MessageSource == "" && copy.Message != o.Message { + copy.MessageSource = o.Message + } + return © +} + +// RewriteImageURLs takes a message and returns a copy that has all of the image URLs replaced +// according to the function f. For each image URL, f will be invoked, and the resulting markdown +// will contain the URL returned by that invocation instead. +// +// Image URLs are destination URLs used in inline images or reference definitions that are used +// anywhere in the input markdown as an image. +func RewriteImageURLs(message string, f func(string) string) string { + if !strings.Contains(message, "![") { + return message + } + + var ranges []markdown.Range + + markdown.Inspect(message, func(blockOrInline interface{}) bool { + switch v := blockOrInline.(type) { + case *markdown.ReferenceImage: + ranges = append(ranges, v.ReferenceDefinition.RawDestination) + case *markdown.InlineImage: + ranges = append(ranges, v.RawDestination) + default: + return true + } + return true + }) + + if ranges == nil { + return message + } + + sort.Slice(ranges, func(i, j int) bool { + return ranges[i].Position < ranges[j].Position + }) + + copyRanges := make([]markdown.Range, 0, len(ranges)) + urls := make([]string, 0, len(ranges)) + resultLength := len(message) + + start := 0 + for i, r := range ranges { + switch { + case i == 0: + case r.Position != ranges[i-1].Position: + start = ranges[i-1].End + default: + continue + } + original := message[r.Position:r.End] + replacement := markdownDestinationEscaper.Replace(f(markdown.Unescape(original))) + resultLength += len(replacement) - len(original) + copyRanges = append(copyRanges, markdown.Range{Position: start, End: r.Position}) + urls = append(urls, replacement) + } + + result := make([]byte, resultLength) + + offset := 0 + for i, r := range copyRanges { + offset += copy(result[offset:], message[r.Position:r.End]) + offset += copy(result[offset:], urls[i]) + } + copy(result[offset:], message[ranges[len(ranges)-1].End:]) + + return string(result) +} diff --git a/model/post_list.go b/model/post_list.go index 018f7d14f..09cddfdcf 100644 --- a/model/post_list.go +++ b/model/post_list.go @@ -21,6 +21,15 @@ func NewPostList() *PostList { } } +func (o *PostList) WithRewrittenImageURLs(f func(string) string) *PostList { + copy := *o + copy.Posts = make(map[string]*Post) + for id, post := range o.Posts { + copy.Posts[id] = post.WithRewrittenImageURLs(f) + } + return © +} + func (o *PostList) StripActionIntegrations() { posts := o.Posts o.Posts = make(map[string]*Post) diff --git a/model/post_test.go b/model/post_test.go index 6a908887d..5d5e7c9ec 100644 --- a/model/post_test.go +++ b/model/post_test.go @@ -4,6 +4,7 @@ package model import ( + "io/ioutil" "strings" "testing" @@ -173,3 +174,129 @@ func TestPostSanitizeProps(t *testing.T) { t.Fatal("should not be nil") } } + +var markdownSample, markdownSampleWithRewrittenImageURLs string + +func init() { + bytes, err := ioutil.ReadFile("testdata/markdown-sample.md") + if err != nil { + panic(err) + } + markdownSample = string(bytes) + + bytes, err = ioutil.ReadFile("testdata/markdown-sample-with-rewritten-image-urls.md") + if err != nil { + panic(err) + } + markdownSampleWithRewrittenImageURLs = string(bytes) +} + +func TestRewriteImageURLs(t *testing.T) { + for name, tc := range map[string]struct { + Markdown string + Expected string + }{ + "Empty": { + Markdown: ``, + Expected: ``, + }, + "NoImages": { + Markdown: `foo`, + Expected: `foo`, + }, + "Link": { + Markdown: `[foo](/url)`, + Expected: `[foo](/url)`, + }, + "Image": { + Markdown: `![foo](/url)`, + Expected: `![foo](rewritten:/url)`, + }, + "SpacedURL": { + Markdown: `![foo]( /url )`, + Expected: `![foo]( rewritten:/url )`, + }, + "Title": { + Markdown: `![foo](/url "title")`, + Expected: `![foo](rewritten:/url "title")`, + }, + "Parentheses": { + Markdown: `![foo](/url(1) "title")`, + Expected: `![foo](rewritten:/url\(1\) "title")`, + }, + "AngleBrackets": { + Markdown: `![foo](</url\<1\>\\> "title")`, + Expected: `![foo](<rewritten:/url\<1\>\\> "title")`, + }, + "MultipleLines": { + Markdown: `![foo]( + </url\<1\>\\> + "title" + )`, + Expected: `![foo]( + <rewritten:/url\<1\>\\> + "title" + )`, + }, + "ReferenceLink": { + Markdown: `[foo]: </url\<1\>\\> "title" + [foo]`, + Expected: `[foo]: </url\<1\>\\> "title" + [foo]`, + }, + "ReferenceImage": { + Markdown: `[foo]: </url\<1\>\\> "title" + ![foo]`, + Expected: `[foo]: <rewritten:/url\<1\>\\> "title" + ![foo]`, + }, + "MultipleReferenceImages": { + Markdown: `[foo]: </url1> "title" + [bar]: </url2> + [baz]: /url3 "title" + [qux]: /url4 + ![foo]![qux]`, + Expected: `[foo]: <rewritten:/url1> "title" + [bar]: </url2> + [baz]: /url3 "title" + [qux]: rewritten:/url4 + ![foo]![qux]`, + }, + "DuplicateReferences": { + Markdown: `[foo]: </url1> "title" + [foo]: </url2> + [foo]: /url3 "title" + [foo]: /url4 + ![foo]![foo]![foo]`, + Expected: `[foo]: <rewritten:/url1> "title" + [foo]: </url2> + [foo]: /url3 "title" + [foo]: /url4 + ![foo]![foo]![foo]`, + }, + "TrailingURL": { + Markdown: "![foo]\n\n[foo]: /url", + Expected: "![foo]\n\n[foo]: rewritten:/url", + }, + "Sample": { + Markdown: markdownSample, + Expected: markdownSampleWithRewrittenImageURLs, + }, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.Expected, RewriteImageURLs(tc.Markdown, func(url string) string { + return "rewritten:" + url + })) + }) + } +} + +var rewriteImageURLsSink string + +func BenchmarkRewriteImageURLs(b *testing.B) { + for i := 0; i < b.N; i++ { + rewriteImageURLsSink = RewriteImageURLs(markdownSample, func(url string) string { + return "rewritten:" + url + }) + } +} diff --git a/model/testdata/markdown-sample-with-rewritten-image-urls.md b/model/testdata/markdown-sample-with-rewritten-image-urls.md new file mode 100644 index 000000000..6683bc459 --- /dev/null +++ b/model/testdata/markdown-sample-with-rewritten-image-urls.md @@ -0,0 +1,245 @@ +--- +__Advertisement :)__ + +- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image + resize in browser. +- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly + i18n with plurals support and easy syntax. + +You will like those projects! + +--- + +# h1 Heading 8-) +## h2 Heading +### h3 Heading +#### h4 Heading +##### h5 Heading +###### h6 Heading + + +## Horizontal Rules + +___ + +--- + +*** + + +## Typographic replacements + +Enable typographer option to see result. + +(c) (C) (r) (R) (tm) (TM) (p) (P) +- + +test.. test... test..... test?..... test!.... + +!!!!!! ???? ,, -- --- + +"Smartypants, double quotes" and 'single quotes' + + +## Emphasis + +**This is bold text** + +__This is bold text__ + +*This is italic text* + +_This is italic text_ + +~~Strikethrough~~ + + +## Blockquotes + + +> Blockquotes can also be nested... +>> ...by using additional greater-than signs right next to each other... +> > > ...or with spaces between arrows. + + +## Lists + +Unordered + ++ Create a list by starting a line with `+`, `-`, or `*` ++ Sub-lists are made by indenting 2 spaces: + - Marker character change forces new list start: + * Ac tristique libero volutpat at + + Facilisis in pretium nisl aliquet + - Nulla volutpat aliquam velit ++ Very easy! + +Ordered + +1. Lorem ipsum dolor sit amet +2. Consectetur adipiscing elit +3. Integer molestie lorem at massa + + +1. You can use sequential numbers... +1. ...or keep all the numbers as `1.` + +Start numbering with offset: + +57. foo +1. bar + + +## Code + +Inline `code` + +Indented code + + // Some comments + line 1 of code + line 2 of code + line 3 of code + + +Block code "fences" + +``` +Sample text here... +``` + +Syntax highlighting + +``` js +var foo = function (bar) { + return bar++; +}; + +console.log(foo(5)); +``` + +## Tables + +| Option | Description | +| ------ | ----------- | +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + +Right aligned columns + +| Option | Description | +| ------:| -----------:| +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + + +## Links + +[link text](http://dev.nodeca.com) + +[link with title](http://nodeca.github.io/pica/demo/ "title text!") + +Autoconverted link https://github.com/nodeca/pica (enable linkify to see) + + +## Images + +![Minion](rewritten:https://octodex.github.com/images/minion.png) +![Stormtroopocat](rewritten:https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat") + +Like links, Images also have a footnote style syntax + +![Alt text][id] + +With a reference later in the document defining the URL location: + +[id]: rewritten:https://octodex.github.com/images/dojocat.jpg "The Dojocat" + + +## Plugins + +The killer feature of `markdown-it` is very effective support of +[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin). + + +### [Emojies](https://github.com/markdown-it/markdown-it-emoji) + +> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum: +> +> Shortcuts (emoticons): :-) :-( 8-) ;) + +see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji. + + +### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup) + +- 19^th^ +- H~2~O + + +### [\<ins>](https://github.com/markdown-it/markdown-it-ins) + +++Inserted text++ + + +### [\<mark>](https://github.com/markdown-it/markdown-it-mark) + +==Marked text== + + +### [Footnotes](https://github.com/markdown-it/markdown-it-footnote) + +Footnote 1 link[^first]. + +Footnote 2 link[^second]. + +Inline footnote^[Text of inline footnote] definition. + +Duplicated footnote reference[^second]. + +[^first]: Footnote **can have markup** + + and multiple paragraphs. + +[^second]: Footnote text. + + +### [Definition lists](https://github.com/markdown-it/markdown-it-deflist) + +Term 1 + +: Definition 1 +with lazy continuation. + +Term 2 with *inline markup* + +: Definition 2 + + { some code, part of Definition 2 } + + Third paragraph of definition 2. + +_Compact style:_ + +Term 1 + ~ Definition 1 + +Term 2 + ~ Definition 2a + ~ Definition 2b + + +### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr) + +This is HTML abbreviation example. + +It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on. + +*[HTML]: Hyper Text Markup Language + +### [Custom containers](https://github.com/markdown-it/markdown-it-container) + +::: warning +*here be dragons* +::: diff --git a/model/testdata/markdown-sample.md b/model/testdata/markdown-sample.md new file mode 100644 index 000000000..f894c1d35 --- /dev/null +++ b/model/testdata/markdown-sample.md @@ -0,0 +1,245 @@ +--- +__Advertisement :)__ + +- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image + resize in browser. +- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly + i18n with plurals support and easy syntax. + +You will like those projects! + +--- + +# h1 Heading 8-) +## h2 Heading +### h3 Heading +#### h4 Heading +##### h5 Heading +###### h6 Heading + + +## Horizontal Rules + +___ + +--- + +*** + + +## Typographic replacements + +Enable typographer option to see result. + +(c) (C) (r) (R) (tm) (TM) (p) (P) +- + +test.. test... test..... test?..... test!.... + +!!!!!! ???? ,, -- --- + +"Smartypants, double quotes" and 'single quotes' + + +## Emphasis + +**This is bold text** + +__This is bold text__ + +*This is italic text* + +_This is italic text_ + +~~Strikethrough~~ + + +## Blockquotes + + +> Blockquotes can also be nested... +>> ...by using additional greater-than signs right next to each other... +> > > ...or with spaces between arrows. + + +## Lists + +Unordered + ++ Create a list by starting a line with `+`, `-`, or `*` ++ Sub-lists are made by indenting 2 spaces: + - Marker character change forces new list start: + * Ac tristique libero volutpat at + + Facilisis in pretium nisl aliquet + - Nulla volutpat aliquam velit ++ Very easy! + +Ordered + +1. Lorem ipsum dolor sit amet +2. Consectetur adipiscing elit +3. Integer molestie lorem at massa + + +1. You can use sequential numbers... +1. ...or keep all the numbers as `1.` + +Start numbering with offset: + +57. foo +1. bar + + +## Code + +Inline `code` + +Indented code + + // Some comments + line 1 of code + line 2 of code + line 3 of code + + +Block code "fences" + +``` +Sample text here... +``` + +Syntax highlighting + +``` js +var foo = function (bar) { + return bar++; +}; + +console.log(foo(5)); +``` + +## Tables + +| Option | Description | +| ------ | ----------- | +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + +Right aligned columns + +| Option | Description | +| ------:| -----------:| +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + + +## Links + +[link text](http://dev.nodeca.com) + +[link with title](http://nodeca.github.io/pica/demo/ "title text!") + +Autoconverted link https://github.com/nodeca/pica (enable linkify to see) + + +## Images + +![Minion](https://octodex.github.com/images/minion.png) +![Stormtroopocat](https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat") + +Like links, Images also have a footnote style syntax + +![Alt text][id] + +With a reference later in the document defining the URL location: + +[id]: https://octodex.github.com/images/dojocat.jpg "The Dojocat" + + +## Plugins + +The killer feature of `markdown-it` is very effective support of +[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin). + + +### [Emojies](https://github.com/markdown-it/markdown-it-emoji) + +> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum: +> +> Shortcuts (emoticons): :-) :-( 8-) ;) + +see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji. + + +### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup) + +- 19^th^ +- H~2~O + + +### [\<ins>](https://github.com/markdown-it/markdown-it-ins) + +++Inserted text++ + + +### [\<mark>](https://github.com/markdown-it/markdown-it-mark) + +==Marked text== + + +### [Footnotes](https://github.com/markdown-it/markdown-it-footnote) + +Footnote 1 link[^first]. + +Footnote 2 link[^second]. + +Inline footnote^[Text of inline footnote] definition. + +Duplicated footnote reference[^second]. + +[^first]: Footnote **can have markup** + + and multiple paragraphs. + +[^second]: Footnote text. + + +### [Definition lists](https://github.com/markdown-it/markdown-it-deflist) + +Term 1 + +: Definition 1 +with lazy continuation. + +Term 2 with *inline markup* + +: Definition 2 + + { some code, part of Definition 2 } + + Third paragraph of definition 2. + +_Compact style:_ + +Term 1 + ~ Definition 1 + +Term 2 + ~ Definition 2a + ~ Definition 2b + + +### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr) + +This is HTML abbreviation example. + +It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on. + +*[HTML]: Hyper Text Markup Language + +### [Custom containers](https://github.com/markdown-it/markdown-it-container) + +::: warning +*here be dragons* +::: |