From 599991ea731953f772824ce3ed1e591246aa004f Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 22 Jan 2018 15:32:50 -0600 Subject: PLT-3383: image proxy support (#7991) * image proxy support * go vet fix, remove mistakenly added coverage file * fix test compile error * add validation to config settings and documentation to model functions * add message_source field to post --- api/post.go | 2 +- api4/channel.go | 2 +- api4/post.go | 22 +- app/notification.go | 2 +- app/post.go | 132 +- app/post_test.go | 81 + app/reaction.go | 2 +- config/default.json | 4 +- i18n/en.json | 8 + model/config.go | 27 +- model/post.go | 126 +- model/post_list.go | 9 + model/post_test.go | 127 ++ .../markdown-sample-with-rewritten-image-urls.md | 245 +++ model/testdata/markdown-sample.md | 245 +++ utils/markdown/block_quote.go | 62 + utils/markdown/blocks.go | 153 ++ utils/markdown/commonmark_test.go | 1001 +++++++++ utils/markdown/document.go | 22 + utils/markdown/fenced_code.go | 112 + utils/markdown/html.go | 186 ++ utils/markdown/html_entities.go | 2132 ++++++++++++++++++++ utils/markdown/indented_code.go | 98 + utils/markdown/inlines.go | 489 +++++ utils/markdown/inspect.go | 78 + utils/markdown/inspect_test.go | 54 + utils/markdown/lines.go | 27 + utils/markdown/lines_test.go | 36 + utils/markdown/links.go | 130 ++ utils/markdown/list.go | 220 ++ utils/markdown/markdown.go | 132 ++ utils/markdown/paragraph.go | 71 + utils/markdown/reference_definition.go | 75 + 33 files changed, 6080 insertions(+), 32 deletions(-) create mode 100644 model/testdata/markdown-sample-with-rewritten-image-urls.md create mode 100644 model/testdata/markdown-sample.md create mode 100644 utils/markdown/block_quote.go create mode 100644 utils/markdown/blocks.go create mode 100644 utils/markdown/commonmark_test.go create mode 100644 utils/markdown/document.go create mode 100644 utils/markdown/fenced_code.go create mode 100644 utils/markdown/html.go create mode 100644 utils/markdown/html_entities.go create mode 100644 utils/markdown/indented_code.go create mode 100644 utils/markdown/inlines.go create mode 100644 utils/markdown/inspect.go create mode 100644 utils/markdown/inspect_test.go create mode 100644 utils/markdown/lines.go create mode 100644 utils/markdown/lines_test.go create mode 100644 utils/markdown/links.go create mode 100644 utils/markdown/list.go create mode 100644 utils/markdown/markdown.go create mode 100644 utils/markdown/paragraph.go create mode 100644 utils/markdown/reference_definition.go diff --git a/api/post.go b/api/post.go index e0cfb720c..192dc0abc 100644 --- a/api/post.go +++ b/api/post.go @@ -135,7 +135,7 @@ func saveIsPinnedPost(c *Context, w http.ResponseWriter, r *http.Request, isPinn rpost := result.Data.(*model.Post) message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", rpost.ChannelId, "", nil) - message.Add("post", rpost.ToJson()) + message.Add("post", c.App.PostWithProxyAddedToImageURLs(rpost).ToJson()) c.App.Go(func() { c.App.Publish(message) diff --git a/api4/channel.go b/api4/channel.go index f8a069ea0..9801c9dc0 100644 --- a/api4/channel.go +++ b/api4/channel.go @@ -386,7 +386,7 @@ func getPinnedPosts(c *Context, w http.ResponseWriter, r *http.Request) { return } else { w.Header().Set(model.HEADER_ETAG_SERVER, posts.Etag()) - w.Write([]byte(posts.ToJson())) + w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson())) } } diff --git a/api4/post.go b/api4/post.go index 7a365845f..80088d9ef 100644 --- a/api4/post.go +++ b/api4/post.go @@ -56,7 +56,7 @@ func createPost(c *Context, w http.ResponseWriter, r *http.Request) { post.CreateAt = 0 } - rp, err := c.App.CreatePostAsUser(post) + rp, err := c.App.CreatePostAsUser(c.App.PostWithProxyRemovedFromImageURLs(post)) if err != nil { c.Err = err return @@ -66,7 +66,7 @@ func createPost(c *Context, w http.ResponseWriter, r *http.Request) { c.App.UpdateLastActivityAtIfNeeded(c.Session) w.WriteHeader(http.StatusCreated) - w.Write([]byte(rp.ToJson())) + w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(rp).ToJson())) } func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) { @@ -135,7 +135,7 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) { if len(etag) > 0 { w.Header().Set(model.HEADER_ETAG_SERVER, etag) } - w.Write([]byte(list.ToJson())) + w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(list).ToJson())) } func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request) { @@ -168,7 +168,7 @@ func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request) return } - w.Write([]byte(posts.ToJson())) + w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson())) } func getPost(c *Context, w http.ResponseWriter, r *http.Request) { @@ -206,7 +206,7 @@ func getPost(c *Context, w http.ResponseWriter, r *http.Request) { return } else { w.Header().Set(model.HEADER_ETAG_SERVER, post.Etag()) - w.Write([]byte(post.ToJson())) + w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(post).ToJson())) } } @@ -272,7 +272,7 @@ func getPostThread(c *Context, w http.ResponseWriter, r *http.Request) { return } else { w.Header().Set(model.HEADER_ETAG_SERVER, list.Etag()) - w.Write([]byte(list.ToJson())) + w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(list).ToJson())) } } @@ -313,7 +313,7 @@ func searchPosts(c *Context, w http.ResponseWriter, r *http.Request) { } w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") - w.Write([]byte(posts.ToJson())) + w.Write([]byte(c.App.PostListWithProxyAddedToImageURLs(posts).ToJson())) } func updatePost(c *Context, w http.ResponseWriter, r *http.Request) { @@ -341,13 +341,13 @@ func updatePost(c *Context, w http.ResponseWriter, r *http.Request) { post.Id = c.Params.PostId - rpost, err := c.App.UpdatePost(post, false) + rpost, err := c.App.UpdatePost(c.App.PostWithProxyRemovedFromImageURLs(post), false) if err != nil { c.Err = err return } - w.Write([]byte(rpost.ToJson())) + w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(rpost).ToJson())) } func patchPost(c *Context, w http.ResponseWriter, r *http.Request) { @@ -373,13 +373,13 @@ func patchPost(c *Context, w http.ResponseWriter, r *http.Request) { return } - patchedPost, err := c.App.PatchPost(c.Params.PostId, post) + patchedPost, err := c.App.PatchPost(c.Params.PostId, c.App.PostPatchWithProxyRemovedFromImageURLs(post)) if err != nil { c.Err = err return } - w.Write([]byte(patchedPost.ToJson())) + w.Write([]byte(c.App.PostWithProxyAddedToImageURLs(patchedPost).ToJson())) } func saveIsPinnedPost(c *Context, w http.ResponseWriter, r *http.Request, isPinned bool) { diff --git a/app/notification.go b/app/notification.go index 4929d56f4..62aad4c28 100644 --- a/app/notification.go +++ b/app/notification.go @@ -270,7 +270,7 @@ func (a *App) SendNotifications(post *model.Post, team *model.Team, channel *mod } message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POSTED, "", post.ChannelId, "", nil) - message.Add("post", post.ToJson()) + message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson()) message.Add("channel_type", channel.Type) message.Add("channel_display_name", channelName) message.Add("channel_name", channel.Name) diff --git a/app/post.go b/app/post.go index 192c2effb..bf4725e77 100644 --- a/app/post.go +++ b/app/post.go @@ -4,6 +4,11 @@ package app import ( + "crypto/hmac" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -309,7 +314,7 @@ func (a *App) SendEphemeralPost(userId string, post *model.Post) *model.Post { } message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil) - message.Add("post", post.ToJson()) + message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson()) a.Go(func() { a.Publish(message) @@ -419,7 +424,7 @@ func (a *App) PatchPost(postId string, patch *model.PostPatch) (*model.Post, *mo func (a *App) sendUpdatedPostEvent(post *model.Post) { message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil) - message.Add("post", post.ToJson()) + message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson()) a.Go(func() { a.Publish(message) @@ -562,7 +567,7 @@ func (a *App) DeletePost(postId string) (*model.Post, *model.AppError) { } message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_DELETED, "", post.ChannelId, "", nil) - message.Add("post", post.ToJson()) + message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson()) a.Go(func() { a.Publish(message) @@ -823,3 +828,124 @@ func (a *App) DoPostAction(postId string, actionId string, userId string) *model return nil } + +func (a *App) PostListWithProxyAddedToImageURLs(list *model.PostList) *model.PostList { + if f := a.ImageProxyAdder(); f != nil { + return list.WithRewrittenImageURLs(f) + } + return list +} + +func (a *App) PostWithProxyAddedToImageURLs(post *model.Post) *model.Post { + if f := a.ImageProxyAdder(); f != nil { + return post.WithRewrittenImageURLs(f) + } + return post +} + +func (a *App) PostWithProxyRemovedFromImageURLs(post *model.Post) *model.Post { + if f := a.ImageProxyRemover(); f != nil { + return post.WithRewrittenImageURLs(f) + } + return post +} + +func (a *App) PostPatchWithProxyRemovedFromImageURLs(patch *model.PostPatch) *model.PostPatch { + if f := a.ImageProxyRemover(); f != nil { + return patch.WithRewrittenImageURLs(f) + } + return patch +} + +func (a *App) imageProxyConfig() (proxyType, proxyURL, options, siteURL string) { + cfg := a.Config() + + if cfg.ServiceSettings.ImageProxyURL == nil || cfg.ServiceSettings.ImageProxyType == nil || cfg.ServiceSettings.SiteURL == nil { + return + } + + proxyURL = *cfg.ServiceSettings.ImageProxyURL + proxyType = *cfg.ServiceSettings.ImageProxyType + siteURL = *cfg.ServiceSettings.SiteURL + + if proxyURL == "" || proxyType == "" { + return "", "", "", "" + } + + if proxyURL[len(proxyURL)-1] != '/' { + proxyURL += "/" + } + + if cfg.ServiceSettings.ImageProxyOptions != nil { + options = *cfg.ServiceSettings.ImageProxyOptions + } + + return +} + +func (a *App) ImageProxyAdder() func(string) string { + proxyType, proxyURL, options, siteURL := a.imageProxyConfig() + if proxyType == "" { + return nil + } + + return func(url string) string { + if strings.HasPrefix(url, proxyURL) { + return url + } + + if url[0] == '/' { + url = siteURL + url + } + + switch proxyType { + case "atmos/camo": + mac := hmac.New(sha1.New, []byte(options)) + mac.Write([]byte(url)) + digest := hex.EncodeToString(mac.Sum(nil)) + return proxyURL + digest + "/" + hex.EncodeToString([]byte(url)) + case "willnorris/imageproxy": + options := strings.Split(options, "|") + if len(options) > 1 { + mac := hmac.New(sha256.New, []byte(options[1])) + mac.Write([]byte(url)) + digest := base64.URLEncoding.EncodeToString(mac.Sum(nil)) + if options[0] == "" { + return proxyURL + "s" + digest + "/" + url + } + return proxyURL + options[0] + ",s" + digest + "/" + url + } + return proxyURL + options[0] + "/" + url + } + + return url + } +} + +func (a *App) ImageProxyRemover() (f func(string) string) { + proxyType, proxyURL, _, _ := a.imageProxyConfig() + if proxyType == "" { + return nil + } + + return func(url string) string { + switch proxyType { + case "atmos/camo": + if strings.HasPrefix(url, proxyURL) { + if slash := strings.IndexByte(url[len(proxyURL):], '/'); slash >= 0 { + if decoded, err := hex.DecodeString(url[len(proxyURL)+slash+1:]); err == nil { + return string(decoded) + } + } + } + case "willnorris/imageproxy": + if strings.HasPrefix(url, proxyURL) { + if slash := strings.IndexByte(url[len(proxyURL):], '/'); slash >= 0 { + return url[len(proxyURL)+slash+1:] + } + } + } + + return url + } +} diff --git a/app/post_test.go b/app/post_test.go index 82eac3cd1..9854bb707 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -185,3 +185,84 @@ func TestPostChannelMentions(t *testing.T) { }, }, result.Props["channel_mentions"]) } + +func TestImageProxy(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + + for name, tc := range map[string]struct { + ProxyType string + ProxyURL string + ProxyOptions string + ImageURL string + ProxiedImageURL string + }{ + "atmos/camo": { + ProxyType: "atmos/camo", + ProxyURL: "https://127.0.0.1", + ProxyOptions: "foo", + ImageURL: "http://mydomain.com/myimage", + ProxiedImageURL: "https://127.0.0.1/f8dace906d23689e8d5b12c3cefbedbf7b9b72f5/687474703a2f2f6d79646f6d61696e2e636f6d2f6d79696d616765", + }, + "willnorris/imageproxy": { + ProxyType: "willnorris/imageproxy", + ProxyURL: "https://127.0.0.1", + ProxyOptions: "x1000", + ImageURL: "http://mydomain.com/myimage", + ProxiedImageURL: "https://127.0.0.1/x1000/http://mydomain.com/myimage", + }, + "willnorris/imageproxy_WithSigning": { + ProxyType: "willnorris/imageproxy", + ProxyURL: "https://127.0.0.1", + ProxyOptions: "x1000|foo", + ImageURL: "http://mydomain.com/myimage", + ProxiedImageURL: "https://127.0.0.1/x1000,sbhHVoG5d60UvnNtGh6Iy6x4PaMmnsh8JfZ7JfErKjGU=/http://mydomain.com/myimage", + }, + } { + t.Run(name, func(t *testing.T) { + th.App.UpdateConfig(func(cfg *model.Config) { + cfg.ServiceSettings.ImageProxyType = model.NewString(tc.ProxyType) + cfg.ServiceSettings.ImageProxyOptions = model.NewString(tc.ProxyOptions) + cfg.ServiceSettings.ImageProxyURL = model.NewString(tc.ProxyURL) + }) + + post := &model.Post{ + Id: model.NewId(), + Message: "![foo](" + tc.ImageURL + ")", + } + + list := model.NewPostList() + list.Posts[post.Id] = post + + assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostListWithProxyAddedToImageURLs(list).Posts[post.Id].Message) + assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostWithProxyAddedToImageURLs(post).Message) + + assert.Equal(t, "![foo]("+tc.ImageURL+")", th.App.PostWithProxyRemovedFromImageURLs(post).Message) + post.Message = "![foo](" + tc.ProxiedImageURL + ")" + assert.Equal(t, "![foo]("+tc.ImageURL+")", th.App.PostWithProxyRemovedFromImageURLs(post).Message) + }) + } +} + +var imageProxyBenchmarkSink *model.Post + +func BenchmarkPostWithProxyRemovedFromImageURLs(b *testing.B) { + th := Setup().InitBasic() + defer th.TearDown() + + th.App.UpdateConfig(func(cfg *model.Config) { + cfg.ServiceSettings.ImageProxyType = model.NewString("willnorris/imageproxy") + cfg.ServiceSettings.ImageProxyOptions = model.NewString("x1000|foo") + cfg.ServiceSettings.ImageProxyURL = model.NewString("https://127.0.0.1") + }) + + post := &model.Post{ + Message: "![foo](http://mydomain.com/myimage)", + } + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + imageProxyBenchmarkSink = th.App.PostWithProxyAddedToImageURLs(post) + } +} diff --git a/app/reaction.go b/app/reaction.go index bf0d20e2b..062622f34 100644 --- a/app/reaction.go +++ b/app/reaction.go @@ -62,6 +62,6 @@ func (a *App) sendReactionEvent(event string, reaction *model.Reaction, post *mo post.HasReactions = true post.UpdateAt = model.GetMillis() umessage := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil) - umessage.Add("post", post.ToJson()) + umessage.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson()) a.Publish(umessage) } diff --git a/config/default.json b/config/default.json index 6ebf29789..6474043cb 100644 --- a/config/default.json +++ b/config/default.json @@ -56,7 +56,9 @@ "EnablePreviewFeatures": true, "CloseUnusedDirectMessages": false, "EnableTutorial": true, - "ExperimentalEnableDefaultChannelLeaveJoinMessages": true + "ExperimentalEnableDefaultChannelLeaveJoinMessages": true, + "ImageProxyType": "", + "ImageProxyURL": "" }, "TeamSettings": { "SiteName": "Mattermost", diff --git a/i18n/en.json b/i18n/en.json index cf82f6644..e40d4bc67 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -4814,6 +4814,14 @@ "id": "model.config.is_valid.listen_address.app_error", "translation": "Invalid listen address for service settings Must be set." }, + { + "id": "model.config.is_valid.image_proxy_type.app_error", + "translation": "Invalid image proxy type for service settings." + }, + { + "id": "model.config.is_valid.atmos_camo_image_proxy_options.app_error", + "translation": "Invalid atmos/camo image proxy options for service settings. Must be set to your shared key." + }, { "id": "model.config.is_valid.localization.available_locales.app_error", "translation": "Available Languages must contain Default Client Language" diff --git a/model/config.go b/model/config.go index d1bb0a41c..6e3e8859e 100644 --- a/model/config.go +++ b/model/config.go @@ -214,6 +214,9 @@ type ServiceSettings struct { EnablePreviewFeatures *bool EnableTutorial *bool ExperimentalEnableDefaultChannelLeaveJoinMessages *bool + ImageProxyType *string + ImageProxyURL *string + ImageProxyOptions *string } func (s *ServiceSettings) SetDefaults() { @@ -250,7 +253,7 @@ func (s *ServiceSettings) SetDefaults() { } if s.AllowedUntrustedInternalConnections == nil { - s.AllowedUntrustedInternalConnections = new(string) + s.AllowedUntrustedInternalConnections = NewString("") } if s.EnableMultifactorAuthentication == nil { @@ -418,6 +421,18 @@ func (s *ServiceSettings) SetDefaults() { if s.ExperimentalEnableDefaultChannelLeaveJoinMessages == nil { s.ExperimentalEnableDefaultChannelLeaveJoinMessages = NewBool(true) } + + if s.ImageProxyType == nil { + s.ImageProxyType = NewString("") + } + + if s.ImageProxyURL == nil { + s.ImageProxyURL = NewString("") + } + + if s.ImageProxyOptions == nil { + s.ImageProxyOptions = NewString("") + } } type ClusterSettings struct { @@ -2050,6 +2065,16 @@ func (ss *ServiceSettings) isValid() *AppError { return NewAppError("Config.IsValid", "model.config.is_valid.listen_address.app_error", nil, "", http.StatusBadRequest) } + switch *ss.ImageProxyType { + case "", "willnorris/imageproxy": + case "atmos/camo": + if *ss.ImageProxyOptions == "" { + return NewAppError("Config.IsValid", "model.config.is_valid.atmos_camo_image_proxy_options.app_error", nil, "", http.StatusBadRequest) + } + default: + return NewAppError("Config.IsValid", "model.config.is_valid.image_proxy_type.app_error", nil, "", http.StatusBadRequest) + } + return nil } diff --git a/model/post.go b/model/post.go index 2ae8d902d..950bf401c 100644 --- a/model/post.go +++ b/model/post.go @@ -8,8 +8,11 @@ import ( "io" "net/http" "regexp" + "sort" "strings" "unicode/utf8" + + "github.com/mattermost/mattermost-server/utils/markdown" ) const ( @@ -43,18 +46,25 @@ const ( ) type Post struct { - Id string `json:"id"` - CreateAt int64 `json:"create_at"` - UpdateAt int64 `json:"update_at"` - EditAt int64 `json:"edit_at"` - DeleteAt int64 `json:"delete_at"` - IsPinned bool `json:"is_pinned"` - UserId string `json:"user_id"` - ChannelId string `json:"channel_id"` - RootId string `json:"root_id"` - ParentId string `json:"parent_id"` - OriginalId string `json:"original_id"` - Message string `json:"message"` + Id string `json:"id"` + CreateAt int64 `json:"create_at"` + UpdateAt int64 `json:"update_at"` + EditAt int64 `json:"edit_at"` + DeleteAt int64 `json:"delete_at"` + IsPinned bool `json:"is_pinned"` + UserId string `json:"user_id"` + ChannelId string `json:"channel_id"` + RootId string `json:"root_id"` + ParentId string `json:"parent_id"` + OriginalId string `json:"original_id"` + + Message string `json:"message"` + + // MessageSource will contain the message as submitted by the user if Message has been modified + // by Mattermost for presentation (e.g if an image proxy is being used). It should be used to + // populate edit boxes if present. + MessageSource string `json:"message_source,omitempty" db:"-"` + Type string `json:"type"` Props StringInterface `json:"props"` Hashtags string `json:"hashtags"` @@ -72,6 +82,14 @@ type PostPatch struct { HasReactions *bool `json:"has_reactions"` } +func (o *PostPatch) WithRewrittenImageURLs(f func(string) string) *PostPatch { + copy := *o + if copy.Message != nil { + *copy.Message = RewriteImageURLs(*o.Message, f) + } + return © +} + type PostForIndexing struct { Post TeamId string `json:"team_id"` @@ -392,3 +410,87 @@ func (o *Post) GenerateActionIds() { } } } + +var markdownDestinationEscaper = strings.NewReplacer( + `\`, `\\`, + `<`, `\<`, + `>`, `\>`, + `(`, `\(`, + `)`, `\)`, +) + +// WithRewrittenImageURLs returns a new shallow copy of the post where the message has been +// rewritten via RewriteImageURLs. +func (o *Post) WithRewrittenImageURLs(f func(string) string) *Post { + copy := *o + copy.Message = RewriteImageURLs(o.Message, f) + if copy.MessageSource == "" && copy.Message != o.Message { + copy.MessageSource = o.Message + } + return © +} + +// RewriteImageURLs takes a message and returns a copy that has all of the image URLs replaced +// according to the function f. For each image URL, f will be invoked, and the resulting markdown +// will contain the URL returned by that invocation instead. +// +// Image URLs are destination URLs used in inline images or reference definitions that are used +// anywhere in the input markdown as an image. +func RewriteImageURLs(message string, f func(string) string) string { + if !strings.Contains(message, "![") { + return message + } + + var ranges []markdown.Range + + markdown.Inspect(message, func(blockOrInline interface{}) bool { + switch v := blockOrInline.(type) { + case *markdown.ReferenceImage: + ranges = append(ranges, v.ReferenceDefinition.RawDestination) + case *markdown.InlineImage: + ranges = append(ranges, v.RawDestination) + default: + return true + } + return true + }) + + if ranges == nil { + return message + } + + sort.Slice(ranges, func(i, j int) bool { + return ranges[i].Position < ranges[j].Position + }) + + copyRanges := make([]markdown.Range, 0, len(ranges)) + urls := make([]string, 0, len(ranges)) + resultLength := len(message) + + start := 0 + for i, r := range ranges { + switch { + case i == 0: + case r.Position != ranges[i-1].Position: + start = ranges[i-1].End + default: + continue + } + original := message[r.Position:r.End] + replacement := markdownDestinationEscaper.Replace(f(markdown.Unescape(original))) + resultLength += len(replacement) - len(original) + copyRanges = append(copyRanges, markdown.Range{Position: start, End: r.Position}) + urls = append(urls, replacement) + } + + result := make([]byte, resultLength) + + offset := 0 + for i, r := range copyRanges { + offset += copy(result[offset:], message[r.Position:r.End]) + offset += copy(result[offset:], urls[i]) + } + copy(result[offset:], message[ranges[len(ranges)-1].End:]) + + return string(result) +} diff --git a/model/post_list.go b/model/post_list.go index 018f7d14f..09cddfdcf 100644 --- a/model/post_list.go +++ b/model/post_list.go @@ -21,6 +21,15 @@ func NewPostList() *PostList { } } +func (o *PostList) WithRewrittenImageURLs(f func(string) string) *PostList { + copy := *o + copy.Posts = make(map[string]*Post) + for id, post := range o.Posts { + copy.Posts[id] = post.WithRewrittenImageURLs(f) + } + return © +} + func (o *PostList) StripActionIntegrations() { posts := o.Posts o.Posts = make(map[string]*Post) diff --git a/model/post_test.go b/model/post_test.go index 6a908887d..5d5e7c9ec 100644 --- a/model/post_test.go +++ b/model/post_test.go @@ -4,6 +4,7 @@ package model import ( + "io/ioutil" "strings" "testing" @@ -173,3 +174,129 @@ func TestPostSanitizeProps(t *testing.T) { t.Fatal("should not be nil") } } + +var markdownSample, markdownSampleWithRewrittenImageURLs string + +func init() { + bytes, err := ioutil.ReadFile("testdata/markdown-sample.md") + if err != nil { + panic(err) + } + markdownSample = string(bytes) + + bytes, err = ioutil.ReadFile("testdata/markdown-sample-with-rewritten-image-urls.md") + if err != nil { + panic(err) + } + markdownSampleWithRewrittenImageURLs = string(bytes) +} + +func TestRewriteImageURLs(t *testing.T) { + for name, tc := range map[string]struct { + Markdown string + Expected string + }{ + "Empty": { + Markdown: ``, + Expected: ``, + }, + "NoImages": { + Markdown: `foo`, + Expected: `foo`, + }, + "Link": { + Markdown: `[foo](/url)`, + Expected: `[foo](/url)`, + }, + "Image": { + Markdown: `![foo](/url)`, + Expected: `![foo](rewritten:/url)`, + }, + "SpacedURL": { + Markdown: `![foo]( /url )`, + Expected: `![foo]( rewritten:/url )`, + }, + "Title": { + Markdown: `![foo](/url "title")`, + Expected: `![foo](rewritten:/url "title")`, + }, + "Parentheses": { + Markdown: `![foo](/url(1) "title")`, + Expected: `![foo](rewritten:/url\(1\) "title")`, + }, + "AngleBrackets": { + Markdown: `![foo](\\> "title")`, + Expected: `![foo](\\> "title")`, + }, + "MultipleLines": { + Markdown: `![foo]( + \\> + "title" + )`, + Expected: `![foo]( + \\> + "title" + )`, + }, + "ReferenceLink": { + Markdown: `[foo]: \\> "title" + [foo]`, + Expected: `[foo]: \\> "title" + [foo]`, + }, + "ReferenceImage": { + Markdown: `[foo]: \\> "title" + ![foo]`, + Expected: `[foo]: \\> "title" + ![foo]`, + }, + "MultipleReferenceImages": { + Markdown: `[foo]: "title" + [bar]: + [baz]: /url3 "title" + [qux]: /url4 + ![foo]![qux]`, + Expected: `[foo]: "title" + [bar]: + [baz]: /url3 "title" + [qux]: rewritten:/url4 + ![foo]![qux]`, + }, + "DuplicateReferences": { + Markdown: `[foo]: "title" + [foo]: + [foo]: /url3 "title" + [foo]: /url4 + ![foo]![foo]![foo]`, + Expected: `[foo]: "title" + [foo]: + [foo]: /url3 "title" + [foo]: /url4 + ![foo]![foo]![foo]`, + }, + "TrailingURL": { + Markdown: "![foo]\n\n[foo]: /url", + Expected: "![foo]\n\n[foo]: rewritten:/url", + }, + "Sample": { + Markdown: markdownSample, + Expected: markdownSampleWithRewrittenImageURLs, + }, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.Expected, RewriteImageURLs(tc.Markdown, func(url string) string { + return "rewritten:" + url + })) + }) + } +} + +var rewriteImageURLsSink string + +func BenchmarkRewriteImageURLs(b *testing.B) { + for i := 0; i < b.N; i++ { + rewriteImageURLsSink = RewriteImageURLs(markdownSample, func(url string) string { + return "rewritten:" + url + }) + } +} diff --git a/model/testdata/markdown-sample-with-rewritten-image-urls.md b/model/testdata/markdown-sample-with-rewritten-image-urls.md new file mode 100644 index 000000000..6683bc459 --- /dev/null +++ b/model/testdata/markdown-sample-with-rewritten-image-urls.md @@ -0,0 +1,245 @@ +--- +__Advertisement :)__ + +- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image + resize in browser. +- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly + i18n with plurals support and easy syntax. + +You will like those projects! + +--- + +# h1 Heading 8-) +## h2 Heading +### h3 Heading +#### h4 Heading +##### h5 Heading +###### h6 Heading + + +## Horizontal Rules + +___ + +--- + +*** + + +## Typographic replacements + +Enable typographer option to see result. + +(c) (C) (r) (R) (tm) (TM) (p) (P) +- + +test.. test... test..... test?..... test!.... + +!!!!!! ???? ,, -- --- + +"Smartypants, double quotes" and 'single quotes' + + +## Emphasis + +**This is bold text** + +__This is bold text__ + +*This is italic text* + +_This is italic text_ + +~~Strikethrough~~ + + +## Blockquotes + + +> Blockquotes can also be nested... +>> ...by using additional greater-than signs right next to each other... +> > > ...or with spaces between arrows. + + +## Lists + +Unordered + ++ Create a list by starting a line with `+`, `-`, or `*` ++ Sub-lists are made by indenting 2 spaces: + - Marker character change forces new list start: + * Ac tristique libero volutpat at + + Facilisis in pretium nisl aliquet + - Nulla volutpat aliquam velit ++ Very easy! + +Ordered + +1. Lorem ipsum dolor sit amet +2. Consectetur adipiscing elit +3. Integer molestie lorem at massa + + +1. You can use sequential numbers... +1. ...or keep all the numbers as `1.` + +Start numbering with offset: + +57. foo +1. bar + + +## Code + +Inline `code` + +Indented code + + // Some comments + line 1 of code + line 2 of code + line 3 of code + + +Block code "fences" + +``` +Sample text here... +``` + +Syntax highlighting + +``` js +var foo = function (bar) { + return bar++; +}; + +console.log(foo(5)); +``` + +## Tables + +| Option | Description | +| ------ | ----------- | +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + +Right aligned columns + +| Option | Description | +| ------:| -----------:| +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + + +## Links + +[link text](http://dev.nodeca.com) + +[link with title](http://nodeca.github.io/pica/demo/ "title text!") + +Autoconverted link https://github.com/nodeca/pica (enable linkify to see) + + +## Images + +![Minion](rewritten:https://octodex.github.com/images/minion.png) +![Stormtroopocat](rewritten:https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat") + +Like links, Images also have a footnote style syntax + +![Alt text][id] + +With a reference later in the document defining the URL location: + +[id]: rewritten:https://octodex.github.com/images/dojocat.jpg "The Dojocat" + + +## Plugins + +The killer feature of `markdown-it` is very effective support of +[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin). + + +### [Emojies](https://github.com/markdown-it/markdown-it-emoji) + +> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum: +> +> Shortcuts (emoticons): :-) :-( 8-) ;) + +see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji. + + +### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup) + +- 19^th^ +- H~2~O + + +### [\](https://github.com/markdown-it/markdown-it-ins) + +++Inserted text++ + + +### [\](https://github.com/markdown-it/markdown-it-mark) + +==Marked text== + + +### [Footnotes](https://github.com/markdown-it/markdown-it-footnote) + +Footnote 1 link[^first]. + +Footnote 2 link[^second]. + +Inline footnote^[Text of inline footnote] definition. + +Duplicated footnote reference[^second]. + +[^first]: Footnote **can have markup** + + and multiple paragraphs. + +[^second]: Footnote text. + + +### [Definition lists](https://github.com/markdown-it/markdown-it-deflist) + +Term 1 + +: Definition 1 +with lazy continuation. + +Term 2 with *inline markup* + +: Definition 2 + + { some code, part of Definition 2 } + + Third paragraph of definition 2. + +_Compact style:_ + +Term 1 + ~ Definition 1 + +Term 2 + ~ Definition 2a + ~ Definition 2b + + +### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr) + +This is HTML abbreviation example. + +It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on. + +*[HTML]: Hyper Text Markup Language + +### [Custom containers](https://github.com/markdown-it/markdown-it-container) + +::: warning +*here be dragons* +::: diff --git a/model/testdata/markdown-sample.md b/model/testdata/markdown-sample.md new file mode 100644 index 000000000..f894c1d35 --- /dev/null +++ b/model/testdata/markdown-sample.md @@ -0,0 +1,245 @@ +--- +__Advertisement :)__ + +- __[pica](https://nodeca.github.io/pica/demo/)__ - high quality and fast image + resize in browser. +- __[babelfish](https://github.com/nodeca/babelfish/)__ - developer friendly + i18n with plurals support and easy syntax. + +You will like those projects! + +--- + +# h1 Heading 8-) +## h2 Heading +### h3 Heading +#### h4 Heading +##### h5 Heading +###### h6 Heading + + +## Horizontal Rules + +___ + +--- + +*** + + +## Typographic replacements + +Enable typographer option to see result. + +(c) (C) (r) (R) (tm) (TM) (p) (P) +- + +test.. test... test..... test?..... test!.... + +!!!!!! ???? ,, -- --- + +"Smartypants, double quotes" and 'single quotes' + + +## Emphasis + +**This is bold text** + +__This is bold text__ + +*This is italic text* + +_This is italic text_ + +~~Strikethrough~~ + + +## Blockquotes + + +> Blockquotes can also be nested... +>> ...by using additional greater-than signs right next to each other... +> > > ...or with spaces between arrows. + + +## Lists + +Unordered + ++ Create a list by starting a line with `+`, `-`, or `*` ++ Sub-lists are made by indenting 2 spaces: + - Marker character change forces new list start: + * Ac tristique libero volutpat at + + Facilisis in pretium nisl aliquet + - Nulla volutpat aliquam velit ++ Very easy! + +Ordered + +1. Lorem ipsum dolor sit amet +2. Consectetur adipiscing elit +3. Integer molestie lorem at massa + + +1. You can use sequential numbers... +1. ...or keep all the numbers as `1.` + +Start numbering with offset: + +57. foo +1. bar + + +## Code + +Inline `code` + +Indented code + + // Some comments + line 1 of code + line 2 of code + line 3 of code + + +Block code "fences" + +``` +Sample text here... +``` + +Syntax highlighting + +``` js +var foo = function (bar) { + return bar++; +}; + +console.log(foo(5)); +``` + +## Tables + +| Option | Description | +| ------ | ----------- | +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + +Right aligned columns + +| Option | Description | +| ------:| -----------:| +| data | path to data files to supply the data that will be passed into templates. | +| engine | engine to be used for processing templates. Handlebars is the default. | +| ext | extension to be used for dest files. | + + +## Links + +[link text](http://dev.nodeca.com) + +[link with title](http://nodeca.github.io/pica/demo/ "title text!") + +Autoconverted link https://github.com/nodeca/pica (enable linkify to see) + + +## Images + +![Minion](https://octodex.github.com/images/minion.png) +![Stormtroopocat](https://octodex.github.com/images/stormtroopocat.jpg "The Stormtroopocat") + +Like links, Images also have a footnote style syntax + +![Alt text][id] + +With a reference later in the document defining the URL location: + +[id]: https://octodex.github.com/images/dojocat.jpg "The Dojocat" + + +## Plugins + +The killer feature of `markdown-it` is very effective support of +[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin). + + +### [Emojies](https://github.com/markdown-it/markdown-it-emoji) + +> Classic markup: :wink: :crush: :cry: :tear: :laughing: :yum: +> +> Shortcuts (emoticons): :-) :-( 8-) ;) + +see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji. + + +### [Subscript](https://github.com/markdown-it/markdown-it-sub) / [Superscript](https://github.com/markdown-it/markdown-it-sup) + +- 19^th^ +- H~2~O + + +### [\](https://github.com/markdown-it/markdown-it-ins) + +++Inserted text++ + + +### [\](https://github.com/markdown-it/markdown-it-mark) + +==Marked text== + + +### [Footnotes](https://github.com/markdown-it/markdown-it-footnote) + +Footnote 1 link[^first]. + +Footnote 2 link[^second]. + +Inline footnote^[Text of inline footnote] definition. + +Duplicated footnote reference[^second]. + +[^first]: Footnote **can have markup** + + and multiple paragraphs. + +[^second]: Footnote text. + + +### [Definition lists](https://github.com/markdown-it/markdown-it-deflist) + +Term 1 + +: Definition 1 +with lazy continuation. + +Term 2 with *inline markup* + +: Definition 2 + + { some code, part of Definition 2 } + + Third paragraph of definition 2. + +_Compact style:_ + +Term 1 + ~ Definition 1 + +Term 2 + ~ Definition 2a + ~ Definition 2b + + +### [Abbreviations](https://github.com/markdown-it/markdown-it-abbr) + +This is HTML abbreviation example. + +It converts "HTML", but keep intact partial entries like "xxxHTMLyyy" and so on. + +*[HTML]: Hyper Text Markup Language + +### [Custom containers](https://github.com/markdown-it/markdown-it-container) + +::: warning +*here be dragons* +::: diff --git a/utils/markdown/block_quote.go b/utils/markdown/block_quote.go new file mode 100644 index 000000000..04a324617 --- /dev/null +++ b/utils/markdown/block_quote.go @@ -0,0 +1,62 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +type BlockQuote struct { + blockBase + markdown string + + Children []Block +} + +func (b *BlockQuote) Continuation(indentation int, r Range) *continuation { + if indentation > 3 { + return nil + } + s := b.markdown[r.Position:r.End] + if s == "" || s[0] != '>' { + return nil + } + remaining := Range{r.Position + 1, r.End} + indentation, indentationBytes := countIndentation(b.markdown, remaining) + if indentation > 0 { + indentation-- + } + return &continuation{ + Indentation: indentation, + Remaining: Range{remaining.Position + indentationBytes, remaining.End}, + } +} + +func (b *BlockQuote) AddChild(openBlocks []Block) []Block { + b.Children = append(b.Children, openBlocks[0]) + return openBlocks +} + +func blockQuoteStart(markdown string, indent int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block { + if indent > 3 { + return nil + } + s := markdown[r.Position:r.End] + if s == "" || s[0] != '>' { + return nil + } + + block := &BlockQuote{ + markdown: markdown, + } + r.Position++ + if len(s) > 1 && s[1] == ' ' { + r.Position++ + } + + indent, bytes := countIndentation(markdown, r) + + ret := []Block{block} + if descendants := blockStartOrParagraph(markdown, indent, Range{r.Position + bytes, r.End}, nil, nil); descendants != nil { + block.Children = append(block.Children, descendants[0]) + ret = append(ret, descendants...) + } + return ret +} diff --git a/utils/markdown/blocks.go b/utils/markdown/blocks.go new file mode 100644 index 000000000..14972f943 --- /dev/null +++ b/utils/markdown/blocks.go @@ -0,0 +1,153 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "strings" +) + +type continuation struct { + Indentation int + Remaining Range +} + +type Block interface { + Continuation(indentation int, r Range) *continuation + AddLine(indentation int, r Range) bool + Close() + AllowsBlockStarts() bool + HasTrailingBlankLine() bool +} + +type blockBase struct{} + +func (*blockBase) AddLine(indentation int, r Range) bool { return false } +func (*blockBase) Close() {} +func (*blockBase) AllowsBlockStarts() bool { return true } +func (*blockBase) HasTrailingBlankLine() bool { return false } + +type ContainerBlock interface { + Block + AddChild(openBlocks []Block) []Block +} + +type Range struct { + Position int + End int +} + +func closeBlocks(blocks []Block, referenceDefinitions *[]*ReferenceDefinition) { + for _, block := range blocks { + block.Close() + if p, ok := block.(*Paragraph); ok && len(p.ReferenceDefinitions) > 0 { + *referenceDefinitions = append(*referenceDefinitions, p.ReferenceDefinitions...) + } + } +} + +func ParseBlocks(markdown string, lines []Line) (*Document, []*ReferenceDefinition) { + document := &Document{} + var referenceDefinitions []*ReferenceDefinition + + openBlocks := []Block{document} + + for _, line := range lines { + r := line.Range + lastMatchIndex := 0 + + indentation, indentationBytes := countIndentation(markdown, r) + r = Range{r.Position + indentationBytes, r.End} + + for i, block := range openBlocks { + if continuation := block.Continuation(indentation, r); continuation != nil { + indentation = continuation.Indentation + r = continuation.Remaining + additionalIndentation, additionalIndentationBytes := countIndentation(markdown, r) + r = Range{r.Position + additionalIndentationBytes, r.End} + indentation += additionalIndentation + lastMatchIndex = i + } else { + break + } + } + + if openBlocks[lastMatchIndex].AllowsBlockStarts() { + if newBlocks := blockStart(markdown, indentation, r, openBlocks[:lastMatchIndex+1], openBlocks[lastMatchIndex+1:]); newBlocks != nil { + didAdd := false + for i := lastMatchIndex; i >= 0; i-- { + if container, ok := openBlocks[i].(ContainerBlock); ok { + if newBlocks := container.AddChild(newBlocks); newBlocks != nil { + closeBlocks(openBlocks[i+1:], &referenceDefinitions) + openBlocks = openBlocks[:i+1] + openBlocks = append(openBlocks, newBlocks...) + didAdd = true + break + } + } + } + if didAdd { + continue + } + } + } + + isBlank := strings.TrimSpace(markdown[r.Position:r.End]) == "" + if paragraph, ok := openBlocks[len(openBlocks)-1].(*Paragraph); ok && !isBlank { + paragraph.Text = append(paragraph.Text, r) + continue + } + + closeBlocks(openBlocks[lastMatchIndex+1:], &referenceDefinitions) + openBlocks = openBlocks[:lastMatchIndex+1] + + if openBlocks[lastMatchIndex].AddLine(indentation, r) { + continue + } + + if paragraph := newParagraph(markdown, r); paragraph != nil { + for i := lastMatchIndex; i >= 0; i-- { + if container, ok := openBlocks[i].(ContainerBlock); ok { + if newBlocks := container.AddChild([]Block{paragraph}); newBlocks != nil { + closeBlocks(openBlocks[i+1:], &referenceDefinitions) + openBlocks = openBlocks[:i+1] + openBlocks = append(openBlocks, newBlocks...) + break + } + } + } + } + } + + closeBlocks(openBlocks, &referenceDefinitions) + + return document, referenceDefinitions +} + +func blockStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block { + if r.Position >= r.End { + return nil + } + + if start := blockQuoteStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil { + return start + } else if start := listStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil { + return start + } else if start := indentedCodeStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil { + return start + } else if start := fencedCodeStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil { + return start + } + + return nil +} + +func blockStartOrParagraph(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block { + if start := blockStart(markdown, indentation, r, matchedBlocks, unmatchedBlocks); start != nil { + return start + } + if paragraph := newParagraph(markdown, r); paragraph != nil { + return []Block{paragraph} + } + return nil +} diff --git a/utils/markdown/commonmark_test.go b/utils/markdown/commonmark_test.go new file mode 100644 index 000000000..0a0959030 --- /dev/null +++ b/utils/markdown/commonmark_test.go @@ -0,0 +1,1001 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCommonMarkReferenceStrings(t *testing.T) { + // For the most part, we aim for CommonMark compliance with the features that we support. We + // also support some GitHub flavored extensions. + // + // You can find most of the references used here: https://github.github.com/gfm/ + + // CommonMark handles leading tabs that aren't on 4-character boundaries differently, so the + // following reference strings will fail. The current implementation is much closer to our + // webapp's behavior though, so I'm leaving it as is for now. It doesn't really impact anything + // we use this package for anyways. + // + // " \tfoo\tbaz\t\tbim\n": "
foo\tbaz\t\tbim\n
", + // ">\t\tfoo": "
   foo
", + + for name, tc := range map[string]struct { + Markdown string + ExpectedHTML string + }{ + "0.28-gfm-1": { + Markdown: "\tfoo\tbaz\t\tbim\n", + ExpectedHTML: "
foo\tbaz\t\tbim\n
", + }, + "0.28-gfm-3": { + Markdown: " a\ta\n ὐ\ta\n", + ExpectedHTML: "
a\ta\nὐ\ta\n
", + }, + "0.28-gfm-4": { + Markdown: " - foo\n\n\tbar\n", + ExpectedHTML: "
  • foo

    bar

", + }, + "0.28-gfm-5": { + Markdown: "- foo\n\n\t\tbar", + ExpectedHTML: "
  • foo

      bar
", + }, + "0.28-gfm-8": { + Markdown: " foo\n\tbar", + ExpectedHTML: "
foo\nbar
", + }, + "0.28-gfm-9": { + Markdown: " - foo\n - bar\n\t - baz", + ExpectedHTML: "
  • foo
    • bar
      • baz
", + }, + "0.28-gfm-12": { + Markdown: "- `one\n- two`", + ExpectedHTML: "
  • `one
  • two`
", + }, + "0.28-gfm-76": { + Markdown: " a simple\n indented code block", + ExpectedHTML: "
a simple\n  indented code block
", + }, + "0.28-gfm-77": { + Markdown: " - foo\n\n bar", + ExpectedHTML: "
  • foo

    bar

", + }, + "0.28-gfm-78": { + Markdown: "1. foo\n\n - bar", + ExpectedHTML: "
  1. foo

    • bar
", + }, + "0.28-gfm-79": { + Markdown: " \n *hi*\n\n - one", + ExpectedHTML: "
<a/>\n*hi*\n\n- one
", + }, + "0.28-gfm-80": { + Markdown: " chunk1\n\n chunk2\n \n \n \n chunk3", + ExpectedHTML: "
chunk1\n\nchunk2\n\n\n\nchunk3
", + }, + "0.28-gfm-81": { + Markdown: " chunk1\n \n chunk2", + ExpectedHTML: "
chunk1\n  \n  chunk2
", + }, + "0.28-gfm-82": { + Markdown: "Foo\n bar", + ExpectedHTML: "

Foo\nbar

", + }, + "0.28-gfm-83": { + Markdown: " foo\nbar", + ExpectedHTML: "
foo\n

bar

", + }, + "0.28-gfm-85": { + Markdown: " foo\n bar", + ExpectedHTML: "
    foo\nbar
", + }, + "0.28-gfm-86": { + Markdown: "\n \n foo\n ", + ExpectedHTML: "
foo\n
", + }, + "0.28-gfm-87": { + Markdown: " foo ", + ExpectedHTML: "
foo  
", + }, + "0.28-gfm-88": { + Markdown: "```\n<\n >\n```", + ExpectedHTML: "
<\n >\n
", + }, + "0.28-gfm-89": { + Markdown: "~~~\n<\n >\n~~~", + ExpectedHTML: "
<\n >\n
", + }, + "0.28-gfm-91": { + Markdown: "```\naaa\n~~~\n```", + ExpectedHTML: "
aaa\n~~~\n
", + }, + "0.28-gfm-92": { + Markdown: "~~~\naaa\n```\n~~~", + ExpectedHTML: "
aaa\n```\n
", + }, + "0.28-gfm-93": { + Markdown: "````\naaa\n```\n``````", + ExpectedHTML: "
aaa\n```\n
", + }, + "0.28-gfm-94": { + Markdown: "~~~~\naaa\n~~~\n~~~~", + ExpectedHTML: "
aaa\n~~~\n
", + }, + "0.28-gfm-95": { + Markdown: "```", + ExpectedHTML: "
", + }, + "0.28-gfm-96": { + Markdown: "`````\n\n```\naaa", + ExpectedHTML: "
\n```\naaa
", + }, + "0.28-gfm-97": { + Markdown: "> ```\n> aaa\n\nbbb", + ExpectedHTML: "
aaa\n

bbb

", + }, + "0.28-gfm-98": { + Markdown: "```\n\n \n```", + ExpectedHTML: "
\n  \n
", + }, + "0.28-gfm-99": { + Markdown: "```\n```", + ExpectedHTML: "
", + }, + "0.28-gfm-100": { + Markdown: " ```\n aaa\naaa\n```", + ExpectedHTML: "
aaa\naaa\n
", + }, + "0.28-gfm-101": { + Markdown: " ```\naaa\n aaa\naaa\n ```", + ExpectedHTML: "
aaa\naaa\naaa\n
", + }, + "0.28-gfm-102": { + Markdown: " ```\n aaa\n aaa\n aaa\n ```", + ExpectedHTML: "
aaa\n aaa\naaa\n
", + }, + "0.28-gfm-103": { + Markdown: " ```\n aaa\n ```", + ExpectedHTML: "
```\naaa\n```
", + }, + "0.28-gfm-104": { + Markdown: "```\naaa\n ```", + ExpectedHTML: "
aaa\n
", + }, + "0.28-gfm-105": { + Markdown: " ```\naaa\n ```", + ExpectedHTML: "
aaa\n
", + }, + "0.28-gfm-106": { + Markdown: "```\naaa\n ```", + ExpectedHTML: "
aaa\n    ```
", + }, + "0.28-gfm-108": { + Markdown: "~~~~~~\naaa\n~~~ ~~", + ExpectedHTML: "
aaa\n~~~ ~~
", + }, + "0.28-gfm-109": { + Markdown: "foo\n```\nbar\n```\nbaz", + ExpectedHTML: "

foo

bar\n

baz

", + }, + "0.28-gfm-111": { + Markdown: "```ruby\ndef foo(x)\n return 3\nend\n```", + ExpectedHTML: "
def foo(x)\n  return 3\nend\n
", + }, + "0.28-gfm-112": { + Markdown: "```ruby startline=3 $%@#$\ndef foo(x)\n return 3\nend\n```", + ExpectedHTML: "
def foo(x)\n  return 3\nend\n
", + }, + "0.28-gfm-113": { + Markdown: "````;\n````", + ExpectedHTML: "
", + }, + "0.28-gfm-115": { + Markdown: "```\n``` aaa\n```", + ExpectedHTML: "
``` aaa\n
", + }, + "0.28-gfm-159": { + Markdown: "[foo]: /url \"title\"\n\n[foo]", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-160": { + Markdown: " [foo]: \n /url \n 'the title' \n\n[foo]", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-161": { + Markdown: "[Foo*bar\\]]:my_(url) 'title (with parens)'\n\n[Foo*bar\\]]", + ExpectedHTML: `

Foo*bar]

`, + }, + "0.28-gfm-162": { + Markdown: "[Foo bar]:\n\n'title'\n\n[Foo bar]", + ExpectedHTML: `

Foo bar

`, + }, + "0.28-gfm-163": { + Markdown: "[foo]: /url '\ntitle\nline1\nline2\n'\n\n[foo]", + ExpectedHTML: "

foo

", + }, + "0.28-gfm-164": { + Markdown: "[foo]: /url 'title\n\nwith blank line'\n\n[foo]", + ExpectedHTML: "

[foo]: /url 'title

with blank line'

[foo]

", + }, + "0.28-gfm-165": { + Markdown: "[foo]:\n/url\n\n[foo]", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-166": { + Markdown: "[foo]:\n\n[foo]", + ExpectedHTML: `

[foo]:

[foo]

`, + }, + "0.28-gfm-167": { + Markdown: "[foo]: /url\\bar\\*baz \"foo\\\"bar\\baz\"\n\n[foo]", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-168": { + Markdown: "[foo]\n\n[foo]: url", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-169": { + Markdown: "[foo]\n\n[foo]: first\n[foo]: second", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-170": { + Markdown: "[FOO]: /url\n\n[Foo]", + ExpectedHTML: `

Foo

`, + }, + "0.28-gfm-171": { + Markdown: "[ΑΓΩ]: /φου\n\n[αγω]", + ExpectedHTML: `

αγω

`, + }, + "0.28-gfm-172": { + Markdown: "[foo]: /url", + ExpectedHTML: ``, + }, + "0.28-gfm-173": { + Markdown: "[\nfoo\n]: /url\nbar", + ExpectedHTML: `

bar

`, + }, + "0.28-gfm-174": { + Markdown: `[foo]: /url "title" ok`, + ExpectedHTML: `

[foo]: /url "title" ok

`, + }, + "0.28-gfm-175": { + Markdown: "[foo]: /url\n\"title\" ok", + ExpectedHTML: `

"title" ok

`, + }, + "0.28-gfm-176": { + Markdown: " [foo]: /url \"title\"\n\n[foo]", + ExpectedHTML: "
[foo]: /url "title"\n

[foo]

", + }, + "0.28-gfm-177": { + Markdown: "```\n[foo]: /url\n```\n\n[foo]", + ExpectedHTML: "
[foo]: /url\n

[foo]

", + }, + "0.28-gfm-178": { + Markdown: "Foo\n[bar]: /baz\n\n[bar]", + ExpectedHTML: "

Foo\n[bar]: /baz

[bar]

", + }, + "0.28-gfm-180": { + Markdown: "[foo]: /foo-url \"foo\"\n[bar]: /bar-url\n\"bar\"\n[baz]: /baz-url\n\n[foo],\n[bar],\n[baz]", + ExpectedHTML: `

foo, +bar, +baz

`, + }, + "0.28-gfm-181": { + Markdown: "[foo]\n\n> [foo]: /url", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-182": { + Markdown: "aaa\n\nbbb", + ExpectedHTML: "

aaa

bbb

", + }, + "0.28-gfm-183": { + Markdown: "aaa\nbbb\n\nccc\nddd", + ExpectedHTML: "

aaa\nbbb

ccc\nddd

", + }, + "0.28-gfm-184": { + Markdown: "aaa\n\n\nbbb", + ExpectedHTML: "

aaa

bbb

", + }, + "0.28-gfm-185": { + Markdown: " aaa\n bbb", + ExpectedHTML: "

aaa\nbbb

", + }, + "0.28-gfm-186": { + Markdown: "aaa\n bbb\n ccc", + ExpectedHTML: "

aaa\nbbb\nccc

", + }, + "0.28-gfm-187": { + Markdown: " aaa\nbbb", + ExpectedHTML: "

aaa\nbbb

", + }, + "0.28-gfm-188": { + Markdown: " aaa\nbbb", + ExpectedHTML: "
aaa\n

bbb

", + }, + "0.28-gfm-189": { + Markdown: "aaa \nbbb \n", + ExpectedHTML: "

aaa
bbb

", + }, + "0.28-gfm-204": { + Markdown: "> bar\nbaz\n> foo", + ExpectedHTML: "

bar\nbaz\nfoo

", + }, + "0.28-gfm-206": { + Markdown: "> - foo\n- bar", + ExpectedHTML: "
  • foo
  • bar
", + }, + "0.28-gfm-207": { + Markdown: "> foo\n bar", + ExpectedHTML: "
foo\n
bar
", + }, + "0.28-gfm-208": { + Markdown: "> ```\nfoo\n```", + ExpectedHTML: "

foo

", + }, + "0.28-gfm-209": { + Markdown: "> foo\n - bar", + ExpectedHTML: "

foo\n- bar

", + }, + "0.28-gfm-210": { + Markdown: ">", + ExpectedHTML: "
", + }, + "0.28-gfm-211": { + Markdown: ">\n> \n> ", + ExpectedHTML: "
", + }, + "0.28-gfm-212": { + Markdown: ">\n> foo\n> ", + ExpectedHTML: "

foo

", + }, + "0.28-gfm-213": { + Markdown: "> foo\n\n> bar", + ExpectedHTML: "

foo

bar

", + }, + "0.28-gfm-214": { + Markdown: "> foo\n> bar", + ExpectedHTML: "

foo\nbar

", + }, + "0.28-gfm-215": { + Markdown: "> foo\n>\n> bar", + ExpectedHTML: "

foo

bar

", + }, + "0.28-gfm-216": { + Markdown: "foo\n> bar", + ExpectedHTML: "

foo

bar

", + }, + "0.28-gfm-218": { + Markdown: "> bar\nbaz", + ExpectedHTML: "

bar\nbaz

", + }, + "0.28-gfm-219": { + Markdown: "> bar\n\nbaz", + ExpectedHTML: "

bar

baz

", + }, + "0.28-gfm-220": { + Markdown: "> bar\n>\nbaz", + ExpectedHTML: "

bar

baz

", + }, + "0.28-gfm-221": { + Markdown: "> > > foo\nbar", + ExpectedHTML: "

foo\nbar

", + }, + "0.28-gfm-222": { + Markdown: ">>> foo\n> bar\n>>baz", + ExpectedHTML: "

foo\nbar\nbaz

", + }, + "0.28-gfm-223": { + Markdown: "> code\n\n> not code", + ExpectedHTML: "
code\n

not code

", + }, + "0.28-gfm-224": { + Markdown: "A paragraph\nwith two lines.\n\n indented code\n\n> A block quote.", + ExpectedHTML: "

A paragraph\nwith two lines.

indented code\n

A block quote.

", + }, + "0.28-gfm-225": { + Markdown: "1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.", + ExpectedHTML: "
  1. A paragraph\nwith two lines.

    indented code\n

    A block quote.

", + }, + "0.28-gfm-226": { + Markdown: "- one\n\n two", + ExpectedHTML: "
  • one

two

", + }, + "0.28-gfm-227": { + Markdown: "- one\n\n two", + ExpectedHTML: "
  • one

    two

", + }, + "0.28-gfm-228": { + Markdown: " - one\n\n two", + ExpectedHTML: "
  • one
 two
", + }, + "0.28-gfm-229": { + Markdown: " - one\n\n two", + ExpectedHTML: "
  • one

    two

", + }, + "0.28-gfm-230": { + Markdown: " > > 1. one\n>>\n>> two", + ExpectedHTML: "
  1. one

    two

", + }, + "0.28-gfm-231": { + Markdown: ">>- one\n>>\n > > two", + ExpectedHTML: "
  • one

two

", + }, + "0.28-gfm-232": { + Markdown: "-one\n\n2.two", + ExpectedHTML: "

-one

2.two

", + }, + "0.28-gfm-233": { + Markdown: "- foo\n\n\n bar", + ExpectedHTML: "
  • foo

    bar

", + }, + "0.28-gfm-234": { + Markdown: "1. foo\n\n ```\n bar\n ```\n\n baz\n\n > bam", + ExpectedHTML: "
  1. foo

    bar\n

    baz

    bam

", + }, + "0.28-gfm-235": { + Markdown: "- Foo\n\n bar\n\n\n baz", + ExpectedHTML: "
  • Foo

    bar\n\n\nbaz
", + }, + "0.28-gfm-236": { + Markdown: "123456789. ok", + ExpectedHTML: `
  1. ok
`, + }, + "0.28-gfm-237": { + Markdown: "1234567890. not ok", + ExpectedHTML: "

1234567890. not ok

", + }, + "0.28-gfm-238": { + Markdown: "0. ok", + ExpectedHTML: `
  1. ok
`, + }, + "0.28-gfm-239": { + Markdown: "003. ok", + ExpectedHTML: `
  1. ok
`, + }, + "0.28-gfm-240": { + Markdown: "-1. not ok", + ExpectedHTML: "

-1. not ok

", + }, + "0.28-gfm-241": { + Markdown: "- foo\n\n bar", + ExpectedHTML: "
  • foo

    bar
", + }, + "0.28-gfm-242": { + Markdown: " 10. foo\n\n bar", + ExpectedHTML: `
  1. foo

    bar
`, + }, + "0.28-gfm-243": { + Markdown: " indented code\n\nparagraph\n\n more code", + ExpectedHTML: "
indented code\n

paragraph

more code
", + }, + "0.28-gfm-244": { + Markdown: "1. indented code\n\n paragraph\n\n more code", + ExpectedHTML: "
  1. indented code\n

    paragraph

    more code
", + }, + "0.28-gfm-245": { + Markdown: "1. indented code\n\n paragraph\n\n more code", + ExpectedHTML: "
  1.  indented code\n

    paragraph

    more code
", + }, + "0.28-gfm-246": { + Markdown: " foo\n\nbar", + ExpectedHTML: "

foo

bar

", + }, + "0.28-gfm-247": { + Markdown: "- foo\n\n bar", + ExpectedHTML: "
  • foo

bar

", + }, + "0.28-gfm-248": { + Markdown: "- foo\n\n bar", + ExpectedHTML: "
  • foo

    bar

", + }, + "0.28-gfm-249": { + Markdown: "-\n foo\n-\n ```\n bar\n ```\n-\n baz", + ExpectedHTML: "
  • foo
  • bar\n
  • baz
", + }, + "0.28-gfm-250": { + Markdown: "- \n foo", + ExpectedHTML: "
  • foo
", + }, + "0.28-gfm-251": { + Markdown: "-\n\n foo", + ExpectedHTML: "

foo

", + }, + "0.28-gfm-252": { + Markdown: "- foo\n-\n- bar", + ExpectedHTML: "
  • foo
  • bar
", + }, + "0.28-gfm-253": { + Markdown: "- foo\n- \n- bar", + ExpectedHTML: "
  • foo
  • bar
", + }, + "0.28-gfm-254": { + Markdown: "1. foo\n2.\n3. bar", + ExpectedHTML: "
  1. foo
  2. bar
", + }, + "0.28-gfm-255": { + Markdown: "*", + ExpectedHTML: "
", + }, + "0.28-gfm-256": { + Markdown: "foo\n*\n\nfoo\n1.", + ExpectedHTML: "

foo\n*

foo\n1.

", + }, + "0.28-gfm-257": { + Markdown: " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.", + ExpectedHTML: "
  1. A paragraph\nwith two lines.

    indented code\n

    A block quote.

", + }, + "0.28-gfm-258": { + Markdown: " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.", + ExpectedHTML: "
  1. A paragraph\nwith two lines.

    indented code\n

    A block quote.

", + }, + "0.28-gfm-259": { + Markdown: " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.", + ExpectedHTML: "
  1. A paragraph\nwith two lines.

    indented code\n

    A block quote.

", + }, + "0.28-gfm-260": { + Markdown: " 1. A paragraph\n with two lines.\n\n indented code\n\n > A block quote.", + ExpectedHTML: "
1.  A paragraph\n    with two lines.\n\n        indented code\n\n    > A block quote.
", + }, + "0.28-gfm-261": { + Markdown: " 1. A paragraph\nwith two lines.\n\n indented code\n\n > A block quote.", + ExpectedHTML: "
  1. A paragraph\nwith two lines.

    indented code\n

    A block quote.

", + }, + "0.28-gfm-262": { + Markdown: " 1. A paragraph\n with two lines.", + ExpectedHTML: "
  1. A paragraph\nwith two lines.
", + }, + "0.28-gfm-263": { + Markdown: "> 1. > Blockquote\ncontinued here.", + ExpectedHTML: "
  1. Blockquote\ncontinued here.

", + }, + "0.28-gfm-264": { + Markdown: "> 1. > Blockquote\n> continued here.", + ExpectedHTML: "
  1. Blockquote\ncontinued here.

", + }, + "0.28-gfm-265": { + Markdown: "- foo\n - bar\n - baz\n - boo", + ExpectedHTML: "
  • foo
    • bar
      • baz
        • boo
", + }, + "0.28-gfm-266": { + Markdown: "- foo\n - bar\n - baz\n - boo", + ExpectedHTML: "
  • foo
  • bar
  • baz
  • boo
", + }, + "0.28-gfm-267": { + Markdown: "10) foo\n - bar", + ExpectedHTML: `
  1. foo
    • bar
`, + }, + "0.28-gfm-268": { + Markdown: "10) foo\n - bar", + ExpectedHTML: `
  1. foo
  • bar
`, + }, + "0.28-gfm-269": { + Markdown: "- - foo", + ExpectedHTML: "
    • foo
", + }, + "0.28-gfm-270": { + Markdown: "1. - 2. foo", + ExpectedHTML: `
      1. foo
`, + }, + "0.28-gfm-274": { + Markdown: "- foo\n- bar\n+ baz", + ExpectedHTML: "
  • foo
  • bar
  • baz
", + }, + "0.28-gfm-275": { + Markdown: "1. foo\n2. bar\n3) baz", + ExpectedHTML: `
  1. foo
  2. bar
  1. baz
`, + }, + "0.28-gfm-276": { + Markdown: "Foo\n- bar\n- baz", + ExpectedHTML: "

Foo

  • bar
  • baz
", + }, + "0.28-gfm-277": { + Markdown: "The number of windows in my house is\n14. The number of doors is 6.", + ExpectedHTML: "

The number of windows in my house is\n14. The number of doors is 6.

", + }, + "0.28-gfm-278": { + Markdown: "The number of windows in my house is\n1. The number of doors is 6.", + ExpectedHTML: "

The number of windows in my house is

  1. The number of doors is 6.
", + }, + "0.28-gfm-279": { + Markdown: "- foo\n\n- bar\n\n\n- baz", + ExpectedHTML: "
  • foo

  • bar

  • baz

", + }, + "0.28-gfm-280": { + Markdown: "- foo\n - bar\n - baz\n\n\n bim", + ExpectedHTML: "
  • foo
    • bar
      • baz

        bim

", + }, + "0.28-gfm-283": { + Markdown: "- a\n - b\n - c\n - d\n - e\n - f\n - g\n - h\n- i", + ExpectedHTML: "
  • a
  • b
  • c
  • d
  • e
  • f
  • g
  • h
  • i
", + }, + "0.28-gfm-284": { + Markdown: "1. a\n\n 2. b\n\n 3. c", + ExpectedHTML: "
  1. a

  2. b

  3. c

", + }, + "0.28-gfm-285": { + Markdown: "- a\n- b\n\n- c", + ExpectedHTML: "
  • a

  • b

  • c

", + }, + "0.28-gfm-286": { + Markdown: "* a\n*\n\n* c", + ExpectedHTML: "
  • a

  • c

", + }, + "0.28-gfm-287": { + Markdown: "- a\n- b\n\n c\n- d", + ExpectedHTML: "
  • a

  • b

    c

  • d

", + }, + "0.28-gfm-288": { + Markdown: "- a\n- b\n\n [ref]: /url\n- d", + ExpectedHTML: "
  • a

  • b

  • d

", + }, + "0.28-gfm-289": { + Markdown: "- a\n- ```\n b\n\n\n ```\n- c", + ExpectedHTML: "
  • a
  • b\n\n\n
  • c
", + }, + "0.28-gfm-290": { + Markdown: "- a\n - b\n\n c\n- d", + ExpectedHTML: "
  • a
    • b

      c

  • d
", + }, + "0.28-gfm-291": { + Markdown: "* a\n > b\n >\n* c", + ExpectedHTML: "
  • a

    b

  • c
", + }, + "0.28-gfm-292": { + Markdown: "- a\n > b\n ```\n c\n ```\n- d", + ExpectedHTML: "
  • a

    b

    c\n
  • d
", + }, + "0.28-gfm-293": { + Markdown: "- a", + ExpectedHTML: "
  • a
", + }, + "0.28-gfm-294": { + Markdown: "- a\n - b", + ExpectedHTML: "
  • a
    • b
", + }, + "0.28-gfm-295": { + Markdown: "1. ```\n foo\n ```\n\n bar", + ExpectedHTML: "
  1. foo\n

    bar

", + }, + "0.28-gfm-296": { + Markdown: "* foo\n * bar\n\n baz", + ExpectedHTML: "
  • foo

    • bar

    baz

", + }, + "0.28-gfm-297": { + Markdown: "- a\n - b\n - c\n\n- d\n - e\n - f", + ExpectedHTML: "
  • a

    • b
    • c
  • d

    • e
    • f
", + }, + "0.28-gfm-298": { + Markdown: "`hi`lo`", + ExpectedHTML: "

hilo`

", + }, + "0.28-gfm-299": { + Markdown: `\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_` + "\\`" + `\{\|\}\~`, + ExpectedHTML: "

!"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~

", + }, + "0.28-gfm-300": { + Markdown: `\→\A\a\ \3\φ\«`, + ExpectedHTML: `

\→\A\a\ \3\φ\«

`, + }, + "0.28-gfm-301": { + Markdown: `\*not emphasized* +\
not a tag +\[not a link](/foo) +\` + "`not code`" + ` +1\. not a list +\* not a list +\# not a heading +\[foo]: /url "not a reference"`, + ExpectedHTML: `

*not emphasized* +<br/> not a tag +[not a link](/foo) +` + "`not code`" + ` +1. not a list +* not a list +# not a heading +[foo]: /url "not a reference"

`, + }, + "0.28-gfm-304": { + Markdown: "`` \\[\\` ``", + ExpectedHTML: "

\\[\\`

", + }, + "0.28-gfm-305": { + Markdown: ` \[\]`, + ExpectedHTML: `
\[\]
`, + }, + "0.28-gfm-306": { + Markdown: "~~~\n\\[\\]\n~~~", + ExpectedHTML: "
\\[\\]\n
", + }, + "0.28-gfm-309": { + Markdown: `[foo](/bar\* "ti\*tle")`, + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-310": { + Markdown: `[foo] + +[foo]: /bar\* "ti\*tle"`, + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-311": { + Markdown: "``` foo\\+bar\nfoo\n```", + ExpectedHTML: "
foo\n
", + }, + "0.28-gfm-312": { + Markdown: "  & © Æ Ď\n¾ ℋ ⅆ\n∲ ≧̸", + ExpectedHTML: "

\u00a0 & © Æ Ď\n¾ ℋ ⅆ\n∲ ≧̸

", + }, + "0.28-gfm-313": { + Markdown: "# Ӓ Ϡ � �", + ExpectedHTML: "

# Ӓ Ϡ � �

", + }, + "0.28-gfm-314": { + Markdown: "" ആ ಫ", + ExpectedHTML: "

" ആ ಫ

", + }, + "0.28-gfm-315": { + Markdown: "  &x; &#; &#x;\n&ThisIsNotDefined; &hi?;", + ExpectedHTML: "

&nbsp &x; &#; &#x;\n&ThisIsNotDefined; &hi?;

", + }, + "0.28-gfm-316": { + Markdown: "©", + ExpectedHTML: "

&copy

", + }, + "0.28-gfm-317": { + Markdown: "&MadeUpEntity;", + ExpectedHTML: "

&MadeUpEntity;

", + }, + "0.28-gfm-319": { + Markdown: `[foo](/föö "föö")`, + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-320": { + Markdown: "[foo]\n\n[foo]: /föö \"föö\"", + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-321": { + Markdown: "``` föö\nfoo\n```", + ExpectedHTML: "
foo\n
", + }, + "0.28-gfm-322": { + Markdown: "`föö`", + ExpectedHTML: "

f&ouml;&ouml;

", + }, + "0.28-gfm-323": { + Markdown: " föfö", + ExpectedHTML: "
f&ouml;f&ouml;
", + }, + "0.28-gfm-324": { + Markdown: "`foo`", + ExpectedHTML: "

foo

", + }, + "0.28-gfm-325": { + Markdown: "`` foo ` bar ``", + ExpectedHTML: "

foo ` bar

", + }, + "0.28-gfm-326": { + Markdown: "` `` `", + ExpectedHTML: "

``

", + }, + "0.28-gfm-327": { + Markdown: "``\nfoo\n``", + ExpectedHTML: "

foo

", + }, + "0.28-gfm-328": { + Markdown: "`foo bar\n baz`", + ExpectedHTML: "

foo bar baz

", + }, + "0.28-gfm-329": { + Markdown: "`a\xa0\xa0b`", + ExpectedHTML: "

a\xa0\xa0b

", + }, + "0.28-gfm-330": { + Markdown: "`foo `` bar`", + ExpectedHTML: "

foo `` bar

", + }, + "0.28-gfm-331": { + Markdown: "`foo\\`bar`", + ExpectedHTML: "

foo\\bar`

", + }, + "0.28-gfm-332": { + Markdown: "*foo`*`", + ExpectedHTML: "

*foo*

", + }, + "0.28-gfm-333": { + Markdown: "[not a `link](/foo`)", + ExpectedHTML: "

[not a link](/foo)

", + }, + "0.28-gfm-334": { + Markdown: "``", + ExpectedHTML: "

<a href="">`

", + }, + "0.28-gfm-336": { + Markdown: "``", + ExpectedHTML: "

<http://foo.bar.baz>`

", + }, + "0.28-gfm-338": { + Markdown: "```foo``", + ExpectedHTML: "

```foo``

", + }, + "0.28-gfm-339": { + Markdown: "`foo", + ExpectedHTML: "

`foo

", + }, + "0.28-gfm-340": { + Markdown: "`foo``bar``", + ExpectedHTML: "

`foobar

", + }, + "0.28-gfm-472": { + Markdown: `[link](/uri "title")`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-473": { + Markdown: `[link](/uri)`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-474": { + Markdown: `[link]()`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-475": { + Markdown: `[link](<>)`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-476": { + Markdown: `[link](/my uri)`, + ExpectedHTML: `

[link](/my uri)

`, + }, + "0.28-gfm-477": { + Markdown: `[link]()`, + ExpectedHTML: `

[link](</my uri>)

`, + }, + "0.28-gfm-478": { + Markdown: "[link](foo\nbar)", + ExpectedHTML: "

[link](foo\nbar)

", + }, + "0.28-gfm-480": { + Markdown: `[link](\(foo\))`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-481": { + Markdown: `[link](foo(and(bar)))`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-482": { + Markdown: `[link](foo\(and\(bar\))`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-483": { + Markdown: `[link]()`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-484": { + Markdown: `[link](foo\)\:)`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-485": { + Markdown: "[link](#fragment)\n\n[link](http://example.com#fragment)\n\n[link](http://example.com?foo=3#frag)", + ExpectedHTML: `

link

link

link

`, + }, + "0.28-gfm-486": { + Markdown: `[link](foo\bar)`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-488": { + Markdown: `[link]("title")`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-489": { + Markdown: "[link](/url \"title\")\n[link](/url 'title')\n[link](/url (title))", + ExpectedHTML: "

link\nlink\nlink

", + }, + "0.28-gfm-490": { + Markdown: `[link](/url "title \""")`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-491": { + Markdown: "[link](/url\u00a0\"title\")", + ExpectedHTML: `

link

`, + }, + "0.28-gfm-492": { + Markdown: `[link](/url "title "and" title")`, + ExpectedHTML: `

[link](/url "title "and" title")

`, + }, + "0.28-gfm-493": { + Markdown: `[link](/url 'title "and" title')`, + ExpectedHTML: `

link

`, + }, + "0.28-gfm-494": { + Markdown: "[link]( /uri\n \"title\" )", + ExpectedHTML: `

link

`, + }, + "0.28-gfm-495": { + Markdown: "[link] (/uri)", + ExpectedHTML: `

[link] (/uri)

`, + }, + "0.28-gfm-496": { + Markdown: "[link [foo [bar]]](/uri)", + ExpectedHTML: `

link [foo [bar]]

`, + }, + "0.28-gfm-497": { + Markdown: "[link] bar](/uri)", + ExpectedHTML: `

[link] bar](/uri)

`, + }, + "0.28-gfm-498": { + Markdown: "[link [bar](/uri)", + ExpectedHTML: `

[link bar

`, + }, + "0.28-gfm-499": { + Markdown: `[link \[bar](/uri)`, + ExpectedHTML: `

link [bar

`, + }, + "0.28-gfm-501": { + Markdown: "[![moon](moon.jpg)](/uri)", + ExpectedHTML: `

moon

`, + }, + "0.28-gfm-502": { + Markdown: "[foo [bar](/uri)](/uri)", + ExpectedHTML: `

[foo bar](/uri)

`, + }, + "0.28-gfm-504": { + Markdown: "![[[foo](uri1)](uri2)](uri3)", + ExpectedHTML: `

[foo](uri2)

`, + }, + "0.28-gfm-505": { + Markdown: "*[foo*](/uri)", + ExpectedHTML: `

*foo*

`, + }, + "0.28-gfm-506": { + Markdown: "[foo *bar](baz*)", + ExpectedHTML: `

foo *bar

`, + }, + "0.28-gfm-509": { + Markdown: "[foo`](/uri)`", + ExpectedHTML: `

[foo](/uri)

`, + }, + "0.28-gfm-556": { + Markdown: `![foo](/url "title")`, + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-558": { + Markdown: `![foo ![bar](/url)](/url2)`, + ExpectedHTML: `

foo bar

`, + }, + "0.28-gfm-559": { + Markdown: `![foo [bar](/url)](/url2)`, + ExpectedHTML: `

foo bar

`, + }, + "0.28-gfm-562": { + Markdown: `![foo](train.jpg)`, + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-563": { + Markdown: `My ![foo bar](/path/to/train.jpg "title" )`, + ExpectedHTML: `

My foo bar

`, + }, + "0.28-gfm-564": { + Markdown: `![foo]()`, + ExpectedHTML: `

foo

`, + }, + "0.28-gfm-565": { + Markdown: `![](/url)`, + ExpectedHTML: `

`, + }, + "0.28-gfm-647": { + Markdown: "hello $.;'there", + ExpectedHTML: "

hello $.;'there

", + }, + "0.28-gfm-648": { + Markdown: "Foo χρῆν", + ExpectedHTML: "

Foo χρῆν

", + }, + "0.28-gfm-649": { + Markdown: "Multiple spaces", + ExpectedHTML: "

Multiple spaces

", + }, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.ExpectedHTML, RenderHTML(tc.Markdown)) + }) + } +} diff --git a/utils/markdown/document.go b/utils/markdown/document.go new file mode 100644 index 000000000..224b5d215 --- /dev/null +++ b/utils/markdown/document.go @@ -0,0 +1,22 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +type Document struct { + blockBase + + Children []Block +} + +func (b *Document) Continuation(indentation int, r Range) *continuation { + return &continuation{ + Indentation: indentation, + Remaining: r, + } +} + +func (b *Document) AddChild(openBlocks []Block) []Block { + b.Children = append(b.Children, openBlocks[0]) + return openBlocks +} diff --git a/utils/markdown/fenced_code.go b/utils/markdown/fenced_code.go new file mode 100644 index 000000000..8b2ebd4fe --- /dev/null +++ b/utils/markdown/fenced_code.go @@ -0,0 +1,112 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "strings" +) + +type FencedCodeLine struct { + Indentation int + Range Range +} + +type FencedCode struct { + blockBase + markdown string + didSeeClosingFence bool + + Indentation int + OpeningFence Range + RawInfo Range + RawCode []FencedCodeLine +} + +func (b *FencedCode) Code() (result string) { + for _, code := range b.RawCode { + result += strings.Repeat(" ", code.Indentation) + b.markdown[code.Range.Position:code.Range.End] + } + return +} + +func (b *FencedCode) Info() string { + return Unescape(b.markdown[b.RawInfo.Position:b.RawInfo.End]) +} + +func (b *FencedCode) Continuation(indentation int, r Range) *continuation { + if b.didSeeClosingFence { + return nil + } + return &continuation{ + Indentation: indentation, + Remaining: r, + } +} + +func (b *FencedCode) AddLine(indentation int, r Range) bool { + s := b.markdown[r.Position:r.End] + if indentation <= 3 && strings.HasPrefix(s, b.markdown[b.OpeningFence.Position:b.OpeningFence.End]) { + suffix := strings.TrimSpace(s[b.OpeningFence.End-b.OpeningFence.Position:]) + isClosingFence := true + for _, c := range suffix { + if c != rune(s[0]) { + isClosingFence = false + break + } + } + if isClosingFence { + b.didSeeClosingFence = true + return true + } + } + + if indentation >= b.Indentation { + indentation -= b.Indentation + } else { + indentation = 0 + } + + b.RawCode = append(b.RawCode, FencedCodeLine{ + Indentation: indentation, + Range: r, + }) + return true +} + +func (b *FencedCode) AllowsBlockStarts() bool { + return false +} + +func fencedCodeStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block { + s := markdown[r.Position:r.End] + + if !strings.HasPrefix(s, "```") && !strings.HasPrefix(s, "~~~") { + return nil + } + + fenceCharacter := rune(s[0]) + fenceLength := 3 + for _, c := range s[3:] { + if c == fenceCharacter { + fenceLength++ + } else { + break + } + } + + for i := r.Position + fenceLength; i < r.End; i++ { + if markdown[i] == '`' { + return nil + } + } + + return []Block{ + &FencedCode{ + markdown: markdown, + Indentation: indentation, + RawInfo: trimRightSpace(markdown, Range{r.Position + fenceLength, r.End}), + OpeningFence: Range{r.Position, r.Position + fenceLength}, + }, + } +} diff --git a/utils/markdown/html.go b/utils/markdown/html.go new file mode 100644 index 000000000..8d8e02c55 --- /dev/null +++ b/utils/markdown/html.go @@ -0,0 +1,186 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "fmt" + "strings" +) + +var htmlEscaper = strings.NewReplacer( + `&`, "&", + `<`, "<", + `>`, ">", + `"`, """, +) + +// RenderHTML produces HTML with the same behavior as the example renderer used in the CommonMark +// reference materials except for one slight difference: for brevity, no unnecessary whitespace is +// inserted between elements. The output is not defined by the CommonMark spec, and it exists +// primarily as an aid in testing. +func RenderHTML(markdown string) string { + return RenderBlockHTML(Parse(markdown)) +} + +func RenderBlockHTML(block Block, referenceDefinitions []*ReferenceDefinition) (result string) { + return renderBlockHTML(block, referenceDefinitions, false) +} + +func renderBlockHTML(block Block, referenceDefinitions []*ReferenceDefinition, isTightList bool) (result string) { + switch v := block.(type) { + case *Document: + for _, block := range v.Children { + result += RenderBlockHTML(block, referenceDefinitions) + } + case *Paragraph: + if len(v.Text) == 0 { + return + } + if !isTightList { + result += "

" + } + for _, inline := range v.ParseInlines(referenceDefinitions) { + result += RenderInlineHTML(inline) + } + if !isTightList { + result += "

" + } + case *List: + if v.IsOrdered { + if v.OrderedStart != 1 { + result += fmt.Sprintf(`
    `, v.OrderedStart) + } else { + result += "
      " + } + } else { + result += "
        " + } + for _, block := range v.Children { + result += renderBlockHTML(block, referenceDefinitions, !v.IsLoose) + } + if v.IsOrdered { + result += "
    " + } else { + result += "" + } + case *ListItem: + result += "
  1. " + for _, block := range v.Children { + result += renderBlockHTML(block, referenceDefinitions, isTightList) + } + result += "
  2. " + case *BlockQuote: + result += "
    " + for _, block := range v.Children { + result += RenderBlockHTML(block, referenceDefinitions) + } + result += "
    " + case *FencedCode: + if info := v.Info(); info != "" { + language := strings.Fields(info)[0] + result += `
    `
    +		} else {
    +			result += "
    "
    +		}
    +		result += htmlEscaper.Replace(v.Code()) + "
    " + case *IndentedCode: + result += "
    " + htmlEscaper.Replace(v.Code()) + "
    " + default: + panic(fmt.Sprintf("missing case for type %T", v)) + } + return +} + +func escapeURL(url string) (result string) { + for i := 0; i < len(url); { + switch b := url[i]; b { + case ';', '/', '?', ':', '@', '&', '=', '+', '$', ',', '-', '_', '.', '!', '~', '*', '\'', '(', ')', '#': + result += string(b) + i++ + default: + if b == '%' && i+2 < len(url) && isHexByte(url[i+1]) && isHexByte(url[i+2]) { + result += url[i : i+3] + i += 3 + } else if (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') { + result += string(b) + i++ + } else { + result += fmt.Sprintf("%%%0X", b) + i++ + } + } + } + return +} + +func RenderInlineHTML(inline Inline) (result string) { + switch v := inline.(type) { + case *Text: + return htmlEscaper.Replace(v.Text) + case *HardLineBreak: + return "
    " + case *SoftLineBreak: + return "\n" + case *CodeSpan: + return "" + htmlEscaper.Replace(v.Code) + "" + case *InlineImage: + result += `` + htmlEscaper.Replace(renderImageAltText(v.Children)) + `` + case *ReferenceImage: + result += `` + htmlEscaper.Replace(renderImageAltText(v.Children)) + `` + case *InlineLink: + result += `` + for _, inline := range v.Children { + result += RenderInlineHTML(inline) + } + result += "" + case *ReferenceLink: + result += `` + for _, inline := range v.Children { + result += RenderInlineHTML(inline) + } + result += "" + default: + panic(fmt.Sprintf("missing case for type %T", v)) + } + return +} + +func renderImageAltText(children []Inline) (result string) { + for _, inline := range children { + result += renderImageChildAltText(inline) + } + return +} + +func renderImageChildAltText(inline Inline) (result string) { + switch v := inline.(type) { + case *Text: + return v.Text + case *InlineImage: + for _, inline := range v.Children { + result += renderImageChildAltText(inline) + } + case *InlineLink: + for _, inline := range v.Children { + result += renderImageChildAltText(inline) + } + } + return +} diff --git a/utils/markdown/html_entities.go b/utils/markdown/html_entities.go new file mode 100644 index 000000000..797741de7 --- /dev/null +++ b/utils/markdown/html_entities.go @@ -0,0 +1,2132 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +var htmlEntities = map[string]string{ + "AElig": "\u00C6", + "AMP": "\u0026", + "Aacute": "\u00C1", + "Abreve": "\u0102", + "Acirc": "\u00C2", + "Acy": "\u0410", + "Afr": "\U0001D504", + "Agrave": "\u00C0", + "Alpha": "\u0391", + "Amacr": "\u0100", + "And": "\u2A53", + "Aogon": "\u0104", + "Aopf": "\U0001D538", + "ApplyFunction": "\u2061", + "Aring": "\u00C5", + "Ascr": "\U0001D49C", + "Assign": "\u2254", + "Atilde": "\u00C3", + "Auml": "\u00C4", + "Backslash": "\u2216", + "Barv": "\u2AE7", + "Barwed": "\u2306", + "Bcy": "\u0411", + "Because": "\u2235", + "Bernoullis": "\u212C", + "Beta": "\u0392", + "Bfr": "\U0001D505", + "Bopf": "\U0001D539", + "Breve": "\u02D8", + "Bscr": "\u212C", + "Bumpeq": "\u224E", + "CHcy": "\u0427", + "COPY": "\u00A9", + "Cacute": "\u0106", + "Cap": "\u22D2", + "CapitalDifferentialD": "\u2145", + "Cayleys": "\u212D", + "Ccaron": "\u010C", + "Ccedil": "\u00C7", + "Ccirc": "\u0108", + "Cconint": "\u2230", + "Cdot": "\u010A", + "Cedilla": "\u00B8", + "CenterDot": "\u00B7", + "Cfr": "\u212D", + "Chi": "\u03A7", + "CircleDot": "\u2299", + "CircleMinus": "\u2296", + "CirclePlus": "\u2295", + "CircleTimes": "\u2297", + "ClockwiseContourIntegral": "\u2232", + "CloseCurlyDoubleQuote": "\u201D", + "CloseCurlyQuote": "\u2019", + "Colon": "\u2237", + "Colone": "\u2A74", + "Congruent": "\u2261", + "Conint": "\u222F", + "ContourIntegral": "\u222E", + "Copf": "\u2102", + "Coproduct": "\u2210", + "CounterClockwiseContourIntegral": "\u2233", + "Cross": "\u2A2F", + "Cscr": "\U0001D49E", + "Cup": "\u22D3", + "CupCap": "\u224D", + "DD": "\u2145", + "DDotrahd": "\u2911", + "DJcy": "\u0402", + "DScy": "\u0405", + "DZcy": "\u040F", + "Dagger": "\u2021", + "Darr": "\u21A1", + "Dashv": "\u2AE4", + "Dcaron": "\u010E", + "Dcy": "\u0414", + "Del": "\u2207", + "Delta": "\u0394", + "Dfr": "\U0001D507", + "DiacriticalAcute": "\u00B4", + "DiacriticalDot": "\u02D9", + "DiacriticalDoubleAcute": "\u02DD", + "DiacriticalGrave": "\u0060", + "DiacriticalTilde": "\u02DC", + "Diamond": "\u22C4", + "DifferentialD": "\u2146", + "Dopf": "\U0001D53B", + "Dot": "\u00A8", + "DotDot": "\u20DC", + "DotEqual": "\u2250", + "DoubleContourIntegral": "\u222F", + "DoubleDot": "\u00A8", + "DoubleDownArrow": "\u21D3", + "DoubleLeftArrow": "\u21D0", + "DoubleLeftRightArrow": "\u21D4", + "DoubleLeftTee": "\u2AE4", + "DoubleLongLeftArrow": "\u27F8", + "DoubleLongLeftRightArrow": "\u27FA", + "DoubleLongRightArrow": "\u27F9", + "DoubleRightArrow": "\u21D2", + "DoubleRightTee": "\u22A8", + "DoubleUpArrow": "\u21D1", + "DoubleUpDownArrow": "\u21D5", + "DoubleVerticalBar": "\u2225", + "DownArrow": "\u2193", + "DownArrowBar": "\u2913", + "DownArrowUpArrow": "\u21F5", + "DownBreve": "\u0311", + "DownLeftRightVector": "\u2950", + "DownLeftTeeVector": "\u295E", + "DownLeftVector": "\u21BD", + "DownLeftVectorBar": "\u2956", + "DownRightTeeVector": "\u295F", + "DownRightVector": "\u21C1", + "DownRightVectorBar": "\u2957", + "DownTee": "\u22A4", + "DownTeeArrow": "\u21A7", + "Downarrow": "\u21D3", + "Dscr": "\U0001D49F", + "Dstrok": "\u0110", + "ENG": "\u014A", + "ETH": "\u00D0", + "Eacute": "\u00C9", + "Ecaron": "\u011A", + "Ecirc": "\u00CA", + "Ecy": "\u042D", + "Edot": "\u0116", + "Efr": "\U0001D508", + "Egrave": "\u00C8", + "Element": "\u2208", + "Emacr": "\u0112", + "EmptySmallSquare": "\u25FB", + "EmptyVerySmallSquare": "\u25AB", + "Eogon": "\u0118", + "Eopf": "\U0001D53C", + "Epsilon": "\u0395", + "Equal": "\u2A75", + "EqualTilde": "\u2242", + "Equilibrium": "\u21CC", + "Escr": "\u2130", + "Esim": "\u2A73", + "Eta": "\u0397", + "Euml": "\u00CB", + "Exists": "\u2203", + "ExponentialE": "\u2147", + "Fcy": "\u0424", + "Ffr": "\U0001D509", + "FilledSmallSquare": "\u25FC", + "FilledVerySmallSquare": "\u25AA", + "Fopf": "\U0001D53D", + "ForAll": "\u2200", + "Fouriertrf": "\u2131", + "Fscr": "\u2131", + "GJcy": "\u0403", + "GT": "\u003E", + "Gamma": "\u0393", + "Gammad": "\u03DC", + "Gbreve": "\u011E", + "Gcedil": "\u0122", + "Gcirc": "\u011C", + "Gcy": "\u0413", + "Gdot": "\u0120", + "Gfr": "\U0001D50A", + "Gg": "\u22D9", + "Gopf": "\U0001D53E", + "GreaterEqual": "\u2265", + "GreaterEqualLess": "\u22DB", + "GreaterFullEqual": "\u2267", + "GreaterGreater": "\u2AA2", + "GreaterLess": "\u2277", + "GreaterSlantEqual": "\u2A7E", + "GreaterTilde": "\u2273", + "Gscr": "\U0001D4A2", + "Gt": "\u226B", + "HARDcy": "\u042A", + "Hacek": "\u02C7", + "Hat": "\u005E", + "Hcirc": "\u0124", + "Hfr": "\u210C", + "HilbertSpace": "\u210B", + "Hopf": "\u210D", + "HorizontalLine": "\u2500", + "Hscr": "\u210B", + "Hstrok": "\u0126", + "HumpDownHump": "\u224E", + "HumpEqual": "\u224F", + "IEcy": "\u0415", + "IJlig": "\u0132", + "IOcy": "\u0401", + "Iacute": "\u00CD", + "Icirc": "\u00CE", + "Icy": "\u0418", + "Idot": "\u0130", + "Ifr": "\u2111", + "Igrave": "\u00CC", + "Im": "\u2111", + "Imacr": "\u012A", + "ImaginaryI": "\u2148", + "Implies": "\u21D2", + "Int": "\u222C", + "Integral": "\u222B", + "Intersection": "\u22C2", + "InvisibleComma": "\u2063", + "InvisibleTimes": "\u2062", + "Iogon": "\u012E", + "Iopf": "\U0001D540", + "Iota": "\u0399", + "Iscr": "\u2110", + "Itilde": "\u0128", + "Iukcy": "\u0406", + "Iuml": "\u00CF", + "Jcirc": "\u0134", + "Jcy": "\u0419", + "Jfr": "\U0001D50D", + "Jopf": "\U0001D541", + "Jscr": "\U0001D4A5", + "Jsercy": "\u0408", + "Jukcy": "\u0404", + "KHcy": "\u0425", + "KJcy": "\u040C", + "Kappa": "\u039A", + "Kcedil": "\u0136", + "Kcy": "\u041A", + "Kfr": "\U0001D50E", + "Kopf": "\U0001D542", + "Kscr": "\U0001D4A6", + "LJcy": "\u0409", + "LT": "\u003C", + "Lacute": "\u0139", + "Lambda": "\u039B", + "Lang": "\u27EA", + "Laplacetrf": "\u2112", + "Larr": "\u219E", + "Lcaron": "\u013D", + "Lcedil": "\u013B", + "Lcy": "\u041B", + "LeftAngleBracket": "\u27E8", + "LeftArrow": "\u2190", + "LeftArrowBar": "\u21E4", + "LeftArrowRightArrow": "\u21C6", + "LeftCeiling": "\u2308", + "LeftDoubleBracket": "\u27E6", + "LeftDownTeeVector": "\u2961", + "LeftDownVector": "\u21C3", + "LeftDownVectorBar": "\u2959", + "LeftFloor": "\u230A", + "LeftRightArrow": "\u2194", + "LeftRightVector": "\u294E", + "LeftTee": "\u22A3", + "LeftTeeArrow": "\u21A4", + "LeftTeeVector": "\u295A", + "LeftTriangle": "\u22B2", + "LeftTriangleBar": "\u29CF", + "LeftTriangleEqual": "\u22B4", + "LeftUpDownVector": "\u2951", + "LeftUpTeeVector": "\u2960", + "LeftUpVector": "\u21BF", + "LeftUpVectorBar": "\u2958", + "LeftVector": "\u21BC", + "LeftVectorBar": "\u2952", + "Leftarrow": "\u21D0", + "Leftrightarrow": "\u21D4", + "LessEqualGreater": "\u22DA", + "LessFullEqual": "\u2266", + "LessGreater": "\u2276", + "LessLess": "\u2AA1", + "LessSlantEqual": "\u2A7D", + "LessTilde": "\u2272", + "Lfr": "\U0001D50F", + "Ll": "\u22D8", + "Lleftarrow": "\u21DA", + "Lmidot": "\u013F", + "LongLeftArrow": "\u27F5", + "LongLeftRightArrow": "\u27F7", + "LongRightArrow": "\u27F6", + "Longleftarrow": "\u27F8", + "Longleftrightarrow": "\u27FA", + "Longrightarrow": "\u27F9", + "Lopf": "\U0001D543", + "LowerLeftArrow": "\u2199", + "LowerRightArrow": "\u2198", + "Lscr": "\u2112", + "Lsh": "\u21B0", + "Lstrok": "\u0141", + "Lt": "\u226A", + "Map": "\u2905", + "Mcy": "\u041C", + "MediumSpace": "\u205F", + "Mellintrf": "\u2133", + "Mfr": "\U0001D510", + "MinusPlus": "\u2213", + "Mopf": "\U0001D544", + "Mscr": "\u2133", + "Mu": "\u039C", + "NJcy": "\u040A", + "Nacute": "\u0143", + "Ncaron": "\u0147", + "Ncedil": "\u0145", + "Ncy": "\u041D", + "NegativeMediumSpace": "\u200B", + "NegativeThickSpace": "\u200B", + "NegativeThinSpace": "\u200B", + "NegativeVeryThinSpace": "\u200B", + "NestedGreaterGreater": "\u226B", + "NestedLessLess": "\u226A", + "NewLine": "\u000A", + "Nfr": "\U0001D511", + "NoBreak": "\u2060", + "NonBreakingSpace": "\u00A0", + "Nopf": "\u2115", + "Not": "\u2AEC", + "NotCongruent": "\u2262", + "NotCupCap": "\u226D", + "NotDoubleVerticalBar": "\u2226", + "NotElement": "\u2209", + "NotEqual": "\u2260", + "NotEqualTilde": "\u2242\u0338", + "NotExists": "\u2204", + "NotGreater": "\u226F", + "NotGreaterEqual": "\u2271", + "NotGreaterFullEqual": "\u2267\u0338", + "NotGreaterGreater": "\u226B\u0338", + "NotGreaterLess": "\u2279", + "NotGreaterSlantEqual": "\u2A7E\u0338", + "NotGreaterTilde": "\u2275", + "NotHumpDownHump": "\u224E\u0338", + "NotHumpEqual": "\u224F\u0338", + "NotLeftTriangle": "\u22EA", + "NotLeftTriangleBar": "\u29CF\u0338", + "NotLeftTriangleEqual": "\u22EC", + "NotLess": "\u226E", + "NotLessEqual": "\u2270", + "NotLessGreater": "\u2278", + "NotLessLess": "\u226A\u0338", + "NotLessSlantEqual": "\u2A7D\u0338", + "NotLessTilde": "\u2274", + "NotNestedGreaterGreater": "\u2AA2\u0338", + "NotNestedLessLess": "\u2AA1\u0338", + "NotPrecedes": "\u2280", + "NotPrecedesEqual": "\u2AAF\u0338", + "NotPrecedesSlantEqual": "\u22E0", + "NotReverseElement": "\u220C", + "NotRightTriangle": "\u22EB", + "NotRightTriangleBar": "\u29D0\u0338", + "NotRightTriangleEqual": "\u22ED", + "NotSquareSubset": "\u228F\u0338", + "NotSquareSubsetEqual": "\u22E2", + "NotSquareSuperset": "\u2290\u0338", + "NotSquareSupersetEqual": "\u22E3", + "NotSubset": "\u2282\u20D2", + "NotSubsetEqual": "\u2288", + "NotSucceeds": "\u2281", + "NotSucceedsEqual": "\u2AB0\u0338", + "NotSucceedsSlantEqual": "\u22E1", + "NotSucceedsTilde": "\u227F\u0338", + "NotSuperset": "\u2283\u20D2", + "NotSupersetEqual": "\u2289", + "NotTilde": "\u2241", + "NotTildeEqual": "\u2244", + "NotTildeFullEqual": "\u2247", + "NotTildeTilde": "\u2249", + "NotVerticalBar": "\u2224", + "Nscr": "\U0001D4A9", + "Ntilde": "\u00D1", + "Nu": "\u039D", + "OElig": "\u0152", + "Oacute": "\u00D3", + "Ocirc": "\u00D4", + "Ocy": "\u041E", + "Odblac": "\u0150", + "Ofr": "\U0001D512", + "Ograve": "\u00D2", + "Omacr": "\u014C", + "Omega": "\u03A9", + "Omicron": "\u039F", + "Oopf": "\U0001D546", + "OpenCurlyDoubleQuote": "\u201C", + "OpenCurlyQuote": "\u2018", + "Or": "\u2A54", + "Oscr": "\U0001D4AA", + "Oslash": "\u00D8", + "Otilde": "\u00D5", + "Otimes": "\u2A37", + "Ouml": "\u00D6", + "OverBar": "\u203E", + "OverBrace": "\u23DE", + "OverBracket": "\u23B4", + "OverParenthesis": "\u23DC", + "PartialD": "\u2202", + "Pcy": "\u041F", + "Pfr": "\U0001D513", + "Phi": "\u03A6", + "Pi": "\u03A0", + "PlusMinus": "\u00B1", + "Poincareplane": "\u210C", + "Popf": "\u2119", + "Pr": "\u2ABB", + "Precedes": "\u227A", + "PrecedesEqual": "\u2AAF", + "PrecedesSlantEqual": "\u227C", + "PrecedesTilde": "\u227E", + "Prime": "\u2033", + "Product": "\u220F", + "Proportion": "\u2237", + "Proportional": "\u221D", + "Pscr": "\U0001D4AB", + "Psi": "\u03A8", + "QUOT": "\u0022", + "Qfr": "\U0001D514", + "Qopf": "\u211A", + "Qscr": "\U0001D4AC", + "RBarr": "\u2910", + "REG": "\u00AE", + "Racute": "\u0154", + "Rang": "\u27EB", + "Rarr": "\u21A0", + "Rarrtl": "\u2916", + "Rcaron": "\u0158", + "Rcedil": "\u0156", + "Rcy": "\u0420", + "Re": "\u211C", + "ReverseElement": "\u220B", + "ReverseEquilibrium": "\u21CB", + "ReverseUpEquilibrium": "\u296F", + "Rfr": "\u211C", + "Rho": "\u03A1", + "RightAngleBracket": "\u27E9", + "RightArrow": "\u2192", + "RightArrowBar": "\u21E5", + "RightArrowLeftArrow": "\u21C4", + "RightCeiling": "\u2309", + "RightDoubleBracket": "\u27E7", + "RightDownTeeVector": "\u295D", + "RightDownVector": "\u21C2", + "RightDownVectorBar": "\u2955", + "RightFloor": "\u230B", + "RightTee": "\u22A2", + "RightTeeArrow": "\u21A6", + "RightTeeVector": "\u295B", + "RightTriangle": "\u22B3", + "RightTriangleBar": "\u29D0", + "RightTriangleEqual": "\u22B5", + "RightUpDownVector": "\u294F", + "RightUpTeeVector": "\u295C", + "RightUpVector": "\u21BE", + "RightUpVectorBar": "\u2954", + "RightVector": "\u21C0", + "RightVectorBar": "\u2953", + "Rightarrow": "\u21D2", + "Ropf": "\u211D", + "RoundImplies": "\u2970", + "Rrightarrow": "\u21DB", + "Rscr": "\u211B", + "Rsh": "\u21B1", + "RuleDelayed": "\u29F4", + "SHCHcy": "\u0429", + "SHcy": "\u0428", + "SOFTcy": "\u042C", + "Sacute": "\u015A", + "Sc": "\u2ABC", + "Scaron": "\u0160", + "Scedil": "\u015E", + "Scirc": "\u015C", + "Scy": "\u0421", + "Sfr": "\U0001D516", + "ShortDownArrow": "\u2193", + "ShortLeftArrow": "\u2190", + "ShortRightArrow": "\u2192", + "ShortUpArrow": "\u2191", + "Sigma": "\u03A3", + "SmallCircle": "\u2218", + "Sopf": "\U0001D54A", + "Sqrt": "\u221A", + "Square": "\u25A1", + "SquareIntersection": "\u2293", + "SquareSubset": "\u228F", + "SquareSubsetEqual": "\u2291", + "SquareSuperset": "\u2290", + "SquareSupersetEqual": "\u2292", + "SquareUnion": "\u2294", + "Sscr": "\U0001D4AE", + "Star": "\u22C6", + "Sub": "\u22D0", + "Subset": "\u22D0", + "SubsetEqual": "\u2286", + "Succeeds": "\u227B", + "SucceedsEqual": "\u2AB0", + "SucceedsSlantEqual": "\u227D", + "SucceedsTilde": "\u227F", + "SuchThat": "\u220B", + "Sum": "\u2211", + "Sup": "\u22D1", + "Superset": "\u2283", + "SupersetEqual": "\u2287", + "Supset": "\u22D1", + "THORN": "\u00DE", + "TRADE": "\u2122", + "TSHcy": "\u040B", + "TScy": "\u0426", + "Tab": "\u0009", + "Tau": "\u03A4", + "Tcaron": "\u0164", + "Tcedil": "\u0162", + "Tcy": "\u0422", + "Tfr": "\U0001D517", + "Therefore": "\u2234", + "Theta": "\u0398", + "ThickSpace": "\u205F\u200A", + "ThinSpace": "\u2009", + "Tilde": "\u223C", + "TildeEqual": "\u2243", + "TildeFullEqual": "\u2245", + "TildeTilde": "\u2248", + "Topf": "\U0001D54B", + "TripleDot": "\u20DB", + "Tscr": "\U0001D4AF", + "Tstrok": "\u0166", + "Uacute": "\u00DA", + "Uarr": "\u219F", + "Uarrocir": "\u2949", + "Ubrcy": "\u040E", + "Ubreve": "\u016C", + "Ucirc": "\u00DB", + "Ucy": "\u0423", + "Udblac": "\u0170", + "Ufr": "\U0001D518", + "Ugrave": "\u00D9", + "Umacr": "\u016A", + "UnderBar": "\u005F", + "UnderBrace": "\u23DF", + "UnderBracket": "\u23B5", + "UnderParenthesis": "\u23DD", + "Union": "\u22C3", + "UnionPlus": "\u228E", + "Uogon": "\u0172", + "Uopf": "\U0001D54C", + "UpArrow": "\u2191", + "UpArrowBar": "\u2912", + "UpArrowDownArrow": "\u21C5", + "UpDownArrow": "\u2195", + "UpEquilibrium": "\u296E", + "UpTee": "\u22A5", + "UpTeeArrow": "\u21A5", + "Uparrow": "\u21D1", + "Updownarrow": "\u21D5", + "UpperLeftArrow": "\u2196", + "UpperRightArrow": "\u2197", + "Upsi": "\u03D2", + "Upsilon": "\u03A5", + "Uring": "\u016E", + "Uscr": "\U0001D4B0", + "Utilde": "\u0168", + "Uuml": "\u00DC", + "VDash": "\u22AB", + "Vbar": "\u2AEB", + "Vcy": "\u0412", + "Vdash": "\u22A9", + "Vdashl": "\u2AE6", + "Vee": "\u22C1", + "Verbar": "\u2016", + "Vert": "\u2016", + "VerticalBar": "\u2223", + "VerticalLine": "\u007C", + "VerticalSeparator": "\u2758", + "VerticalTilde": "\u2240", + "VeryThinSpace": "\u200A", + "Vfr": "\U0001D519", + "Vopf": "\U0001D54D", + "Vscr": "\U0001D4B1", + "Vvdash": "\u22AA", + "Wcirc": "\u0174", + "Wedge": "\u22C0", + "Wfr": "\U0001D51A", + "Wopf": "\U0001D54E", + "Wscr": "\U0001D4B2", + "Xfr": "\U0001D51B", + "Xi": "\u039E", + "Xopf": "\U0001D54F", + "Xscr": "\U0001D4B3", + "YAcy": "\u042F", + "YIcy": "\u0407", + "YUcy": "\u042E", + "Yacute": "\u00DD", + "Ycirc": "\u0176", + "Ycy": "\u042B", + "Yfr": "\U0001D51C", + "Yopf": "\U0001D550", + "Yscr": "\U0001D4B4", + "Yuml": "\u0178", + "ZHcy": "\u0416", + "Zacute": "\u0179", + "Zcaron": "\u017D", + "Zcy": "\u0417", + "Zdot": "\u017B", + "ZeroWidthSpace": "\u200B", + "Zeta": "\u0396", + "Zfr": "\u2128", + "Zopf": "\u2124", + "Zscr": "\U0001D4B5", + "aacute": "\u00E1", + "abreve": "\u0103", + "ac": "\u223E", + "acE": "\u223E\u0333", + "acd": "\u223F", + "acirc": "\u00E2", + "acute": "\u00B4", + "acy": "\u0430", + "aelig": "\u00E6", + "af": "\u2061", + "afr": "\U0001D51E", + "agrave": "\u00E0", + "alefsym": "\u2135", + "aleph": "\u2135", + "alpha": "\u03B1", + "amacr": "\u0101", + "amalg": "\u2A3F", + "amp": "\u0026", + "and": "\u2227", + "andand": "\u2A55", + "andd": "\u2A5C", + "andslope": "\u2A58", + "andv": "\u2A5A", + "ang": "\u2220", + "ange": "\u29A4", + "angle": "\u2220", + "angmsd": "\u2221", + "angmsdaa": "\u29A8", + "angmsdab": "\u29A9", + "angmsdac": "\u29AA", + "angmsdad": "\u29AB", + "angmsdae": "\u29AC", + "angmsdaf": "\u29AD", + "angmsdag": "\u29AE", + "angmsdah": "\u29AF", + "angrt": "\u221F", + "angrtvb": "\u22BE", + "angrtvbd": "\u299D", + "angsph": "\u2222", + "angst": "\u00C5", + "angzarr": "\u237C", + "aogon": "\u0105", + "aopf": "\U0001D552", + "ap": "\u2248", + "apE": "\u2A70", + "apacir": "\u2A6F", + "ape": "\u224A", + "apid": "\u224B", + "apos": "\u0027", + "approx": "\u2248", + "approxeq": "\u224A", + "aring": "\u00E5", + "ascr": "\U0001D4B6", + "ast": "\u002A", + "asymp": "\u2248", + "asympeq": "\u224D", + "atilde": "\u00E3", + "auml": "\u00E4", + "awconint": "\u2233", + "awint": "\u2A11", + "bNot": "\u2AED", + "backcong": "\u224C", + "backepsilon": "\u03F6", + "backprime": "\u2035", + "backsim": "\u223D", + "backsimeq": "\u22CD", + "barvee": "\u22BD", + "barwed": "\u2305", + "barwedge": "\u2305", + "bbrk": "\u23B5", + "bbrktbrk": "\u23B6", + "bcong": "\u224C", + "bcy": "\u0431", + "bdquo": "\u201E", + "becaus": "\u2235", + "because": "\u2235", + "bemptyv": "\u29B0", + "bepsi": "\u03F6", + "bernou": "\u212C", + "beta": "\u03B2", + "beth": "\u2136", + "between": "\u226C", + "bfr": "\U0001D51F", + "bigcap": "\u22C2", + "bigcirc": "\u25EF", + "bigcup": "\u22C3", + "bigodot": "\u2A00", + "bigoplus": "\u2A01", + "bigotimes": "\u2A02", + "bigsqcup": "\u2A06", + "bigstar": "\u2605", + "bigtriangledown": "\u25BD", + "bigtriangleup": "\u25B3", + "biguplus": "\u2A04", + "bigvee": "\u22C1", + "bigwedge": "\u22C0", + "bkarow": "\u290D", + "blacklozenge": "\u29EB", + "blacksquare": "\u25AA", + "blacktriangle": "\u25B4", + "blacktriangledown": "\u25BE", + "blacktriangleleft": "\u25C2", + "blacktriangleright": "\u25B8", + "blank": "\u2423", + "blk12": "\u2592", + "blk14": "\u2591", + "blk34": "\u2593", + "block": "\u2588", + "bne": "\u003D\u20E5", + "bnequiv": "\u2261\u20E5", + "bnot": "\u2310", + "bopf": "\U0001D553", + "bot": "\u22A5", + "bottom": "\u22A5", + "bowtie": "\u22C8", + "boxDL": "\u2557", + "boxDR": "\u2554", + "boxDl": "\u2556", + "boxDr": "\u2553", + "boxH": "\u2550", + "boxHD": "\u2566", + "boxHU": "\u2569", + "boxHd": "\u2564", + "boxHu": "\u2567", + "boxUL": "\u255D", + "boxUR": "\u255A", + "boxUl": "\u255C", + "boxUr": "\u2559", + "boxV": "\u2551", + "boxVH": "\u256C", + "boxVL": "\u2563", + "boxVR": "\u2560", + "boxVh": "\u256B", + "boxVl": "\u2562", + "boxVr": "\u255F", + "boxbox": "\u29C9", + "boxdL": "\u2555", + "boxdR": "\u2552", + "boxdl": "\u2510", + "boxdr": "\u250C", + "boxh": "\u2500", + "boxhD": "\u2565", + "boxhU": "\u2568", + "boxhd": "\u252C", + "boxhu": "\u2534", + "boxminus": "\u229F", + "boxplus": "\u229E", + "boxtimes": "\u22A0", + "boxuL": "\u255B", + "boxuR": "\u2558", + "boxul": "\u2518", + "boxur": "\u2514", + "boxv": "\u2502", + "boxvH": "\u256A", + "boxvL": "\u2561", + "boxvR": "\u255E", + "boxvh": "\u253C", + "boxvl": "\u2524", + "boxvr": "\u251C", + "bprime": "\u2035", + "breve": "\u02D8", + "brvbar": "\u00A6", + "bscr": "\U0001D4B7", + "bsemi": "\u204F", + "bsim": "\u223D", + "bsime": "\u22CD", + "bsol": "\u005C", + "bsolb": "\u29C5", + "bsolhsub": "\u27C8", + "bull": "\u2022", + "bullet": "\u2022", + "bump": "\u224E", + "bumpE": "\u2AAE", + "bumpe": "\u224F", + "bumpeq": "\u224F", + "cacute": "\u0107", + "cap": "\u2229", + "capand": "\u2A44", + "capbrcup": "\u2A49", + "capcap": "\u2A4B", + "capcup": "\u2A47", + "capdot": "\u2A40", + "caps": "\u2229\uFE00", + "caret": "\u2041", + "caron": "\u02C7", + "ccaps": "\u2A4D", + "ccaron": "\u010D", + "ccedil": "\u00E7", + "ccirc": "\u0109", + "ccups": "\u2A4C", + "ccupssm": "\u2A50", + "cdot": "\u010B", + "cedil": "\u00B8", + "cemptyv": "\u29B2", + "cent": "\u00A2", + "centerdot": "\u00B7", + "cfr": "\U0001D520", + "chcy": "\u0447", + "check": "\u2713", + "checkmark": "\u2713", + "chi": "\u03C7", + "cir": "\u25CB", + "cirE": "\u29C3", + "circ": "\u02C6", + "circeq": "\u2257", + "circlearrowleft": "\u21BA", + "circlearrowright": "\u21BB", + "circledR": "\u00AE", + "circledS": "\u24C8", + "circledast": "\u229B", + "circledcirc": "\u229A", + "circleddash": "\u229D", + "cire": "\u2257", + "cirfnint": "\u2A10", + "cirmid": "\u2AEF", + "cirscir": "\u29C2", + "clubs": "\u2663", + "clubsuit": "\u2663", + "colon": "\u003A", + "colone": "\u2254", + "coloneq": "\u2254", + "comma": "\u002C", + "commat": "\u0040", + "comp": "\u2201", + "compfn": "\u2218", + "complement": "\u2201", + "complexes": "\u2102", + "cong": "\u2245", + "congdot": "\u2A6D", + "conint": "\u222E", + "copf": "\U0001D554", + "coprod": "\u2210", + "copy": "\u00A9", + "copysr": "\u2117", + "crarr": "\u21B5", + "cross": "\u2717", + "cscr": "\U0001D4B8", + "csub": "\u2ACF", + "csube": "\u2AD1", + "csup": "\u2AD0", + "csupe": "\u2AD2", + "ctdot": "\u22EF", + "cudarrl": "\u2938", + "cudarrr": "\u2935", + "cuepr": "\u22DE", + "cuesc": "\u22DF", + "cularr": "\u21B6", + "cularrp": "\u293D", + "cup": "\u222A", + "cupbrcap": "\u2A48", + "cupcap": "\u2A46", + "cupcup": "\u2A4A", + "cupdot": "\u228D", + "cupor": "\u2A45", + "cups": "\u222A\uFE00", + "curarr": "\u21B7", + "curarrm": "\u293C", + "curlyeqprec": "\u22DE", + "curlyeqsucc": "\u22DF", + "curlyvee": "\u22CE", + "curlywedge": "\u22CF", + "curren": "\u00A4", + "curvearrowleft": "\u21B6", + "curvearrowright": "\u21B7", + "cuvee": "\u22CE", + "cuwed": "\u22CF", + "cwconint": "\u2232", + "cwint": "\u2231", + "cylcty": "\u232D", + "dArr": "\u21D3", + "dHar": "\u2965", + "dagger": "\u2020", + "daleth": "\u2138", + "darr": "\u2193", + "dash": "\u2010", + "dashv": "\u22A3", + "dbkarow": "\u290F", + "dblac": "\u02DD", + "dcaron": "\u010F", + "dcy": "\u0434", + "dd": "\u2146", + "ddagger": "\u2021", + "ddarr": "\u21CA", + "ddotseq": "\u2A77", + "deg": "\u00B0", + "delta": "\u03B4", + "demptyv": "\u29B1", + "dfisht": "\u297F", + "dfr": "\U0001D521", + "dharl": "\u21C3", + "dharr": "\u21C2", + "diam": "\u22C4", + "diamond": "\u22C4", + "diamondsuit": "\u2666", + "diams": "\u2666", + "die": "\u00A8", + "digamma": "\u03DD", + "disin": "\u22F2", + "div": "\u00F7", + "divide": "\u00F7", + "divideontimes": "\u22C7", + "divonx": "\u22C7", + "djcy": "\u0452", + "dlcorn": "\u231E", + "dlcrop": "\u230D", + "dollar": "\u0024", + "dopf": "\U0001D555", + "dot": "\u02D9", + "doteq": "\u2250", + "doteqdot": "\u2251", + "dotminus": "\u2238", + "dotplus": "\u2214", + "dotsquare": "\u22A1", + "doublebarwedge": "\u2306", + "downarrow": "\u2193", + "downdownarrows": "\u21CA", + "downharpoonleft": "\u21C3", + "downharpoonright": "\u21C2", + "drbkarow": "\u2910", + "drcorn": "\u231F", + "drcrop": "\u230C", + "dscr": "\U0001D4B9", + "dscy": "\u0455", + "dsol": "\u29F6", + "dstrok": "\u0111", + "dtdot": "\u22F1", + "dtri": "\u25BF", + "dtrif": "\u25BE", + "duarr": "\u21F5", + "duhar": "\u296F", + "dwangle": "\u29A6", + "dzcy": "\u045F", + "dzigrarr": "\u27FF", + "eDDot": "\u2A77", + "eDot": "\u2251", + "eacute": "\u00E9", + "easter": "\u2A6E", + "ecaron": "\u011B", + "ecir": "\u2256", + "ecirc": "\u00EA", + "ecolon": "\u2255", + "ecy": "\u044D", + "edot": "\u0117", + "ee": "\u2147", + "efDot": "\u2252", + "efr": "\U0001D522", + "eg": "\u2A9A", + "egrave": "\u00E8", + "egs": "\u2A96", + "egsdot": "\u2A98", + "el": "\u2A99", + "elinters": "\u23E7", + "ell": "\u2113", + "els": "\u2A95", + "elsdot": "\u2A97", + "emacr": "\u0113", + "empty": "\u2205", + "emptyset": "\u2205", + "emptyv": "\u2205", + "emsp": "\u2003", + "emsp13": "\u2004", + "emsp14": "\u2005", + "eng": "\u014B", + "ensp": "\u2002", + "eogon": "\u0119", + "eopf": "\U0001D556", + "epar": "\u22D5", + "eparsl": "\u29E3", + "eplus": "\u2A71", + "epsi": "\u03B5", + "epsilon": "\u03B5", + "epsiv": "\u03F5", + "eqcirc": "\u2256", + "eqcolon": "\u2255", + "eqsim": "\u2242", + "eqslantgtr": "\u2A96", + "eqslantless": "\u2A95", + "equals": "\u003D", + "equest": "\u225F", + "equiv": "\u2261", + "equivDD": "\u2A78", + "eqvparsl": "\u29E5", + "erDot": "\u2253", + "erarr": "\u2971", + "escr": "\u212F", + "esdot": "\u2250", + "esim": "\u2242", + "eta": "\u03B7", + "eth": "\u00F0", + "euml": "\u00EB", + "euro": "\u20AC", + "excl": "\u0021", + "exist": "\u2203", + "expectation": "\u2130", + "exponentiale": "\u2147", + "fallingdotseq": "\u2252", + "fcy": "\u0444", + "female": "\u2640", + "ffilig": "\uFB03", + "fflig": "\uFB00", + "ffllig": "\uFB04", + "ffr": "\U0001D523", + "filig": "\uFB01", + "fjlig": "\u0066\u006A", + "flat": "\u266D", + "fllig": "\uFB02", + "fltns": "\u25B1", + "fnof": "\u0192", + "fopf": "\U0001D557", + "forall": "\u2200", + "fork": "\u22D4", + "forkv": "\u2AD9", + "fpartint": "\u2A0D", + "frac12": "\u00BD", + "frac13": "\u2153", + "frac14": "\u00BC", + "frac15": "\u2155", + "frac16": "\u2159", + "frac18": "\u215B", + "frac23": "\u2154", + "frac25": "\u2156", + "frac34": "\u00BE", + "frac35": "\u2157", + "frac38": "\u215C", + "frac45": "\u2158", + "frac56": "\u215A", + "frac58": "\u215D", + "frac78": "\u215E", + "frasl": "\u2044", + "frown": "\u2322", + "fscr": "\U0001D4BB", + "gE": "\u2267", + "gEl": "\u2A8C", + "gacute": "\u01F5", + "gamma": "\u03B3", + "gammad": "\u03DD", + "gap": "\u2A86", + "gbreve": "\u011F", + "gcirc": "\u011D", + "gcy": "\u0433", + "gdot": "\u0121", + "ge": "\u2265", + "gel": "\u22DB", + "geq": "\u2265", + "geqq": "\u2267", + "geqslant": "\u2A7E", + "ges": "\u2A7E", + "gescc": "\u2AA9", + "gesdot": "\u2A80", + "gesdoto": "\u2A82", + "gesdotol": "\u2A84", + "gesl": "\u22DB\uFE00", + "gesles": "\u2A94", + "gfr": "\U0001D524", + "gg": "\u226B", + "ggg": "\u22D9", + "gimel": "\u2137", + "gjcy": "\u0453", + "gl": "\u2277", + "glE": "\u2A92", + "gla": "\u2AA5", + "glj": "\u2AA4", + "gnE": "\u2269", + "gnap": "\u2A8A", + "gnapprox": "\u2A8A", + "gne": "\u2A88", + "gneq": "\u2A88", + "gneqq": "\u2269", + "gnsim": "\u22E7", + "gopf": "\U0001D558", + "grave": "\u0060", + "gscr": "\u210A", + "gsim": "\u2273", + "gsime": "\u2A8E", + "gsiml": "\u2A90", + "gt": "\u003E", + "gtcc": "\u2AA7", + "gtcir": "\u2A7A", + "gtdot": "\u22D7", + "gtlPar": "\u2995", + "gtquest": "\u2A7C", + "gtrapprox": "\u2A86", + "gtrarr": "\u2978", + "gtrdot": "\u22D7", + "gtreqless": "\u22DB", + "gtreqqless": "\u2A8C", + "gtrless": "\u2277", + "gtrsim": "\u2273", + "gvertneqq": "\u2269\uFE00", + "gvnE": "\u2269\uFE00", + "hArr": "\u21D4", + "hairsp": "\u200A", + "half": "\u00BD", + "hamilt": "\u210B", + "hardcy": "\u044A", + "harr": "\u2194", + "harrcir": "\u2948", + "harrw": "\u21AD", + "hbar": "\u210F", + "hcirc": "\u0125", + "hearts": "\u2665", + "heartsuit": "\u2665", + "hellip": "\u2026", + "hercon": "\u22B9", + "hfr": "\U0001D525", + "hksearow": "\u2925", + "hkswarow": "\u2926", + "hoarr": "\u21FF", + "homtht": "\u223B", + "hookleftarrow": "\u21A9", + "hookrightarrow": "\u21AA", + "hopf": "\U0001D559", + "horbar": "\u2015", + "hscr": "\U0001D4BD", + "hslash": "\u210F", + "hstrok": "\u0127", + "hybull": "\u2043", + "hyphen": "\u2010", + "iacute": "\u00ED", + "ic": "\u2063", + "icirc": "\u00EE", + "icy": "\u0438", + "iecy": "\u0435", + "iexcl": "\u00A1", + "iff": "\u21D4", + "ifr": "\U0001D526", + "igrave": "\u00EC", + "ii": "\u2148", + "iiiint": "\u2A0C", + "iiint": "\u222D", + "iinfin": "\u29DC", + "iiota": "\u2129", + "ijlig": "\u0133", + "imacr": "\u012B", + "image": "\u2111", + "imagline": "\u2110", + "imagpart": "\u2111", + "imath": "\u0131", + "imof": "\u22B7", + "imped": "\u01B5", + "in": "\u2208", + "incare": "\u2105", + "infin": "\u221E", + "infintie": "\u29DD", + "inodot": "\u0131", + "int": "\u222B", + "intcal": "\u22BA", + "integers": "\u2124", + "intercal": "\u22BA", + "intlarhk": "\u2A17", + "intprod": "\u2A3C", + "iocy": "\u0451", + "iogon": "\u012F", + "iopf": "\U0001D55A", + "iota": "\u03B9", + "iprod": "\u2A3C", + "iquest": "\u00BF", + "iscr": "\U0001D4BE", + "isin": "\u2208", + "isinE": "\u22F9", + "isindot": "\u22F5", + "isins": "\u22F4", + "isinsv": "\u22F3", + "isinv": "\u2208", + "it": "\u2062", + "itilde": "\u0129", + "iukcy": "\u0456", + "iuml": "\u00EF", + "jcirc": "\u0135", + "jcy": "\u0439", + "jfr": "\U0001D527", + "jmath": "\u0237", + "jopf": "\U0001D55B", + "jscr": "\U0001D4BF", + "jsercy": "\u0458", + "jukcy": "\u0454", + "kappa": "\u03BA", + "kappav": "\u03F0", + "kcedil": "\u0137", + "kcy": "\u043A", + "kfr": "\U0001D528", + "kgreen": "\u0138", + "khcy": "\u0445", + "kjcy": "\u045C", + "kopf": "\U0001D55C", + "kscr": "\U0001D4C0", + "lAarr": "\u21DA", + "lArr": "\u21D0", + "lAtail": "\u291B", + "lBarr": "\u290E", + "lE": "\u2266", + "lEg": "\u2A8B", + "lHar": "\u2962", + "lacute": "\u013A", + "laemptyv": "\u29B4", + "lagran": "\u2112", + "lambda": "\u03BB", + "lang": "\u27E8", + "langd": "\u2991", + "langle": "\u27E8", + "lap": "\u2A85", + "laquo": "\u00AB", + "larr": "\u2190", + "larrb": "\u21E4", + "larrbfs": "\u291F", + "larrfs": "\u291D", + "larrhk": "\u21A9", + "larrlp": "\u21AB", + "larrpl": "\u2939", + "larrsim": "\u2973", + "larrtl": "\u21A2", + "lat": "\u2AAB", + "latail": "\u2919", + "late": "\u2AAD", + "lates": "\u2AAD\uFE00", + "lbarr": "\u290C", + "lbbrk": "\u2772", + "lbrace": "\u007B", + "lbrack": "\u005B", + "lbrke": "\u298B", + "lbrksld": "\u298F", + "lbrkslu": "\u298D", + "lcaron": "\u013E", + "lcedil": "\u013C", + "lceil": "\u2308", + "lcub": "\u007B", + "lcy": "\u043B", + "ldca": "\u2936", + "ldquo": "\u201C", + "ldquor": "\u201E", + "ldrdhar": "\u2967", + "ldrushar": "\u294B", + "ldsh": "\u21B2", + "le": "\u2264", + "leftarrow": "\u2190", + "leftarrowtail": "\u21A2", + "leftharpoondown": "\u21BD", + "leftharpoonup": "\u21BC", + "leftleftarrows": "\u21C7", + "leftrightarrow": "\u2194", + "leftrightarrows": "\u21C6", + "leftrightharpoons": "\u21CB", + "leftrightsquigarrow": "\u21AD", + "leftthreetimes": "\u22CB", + "leg": "\u22DA", + "leq": "\u2264", + "leqq": "\u2266", + "leqslant": "\u2A7D", + "les": "\u2A7D", + "lescc": "\u2AA8", + "lesdot": "\u2A7F", + "lesdoto": "\u2A81", + "lesdotor": "\u2A83", + "lesg": "\u22DA\uFE00", + "lesges": "\u2A93", + "lessapprox": "\u2A85", + "lessdot": "\u22D6", + "lesseqgtr": "\u22DA", + "lesseqqgtr": "\u2A8B", + "lessgtr": "\u2276", + "lesssim": "\u2272", + "lfisht": "\u297C", + "lfloor": "\u230A", + "lfr": "\U0001D529", + "lg": "\u2276", + "lgE": "\u2A91", + "lhard": "\u21BD", + "lharu": "\u21BC", + "lharul": "\u296A", + "lhblk": "\u2584", + "ljcy": "\u0459", + "ll": "\u226A", + "llarr": "\u21C7", + "llcorner": "\u231E", + "llhard": "\u296B", + "lltri": "\u25FA", + "lmidot": "\u0140", + "lmoust": "\u23B0", + "lmoustache": "\u23B0", + "lnE": "\u2268", + "lnap": "\u2A89", + "lnapprox": "\u2A89", + "lne": "\u2A87", + "lneq": "\u2A87", + "lneqq": "\u2268", + "lnsim": "\u22E6", + "loang": "\u27EC", + "loarr": "\u21FD", + "lobrk": "\u27E6", + "longleftarrow": "\u27F5", + "longleftrightarrow": "\u27F7", + "longmapsto": "\u27FC", + "longrightarrow": "\u27F6", + "looparrowleft": "\u21AB", + "looparrowright": "\u21AC", + "lopar": "\u2985", + "lopf": "\U0001D55D", + "loplus": "\u2A2D", + "lotimes": "\u2A34", + "lowast": "\u2217", + "lowbar": "\u005F", + "loz": "\u25CA", + "lozenge": "\u25CA", + "lozf": "\u29EB", + "lpar": "\u0028", + "lparlt": "\u2993", + "lrarr": "\u21C6", + "lrcorner": "\u231F", + "lrhar": "\u21CB", + "lrhard": "\u296D", + "lrm": "\u200E", + "lrtri": "\u22BF", + "lsaquo": "\u2039", + "lscr": "\U0001D4C1", + "lsh": "\u21B0", + "lsim": "\u2272", + "lsime": "\u2A8D", + "lsimg": "\u2A8F", + "lsqb": "\u005B", + "lsquo": "\u2018", + "lsquor": "\u201A", + "lstrok": "\u0142", + "lt": "\u003C", + "ltcc": "\u2AA6", + "ltcir": "\u2A79", + "ltdot": "\u22D6", + "lthree": "\u22CB", + "ltimes": "\u22C9", + "ltlarr": "\u2976", + "ltquest": "\u2A7B", + "ltrPar": "\u2996", + "ltri": "\u25C3", + "ltrie": "\u22B4", + "ltrif": "\u25C2", + "lurdshar": "\u294A", + "luruhar": "\u2966", + "lvertneqq": "\u2268\uFE00", + "lvnE": "\u2268\uFE00", + "mDDot": "\u223A", + "macr": "\u00AF", + "male": "\u2642", + "malt": "\u2720", + "maltese": "\u2720", + "map": "\u21A6", + "mapsto": "\u21A6", + "mapstodown": "\u21A7", + "mapstoleft": "\u21A4", + "mapstoup": "\u21A5", + "marker": "\u25AE", + "mcomma": "\u2A29", + "mcy": "\u043C", + "mdash": "\u2014", + "measuredangle": "\u2221", + "mfr": "\U0001D52A", + "mho": "\u2127", + "micro": "\u00B5", + "mid": "\u2223", + "midast": "\u002A", + "midcir": "\u2AF0", + "middot": "\u00B7", + "minus": "\u2212", + "minusb": "\u229F", + "minusd": "\u2238", + "minusdu": "\u2A2A", + "mlcp": "\u2ADB", + "mldr": "\u2026", + "mnplus": "\u2213", + "models": "\u22A7", + "mopf": "\U0001D55E", + "mp": "\u2213", + "mscr": "\U0001D4C2", + "mstpos": "\u223E", + "mu": "\u03BC", + "multimap": "\u22B8", + "mumap": "\u22B8", + "nGg": "\u22D9\u0338", + "nGt": "\u226B\u20D2", + "nGtv": "\u226B\u0338", + "nLeftarrow": "\u21CD", + "nLeftrightarrow": "\u21CE", + "nLl": "\u22D8\u0338", + "nLt": "\u226A\u20D2", + "nLtv": "\u226A\u0338", + "nRightarrow": "\u21CF", + "nVDash": "\u22AF", + "nVdash": "\u22AE", + "nabla": "\u2207", + "nacute": "\u0144", + "nang": "\u2220\u20D2", + "nap": "\u2249", + "napE": "\u2A70\u0338", + "napid": "\u224B\u0338", + "napos": "\u0149", + "napprox": "\u2249", + "natur": "\u266E", + "natural": "\u266E", + "naturals": "\u2115", + "nbsp": "\u00A0", + "nbump": "\u224E\u0338", + "nbumpe": "\u224F\u0338", + "ncap": "\u2A43", + "ncaron": "\u0148", + "ncedil": "\u0146", + "ncong": "\u2247", + "ncongdot": "\u2A6D\u0338", + "ncup": "\u2A42", + "ncy": "\u043D", + "ndash": "\u2013", + "ne": "\u2260", + "neArr": "\u21D7", + "nearhk": "\u2924", + "nearr": "\u2197", + "nearrow": "\u2197", + "nedot": "\u2250\u0338", + "nequiv": "\u2262", + "nesear": "\u2928", + "nesim": "\u2242\u0338", + "nexist": "\u2204", + "nexists": "\u2204", + "nfr": "\U0001D52B", + "ngE": "\u2267\u0338", + "nge": "\u2271", + "ngeq": "\u2271", + "ngeqq": "\u2267\u0338", + "ngeqslant": "\u2A7E\u0338", + "nges": "\u2A7E\u0338", + "ngsim": "\u2275", + "ngt": "\u226F", + "ngtr": "\u226F", + "nhArr": "\u21CE", + "nharr": "\u21AE", + "nhpar": "\u2AF2", + "ni": "\u220B", + "nis": "\u22FC", + "nisd": "\u22FA", + "niv": "\u220B", + "njcy": "\u045A", + "nlArr": "\u21CD", + "nlE": "\u2266\u0338", + "nlarr": "\u219A", + "nldr": "\u2025", + "nle": "\u2270", + "nleftarrow": "\u219A", + "nleftrightarrow": "\u21AE", + "nleq": "\u2270", + "nleqq": "\u2266\u0338", + "nleqslant": "\u2A7D\u0338", + "nles": "\u2A7D\u0338", + "nless": "\u226E", + "nlsim": "\u2274", + "nlt": "\u226E", + "nltri": "\u22EA", + "nltrie": "\u22EC", + "nmid": "\u2224", + "nopf": "\U0001D55F", + "not": "\u00AC", + "notin": "\u2209", + "notinE": "\u22F9\u0338", + "notindot": "\u22F5\u0338", + "notinva": "\u2209", + "notinvb": "\u22F7", + "notinvc": "\u22F6", + "notni": "\u220C", + "notniva": "\u220C", + "notnivb": "\u22FE", + "notnivc": "\u22FD", + "npar": "\u2226", + "nparallel": "\u2226", + "nparsl": "\u2AFD\u20E5", + "npart": "\u2202\u0338", + "npolint": "\u2A14", + "npr": "\u2280", + "nprcue": "\u22E0", + "npre": "\u2AAF\u0338", + "nprec": "\u2280", + "npreceq": "\u2AAF\u0338", + "nrArr": "\u21CF", + "nrarr": "\u219B", + "nrarrc": "\u2933\u0338", + "nrarrw": "\u219D\u0338", + "nrightarrow": "\u219B", + "nrtri": "\u22EB", + "nrtrie": "\u22ED", + "nsc": "\u2281", + "nsccue": "\u22E1", + "nsce": "\u2AB0\u0338", + "nscr": "\U0001D4C3", + "nshortmid": "\u2224", + "nshortparallel": "\u2226", + "nsim": "\u2241", + "nsime": "\u2244", + "nsimeq": "\u2244", + "nsmid": "\u2224", + "nspar": "\u2226", + "nsqsube": "\u22E2", + "nsqsupe": "\u22E3", + "nsub": "\u2284", + "nsubE": "\u2AC5\u0338", + "nsube": "\u2288", + "nsubset": "\u2282\u20D2", + "nsubseteq": "\u2288", + "nsubseteqq": "\u2AC5\u0338", + "nsucc": "\u2281", + "nsucceq": "\u2AB0\u0338", + "nsup": "\u2285", + "nsupE": "\u2AC6\u0338", + "nsupe": "\u2289", + "nsupset": "\u2283\u20D2", + "nsupseteq": "\u2289", + "nsupseteqq": "\u2AC6\u0338", + "ntgl": "\u2279", + "ntilde": "\u00F1", + "ntlg": "\u2278", + "ntriangleleft": "\u22EA", + "ntrianglelefteq": "\u22EC", + "ntriangleright": "\u22EB", + "ntrianglerighteq": "\u22ED", + "nu": "\u03BD", + "num": "\u0023", + "numero": "\u2116", + "numsp": "\u2007", + "nvDash": "\u22AD", + "nvHarr": "\u2904", + "nvap": "\u224D\u20D2", + "nvdash": "\u22AC", + "nvge": "\u2265\u20D2", + "nvgt": "\u003E\u20D2", + "nvinfin": "\u29DE", + "nvlArr": "\u2902", + "nvle": "\u2264\u20D2", + "nvlt": "\u003C\u20D2", + "nvltrie": "\u22B4\u20D2", + "nvrArr": "\u2903", + "nvrtrie": "\u22B5\u20D2", + "nvsim": "\u223C\u20D2", + "nwArr": "\u21D6", + "nwarhk": "\u2923", + "nwarr": "\u2196", + "nwarrow": "\u2196", + "nwnear": "\u2927", + "oS": "\u24C8", + "oacute": "\u00F3", + "oast": "\u229B", + "ocir": "\u229A", + "ocirc": "\u00F4", + "ocy": "\u043E", + "odash": "\u229D", + "odblac": "\u0151", + "odiv": "\u2A38", + "odot": "\u2299", + "odsold": "\u29BC", + "oelig": "\u0153", + "ofcir": "\u29BF", + "ofr": "\U0001D52C", + "ogon": "\u02DB", + "ograve": "\u00F2", + "ogt": "\u29C1", + "ohbar": "\u29B5", + "ohm": "\u03A9", + "oint": "\u222E", + "olarr": "\u21BA", + "olcir": "\u29BE", + "olcross": "\u29BB", + "oline": "\u203E", + "olt": "\u29C0", + "omacr": "\u014D", + "omega": "\u03C9", + "omicron": "\u03BF", + "omid": "\u29B6", + "ominus": "\u2296", + "oopf": "\U0001D560", + "opar": "\u29B7", + "operp": "\u29B9", + "oplus": "\u2295", + "or": "\u2228", + "orarr": "\u21BB", + "ord": "\u2A5D", + "order": "\u2134", + "orderof": "\u2134", + "ordf": "\u00AA", + "ordm": "\u00BA", + "origof": "\u22B6", + "oror": "\u2A56", + "orslope": "\u2A57", + "orv": "\u2A5B", + "oscr": "\u2134", + "oslash": "\u00F8", + "osol": "\u2298", + "otilde": "\u00F5", + "otimes": "\u2297", + "otimesas": "\u2A36", + "ouml": "\u00F6", + "ovbar": "\u233D", + "par": "\u2225", + "para": "\u00B6", + "parallel": "\u2225", + "parsim": "\u2AF3", + "parsl": "\u2AFD", + "part": "\u2202", + "pcy": "\u043F", + "percnt": "\u0025", + "period": "\u002E", + "permil": "\u2030", + "perp": "\u22A5", + "pertenk": "\u2031", + "pfr": "\U0001D52D", + "phi": "\u03C6", + "phiv": "\u03D5", + "phmmat": "\u2133", + "phone": "\u260E", + "pi": "\u03C0", + "pitchfork": "\u22D4", + "piv": "\u03D6", + "planck": "\u210F", + "planckh": "\u210E", + "plankv": "\u210F", + "plus": "\u002B", + "plusacir": "\u2A23", + "plusb": "\u229E", + "pluscir": "\u2A22", + "plusdo": "\u2214", + "plusdu": "\u2A25", + "pluse": "\u2A72", + "plusmn": "\u00B1", + "plussim": "\u2A26", + "plustwo": "\u2A27", + "pm": "\u00B1", + "pointint": "\u2A15", + "popf": "\U0001D561", + "pound": "\u00A3", + "pr": "\u227A", + "prE": "\u2AB3", + "prap": "\u2AB7", + "prcue": "\u227C", + "pre": "\u2AAF", + "prec": "\u227A", + "precapprox": "\u2AB7", + "preccurlyeq": "\u227C", + "preceq": "\u2AAF", + "precnapprox": "\u2AB9", + "precneqq": "\u2AB5", + "precnsim": "\u22E8", + "precsim": "\u227E", + "prime": "\u2032", + "primes": "\u2119", + "prnE": "\u2AB5", + "prnap": "\u2AB9", + "prnsim": "\u22E8", + "prod": "\u220F", + "profalar": "\u232E", + "profline": "\u2312", + "profsurf": "\u2313", + "prop": "\u221D", + "propto": "\u221D", + "prsim": "\u227E", + "prurel": "\u22B0", + "pscr": "\U0001D4C5", + "psi": "\u03C8", + "puncsp": "\u2008", + "qfr": "\U0001D52E", + "qint": "\u2A0C", + "qopf": "\U0001D562", + "qprime": "\u2057", + "qscr": "\U0001D4C6", + "quaternions": "\u210D", + "quatint": "\u2A16", + "quest": "\u003F", + "questeq": "\u225F", + "quot": "\u0022", + "rAarr": "\u21DB", + "rArr": "\u21D2", + "rAtail": "\u291C", + "rBarr": "\u290F", + "rHar": "\u2964", + "race": "\u223D\u0331", + "racute": "\u0155", + "radic": "\u221A", + "raemptyv": "\u29B3", + "rang": "\u27E9", + "rangd": "\u2992", + "range": "\u29A5", + "rangle": "\u27E9", + "raquo": "\u00BB", + "rarr": "\u2192", + "rarrap": "\u2975", + "rarrb": "\u21E5", + "rarrbfs": "\u2920", + "rarrc": "\u2933", + "rarrfs": "\u291E", + "rarrhk": "\u21AA", + "rarrlp": "\u21AC", + "rarrpl": "\u2945", + "rarrsim": "\u2974", + "rarrtl": "\u21A3", + "rarrw": "\u219D", + "ratail": "\u291A", + "ratio": "\u2236", + "rationals": "\u211A", + "rbarr": "\u290D", + "rbbrk": "\u2773", + "rbrace": "\u007D", + "rbrack": "\u005D", + "rbrke": "\u298C", + "rbrksld": "\u298E", + "rbrkslu": "\u2990", + "rcaron": "\u0159", + "rcedil": "\u0157", + "rceil": "\u2309", + "rcub": "\u007D", + "rcy": "\u0440", + "rdca": "\u2937", + "rdldhar": "\u2969", + "rdquo": "\u201D", + "rdquor": "\u201D", + "rdsh": "\u21B3", + "real": "\u211C", + "realine": "\u211B", + "realpart": "\u211C", + "reals": "\u211D", + "rect": "\u25AD", + "reg": "\u00AE", + "rfisht": "\u297D", + "rfloor": "\u230B", + "rfr": "\U0001D52F", + "rhard": "\u21C1", + "rharu": "\u21C0", + "rharul": "\u296C", + "rho": "\u03C1", + "rhov": "\u03F1", + "rightarrow": "\u2192", + "rightarrowtail": "\u21A3", + "rightharpoondown": "\u21C1", + "rightharpoonup": "\u21C0", + "rightleftarrows": "\u21C4", + "rightleftharpoons": "\u21CC", + "rightrightarrows": "\u21C9", + "rightsquigarrow": "\u219D", + "rightthreetimes": "\u22CC", + "ring": "\u02DA", + "risingdotseq": "\u2253", + "rlarr": "\u21C4", + "rlhar": "\u21CC", + "rlm": "\u200F", + "rmoust": "\u23B1", + "rmoustache": "\u23B1", + "rnmid": "\u2AEE", + "roang": "\u27ED", + "roarr": "\u21FE", + "robrk": "\u27E7", + "ropar": "\u2986", + "ropf": "\U0001D563", + "roplus": "\u2A2E", + "rotimes": "\u2A35", + "rpar": "\u0029", + "rpargt": "\u2994", + "rppolint": "\u2A12", + "rrarr": "\u21C9", + "rsaquo": "\u203A", + "rscr": "\U0001D4C7", + "rsh": "\u21B1", + "rsqb": "\u005D", + "rsquo": "\u2019", + "rsquor": "\u2019", + "rthree": "\u22CC", + "rtimes": "\u22CA", + "rtri": "\u25B9", + "rtrie": "\u22B5", + "rtrif": "\u25B8", + "rtriltri": "\u29CE", + "ruluhar": "\u2968", + "rx": "\u211E", + "sacute": "\u015B", + "sbquo": "\u201A", + "sc": "\u227B", + "scE": "\u2AB4", + "scap": "\u2AB8", + "scaron": "\u0161", + "sccue": "\u227D", + "sce": "\u2AB0", + "scedil": "\u015F", + "scirc": "\u015D", + "scnE": "\u2AB6", + "scnap": "\u2ABA", + "scnsim": "\u22E9", + "scpolint": "\u2A13", + "scsim": "\u227F", + "scy": "\u0441", + "sdot": "\u22C5", + "sdotb": "\u22A1", + "sdote": "\u2A66", + "seArr": "\u21D8", + "searhk": "\u2925", + "searr": "\u2198", + "searrow": "\u2198", + "sect": "\u00A7", + "semi": "\u003B", + "seswar": "\u2929", + "setminus": "\u2216", + "setmn": "\u2216", + "sext": "\u2736", + "sfr": "\U0001D530", + "sfrown": "\u2322", + "sharp": "\u266F", + "shchcy": "\u0449", + "shcy": "\u0448", + "shortmid": "\u2223", + "shortparallel": "\u2225", + "shy": "\u00AD", + "sigma": "\u03C3", + "sigmaf": "\u03C2", + "sigmav": "\u03C2", + "sim": "\u223C", + "simdot": "\u2A6A", + "sime": "\u2243", + "simeq": "\u2243", + "simg": "\u2A9E", + "simgE": "\u2AA0", + "siml": "\u2A9D", + "simlE": "\u2A9F", + "simne": "\u2246", + "simplus": "\u2A24", + "simrarr": "\u2972", + "slarr": "\u2190", + "smallsetminus": "\u2216", + "smashp": "\u2A33", + "smeparsl": "\u29E4", + "smid": "\u2223", + "smile": "\u2323", + "smt": "\u2AAA", + "smte": "\u2AAC", + "smtes": "\u2AAC\uFE00", + "softcy": "\u044C", + "sol": "\u002F", + "solb": "\u29C4", + "solbar": "\u233F", + "sopf": "\U0001D564", + "spades": "\u2660", + "spadesuit": "\u2660", + "spar": "\u2225", + "sqcap": "\u2293", + "sqcaps": "\u2293\uFE00", + "sqcup": "\u2294", + "sqcups": "\u2294\uFE00", + "sqsub": "\u228F", + "sqsube": "\u2291", + "sqsubset": "\u228F", + "sqsubseteq": "\u2291", + "sqsup": "\u2290", + "sqsupe": "\u2292", + "sqsupset": "\u2290", + "sqsupseteq": "\u2292", + "squ": "\u25A1", + "square": "\u25A1", + "squarf": "\u25AA", + "squf": "\u25AA", + "srarr": "\u2192", + "sscr": "\U0001D4C8", + "ssetmn": "\u2216", + "ssmile": "\u2323", + "sstarf": "\u22C6", + "star": "\u2606", + "starf": "\u2605", + "straightepsilon": "\u03F5", + "straightphi": "\u03D5", + "strns": "\u00AF", + "sub": "\u2282", + "subE": "\u2AC5", + "subdot": "\u2ABD", + "sube": "\u2286", + "subedot": "\u2AC3", + "submult": "\u2AC1", + "subnE": "\u2ACB", + "subne": "\u228A", + "subplus": "\u2ABF", + "subrarr": "\u2979", + "subset": "\u2282", + "subseteq": "\u2286", + "subseteqq": "\u2AC5", + "subsetneq": "\u228A", + "subsetneqq": "\u2ACB", + "subsim": "\u2AC7", + "subsub": "\u2AD5", + "subsup": "\u2AD3", + "succ": "\u227B", + "succapprox": "\u2AB8", + "succcurlyeq": "\u227D", + "succeq": "\u2AB0", + "succnapprox": "\u2ABA", + "succneqq": "\u2AB6", + "succnsim": "\u22E9", + "succsim": "\u227F", + "sum": "\u2211", + "sung": "\u266A", + "sup": "\u2283", + "sup1": "\u00B9", + "sup2": "\u00B2", + "sup3": "\u00B3", + "supE": "\u2AC6", + "supdot": "\u2ABE", + "supdsub": "\u2AD8", + "supe": "\u2287", + "supedot": "\u2AC4", + "suphsol": "\u27C9", + "suphsub": "\u2AD7", + "suplarr": "\u297B", + "supmult": "\u2AC2", + "supnE": "\u2ACC", + "supne": "\u228B", + "supplus": "\u2AC0", + "supset": "\u2283", + "supseteq": "\u2287", + "supseteqq": "\u2AC6", + "supsetneq": "\u228B", + "supsetneqq": "\u2ACC", + "supsim": "\u2AC8", + "supsub": "\u2AD4", + "supsup": "\u2AD6", + "swArr": "\u21D9", + "swarhk": "\u2926", + "swarr": "\u2199", + "swarrow": "\u2199", + "swnwar": "\u292A", + "szlig": "\u00DF", + "target": "\u2316", + "tau": "\u03C4", + "tbrk": "\u23B4", + "tcaron": "\u0165", + "tcedil": "\u0163", + "tcy": "\u0442", + "tdot": "\u20DB", + "telrec": "\u2315", + "tfr": "\U0001D531", + "there4": "\u2234", + "therefore": "\u2234", + "theta": "\u03B8", + "thetasym": "\u03D1", + "thetav": "\u03D1", + "thickapprox": "\u2248", + "thicksim": "\u223C", + "thinsp": "\u2009", + "thkap": "\u2248", + "thksim": "\u223C", + "thorn": "\u00FE", + "tilde": "\u02DC", + "times": "\u00D7", + "timesb": "\u22A0", + "timesbar": "\u2A31", + "timesd": "\u2A30", + "tint": "\u222D", + "toea": "\u2928", + "top": "\u22A4", + "topbot": "\u2336", + "topcir": "\u2AF1", + "topf": "\U0001D565", + "topfork": "\u2ADA", + "tosa": "\u2929", + "tprime": "\u2034", + "trade": "\u2122", + "triangle": "\u25B5", + "triangledown": "\u25BF", + "triangleleft": "\u25C3", + "trianglelefteq": "\u22B4", + "triangleq": "\u225C", + "triangleright": "\u25B9", + "trianglerighteq": "\u22B5", + "tridot": "\u25EC", + "trie": "\u225C", + "triminus": "\u2A3A", + "triplus": "\u2A39", + "trisb": "\u29CD", + "tritime": "\u2A3B", + "trpezium": "\u23E2", + "tscr": "\U0001D4C9", + "tscy": "\u0446", + "tshcy": "\u045B", + "tstrok": "\u0167", + "twixt": "\u226C", + "twoheadleftarrow": "\u219E", + "twoheadrightarrow": "\u21A0", + "uArr": "\u21D1", + "uHar": "\u2963", + "uacute": "\u00FA", + "uarr": "\u2191", + "ubrcy": "\u045E", + "ubreve": "\u016D", + "ucirc": "\u00FB", + "ucy": "\u0443", + "udarr": "\u21C5", + "udblac": "\u0171", + "udhar": "\u296E", + "ufisht": "\u297E", + "ufr": "\U0001D532", + "ugrave": "\u00F9", + "uharl": "\u21BF", + "uharr": "\u21BE", + "uhblk": "\u2580", + "ulcorn": "\u231C", + "ulcorner": "\u231C", + "ulcrop": "\u230F", + "ultri": "\u25F8", + "umacr": "\u016B", + "uml": "\u00A8", + "uogon": "\u0173", + "uopf": "\U0001D566", + "uparrow": "\u2191", + "updownarrow": "\u2195", + "upharpoonleft": "\u21BF", + "upharpoonright": "\u21BE", + "uplus": "\u228E", + "upsi": "\u03C5", + "upsih": "\u03D2", + "upsilon": "\u03C5", + "upuparrows": "\u21C8", + "urcorn": "\u231D", + "urcorner": "\u231D", + "urcrop": "\u230E", + "uring": "\u016F", + "urtri": "\u25F9", + "uscr": "\U0001D4CA", + "utdot": "\u22F0", + "utilde": "\u0169", + "utri": "\u25B5", + "utrif": "\u25B4", + "uuarr": "\u21C8", + "uuml": "\u00FC", + "uwangle": "\u29A7", + "vArr": "\u21D5", + "vBar": "\u2AE8", + "vBarv": "\u2AE9", + "vDash": "\u22A8", + "vangrt": "\u299C", + "varepsilon": "\u03F5", + "varkappa": "\u03F0", + "varnothing": "\u2205", + "varphi": "\u03D5", + "varpi": "\u03D6", + "varpropto": "\u221D", + "varr": "\u2195", + "varrho": "\u03F1", + "varsigma": "\u03C2", + "varsubsetneq": "\u228A\uFE00", + "varsubsetneqq": "\u2ACB\uFE00", + "varsupsetneq": "\u228B\uFE00", + "varsupsetneqq": "\u2ACC\uFE00", + "vartheta": "\u03D1", + "vartriangleleft": "\u22B2", + "vartriangleright": "\u22B3", + "vcy": "\u0432", + "vdash": "\u22A2", + "vee": "\u2228", + "veebar": "\u22BB", + "veeeq": "\u225A", + "vellip": "\u22EE", + "verbar": "\u007C", + "vert": "\u007C", + "vfr": "\U0001D533", + "vltri": "\u22B2", + "vnsub": "\u2282\u20D2", + "vnsup": "\u2283\u20D2", + "vopf": "\U0001D567", + "vprop": "\u221D", + "vrtri": "\u22B3", + "vscr": "\U0001D4CB", + "vsubnE": "\u2ACB\uFE00", + "vsubne": "\u228A\uFE00", + "vsupnE": "\u2ACC\uFE00", + "vsupne": "\u228B\uFE00", + "vzigzag": "\u299A", + "wcirc": "\u0175", + "wedbar": "\u2A5F", + "wedge": "\u2227", + "wedgeq": "\u2259", + "weierp": "\u2118", + "wfr": "\U0001D534", + "wopf": "\U0001D568", + "wp": "\u2118", + "wr": "\u2240", + "wreath": "\u2240", + "wscr": "\U0001D4CC", + "xcap": "\u22C2", + "xcirc": "\u25EF", + "xcup": "\u22C3", + "xdtri": "\u25BD", + "xfr": "\U0001D535", + "xhArr": "\u27FA", + "xharr": "\u27F7", + "xi": "\u03BE", + "xlArr": "\u27F8", + "xlarr": "\u27F5", + "xmap": "\u27FC", + "xnis": "\u22FB", + "xodot": "\u2A00", + "xopf": "\U0001D569", + "xoplus": "\u2A01", + "xotime": "\u2A02", + "xrArr": "\u27F9", + "xrarr": "\u27F6", + "xscr": "\U0001D4CD", + "xsqcup": "\u2A06", + "xuplus": "\u2A04", + "xutri": "\u25B3", + "xvee": "\u22C1", + "xwedge": "\u22C0", + "yacute": "\u00FD", + "yacy": "\u044F", + "ycirc": "\u0177", + "ycy": "\u044B", + "yen": "\u00A5", + "yfr": "\U0001D536", + "yicy": "\u0457", + "yopf": "\U0001D56A", + "yscr": "\U0001D4CE", + "yucy": "\u044E", + "yuml": "\u00FF", + "zacute": "\u017A", + "zcaron": "\u017E", + "zcy": "\u0437", + "zdot": "\u017C", + "zeetrf": "\u2128", + "zeta": "\u03B6", + "zfr": "\U0001D537", + "zhcy": "\u0436", + "zigrarr": "\u21DD", + "zopf": "\U0001D56B", + "zscr": "\U0001D4CF", + "zwj": "\u200D", + "zwnj": "\u200C", +} diff --git a/utils/markdown/indented_code.go b/utils/markdown/indented_code.go new file mode 100644 index 000000000..dc5dce1ac --- /dev/null +++ b/utils/markdown/indented_code.go @@ -0,0 +1,98 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "strings" +) + +type IndentedCodeLine struct { + Indentation int + Range Range +} + +type IndentedCode struct { + blockBase + markdown string + + RawCode []IndentedCodeLine +} + +func (b *IndentedCode) Code() (result string) { + for _, code := range b.RawCode { + result += strings.Repeat(" ", code.Indentation) + b.markdown[code.Range.Position:code.Range.End] + } + return +} + +func (b *IndentedCode) Continuation(indentation int, r Range) *continuation { + if indentation >= 4 { + return &continuation{ + Indentation: indentation - 4, + Remaining: r, + } + } + s := b.markdown[r.Position:r.End] + if strings.TrimSpace(s) == "" { + return &continuation{ + Remaining: r, + } + } + return nil +} + +func (b *IndentedCode) AddLine(indentation int, r Range) bool { + b.RawCode = append(b.RawCode, IndentedCodeLine{ + Indentation: indentation, + Range: r, + }) + return true +} + +func (b *IndentedCode) Close() { + for { + last := b.RawCode[len(b.RawCode)-1] + s := b.markdown[last.Range.Position:last.Range.End] + if strings.TrimRight(s, "\r\n") == "" { + b.RawCode = b.RawCode[:len(b.RawCode)-1] + } else { + break + } + } +} + +func (b *IndentedCode) AllowsBlockStarts() bool { + return false +} + +func indentedCodeStart(markdown string, indentation int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block { + if len(unmatchedBlocks) > 0 { + if _, ok := unmatchedBlocks[len(unmatchedBlocks)-1].(*Paragraph); ok { + return nil + } + } else if len(matchedBlocks) > 0 { + if _, ok := matchedBlocks[len(matchedBlocks)-1].(*Paragraph); ok { + return nil + } + } + + if indentation < 4 { + return nil + } + + s := markdown[r.Position:r.End] + if strings.TrimSpace(s) == "" { + return nil + } + + return []Block{ + &IndentedCode{ + markdown: markdown, + RawCode: []IndentedCodeLine{{ + Indentation: indentation - 4, + Range: r, + }}, + }, + } +} diff --git a/utils/markdown/inlines.go b/utils/markdown/inlines.go new file mode 100644 index 000000000..03da2f15c --- /dev/null +++ b/utils/markdown/inlines.go @@ -0,0 +1,489 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "container/list" + "strings" + "unicode" + "unicode/utf8" +) + +type Inline interface { + IsInline() bool +} + +type inlineBase struct{} + +func (inlineBase) IsInline() bool { return true } + +type Text struct { + inlineBase + + Text string +} + +type CodeSpan struct { + inlineBase + + Code string +} + +type HardLineBreak struct { + inlineBase +} + +type SoftLineBreak struct { + inlineBase +} + +type InlineLinkOrImage struct { + inlineBase + + Children []Inline + + RawDestination Range + + markdown string + rawTitle string +} + +func (i *InlineLinkOrImage) Destination() string { + return Unescape(i.markdown[i.RawDestination.Position:i.RawDestination.End]) +} + +func (i *InlineLinkOrImage) Title() string { + return Unescape(i.rawTitle) +} + +type InlineLink struct { + InlineLinkOrImage +} + +type InlineImage struct { + InlineLinkOrImage +} + +type ReferenceLinkOrImage struct { + inlineBase + *ReferenceDefinition + + Children []Inline +} + +type ReferenceLink struct { + ReferenceLinkOrImage +} + +type ReferenceImage struct { + ReferenceLinkOrImage +} + +type delimiterType int + +const ( + linkOpeningDelimiter delimiterType = iota + imageOpeningDelimiter +) + +type delimiter struct { + Type delimiterType + IsInactive bool + TextNode int + Range Range +} + +type inlineParser struct { + markdown string + ranges []Range + referenceDefinitions []*ReferenceDefinition + + raw string + position int + inlines []Inline + delimiterStack *list.List +} + +func newInlineParser(markdown string, ranges []Range, referenceDefinitions []*ReferenceDefinition) *inlineParser { + return &inlineParser{ + markdown: markdown, + ranges: ranges, + referenceDefinitions: referenceDefinitions, + delimiterStack: list.New(), + } +} + +func (p *inlineParser) parseBackticks() { + count := 1 + for i := p.position + 1; i < len(p.raw) && p.raw[i] == '`'; i++ { + count++ + } + opening := p.raw[p.position : p.position+count] + search := p.position + count + for search < len(p.raw) { + end := strings.Index(p.raw[search:], opening) + if end == -1 { + break + } + if search+end+count < len(p.raw) && p.raw[search+end+count] == '`' { + search += end + count + for search < len(p.raw) && p.raw[search] == '`' { + search++ + } + continue + } + code := strings.Join(strings.Fields(p.raw[p.position+count:search+end]), " ") + p.position = search + end + count + p.inlines = append(p.inlines, &CodeSpan{ + Code: code, + }) + return + } + p.position += len(opening) + p.inlines = append(p.inlines, &Text{ + Text: opening, + }) +} + +func (p *inlineParser) parseLineEnding() { + if p.position >= 1 && p.raw[p.position-1] == '\t' { + p.inlines = append(p.inlines, &HardLineBreak{}) + } else if p.position >= 2 && p.raw[p.position-1] == ' ' && (p.raw[p.position-2] == '\t' || p.raw[p.position-1] == ' ') { + p.inlines = append(p.inlines, &HardLineBreak{}) + } else { + p.inlines = append(p.inlines, &SoftLineBreak{}) + } + p.position++ + if p.position < len(p.raw) && p.raw[p.position] == '\n' { + p.position++ + } +} + +func (p *inlineParser) parseEscapeCharacter() { + if p.position+1 < len(p.raw) && isEscapableByte(p.raw[p.position+1]) { + p.inlines = append(p.inlines, &Text{ + Text: string(p.raw[p.position+1]), + }) + p.position += 2 + } else { + p.inlines = append(p.inlines, &Text{ + Text: `\`, + }) + p.position++ + } +} + +func (p *inlineParser) parseText() { + if next := strings.IndexAny(p.raw[p.position:], "\r\n\\`&![]"); next == -1 { + p.inlines = append(p.inlines, &Text{ + Text: strings.TrimRightFunc(p.raw[p.position:], isWhitespace), + }) + p.position = len(p.raw) + } else { + if p.raw[p.position+next] == '\r' || p.raw[p.position+next] == '\n' { + p.inlines = append(p.inlines, &Text{ + Text: strings.TrimRightFunc(p.raw[p.position:p.position+next], isWhitespace), + }) + } else { + p.inlines = append(p.inlines, &Text{ + Text: p.raw[p.position : p.position+next], + }) + } + p.position += next + } +} + +func (p *inlineParser) parseLinkOrImageDelimiter() { + if p.raw[p.position] == '[' { + p.inlines = append(p.inlines, &Text{ + Text: "[", + }) + p.delimiterStack.PushBack(&delimiter{ + Type: linkOpeningDelimiter, + TextNode: len(p.inlines) - 1, + Range: Range{p.position, p.position + 1}, + }) + p.position++ + } else if p.raw[p.position] == '!' && p.position+1 < len(p.raw) && p.raw[p.position+1] == '[' { + p.inlines = append(p.inlines, &Text{ + Text: "![", + }) + p.delimiterStack.PushBack(&delimiter{ + Type: imageOpeningDelimiter, + TextNode: len(p.inlines) - 1, + Range: Range{p.position, p.position + 2}, + }) + p.position += 2 + } else { + p.inlines = append(p.inlines, &Text{ + Text: "!", + }) + p.position++ + } +} + +func (p *inlineParser) peekAtInlineLinkDestinationAndTitle(position int) (destination, title Range, end int, ok bool) { + if position >= len(p.raw) || p.raw[position] != '(' { + return + } + position++ + + destinationStart := nextNonWhitespace(p.raw, position) + if destinationStart >= len(p.raw) { + return + } else if p.raw[destinationStart] == ')' { + return Range{destinationStart, destinationStart}, Range{destinationStart, destinationStart}, destinationStart + 1, true + } + + destination, end, ok = parseLinkDestination(p.raw, destinationStart) + if !ok { + return + } + position = end + + if position < len(p.raw) && isWhitespaceByte(p.raw[position]) { + titleStart := nextNonWhitespace(p.raw, position) + if titleStart >= len(p.raw) { + return + } else if p.raw[titleStart] == ')' { + return destination, Range{titleStart, titleStart}, titleStart + 1, true + } + + title, end, ok = parseLinkTitle(p.raw, titleStart) + if !ok { + return + } + position = end + } + + closingPosition := nextNonWhitespace(p.raw, position) + if closingPosition >= len(p.raw) || p.raw[closingPosition] != ')' { + return Range{}, Range{}, 0, false + } + + return destination, title, closingPosition + 1, true +} + +func (p *inlineParser) referenceDefinition(label string) *ReferenceDefinition { + clean := strings.Join(strings.Fields(label), " ") + for _, d := range p.referenceDefinitions { + if strings.EqualFold(clean, strings.Join(strings.Fields(d.Label()), " ")) { + return d + } + } + return nil +} + +func (p *inlineParser) lookForLinkOrImage() { + for element := p.delimiterStack.Back(); element != nil; element = element.Prev() { + d := element.Value.(*delimiter) + if d.Type != imageOpeningDelimiter && d.Type != linkOpeningDelimiter { + continue + } + if d.IsInactive { + p.delimiterStack.Remove(element) + break + } + + var inline Inline + + if destination, title, next, ok := p.peekAtInlineLinkDestinationAndTitle(p.position + 1); ok { + destinationMarkdownPosition := relativeToAbsolutePosition(p.ranges, destination.Position) + linkOrImage := InlineLinkOrImage{ + Children: append([]Inline(nil), p.inlines[d.TextNode+1:]...), + RawDestination: Range{destinationMarkdownPosition, destinationMarkdownPosition + destination.End - destination.Position}, + markdown: p.markdown, + rawTitle: p.raw[title.Position:title.End], + } + if d.Type == imageOpeningDelimiter { + inline = &InlineImage{linkOrImage} + } else { + inline = &InlineLink{linkOrImage} + } + p.position = next + } else { + referenceLabel := "" + label, next, hasLinkLabel := parseLinkLabel(p.raw, p.position+1) + if hasLinkLabel && label.End > label.Position { + referenceLabel = p.raw[label.Position:label.End] + } else { + referenceLabel = p.raw[d.Range.End:p.position] + if !hasLinkLabel { + next = p.position + 1 + } + } + if referenceLabel != "" { + if reference := p.referenceDefinition(referenceLabel); reference != nil { + linkOrImage := ReferenceLinkOrImage{ + ReferenceDefinition: reference, + Children: append([]Inline(nil), p.inlines[d.TextNode+1:]...), + } + if d.Type == imageOpeningDelimiter { + inline = &ReferenceImage{linkOrImage} + } else { + inline = &ReferenceLink{linkOrImage} + } + p.position = next + } + } + } + + if inline != nil { + if d.Type == imageOpeningDelimiter { + p.inlines = append(p.inlines[:d.TextNode], inline) + } else { + p.inlines = append(p.inlines[:d.TextNode], inline) + for element := element.Prev(); element != nil; element = element.Prev() { + if d := element.Value.(*delimiter); d.Type == linkOpeningDelimiter { + d.IsInactive = true + } + } + } + p.delimiterStack.Remove(element) + return + } else { + p.delimiterStack.Remove(element) + break + } + } + p.inlines = append(p.inlines, &Text{ + Text: "]", + }) + p.position++ +} + +func CharacterReference(ref string) string { + if ref == "" { + return "" + } + if ref[0] == '#' { + if len(ref) < 2 { + return "" + } + n := 0 + if ref[1] == 'X' || ref[1] == 'x' { + if len(ref) < 3 { + return "" + } + for i := 2; i < len(ref); i++ { + if i > 9 { + return "" + } + d := ref[i] + switch { + case d >= '0' && d <= '9': + n = n*16 + int(d-'0') + case d >= 'a' && d <= 'f': + n = n*16 + 10 + int(d-'a') + case d >= 'A' && d <= 'F': + n = n*16 + 10 + int(d-'A') + default: + return "" + } + } + } else { + for i := 1; i < len(ref); i++ { + if i > 8 || ref[i] < '0' || ref[i] > '9' { + return "" + } + n = n*10 + int(ref[i]-'0') + } + } + c := rune(n) + if c == '\u0000' || !utf8.ValidRune(c) { + return string(unicode.ReplacementChar) + } + return string(c) + } + if entity, ok := htmlEntities[ref]; ok { + return entity + } + return "" +} + +func (p *inlineParser) parseCharacterReference() { + p.position++ + if semicolon := strings.IndexByte(p.raw[p.position:], ';'); semicolon == -1 { + p.inlines = append(p.inlines, &Text{ + Text: "&", + }) + } else if s := CharacterReference(p.raw[p.position : p.position+semicolon]); s != "" { + p.position += semicolon + 1 + p.inlines = append(p.inlines, &Text{ + Text: s, + }) + } else { + p.inlines = append(p.inlines, &Text{ + Text: "&", + }) + } +} + +func (p *inlineParser) Parse() []Inline { + for _, r := range p.ranges { + p.raw += p.markdown[r.Position:r.End] + } + + for p.position < len(p.raw) { + c, _ := utf8.DecodeRuneInString(p.raw[p.position:]) + + switch c { + case '\r', '\n': + p.parseLineEnding() + case '\\': + p.parseEscapeCharacter() + case '`': + p.parseBackticks() + case '&': + p.parseCharacterReference() + case '!', '[': + p.parseLinkOrImageDelimiter() + case ']': + p.lookForLinkOrImage() + default: + p.parseText() + } + } + + return p.inlines +} + +func ParseInlines(markdown string, ranges []Range, referenceDefinitions []*ReferenceDefinition) (inlines []Inline) { + return newInlineParser(markdown, ranges, referenceDefinitions).Parse() +} + +func Unescape(markdown string) string { + ret := "" + + position := 0 + for position < len(markdown) { + c, cSize := utf8.DecodeRuneInString(markdown[position:]) + + switch c { + case '\\': + if position+1 < len(markdown) && isEscapableByte(markdown[position+1]) { + ret += string(markdown[position+1]) + position += 2 + } else { + ret += `\` + position++ + } + case '&': + position++ + if semicolon := strings.IndexByte(markdown[position:], ';'); semicolon == -1 { + ret += "&" + } else if s := CharacterReference(markdown[position : position+semicolon]); s != "" { + position += semicolon + 1 + ret += s + } else { + ret += "&" + } + default: + ret += string(c) + position += cSize + } + } + + return ret +} diff --git a/utils/markdown/inspect.go b/utils/markdown/inspect.go new file mode 100644 index 000000000..c4e3bf1ac --- /dev/null +++ b/utils/markdown/inspect.go @@ -0,0 +1,78 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +// Inspect traverses the markdown tree in depth-first order. If f returns true, Inspect invokes f +// recursively for each child of the block or inline, followed by a call of f(nil). +func Inspect(markdown string, f func(interface{}) bool) { + document, referenceDefinitions := Parse(markdown) + InspectBlock(document, func(block Block) bool { + if !f(block) { + return false + } + switch v := block.(type) { + case *Paragraph: + for _, inline := range v.ParseInlines(referenceDefinitions) { + InspectInline(inline, func(inline Inline) bool { + return f(inline) + }) + } + } + return true + }) +} + +// InspectBlock traverses the blocks in depth-first order, starting with block. If f returns true, +// InspectBlock invokes f recursively for each child of the block, followed by a call of f(nil). +func InspectBlock(block Block, f func(Block) bool) { + if !f(block) { + return + } + switch v := block.(type) { + case *Document: + for _, child := range v.Children { + InspectBlock(child, f) + } + case *List: + for _, child := range v.Children { + InspectBlock(child, f) + } + case *ListItem: + for _, child := range v.Children { + InspectBlock(child, f) + } + case *BlockQuote: + for _, child := range v.Children { + InspectBlock(child, f) + } + } + f(nil) +} + +// InspectInline traverses the blocks in depth-first order, starting with block. If f returns true, +// InspectInline invokes f recursively for each child of the block, followed by a call of f(nil). +func InspectInline(inline Inline, f func(Inline) bool) { + if !f(inline) { + return + } + switch v := inline.(type) { + case *InlineImage: + for _, child := range v.Children { + InspectInline(child, f) + } + case *InlineLink: + for _, child := range v.Children { + InspectInline(child, f) + } + case *ReferenceImage: + for _, child := range v.Children { + InspectInline(child, f) + } + case *ReferenceLink: + for _, child := range v.Children { + InspectInline(child, f) + } + } + f(nil) +} diff --git a/utils/markdown/inspect_test.go b/utils/markdown/inspect_test.go new file mode 100644 index 000000000..0c5032f2d --- /dev/null +++ b/utils/markdown/inspect_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInspect(t *testing.T) { + markdown := ` +[foo]: bar +- a + > [![]()]() + > [![foo]][foo] +- d +` + + visited := []string{} + level := 0 + Inspect(markdown, func(blockOrInline interface{}) bool { + if blockOrInline == nil { + level-- + } else { + visited = append(visited, strings.Repeat(" ", level*4)+strings.TrimPrefix(fmt.Sprintf("%T", blockOrInline), "*markdown.")) + level++ + } + return true + }) + + assert.Equal(t, []string{ + "Document", + " Paragraph", + " List", + " ListItem", + " Paragraph", + " Text", + " BlockQuote", + " Paragraph", + " InlineLink", + " InlineImage", + " SoftLineBreak", + " ReferenceLink", + " ReferenceImage", + " Text", + " ListItem", + " Paragraph", + " Text", + }, visited) +} diff --git a/utils/markdown/lines.go b/utils/markdown/lines.go new file mode 100644 index 000000000..a38b5164c --- /dev/null +++ b/utils/markdown/lines.go @@ -0,0 +1,27 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +type Line struct { + Range +} + +func ParseLines(markdown string) (lines []Line) { + lineStartPosition := 0 + isAfterCarriageReturn := false + for position, r := range markdown { + if r == '\n' { + lines = append(lines, Line{Range{lineStartPosition, position + 1}}) + lineStartPosition = position + 1 + } else if isAfterCarriageReturn { + lines = append(lines, Line{Range{lineStartPosition, position}}) + lineStartPosition = position + } + isAfterCarriageReturn = r == '\r' + } + if lineStartPosition < len(markdown) { + lines = append(lines, Line{Range{lineStartPosition, len(markdown)}}) + } + return +} diff --git a/utils/markdown/lines_test.go b/utils/markdown/lines_test.go new file mode 100644 index 000000000..78603c5ea --- /dev/null +++ b/utils/markdown/lines_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseLines(t *testing.T) { + assert.Equal(t, []Line{ + {Range{0, 4}}, {Range{4, 7}}, + }, ParseLines("foo\nbar")) + + assert.Equal(t, []Line{ + {Range{0, 5}}, {Range{5, 8}}, + }, ParseLines("foo\r\nbar")) + + assert.Equal(t, []Line{ + {Range{0, 4}}, {Range{4, 6}}, {Range{6, 9}}, + }, ParseLines("foo\r\r\nbar")) + + assert.Equal(t, []Line{ + {Range{0, 4}}, + }, ParseLines("foo\n")) + + assert.Equal(t, []Line{ + {Range{0, 4}}, + }, ParseLines("foo\r")) + + assert.Equal(t, []Line{ + {Range{0, 5}}, + }, ParseLines("foo\r\n")) +} diff --git a/utils/markdown/links.go b/utils/markdown/links.go new file mode 100644 index 000000000..419797cb9 --- /dev/null +++ b/utils/markdown/links.go @@ -0,0 +1,130 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "unicode/utf8" +) + +func parseLinkDestination(markdown string, position int) (raw Range, next int, ok bool) { + if position >= len(markdown) { + return + } + + if markdown[position] == '<' { + isEscaped := false + + for offset, c := range []byte(markdown[position+1:]) { + if isEscaped { + isEscaped = false + if isEscapableByte(c) { + continue + } + } + + if c == '\\' { + isEscaped = true + } else if c == '<' { + break + } else if c == '>' { + return Range{position + 1, position + 1 + offset}, position + 1 + offset + 1, true + } else if isWhitespaceByte(c) { + break + } + } + } + + openCount := 0 + isEscaped := false + for offset, c := range []byte(markdown[position:]) { + if isEscaped { + isEscaped = false + if isEscapableByte(c) { + continue + } + } + + switch c { + case '\\': + isEscaped = true + case '(': + openCount++ + case ')': + if openCount < 1 { + return Range{position, position + offset}, position + offset, true + } + openCount-- + default: + if isWhitespaceByte(c) { + return Range{position, position + offset}, position + offset, true + } + } + } + return Range{position, len(markdown)}, len(markdown), true +} + +func parseLinkTitle(markdown string, position int) (raw Range, next int, ok bool) { + if position >= len(markdown) { + return + } + + originalPosition := position + + var closer byte + switch markdown[position] { + case '"', '\'': + closer = markdown[position] + case '(': + closer = ')' + default: + return + } + position++ + + for position < len(markdown) { + switch markdown[position] { + case '\\': + position++ + if position < len(markdown) && isEscapableByte(markdown[position]) { + position++ + } + case closer: + return Range{originalPosition + 1, position}, position + 1, true + default: + position++ + } + } + + return +} + +func parseLinkLabel(markdown string, position int) (raw Range, next int, ok bool) { + if position >= len(markdown) || markdown[position] != '[' { + return + } + + originalPosition := position + position++ + + for position < len(markdown) { + switch markdown[position] { + case '\\': + position++ + if position < len(markdown) && isEscapableByte(markdown[position]) { + position++ + } + case '[': + return + case ']': + if position-originalPosition >= 1000 && utf8.RuneCountInString(markdown[originalPosition:position]) >= 1000 { + return + } + return Range{originalPosition + 1, position}, position + 1, true + default: + position++ + } + } + + return +} diff --git a/utils/markdown/list.go b/utils/markdown/list.go new file mode 100644 index 000000000..aea71156c --- /dev/null +++ b/utils/markdown/list.go @@ -0,0 +1,220 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "strings" +) + +type ListItem struct { + blockBase + markdown string + hasTrailingBlankLine bool + hasBlankLineBetweenChildren bool + + Indentation int + Children []Block +} + +func (b *ListItem) Continuation(indentation int, r Range) *continuation { + s := b.markdown[r.Position:r.End] + if strings.TrimSpace(s) == "" { + if b.Children == nil { + return nil + } + return &continuation{ + Remaining: r, + } + } + if indentation < b.Indentation { + return nil + } + return &continuation{ + Indentation: indentation - b.Indentation, + Remaining: r, + } +} + +func (b *ListItem) AddChild(openBlocks []Block) []Block { + b.Children = append(b.Children, openBlocks[0]) + if b.hasTrailingBlankLine { + b.hasBlankLineBetweenChildren = true + } + b.hasTrailingBlankLine = false + return openBlocks +} + +func (b *ListItem) AddLine(indentation int, r Range) bool { + isBlank := strings.TrimSpace(b.markdown[r.Position:r.End]) == "" + if isBlank { + b.hasTrailingBlankLine = true + } + return false +} + +func (b *ListItem) HasTrailingBlankLine() bool { + return b.hasTrailingBlankLine || (len(b.Children) > 0 && b.Children[len(b.Children)-1].HasTrailingBlankLine()) +} + +func (b *ListItem) isLoose() bool { + if b.hasBlankLineBetweenChildren { + return true + } + for i, child := range b.Children { + if i < len(b.Children)-1 && child.HasTrailingBlankLine() { + return true + } + } + return false +} + +type List struct { + blockBase + markdown string + hasTrailingBlankLine bool + hasBlankLineBetweenChildren bool + + IsLoose bool + IsOrdered bool + OrderedStart int + BulletOrDelimiter byte + Children []*ListItem +} + +func (b *List) Continuation(indentation int, r Range) *continuation { + s := b.markdown[r.Position:r.End] + if strings.TrimSpace(s) == "" { + return &continuation{ + Remaining: r, + } + } + return &continuation{ + Indentation: indentation, + Remaining: r, + } +} + +func (b *List) AddChild(openBlocks []Block) []Block { + if item, ok := openBlocks[0].(*ListItem); ok { + b.Children = append(b.Children, item) + if b.hasTrailingBlankLine { + b.hasBlankLineBetweenChildren = true + } + b.hasTrailingBlankLine = false + return openBlocks + } else if list, ok := openBlocks[0].(*List); ok { + if len(list.Children) == 1 && list.IsOrdered == b.IsOrdered && list.BulletOrDelimiter == b.BulletOrDelimiter { + return b.AddChild(openBlocks[1:]) + } + } + return nil +} + +func (b *List) AddLine(indentation int, r Range) bool { + isBlank := strings.TrimSpace(b.markdown[r.Position:r.End]) == "" + if isBlank { + b.hasTrailingBlankLine = true + } + return false +} + +func (b *List) HasTrailingBlankLine() bool { + return b.hasTrailingBlankLine || (len(b.Children) > 0 && b.Children[len(b.Children)-1].HasTrailingBlankLine()) +} + +func (b *List) isLoose() bool { + if b.hasBlankLineBetweenChildren { + return true + } + for i, child := range b.Children { + if child.isLoose() || (i < len(b.Children)-1 && child.HasTrailingBlankLine()) { + return true + } + } + return false +} + +func (b *List) Close() { + b.IsLoose = b.isLoose() +} + +func parseListMarker(markdown string, r Range) (success, isOrdered bool, orderedStart int, bulletOrDelimiter byte, markerWidth int, remaining Range) { + digits := 0 + n := 0 + for i := r.Position; i < r.End && markdown[i] >= '0' && markdown[i] <= '9'; i++ { + digits++ + n = n*10 + int(markdown[i]-'0') + } + if digits > 0 { + if digits > 9 || r.Position+digits >= r.End { + return + } + next := markdown[r.Position+digits] + if next != '.' && next != ')' { + return + } + return true, true, n, next, digits + 1, Range{r.Position + digits + 1, r.End} + } + if r.Position >= r.End { + return + } + next := markdown[r.Position] + if next != '-' && next != '+' && next != '*' { + return + } + return true, false, 0, next, 1, Range{r.Position + 1, r.End} +} + +func listStart(markdown string, indent int, r Range, matchedBlocks, unmatchedBlocks []Block) []Block { + afterList := false + if len(matchedBlocks) > 0 { + _, afterList = matchedBlocks[len(matchedBlocks)-1].(*List) + } + if !afterList && indent > 3 { + return nil + } + + success, isOrdered, orderedStart, bulletOrDelimiter, markerWidth, remaining := parseListMarker(markdown, r) + if !success { + return nil + } + + isBlank := strings.TrimSpace(markdown[remaining.Position:remaining.End]) == "" + if len(matchedBlocks) > 0 && len(unmatchedBlocks) == 0 { + if _, ok := matchedBlocks[len(matchedBlocks)-1].(*Paragraph); ok { + if isBlank || (isOrdered && orderedStart != 1) { + return nil + } + } + } + + indentAfterMarker, indentBytesAfterMarker := countIndentation(markdown, remaining) + if !isBlank && indentAfterMarker < 1 { + return nil + } + + remaining = Range{remaining.Position + indentBytesAfterMarker, remaining.End} + consumedIndentAfterMarker := indentAfterMarker + if isBlank || indentAfterMarker >= 5 { + consumedIndentAfterMarker = 1 + } + + listItem := &ListItem{ + markdown: markdown, + Indentation: indent + markerWidth + consumedIndentAfterMarker, + } + list := &List{ + markdown: markdown, + IsOrdered: isOrdered, + OrderedStart: orderedStart, + BulletOrDelimiter: bulletOrDelimiter, + Children: []*ListItem{listItem}, + } + ret := []Block{list, listItem} + if descendants := blockStartOrParagraph(markdown, indentAfterMarker-consumedIndentAfterMarker, remaining, nil, nil); descendants != nil { + listItem.Children = append(listItem.Children, descendants[0]) + ret = append(ret, descendants...) + } + return ret +} diff --git a/utils/markdown/markdown.go b/utils/markdown/markdown.go new file mode 100644 index 000000000..3061ba4bb --- /dev/null +++ b/utils/markdown/markdown.go @@ -0,0 +1,132 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +// This package implements a parser for the subset of the CommonMark spec necessary for us to do +// server-side processing. It is not a full implementation and lacks many features. But it is +// complete enough to efficiently and accurately allow us to do what we need to like rewrite image +// URLs for proxying. +package markdown + +import ( + "strings" +) + +func isEscapable(c rune) bool { + return c > ' ' && (c < '0' || (c > '9' && (c < 'A' || (c > 'Z' && (c < 'a' || (c > 'z' && c <= '~')))))) +} + +func isEscapableByte(c byte) bool { + return isEscapable(rune(c)) +} + +func isWhitespace(c rune) bool { + switch c { + case ' ', '\t', '\n', '\u000b', '\u000c', '\r': + return true + default: + return false + } +} + +func isWhitespaceByte(c byte) bool { + return isWhitespace(rune(c)) +} + +func isHex(c rune) bool { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') +} + +func isHexByte(c byte) bool { + return isHex(rune(c)) +} + +func nextNonWhitespace(markdown string, position int) int { + for offset, c := range []byte(markdown[position:]) { + if !isWhitespaceByte(c) { + return position + offset + } + } + return len(markdown) +} + +func nextLine(markdown string, position int) (linePosition int, skippedNonWhitespace bool) { + for i := position; i < len(markdown); i++ { + c := markdown[i] + if c == '\r' { + if i+1 < len(markdown) && markdown[i+1] == '\n' { + return i + 2, skippedNonWhitespace + } + return i + 1, skippedNonWhitespace + } else if c == '\n' { + return i + 1, skippedNonWhitespace + } else if !isWhitespaceByte(c) { + skippedNonWhitespace = true + } + } + return len(markdown), skippedNonWhitespace +} + +func countIndentation(markdown string, r Range) (spaces, bytes int) { + for i := r.Position; i < r.End; i++ { + if markdown[i] == ' ' { + spaces++ + bytes++ + } else if markdown[i] == '\t' { + spaces += 4 + bytes++ + } else { + break + } + } + return +} + +func trimLeftSpace(markdown string, r Range) Range { + s := markdown[r.Position:r.End] + trimmed := strings.TrimLeftFunc(s, isWhitespace) + return Range{r.Position, r.End - (len(s) - len(trimmed))} +} + +func trimRightSpace(markdown string, r Range) Range { + s := markdown[r.Position:r.End] + trimmed := strings.TrimRightFunc(s, isWhitespace) + return Range{r.Position, r.End - (len(s) - len(trimmed))} +} + +func relativeToAbsolutePosition(ranges []Range, position int) int { + rem := position + for _, r := range ranges { + l := r.End - r.Position + if rem < l { + return r.Position + rem + } + rem -= l + } + if len(ranges) == 0 { + return 0 + } + return ranges[len(ranges)-1].End +} + +func trimBytesFromRanges(ranges []Range, bytes int) (result []Range) { + rem := bytes + for _, r := range ranges { + if rem == 0 { + result = append(result, r) + continue + } + l := r.End - r.Position + if rem < l { + result = append(result, Range{r.Position + rem, r.End}) + rem = 0 + continue + } + rem -= l + } + return +} + +func Parse(markdown string) (*Document, []*ReferenceDefinition) { + lines := ParseLines(markdown) + return ParseBlocks(markdown, lines) +} diff --git a/utils/markdown/paragraph.go b/utils/markdown/paragraph.go new file mode 100644 index 000000000..6a40fdf89 --- /dev/null +++ b/utils/markdown/paragraph.go @@ -0,0 +1,71 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +import ( + "strings" +) + +type Paragraph struct { + blockBase + markdown string + + Text []Range + ReferenceDefinitions []*ReferenceDefinition +} + +func (b *Paragraph) ParseInlines(referenceDefinitions []*ReferenceDefinition) []Inline { + return ParseInlines(b.markdown, b.Text, referenceDefinitions) +} + +func (b *Paragraph) Continuation(indentation int, r Range) *continuation { + s := b.markdown[r.Position:r.End] + if strings.TrimSpace(s) == "" { + return nil + } + return &continuation{ + Indentation: indentation, + Remaining: r, + } +} + +func (b *Paragraph) Close() { + for { + for i := 0; i < len(b.Text); i++ { + b.Text[i] = trimLeftSpace(b.markdown, b.Text[i]) + if b.Text[i].Position < b.Text[i].End { + break + } + } + + if len(b.Text) == 0 || b.Text[0].Position < b.Text[0].End && b.markdown[b.Text[0].Position] != '[' { + break + } + + definition, remaining := parseReferenceDefinition(b.markdown, b.Text) + if definition == nil { + break + } + b.ReferenceDefinitions = append(b.ReferenceDefinitions, definition) + b.Text = remaining + } + + for i := len(b.Text) - 1; i >= 0; i-- { + b.Text[i] = trimRightSpace(b.markdown, b.Text[i]) + if b.Text[i].Position < b.Text[i].End { + break + } + } +} + +func newParagraph(markdown string, r Range) *Paragraph { + s := markdown[r.Position:r.End] + if strings.TrimSpace(s) == "" { + return nil + } + return &Paragraph{ + markdown: markdown, + Text: []Range{r}, + } +} diff --git a/utils/markdown/reference_definition.go b/utils/markdown/reference_definition.go new file mode 100644 index 000000000..e2d0be350 --- /dev/null +++ b/utils/markdown/reference_definition.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package markdown + +type ReferenceDefinition struct { + RawDestination Range + + markdown string + rawLabel string + rawTitle string +} + +func (d *ReferenceDefinition) Destination() string { + return Unescape(d.markdown[d.RawDestination.Position:d.RawDestination.End]) +} + +func (d *ReferenceDefinition) Label() string { + return d.rawLabel +} + +func (d *ReferenceDefinition) Title() string { + return Unescape(d.rawTitle) +} + +func parseReferenceDefinition(markdown string, ranges []Range) (*ReferenceDefinition, []Range) { + raw := "" + for _, r := range ranges { + raw += markdown[r.Position:r.End] + } + + label, next, ok := parseLinkLabel(raw, 0) + if !ok { + return nil, nil + } + position := next + + if position >= len(raw) || raw[position] != ':' { + return nil, nil + } + position++ + + destination, next, ok := parseLinkDestination(raw, nextNonWhitespace(raw, position)) + if !ok { + return nil, nil + } + position = next + + absoluteDestination := relativeToAbsolutePosition(ranges, destination.Position) + ret := &ReferenceDefinition{ + RawDestination: Range{absoluteDestination, absoluteDestination + destination.End - destination.Position}, + markdown: markdown, + rawLabel: raw[label.Position:label.End], + } + + if position < len(raw) && isWhitespaceByte(raw[position]) { + title, next, ok := parseLinkTitle(raw, nextNonWhitespace(raw, position)) + if !ok { + if nextLine, skippedNonWhitespace := nextLine(raw, position); !skippedNonWhitespace { + return ret, trimBytesFromRanges(ranges, nextLine) + } + return nil, nil + } + if nextLine, skippedNonWhitespace := nextLine(raw, next); !skippedNonWhitespace { + ret.rawTitle = raw[title.Position:title.End] + return ret, trimBytesFromRanges(ranges, nextLine) + } + } + + if nextLine, skippedNonWhitespace := nextLine(raw, position); !skippedNonWhitespace { + return ret, trimBytesFromRanges(ranges, nextLine) + } + + return nil, nil +} -- cgit v1.2.3-1-g7c22