Merge pull request #1325 from hmhealey/plt618

PLT-618 Ignore formatting markdown lists containing only one element

2 files changed, 350 insertions, 52 deletions

diff --git a/web/react/utils/highlight.jsx b/web/react/utils/highlight.jsx
new file mode 100644
index 000000000..68fef7930
--- /dev/null
+++ b/web/react/utils/highlight.jsx
@@ -0,0 +1,51 @@
+// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+const highlightJs = require('highlight.js/lib/highlight.js');
+const highlightJsDiff = require('highlight.js/lib/languages/diff.js');
+const highlightJsApache = require('highlight.js/lib/languages/apache.js');
+const highlightJsMakefile = require('highlight.js/lib/languages/makefile.js');
+const highlightJsHttp = require('highlight.js/lib/languages/http.js');
+const highlightJsJson = require('highlight.js/lib/languages/json.js');
+const highlightJsMarkdown = require('highlight.js/lib/languages/markdown.js');
+const highlightJsJavascript = require('highlight.js/lib/languages/javascript.js');
+const highlightJsCss = require('highlight.js/lib/languages/css.js');
+const highlightJsNginx = require('highlight.js/lib/languages/nginx.js');
+const highlightJsObjectivec = require('highlight.js/lib/languages/objectivec.js');
+const highlightJsPython = require('highlight.js/lib/languages/python.js');
+const highlightJsXml = require('highlight.js/lib/languages/xml.js');
+const highlightJsPerl = require('highlight.js/lib/languages/perl.js');
+const highlightJsBash = require('highlight.js/lib/languages/bash.js');
+const highlightJsPhp = require('highlight.js/lib/languages/php.js');
+const highlightJsCoffeescript = require('highlight.js/lib/languages/coffeescript.js');
+const highlightJsCs = require('highlight.js/lib/languages/cs.js');
+const highlightJsCpp = require('highlight.js/lib/languages/cpp.js');
+const highlightJsSql = require('highlight.js/lib/languages/sql.js');
+const highlightJsGo = require('highlight.js/lib/languages/go.js');
+const highlightJsRuby = require('highlight.js/lib/languages/ruby.js');
+const highlightJsJava = require('highlight.js/lib/languages/java.js');
+const highlightJsIni = require('highlight.js/lib/languages/ini.js');
+
+highlightJs.registerLanguage('diff', highlightJsDiff);
+highlightJs.registerLanguage('apache', highlightJsApache);
+highlightJs.registerLanguage('makefile', highlightJsMakefile);
+highlightJs.registerLanguage('http', highlightJsHttp);
+highlightJs.registerLanguage('json', highlightJsJson);
+highlightJs.registerLanguage('markdown', highlightJsMarkdown);
+highlightJs.registerLanguage('javascript', highlightJsJavascript);
+highlightJs.registerLanguage('css', highlightJsCss);
+highlightJs.registerLanguage('nginx', highlightJsNginx);
+highlightJs.registerLanguage('objectivec', highlightJsObjectivec);
+highlightJs.registerLanguage('python', highlightJsPython);
+highlightJs.registerLanguage('xml', highlightJsXml);
+highlightJs.registerLanguage('perl', highlightJsPerl);
+highlightJs.registerLanguage('bash', highlightJsBash);
+highlightJs.registerLanguage('php', highlightJsPhp);
+highlightJs.registerLanguage('coffeescript', highlightJsCoffeescript);
+highlightJs.registerLanguage('cs', highlightJsCs);
+highlightJs.registerLanguage('cpp', highlightJsCpp);
+highlightJs.registerLanguage('sql', highlightJsSql);
+highlightJs.registerLanguage('go', highlightJsGo);
+highlightJs.registerLanguage('ruby', highlightJsRuby);
+highlightJs.registerLanguage('java', highlightJsJava);
+highlightJs.registerLanguage('ini', highlightJsIni);
diff --git a/web/react/utils/markdown.jsx b/web/react/utils/markdown.jsx
index 3ef09211f..374caf6dc 100644
--- a/web/react/utils/markdown.jsx
+++ b/web/react/utils/markdown.jsx
@@ -1,38 +1,14 @@
 // Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
 // See License.txt for license information.
 
+require('./highlight.jsx');
 const TextFormatting = require('./text_formatting.jsx');
 const Utils = require('./utils.jsx');
 
+const highlightJs = require('highlight.js/lib/highlight.js');
 const marked = require('marked');
 
-const highlightJs = require('highlight.js/lib/highlight.js');
-const highlightJsDiff = require('highlight.js/lib/languages/diff.js');
-const highlightJsApache = require('highlight.js/lib/languages/apache.js');
-const highlightJsMakefile = require('highlight.js/lib/languages/makefile.js');
-const highlightJsHttp = require('highlight.js/lib/languages/http.js');
-const highlightJsJson = require('highlight.js/lib/languages/json.js');
-const highlightJsMarkdown = require('highlight.js/lib/languages/markdown.js');
-const highlightJsJavascript = require('highlight.js/lib/languages/javascript.js');
-const highlightJsCss = require('highlight.js/lib/languages/css.js');
-const highlightJsNginx = require('highlight.js/lib/languages/nginx.js');
-const highlightJsObjectivec = require('highlight.js/lib/languages/objectivec.js');
-const highlightJsPython = require('highlight.js/lib/languages/python.js');
-const highlightJsXml = require('highlight.js/lib/languages/xml.js');
-const highlightJsPerl = require('highlight.js/lib/languages/perl.js');
-const highlightJsBash = require('highlight.js/lib/languages/bash.js');
-const highlightJsPhp = require('highlight.js/lib/languages/php.js');
-const highlightJsCoffeescript = require('highlight.js/lib/languages/coffeescript.js');
-const highlightJsCs = require('highlight.js/lib/languages/cs.js');
-const highlightJsCpp = require('highlight.js/lib/languages/cpp.js');
-const highlightJsSql = require('highlight.js/lib/languages/sql.js');
-const highlightJsGo = require('highlight.js/lib/languages/go.js');
-const highlightJsRuby = require('highlight.js/lib/languages/ruby.js');
-const highlightJsJava = require('highlight.js/lib/languages/java.js');
-const highlightJsIni = require('highlight.js/lib/languages/ini.js');
-
-const Constants = require('../utils/constants.jsx');
-const HighlightedLanguages = Constants.HighlightedLanguages;
+const HighlightedLanguages = require('../utils/constants.jsx').HighlightedLanguages;
 
 function markdownImageLoaded(image) {
     image.style.height = 'auto';
@@ -84,30 +60,6 @@ class MattermostMarkdownRenderer extends marked.Renderer {
         this.text = this.text.bind(this);
 
         this.formattingOptions = formattingOptions;
-
-        highlightJs.registerLanguage('diff', highlightJsDiff);
-        highlightJs.registerLanguage('apache', highlightJsApache);
-        highlightJs.registerLanguage('makefile', highlightJsMakefile);
-        highlightJs.registerLanguage('http', highlightJsHttp);
-        highlightJs.registerLanguage('json', highlightJsJson);
-        highlightJs.registerLanguage('markdown', highlightJsMarkdown);
-        highlightJs.registerLanguage('javascript', highlightJsJavascript);
-        highlightJs.registerLanguage('css', highlightJsCss);
-        highlightJs.registerLanguage('nginx', highlightJsNginx);
-        highlightJs.registerLanguage('objectivec', highlightJsObjectivec);
-        highlightJs.registerLanguage('python', highlightJsPython);
-        highlightJs.registerLanguage('xml', highlightJsXml);
-        highlightJs.registerLanguage('perl', highlightJsPerl);
-        highlightJs.registerLanguage('bash', highlightJsBash);
-        highlightJs.registerLanguage('php', highlightJsPhp);
-        highlightJs.registerLanguage('coffeescript', highlightJsCoffeescript);
-        highlightJs.registerLanguage('cs', highlightJsCs);
-        highlightJs.registerLanguage('cpp', highlightJsCpp);
-        highlightJs.registerLanguage('sql', highlightJsSql);
-        highlightJs.registerLanguage('go', highlightJsGo);
-        highlightJs.registerLanguage('ruby', highlightJsRuby);
-        highlightJs.registerLanguage('java', highlightJsJava);
-        highlightJs.registerLanguage('ini', highlightJsIni);
     }
 
     code(code, language) {
@@ -204,6 +156,301 @@ class MattermostMarkdownRenderer extends marked.Renderer {
     }
 }
 
+class MattermostLexer extends marked.Lexer {
+    token(originalSrc, top, bq) {
+        let src = originalSrc.replace(/^ +$/gm, '');
+
+        while (src) {
+            // newline
+            let cap = this.rules.newline.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+                if (cap[0].length > 1) {
+                    this.tokens.push({
+                        type: 'space'
+                    });
+                }
+            }
+
+            // code
+            cap = this.rules.code.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+                cap = cap[0].replace(/^ {4}/gm, '');
+                this.tokens.push({
+                    type: 'code',
+                    text: this.options.pedantic ? cap : cap.replace(/\n+$/, '')
+                });
+                continue;
+            }
+
+            // fences (gfm)
+            cap = this.rules.fences.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+                this.tokens.push({
+                    type: 'code',
+                    lang: cap[2],
+                    text: cap[3] || ''
+                });
+                continue;
+            }
+
+            // heading
+            cap = this.rules.heading.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+                this.tokens.push({
+                    type: 'heading',
+                    depth: cap[1].length,
+                    text: cap[2]
+                });
+                continue;
+            }
+
+            // table no leading pipe (gfm)
+            cap = this.rules.nptable.exec(src);
+            if (top && cap) {
+                src = src.substring(cap[0].length);
+
+                const item = {
+                    type: 'table',
+                    header: cap[1].replace(/^ *| *\| *$/g, '').split(/ *\| */),
+                    align: cap[2].replace(/^ *|\| *$/g, '').split(/ *\| */),
+                    cells: cap[3].replace(/\n$/, '').split('\n')
+                };
+
+                for (let i = 0; i < item.align.length; i++) {
+                    if (/^ *-+: *$/.test(item.align[i])) {
+                        item.align[i] = 'right';
+                    } else if (/^ *:-+: *$/.test(item.align[i])) {
+                        item.align[i] = 'center';
+                    } else if (/^ *:-+ *$/.test(item.align[i])) {
+                        item.align[i] = 'left';
+                    } else {
+                        item.align[i] = null;
+                    }
+                }
+
+                for (let i = 0; i < item.cells.length; i++) {
+                    item.cells[i] = item.cells[i].split(/ *\| */);
+                }
+
+                this.tokens.push(item);
+
+                continue;
+            }
+
+            // lheading
+            cap = this.rules.lheading.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+                this.tokens.push({
+                    type: 'heading',
+                    depth: cap[2] === '=' ? 1 : 2,
+                    text: cap[1]
+                });
+                continue;
+            }
+
+            // hr
+            cap = this.rules.hr.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+                this.tokens.push({
+                    type: 'hr'
+                });
+                continue;
+            }
+
+            // blockquote
+            cap = this.rules.blockquote.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+
+                this.tokens.push({
+                    type: 'blockquote_start'
+                });
+
+                cap = cap[0].replace(/^ *> ?/gm, '');
+
+                // Pass `top` to keep the current
+                // "toplevel" state. This is exactly
+                // how markdown.pl works.
+                this.token(cap, top, true);
+
+                this.tokens.push({
+                    type: 'blockquote_end'
+                });
+
+                continue;
+            }
+
+            // list
+            cap = this.rules.list.exec(src);
+            if (cap) {
+                const bull = cap[2];
+                let l = cap[0].length;
+
+                // Get each top-level item.
+                cap = cap[0].match(this.rules.item);
+
+                if (cap.length > 1) {
+                    src = src.substring(l);
+
+                    this.tokens.push({
+                        type: 'list_start',
+                        ordered: bull.length > 1
+                    });
+
+                    let next = false;
+                    l = cap.length;
+
+                    for (let i = 0; i < l; i++) {
+                        let item = cap[i];
+
+                        // Remove the list item's bullet
+                        // so it is seen as the next token.
+                        let space = item.length;
+                        item = item.replace(/^ *([*+-]|\d+\.) +/, '');
+
+                        // Outdent whatever the
+                        // list item contains. Hacky.
+                        if (~item.indexOf('\n ')) {
+                            space -= item.length;
+                            item = this.options.pedantic ? item.replace(/^ {1,4}/gm, '') : item.replace(new RegExp('^ \{1,' + space + '\}', 'gm'), '');
+                        }
+
+                        // Determine whether the next list item belongs here.
+                        // Backpedal if it does not belong in this list.
+                        if (this.options.smartLists && i !== l - 1) {
+                            const bullet = /(?:[*+-]|\d+\.)/;
+                            const b = bullet.exec(cap[i + 1])[0];
+                            if (bull !== b && !(bull.length > 1 && b.length > 1)) {
+                                src = cap.slice(i + 1).join('\n') + src;
+                                i = l - 1;
+                            }
+                        }
+
+                        // Determine whether item is loose or not.
+                        // Use: /(^|\n)(?! )[^\n]+\n\n(?!\s*$)/
+                        // for discount behavior.
+                        let loose = next || (/\n\n(?!\s*$)/).test(item);
+                        if (i !== l - 1) {
+                            next = item.charAt(item.length - 1) === '\n';
+                            if (!loose) {
+                                loose = next;
+                            }
+                        }
+
+                        this.tokens.push({
+                            type: loose ? 'loose_item_start' : 'list_item_start'
+                        });
+
+                        // Recurse.
+                        this.token(item, false, bq);
+
+                        this.tokens.push({
+                            type: 'list_item_end'
+                        });
+                    }
+
+                    this.tokens.push({
+                        type: 'list_end'
+                    });
+
+                    continue;
+                }
+            }
+
+            // html
+            cap = this.rules.html.exec(src);
+            if (cap) {
+                src = src.substring(cap[0].length);
+                this.tokens.push({
+                    type: this.options.sanitize ? 'paragraph' : 'html',
+                    pre: !this.options.sanitizer && (cap[1] === 'pre' || cap[1] === 'script' || cap[1] === 'style'),
+                    text: cap[0]
+                });
+                continue;
+            }
+
+            // def
+            cap = this.rules.def.exec(src);
+            if ((!bq && top) && cap) {
+                src = src.substring(cap[0].length);
+                this.tokens.links[cap[1].toLowerCase()] = {
+                    href: cap[2],
+                    title: cap[3]
+                };
+                continue;
+            }
+
+            // table (gfm)
+            cap = this.rules.table.exec(src);
+            if (top && cap) {
+                src = src.substring(cap[0].length);
+
+                const item = {
+                    type: 'table',
+                    header: cap[1].replace(/^ *| *\| *$/g, '').split(/ *\| */),
+                    align: cap[2].replace(/^ *|\| *$/g, '').split(/ *\| */),
+                    cells: cap[3].replace(/(?: *\| *)?\n$/, '').split('\n')
+                };
+
+                for (let i = 0; i < item.align.length; i++) {
+                    if (/^ *-+: *$/.test(item.align[i])) {
+                        item.align[i] = 'right';
+                    } else if (/^ *:-+: *$/.test(item.align[i])) {
+                        item.align[i] = 'center';
+                    } else if (/^ *:-+ *$/.test(item.align[i])) {
+                        item.align[i] = 'left';
+                    } else {
+                        item.align[i] = null;
+                    }
+                }
+
+                for (let i = 0; i < item.cells.length; i++) {
+                    item.cells[i] = item.cells[i].replace(/^ *\| *| *\| *$/g, '').split(/ *\| */);
+                }
+
+                this.tokens.push(item);
+
+                continue;
+            }
+
+            // top-level paragraph
+            cap = this.rules.paragraph.exec(src);
+            if (top && cap) {
+                src = src.substring(cap[0].length);
+                this.tokens.push({
+                    type: 'paragraph',
+                    text: cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1]
+                });
+                continue;
+            }
+
+            // text
+            cap = this.rules.text.exec(src);
+            if (cap) {
+                // Top-level should never reach here.
+                src = src.substring(cap[0].length);
+                this.tokens.push({
+                    type: 'text',
+                    text: cap[0]
+                });
+                continue;
+            }
+
+            if (src) {
+                throw new Error('Infinite loop on byte: ' + src.charCodeAt(0));
+            }
+        }
+
+        return this.tokens;
+    }
+}
+
 export function format(text, options) {
     const markdownOptions = {
         renderer: new MattermostMarkdownRenderer(null, options),
@@ -212,7 +459,7 @@ export function format(text, options) {
         tables: true
     };
 
-    const tokens = marked.lexer(text, markdownOptions);
+    const tokens = new MattermostLexer(markdownOptions).lex(text);
 
     return new MattermostParser(markdownOptions).parse(tokens);
 }