Skip to content

Commit d600b94

Browse files
dwellemarijnh
dwelle
authored and
marijnh
committed
[markdown mode] improve setext & hr tokenization
1 parent 4d448b2 commit d600b94

File tree

2 files changed

+108
-26
lines changed

2 files changed

+108
-26
lines changed

‎mode/markdown/markdown.js

+46-20
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
9090
, textRE = /^[^#!\[\]*_\\<>` "'(~:]+/
9191
, fencedCodeRE = new RegExp("^(" + (modeCfg.fencedCodeBlocks === true ? "~~~+|```+" : modeCfg.fencedCodeBlocks) +
9292
")[ \\t]*([\\w+#\-]*)")
93+
, linkDefRE = /^\s*\[[^\]]+?\]:\s*\S+(\s*\S*\s*)?$/ // naive link-definition
9394
, punctuation = /[!\"#$%&\'()*+,\-\.\/:;<=>?@\[\\\]^_`{|}~—]/
9495
, expandedTab = " " // CommonMark specifies tab as 4 spaces
9596

@@ -110,6 +111,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
110111
// Blocks
111112

112113
function blankLine(state) {
114+
state.hr = false;
113115
// Reset linkTitle state
114116
state.linkTitle = false;
115117
// Reset EM state
@@ -137,16 +139,18 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
137139

138140
function blockNormal(stream, state) {
139141
var sol = stream.sol();
142+
var prevLineLineIsEmpty = lineIsEmpty(state.prevLine);
143+
var prevLineIsIndentedCode = state.indentedCode;
144+
var prevLineIsHr = state.hr;
145+
var prevLineIsList = state.list !== false;
146+
var maxNonCodeIndentation = (state.listStack[state.listStack.length - 1] || 0) + 3;
140147

141-
var prevLineIsList = state.list !== false,
142-
prevLineIsIndentedCode = state.indentedCode;
143-
148+
state.hr = false;
144149
state.indentedCode = false;
145150

146-
var lineIndentation;
151+
var lineIndentation = state.indentation;
147152
// compute once per line (on first token)
148153
if (state.indentationDiff === null) {
149-
lineIndentation = state.indentation;
150154
state.indentationDiff = state.indentation;
151155
if (prevLineIsList) {
152156
state.list = null;
@@ -168,8 +172,11 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
168172
}
169173
}
170174

175+
var isHr = (state.list === false || prevLineIsHr || prevLineLineIsEmpty) &&
176+
state.indentation <= maxNonCodeIndentation && stream.match(hrRE);
177+
171178
var match = null;
172-
if (state.indentationDiff >= 4 && (prevLineIsIndentedCode || lineIsEmpty(state.prevLine))) {
179+
if (state.indentationDiff >= 4 && (prevLineIsIndentedCode || prevLineLineIsEmpty)) {
173180
stream.skipToEnd();
174181
state.indentedCode = true;
175182
return tokenTypes.code;
@@ -180,23 +187,12 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
180187
if (modeCfg.highlightFormatting) state.formatting = "header";
181188
state.f = state.inline;
182189
return getType(state);
183-
} else if (!lineIsEmpty(state.prevLine) && !state.quote && !prevLineIsList &&
184-
!prevLineIsIndentedCode && (match = stream.match(setextHeaderRE))) {
185-
state.header = match[0].charAt(0) == '=' ? 1 : 2;
186-
if (modeCfg.highlightFormatting) state.formatting = "header";
187-
state.f = state.inline;
188-
return getType(state);
189190
} else if (stream.eat('>')) {
190191
state.quote = sol ? 1 : state.quote + 1;
191192
if (modeCfg.highlightFormatting) state.formatting = "quote";
192193
stream.eatSpace();
193194
return getType(state);
194-
} else if (stream.peek() === '[') {
195-
return switchInline(stream, state, footnoteLink);
196-
} else if (stream.match(hrRE, true)) {
197-
state.hr = true;
198-
return tokenTypes.hr;
199-
} else if (!state.quote && (match = stream.match(listRE))) {
195+
} else if (!isHr && !state.quote && (match = stream.match(listRE))) {
200196
var listType = match[1] ? "ol" : "ul";
201197

202198
state.indentation = lineIndentation + stream.current().length;
@@ -220,6 +216,35 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
220216
if (modeCfg.highlightFormatting) state.formatting = "code-block";
221217
state.code = -1
222218
return getType(state);
219+
// SETEXT has lowest block-scope precedence after HR, so check it after
220+
// the others (code, blockquote, list...)
221+
} else if (
222+
// if setext set, indicates line after ---/===
223+
state.setext || (
224+
// line before ---/===
225+
!state.quote && state.list === false && !state.code && !isHr &&
226+
!prevLineIsList && !linkDefRE.test(stream.string) &&
227+
(match = stream.lookAhead(1)) && (match = match.match(setextHeaderRE))
228+
)
229+
) {
230+
if ( !state.setext ) {
231+
state.header = match[0].charAt(0) == '=' ? 1 : 2;
232+
state.setext = state.header;
233+
} else {
234+
state.header = state.setext;
235+
// has no effect on type so we can reset it now
236+
state.setext = 0;
237+
stream.skipToEnd();
238+
if (modeCfg.highlightFormatting) state.formatting = "header";
239+
}
240+
state.f = state.inline;
241+
return getType(state);
242+
} else if (isHr) {
243+
stream.skipToEnd();
244+
state.hr = true;
245+
return tokenTypes.hr;
246+
} else if (stream.peek() === '[') {
247+
return switchInline(stream, state, footnoteLink);
223248
}
224249

225250
return switchInline(stream, state, state.inline);
@@ -703,6 +728,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
703728
em: false,
704729
strong: false,
705730
header: 0,
731+
setext: 0,
706732
hr: false,
707733
taskList: false,
708734
list: false,
@@ -741,6 +767,7 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
741767
strikethrough: s.strikethrough,
742768
emoji: s.emoji,
743769
header: s.header,
770+
setext: s.setext,
744771
hr: s.hr,
745772
taskList: s.taskList,
746773
list: s.list,
@@ -760,9 +787,8 @@ CodeMirror.defineMode("markdown", function(cmCfg, modeCfg) {
760787
state.formatting = false;
761788

762789
if (stream != state.thisLine) {
763-
// Reset state.header and state.hr
790+
// Reset state.header
764791
state.header = 0;
765-
state.hr = false;
766792

767793
if (stream.match(/^\s*$/, true)) {
768794
blankLine(state);

‎mode/markdown/test.js

+62-6
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
"[header&header-1&formatting&formatting-header&formatting-header-1 # ][header&header-1 foo # bar ][header&header-1&formatting&formatting-header&formatting-header-1 #]");
6868

6969
FT("formatting_setextHeader",
70-
"foo",
70+
"[header&header-1 foo]",
7171
"[header&header-1&formatting&formatting-header&formatting-header-1 =]");
7272

7373
FT("formatting_blockquote",
@@ -237,43 +237,78 @@
237237
//
238238
// Check if single underlining = works
239239
MT("setextH1",
240-
"foo",
240+
"[header&header-1 foo]",
241241
"[header&header-1 =]");
242242

243243
// Check if 3+ ='s work
244244
MT("setextH1",
245-
"foo",
245+
"[header&header-1 foo]",
246246
"[header&header-1 ===]");
247247

248248
// Check if single underlining - works
249249
MT("setextH2",
250-
"foo",
250+
"[header&header-2 foo]",
251251
"[header&header-2 -]");
252252

253253
// Check if 3+ -'s work
254254
MT("setextH2",
255-
"foo",
255+
"[header&header-2 foo]",
256256
"[header&header-2 ---]");
257257

258258
// http://spec.commonmark.org/0.19/#example-45
259259
MT("setextH2AllowSpaces",
260-
"foo",
260+
"[header&header-2 foo]",
261261
" [header&header-2 ---- ]");
262262

263263
// http://spec.commonmark.org/0.19/#example-44
264264
MT("noSetextAfterIndentedCodeBlock",
265265
" [comment foo]",
266266
"[hr ---]");
267267

268+
MT("setextAfterFencedCode",
269+
"[comment ```]",
270+
"[comment foo]",
271+
"[comment ```]",
272+
"[header&header-2 bar]",
273+
"[header&header-2 ---]");
274+
275+
MT("setextAferATX",
276+
"[header&header-1 # foo]",
277+
"[header&header-2 bar]",
278+
"[header&header-2 ---]");
279+
268280
// http://spec.commonmark.org/0.19/#example-51
269281
MT("noSetextAfterQuote",
270282
"[quote&quote-1 > foo]",
283+
"[hr ---]",
284+
"",
285+
"[quote&quote-1 > foo]",
286+
"[quote&quote-1 bar]",
271287
"[hr ---]");
272288

273289
MT("noSetextAfterList",
274290
"[variable-2 - foo]",
291+
"[hr ---]",
292+
"",
293+
"[variable-2 - foo]",
294+
"bar",
295+
"[hr ---]");
296+
297+
MT("setext_nestedInlineMarkup",
298+
"[header&header-1 foo ][em&header&header-1 *bar*]",
299+
"[header&header-1 =]");
300+
301+
MT("setext_linkDef",
302+
"[link [[aaa]]:] [string&url http://google.com 'title']",
275303
"[hr ---]");
276304

305+
// currently, looks max one line ahead, thus won't catch valid CommonMark
306+
// markup
307+
MT("setext_oneLineLookahead",
308+
"foo",
309+
"[header&header-1 bar]",
310+
"[header&header-1 =]");
311+
277312
// Single-line blockquote with trailing space
278313
MT("blockquoteSpace",
279314
"[quote&quote-1 > foo]");
@@ -394,6 +429,27 @@
394429
"[variable-2 - foo]",
395430
"[hr -----]");
396431

432+
MT("hrAfterFencedCode",
433+
"[comment ```]",
434+
"[comment code]",
435+
"[comment ```]",
436+
"[hr ---]");
437+
438+
// allow hr inside lists
439+
// (require prev line to be empty or hr, TODO: non-CommonMark-compliant)
440+
MT("hrInsideList",
441+
"[variable-2 - foo]",
442+
"",
443+
" [hr ---]",
444+
" [hr ---]",
445+
"",
446+
" [comment ---]");
447+
448+
MT("consecutiveHr",
449+
"[hr ---]",
450+
"[hr ---]",
451+
"[hr ---]");
452+
397453
// Formatting in lists (*)
398454
MT("listAsteriskFormatting",
399455
"[variable-2 * ][variable-2&em *foo*][variable-2 bar]",

0 commit comments

Comments
 (0)