Feature: Markdown: Support SetExt Heading (Issue #209)

Like these
==========

See: https://spec.commonmark.org/0.29/#setext-headings

Note: work for follow_link and VimwikiTOC

autoload/vimwiki/base.vim

@@ -727,7 +727,7 @@ function! s:normalize_anchor(anchor, ...) abort
   let anchor = substitute(anchor, punctuation_rx, '', 'g')
 
   " 3 Change any space to a hyphen
-  let anchor = substitute(anchor, ' ', '-', 'g')
+  let anchor = substitute(anchor, ' \+', '-', 'g')
 
   " 4 Append '-1', '-2', '-3',... to make it unique <= If that not unique
   if has_key(previous_anchors, anchor)
@@ -777,7 +777,7 @@ function! s:unnormalize_anchor(anchor) abort
   for char in split(anchor, '\zs')
     " 3 Change any space to a hyphen
     if char ==# '-'
-      let anchor_loop .=  '[ \-]'
+      let anchor_loop .=  '[ \-]\+'
     " 2 Remove anything that is not a letter, number, CJK character, hyphen or space
     " -- So add puncutation regex at each char
     else
@@ -816,13 +816,13 @@ function! s:jump_to_anchor(anchor) abort
 
     let anchor_header = s:safesubstitute(
           \ vimwiki#vars#get_syntaxlocal('header_match'),
-          \ '__Header__', segment_re, '')
+          \ '__Header__', segment_re, 'g')
     let anchor_bold = s:safesubstitute(
           \ vimwiki#vars#get_syntaxlocal('bold_match'),
-          \ '__Text__', segment_re, '')
+          \ '__Text__', segment_re, 'g')
     let anchor_tag = s:safesubstitute(
           \ vimwiki#vars#get_syntaxlocal('tag_match'),
-          \ '__Tag__', segment_re, '')
+          \ '__Tag__', segment_re, 'g')
 
     " Go: Move cursor: maybe more than onces (see markdown suffix)
     let success_nb = 0
@@ -840,7 +840,10 @@ function! s:jump_to_anchor(anchor) abort
         if success_nb < segment_nb-1 | let pos += 1 | endif
         call cursor(pos, 1)
         let success_nb += 1
-      else
+      " Do not move
+      " But maybe suffix -2 is not the segment number but the real header suffix
+      " TODO make this more robust
+      elseif i == 0
         " Next segment (default syntax)
         call setpos('.', oldpos)
         let fail = 1
@@ -1048,6 +1051,8 @@ function! vimwiki#base#edit_file(command, filename, anchor, ...) abort
       call vimwiki#u#ft_set()
     endif
   endif
+
+  " Goto anchor
   if a:anchor !=? ''
     call s:jump_to_anchor(a:anchor)
   endif
@@ -2154,8 +2159,12 @@ endfunction
 " Returns: all the headers in the current buffer as a list of the form
 " [[line_number, header_level, header_text], [...], [...], ...]
 function! s:collect_headers() abort
+  " Init loop variables
   let is_inside_pre_or_math = 0  " 1: inside pre, 2: inside math, 0: outside
   let headers = []
+  let rxHeader = vimwiki#vars#get_syntaxlocal('rxHeader')
+
+  " For all lines in file
   for lnum in range(1, line('$'))
     let line_content = getline(lnum)
     if (is_inside_pre_or_math == 1 && line_content =~# vimwiki#vars#get_syntaxlocal('rxPreEnd')) ||
@@ -2174,17 +2183,31 @@ function! s:collect_headers() abort
       let is_inside_pre_or_math = 2
       continue
     endif
-    if line_content !~# vimwiki#vars#get_syntaxlocal('rxHeader')
+
-      continue
+    " Check SetExt Header
-    endif
+    " TODO mutualise SetExt line (for consistency)
-    if vimwiki#vars#get_wikilocal('syntax') ==# 'markdown'
+    " TODO replace regex with =\+ or -\+
-      if stridx(line_content, vimwiki#vars#get_syntaxlocal('rxH')) > 0
+    if line_content =~# '\s\{0,3}[=-][=-]\+$'
-        continue  " markdown headers must start in the first column
+      let header_level = stridx(line_content, '=') != -1 ? 1 : 2
+      let header_text = getline(lnum-1)
+    " Maybe ATX header
+    else
+      " Clause: Must match rxHeader
+      if line_content !~# rxHeader
+        continue
       endif
+      " Clause: markdown headers must start in the first column
+      if vimwiki#vars#get_wikilocal('syntax') ==# 'markdown'
+            \ && stridx(line_content, vimwiki#vars#get_syntaxlocal('rxH')) > 0
+        continue
+      endif
+      " Get header level && text
+      let header_level = vimwiki#u#count_first_sym(line_content)
+      let header_text = matchstr(line_content, rxHeader)
     endif
-    let header_level = vimwiki#u#count_first_sym(line_content)
+
-    let header_text =
+    " Clean && Append to res
-          \ vimwiki#u#trim(matchstr(line_content, vimwiki#vars#get_syntaxlocal('rxHeader')))
+    let header_text = vimwiki#u#trim(header_text)
     call add(headers, [lnum, header_level, header_text])
   endfor
 
@@ -2294,6 +2317,7 @@ endfunction
 " a:create == 1: creates or updates TOC in current file
 " a:create == 0: update if TOC exists
 function! vimwiki#base#table_of_contents(create) abort
+  " Collect headers
   let headers = s:collect_headers()
   let toc_header_text = vimwiki#vars#get_wikilocal('toc_header')
 

autoload/vimwiki/vars.vim

@@ -638,6 +638,7 @@ function! vimwiki#vars#populate_syntax_vars(syntax) abort
           \ '^\s*\('.header_symbol.'\{1,6}\)\zs[^'.header_symbol.'].*[^'.header_symbol.']\ze\1\s*$'
   else
     " asymmetric
+    " Note: For markdown rxH=# and asymetric
     for i in range(1,6)
       let syntax_dic['rxH'.i.'_Template'] =
             \ repeat(header_symbol, i).' __Header__'
@@ -650,8 +651,11 @@ function! vimwiki#vars#populate_syntax_vars(syntax) abort
       let syntax_dic['rxH'.i.'_End'] =
             \ '^\s*'.header_symbol.'\{1,'.i.'}[^'.header_symbol.'].*$'
     endfor
-    let syntax_dic.rxHeader =
+    " Define header regex
-          \ '^\s*\('.header_symbol.'\{1,6}\)\zs[^'.header_symbol.'].*\ze$'
+    " -- ATX heading := preceed by #*
+    let atx_heading = '^\s*\%('.header_symbol.'\{1,6}\)'
+    let atx_heading .= '\zs[^'.header_symbol.'].*\ze$'
+    let syntax_dic.rxHeader = atx_heading
   endif
 
   let syntax_dic.rxPreStart =

doc/vimwiki.txt

@@ -3717,6 +3717,8 @@ http://code.google.com/p/vimwiki/issues/list. They may be accessible from
 https://github.com/vimwiki-backup/vimwiki/issues.
 
 New:~
+
+    * Issue #209: Feature: Markdown: Support SetExt Heading
     * Issue #847 #640: Feature: Markdown anchor
       normalize and unormalize: better follow_link and |VimwikiTOC|
     * Commit 5408d74b3: Syntax: Html support nested, concealable tag

syntax/vimwiki.vim

@@ -186,6 +186,18 @@ for s:i in range(1,6)
         \ '/me=s-1 transparent fold'
 endfor
 
+" SetExt header
+" TODO mutualise SetExt Regexp
+let setex_header1_re = '^\s\{0,3}[^>].*\n\s\{0,3}==\+$'
+let setex_header2_re = '^\s\{0,3}[^>].*\n\s\{0,3}--\+$'
+execute 'syntax match VimwikiHeader1'
+    \ . ' /'. setex_header1_re . '/ '
+    \ 'contains=VimwikiTodo,VimwikiHeaderChar,VimwikiNoExistsLink,VimwikiCode,'.
+    \ 'VimwikiLink,@Spell'
+execute 'syntax match VimwikiHeader2'
+    \ . ' /'. setex_header2_re . '/ ' .
+    \ 'contains=VimwikiTodo,VimwikiHeaderChar,VimwikiNoExistsLink,VimwikiCode,'.
+    \ 'VimwikiLink,@Spell'
 
 
 let s:options = ' contained transparent contains=NONE'

syntax/vimwiki_markdown.vim

@@ -63,8 +63,25 @@ let s:markdown_syntax.rxMultilineCommentEnd = ''
 let s:markdown_syntax.rxComment = '^\s*%%.*$\|<!--[^>]*-->'
 let s:markdown_syntax.rxTags = '\%(^\|\s\)\@<=:\%([^:[:space:]]\+:\)\+\%(\s\|$\)\@='
 
-let s:markdown_syntax.header_search = '^\s*\(#\{1,6}\)\([^#].*\)$'
+
-let s:markdown_syntax.header_match = '^\s*\(#\{1,6}\)#\@!\s*__Header__\s*$'
+" Used in code (base.vim)
+"""""""""""""""""""""""""
+
+" Header
+" TODO mutualise with rxHeader in vars.vim := Define atx_regex only onces
+" TODO regex_or function  => (1|2)
+let atx_header_search = '^\s*\(#\{1,6}\)\([^#].*\)$'
+let atx_header_match  = '^\s*\(#\{1,6}\)#\@!\s*__Header__\s*$'
+
+let setex_header_search = '^\s\{0,3}\zs[^>].*\ze\n'
+let setex_header_search .= '^\s\{0,3}[=-]\{2,}$'
+
+let setex_header_match = '^\s\{0,3}>\@!__Header__\n'
+let setex_header_match .= '^\s\{0,3}[=-][=-]\+$'
+
+let s:markdown_syntax.header_search = '\%(' . atx_header_search . '\|' . setex_header_search . '\)'
+let s:markdown_syntax.header_match = '\%(' . atx_header_match . '\|' . setex_header_match . '\)'
+
 let s:markdown_syntax.bold_search = '\%(^\|\s\|[[:punct:]]\)\@<=\*\zs'.
       \ '\%([^*`[:space:]][^*`]*[^*`[:space:]]\|[^*`[:space:]]\)\ze\*\%([[:punct:]]\|\s\|$\)\@='
 let s:markdown_syntax.bold_match = '\%(^\|\s\|[[:punct:]]\)\@<=\*__Text__\*'.

test/command_toc.vader

@@ -10,7 +10,7 @@
 # -- 2. remove anything that is not a letter, number, space or hyphen (see the source for how Unicode is handled) => from 'bad characters'
 # -- 3. changes any space to a hyphen => OK: from previous big
 # -- 4. If that is not unique, add "-1", "-2", "-3",... to make it unique => TODO not implemented
-# 
+#
 #
 # TODO if link in heading
 #Given vimwiki (Two same heading {{{1):
@@ -29,6 +29,60 @@
 # Start {{{1
 
 
+Given vimwiki (Underline header (SetExt) (#209) {{{1):
+  First with spaces
+  =====
+
+  toto
+
+  Second
+  -------
+  toto
+
+  Third
+  -----
+  toto
+
+  Four
+  =====
+  toto
+  Last
+  ----
+
+Execute (Set syntax markdown && Set sw=8):
+  call SetSyntax('markdown')
+  set sw=8
+  VimwikiTOC
+
+Expect (Heading SetExt created):
+  # Contents
+
+  - [First with spaces](#first-with-spaces)
+          - [Second](#second)
+          - [Third](#third)
+  - [Four](#four)
+          - [Last](#last)
+
+  First with spaces
+  =====
+
+  toto
+
+  Second
+  -------
+  toto
+
+  Third
+  -----
+  toto
+
+  Four
+  =====
+  toto
+  Last
+  ----
+
+
 
 Given vimwiki (Two same heading (#968) {{{1):
   # One
@@ -69,10 +123,10 @@ Execute (Set syntax markdown && VimwikiTOC):
 
 Expect (Bad characters are removed):
   # Contents
-  
+
   - [One !@#@#(!%#&$^(!@](#one-)
           - [Two !!~!!:"@!>@!>?<](#two-)
-  
+
   # One !@#@#(!%#&$^(!@
   ## Two !!~!!:"@!>@!>?<
 

test/link_markdown_multiple_per_file.vader

@@ -1,8 +1,58 @@
 # Link internal to a file
 # See issue #666 for anchor support (then internal links)
+# Preambule set file onces and for all {{{1
+# Otherwise the bash script is freezing
+Given vimwiki (a):
+  a
 
+Execute (Set filename wiki_test.md):
+  file wiki_test.md
 
-# Link to anchor with spaces {{{!
+Expect (a):
+  a
+
+# Link to anchor in SetExt {{{1
+# Like that
+# -----
+# Issue: #209
+
+Given vimwiki (Anchor SetExt):
+  [jump](#frst-one)
+
+  F!rst    One
+  =========
+
+Execute (Set filename wiki_test.md):
+  call SetSyntax('markdown')
+
+Do (Enter link):
+  \<Cr>
+  A__HERE__\<Esc>
+
+Expect (Cursor jumped SetExt):
+  [jump](#frst-one)
+
+  F!rst    One__HERE__
+  =========
+
+Given vimwiki (Bad Anchor SetExt):
+  [jump](#frst-one)
+
+  F!rst    One
+
+Execute (Set filename wiki_test.md):
+  call SetSyntax('markdown')
+
+Do (Enter link):
+  \<Cr>
+  A__HERE__\<Esc>
+
+Expect (Cursor stayed (not jumped) SetExt):
+  [jump](#frst-one)__HERE__
+
+  F!rst    One
+
+# Link to anchor with spaces {{{1
 # PR #840
 # Issues: #831
 
@@ -10,12 +60,11 @@ Given vimwiki (Internal links zith spaces):
   [Any ! apparent name @#$](#basic-heading-many-spaces)
   One line here
 
-  ## Basic HeAding Many SpacES
+  ##    Basic  HeAding Many   SpacES
 
   One line here
 
 Execute (Set filename wiki_test.md):
-  file wiki_test.md
   call SetSyntax('markdown')
 
 Do (Enter link):
@@ -26,7 +75,7 @@ Expect (Cursor at heading position):
   [Any ! apparent name @#$](#basic-heading-many-spaces)
   One line here
 
-  ## Basic HeAding Many SpacES__HERE__
+  ##    Basic  HeAding Many   SpacES__HERE__
 
   One line here
 

test/syntax.vader

@@ -328,6 +328,20 @@ Execute (Assert Syntax link):
 
 # 3 Header {{{1
 ###############
+Given vimwiki (Markdown SetExt Headers):
+  One
+  ===
+  two
+  ---
+
+Execute (Set syntax markdown):
+  call SetSyntax('markdown')
+
+Execute (Assert Syntax Header SetExt):
+  AssertEqual 'VimwikiHeader1', SyntaxAt(1, 1)
+  AssertEqual 'VimwikiHeader1', SyntaxAt(2, 1)
+  AssertEqual 'VimwikiHeader2', SyntaxAt(3, 1)
+  AssertEqual 'VimwikiHeader2', SyntaxAt(4, 1)
 
 Given vimwiki (Wiki Headers):
   = Header level 1 =