Feature: Markdown anchor normalize and unormalize (Issue #664)

Add normalize anchor and unnormalize to get markdown anchor github
compliant. For respectively follow_link and VimwikiTOC

TODO: Treat the potential -12 suffix in anchor-links: ex:
[got to second link](same-link-2)

autoload/vimwiki/base.vim

@@ -688,8 +688,88 @@ function! vimwiki#base#get_anchors(filename, syntax) abort
 endfunction
 
 
-" Jump to anchor
+" Helper to mutualize
-" Called by edit_file
+" Called: normalize and unnormalize anchor
+function! s:get_punctuaction_regex() abort
+  " From: https://gist.github.com/asabaylus/3071099#gistcomment-2563127
+  return '[^0-9a-zA-Z\u4e00-\u9fff_ \-]'
+endfunction
+
+
+" :param: anchor <string> <= Heading line
+" Return: anchor <string> => link in TOC
+function! s:normalize_anchor(anchor) abort
+  " Note: See unormalize
+  " 0 Get in
+  let anchor = a:anchor
+
+  " A Trim space
+  let anchor = vimwiki#u#trim(anchor)
+
+  " 1 Downcase the string
+  let anchor = tolower(anchor)
+
+  " 2 Remove anything that is not a letter, number, CJK character, hyphen or space
+  " TODO mutualize punctuation_rx with above
+  let punctuation_rx = s:get_punctuaction_regex()
+  let anchor = substitute(anchor, punctuation_rx, '', 'g')
+
+  " 3 Change any space to a hyphen
+  let anchor = substitute(anchor, ' ', '-', 'g')
+
+  " 4.2 TODO anchor: If that is not unique, add '-1', '-2', '-3',... to make it unique
+
+  return anchor
+endfunction
+
+
+" :param: anchor <string> <= link
+" Return: anchor <regex> to look for
+function! s:unnormalize_anchor(anchor) abort
+  " Note:
+  " -- Pandoc keep the '_' in anchor
+  " -- Done after: Add spaces leading and trailing => Later with the template
+  " Link: Inspired from https://gist.github.com/asabaylus/3071099
+  " Issue: #664 => Points to all others
+  let anchor = a:anchor
+  let punctuation_rx = s:get_punctuaction_regex() . '*'
+
+  " 4 Add '-1', '-2', '-3',... to make it unique if not unique
+  " -- Save the trailing -12
+  let sufix = substitute(anchor, '^.*-\(\d\+\)$', '\1', '')
+  if sufix !=# ''
+    let sufix = '[ \-]' . sufix
+  endif
+  " -- Remove it
+  let anchor = substitute(anchor, '\(-\d\+\)$', '', '')
+
+  " For each char
+  let anchor_r = ''
+  for char in split(anchor, '\zs')
+    " 3 Change any space to a hyphen
+    if char ==# '-'
+      let anchor_r .=  '[ \-]'
+    " 2 Remove anything that is not a letter, number, CJK character, hyphen or space
+    " -- So add puncutation regex at each char
+    else
+      let anchor_r .= char . punctuation_rx
+    endif
+  endfor
+  let anchor = punctuation_rx . anchor_r
+
+  " 1 Downcase the string
+  let anchor = '\c' . anchor
+
+  " 4.bis Add the optional suffix
+  let anchor = anchor . '\(' . sufix . '\)\?'
+
+  return anchor
+endfunction
+
+
+" Jump to anchor, doing the oposite of normalize_anchor
+" Called: edit_file
+" TODO treat the sufix: -2 -> Go to second anchor
 function! s:jump_to_anchor(anchor) abort
   let oldpos = getpos('.')
   call cursor(1, 1)
@@ -699,13 +779,9 @@ function! s:jump_to_anchor(anchor) abort
   let segments = split(anchor, '#', 0)
 
   for segment in segments
-
     " Craft segment pattern so that it is case insensitive and also matches dashes
     " in anchor link with spaces in heading
-    " Ignore case
+    let segment = s:unnormalize_anchor(segment)
-    let segment = substitute(segment, '\<\(.\)', '\\c\1', 'g')
-    " Treat - as [- or space]
-    let segment = substitute(segment , '-', '[ -]', 'g')
 
     let anchor_header = s:safesubstitute(
           \ vimwiki#vars#get_syntaxlocal('header_match'),
@@ -2184,13 +2260,15 @@ function! vimwiki#base#table_of_contents(create) abort
   " copy all local variables into dict (add a: if arguments are needed)
   let GeneratorTOC = copy(l:)
   function! GeneratorTOC.f() abort
+    " Gather heading informations
     let numbering = vimwiki#vars#get_global('html_header_numbering')
+    " TODO numbering not used !
     let headers_levels = [['', 0], ['', 0], ['', 0], ['', 0], ['', 0], ['', 0]]
     let complete_header_infos = []
     for header in self.headers
       let h_text = header[2]
       let h_level = header[1]
-      " don't include the TOC's header itself
+      " Don't include the TOC's header itself
       if h_text ==# self.toc_header_text
         continue
       endif
@@ -2198,7 +2276,9 @@ function! vimwiki#base#table_of_contents(create) abort
       for idx in range(h_level, 5) | let headers_levels[idx] = ['', 0] | endfor
 
       let h_complete_id = ''
-      if vimwiki#vars#get_wikilocal('toc_link_format') == 0
+
+      " Add description to the link if toc_link_format == 1 => extended
+      if vimwiki#vars#get_wikilocal('toc_link_format') == 1
         for l in range(h_level-1)
           if headers_levels[l][0] !=? ''
             let h_complete_id .= headers_levels[l][0].'#'
@@ -2210,20 +2290,27 @@ function! vimwiki#base#table_of_contents(create) abort
       call add(complete_header_infos, [h_level, h_complete_id, h_text])
     endfor
 
+    " Insert the information in the Link Template
+    " -- and create line list
     let lines = []
     let startindent = repeat(' ', vimwiki#lst#get_list_margin())
     let indentstring = repeat(' ', vimwiki#u#sw())
     let bullet = vimwiki#lst#default_symbol().' '
-    for [lvl, link, desc] in complete_header_infos
+    for [lvl, anchor, desc] in complete_header_infos
       if vimwiki#vars#get_wikilocal('syntax') ==# 'markdown'
         let link_tpl = vimwiki#vars#get_syntaxlocal('Weblink2Template')
-      elseif vimwiki#vars#get_wikilocal('toc_link_format') == 0
+      elseif vimwiki#vars#get_wikilocal('toc_link_format') == 1
         let link_tpl = vimwiki#vars#get_global('WikiLinkTemplate2')
       else
         let link_tpl = vimwiki#vars#get_global('WikiLinkTemplate1')
       endif
+
+      " Normalize anchor
+      let anchor = s:normalize_anchor(anchor)
+
+      " Insert link in template
       let link = s:safesubstitute(link_tpl, '__LinkUrl__',
-            \ '#'.link, '')
+            \ '#'.anchor, '')
       let link = s:safesubstitute(link, '__LinkDescription__', desc, '')
       call add(lines, startindent.repeat(indentstring, lvl-1).bullet.link)
     endfor

autoload/vimwiki/u.vim

@@ -61,7 +61,6 @@ function! vimwiki#u#count_exe(cmd) abort
 endfunction
 
 
-" Trim spaces: leading and trailing
 function! vimwiki#u#sort_len(list) abort
   function! s:len_compare(s1, s2) abort
     let i1 = len(a:s1)
@@ -72,6 +71,9 @@ function! vimwiki#u#sort_len(list) abort
 endfunction
 
 
+" Trim spaces: leading and trailing
+" :param: string in
+" :param: (1) <string> optional list of character to trim
 function! vimwiki#u#trim(string, ...) abort
   let chars = ''
   if a:0 > 0

doc/vimwiki.txt

@@ -2788,10 +2788,10 @@ The format of the links in the Table of Contents (see |vimwiki-toc|).
 
 
 Value           Description~
-0               Extended: The link contains the description and URL. URL
+0               Brief: The link contains only the URL. URL references only
-                references all levels.
-1               Brief: The link contains only the URL. URL references only
                 the immediate level.
+1               Extended: The link contains the description and URL. URL
+                references all levels.
 
 Default: 0
 
@@ -3718,19 +3718,27 @@ https://github.com/vimwiki-backup/vimwiki/issues.
 
 New:~
     * PR #967: Add multiline comment support via %%+ and +%%
-    * Issue #942: Fixing wrong html link conversion in windows
     * PR #946: Add option |g:vimwiki_commentstring| to customize commentstring
     * Issue #940: Render table header inside thead element and rest under
       tbody element if table header specified in wiki
     * PR #811: Feature: Added handling of absolute path to vimwiki (with //)
-    * PR #919: Fix duplicate helptag
     * PR #907: Cycle bullets
-    * PR #900: conceallevel is now setted locally for vimwiki buffers
     * PR #901: adds multiparagraph blockquotes using email style syntax
     * PR #934: RSS feed generation for diary with :VimwikiRss.
-    * PR #959: Fix :VimwikiNextTask
+
-    * PR #963: Replace VimwikiListChangeLevel references in doc with
+Changed:~
+    * |g:vimwiki_toc_link_format| == 0 (default) means old behavior: short links
+
+Removed:~
+
+Fixed:~
+    * PR #963: Doc: Replace VimwikiListChangeLevel references in doc with
       VimwikiListChangeLvl as defined by command
+    * PR #900: conceallevel is now setted locally for vimwiki buffers
+    * Issue #942: Fixing wrong html link conversion in windows
+    * PR #919: Fix duplicate helptag
+    * PR #959: Fix :VimwikiNextTask
+
 
 2.5 (2020-05-26)~
 

test/command_toc.vader

@@ -1,8 +1,51 @@
-# VimwikiTOC
+# VimwikiTOC {{{1
 #
 # TODO (10min) test if g:vimwiki_to_header well readen
 # TODO (10min) test vimviki_toc_link_format
 # TODO (1h) test if really wiki dependant (for 2 diffrent wikis)
+#
+# Doc: from #664:
+# -- 1. It downcases the string => OK: from previous big collection
+# -- 2. remove anything that is not a letter, number, space or hyphen (see the source for how Unicode is handled) => from 'bad characters'
+# -- 3. changes any space to a hyphen => OK: from previous big
+# -- 4. If that is not unique, add "-1", "-2", "-3",... to make it unique => TODO not implemented
+# 
+# Start {{{1
+
+" TODO
+"Given vimwiki (Two same heading {{{1):
+"  # One
+"  ## two
+"  ## Two
+"
+"Execute (Set syntax markdown && Set sw=8):
+"  call SetSyntax('markdown')
+"  set sw=8
+"  VimwikiTOC
+"
+"Expect (Suffix -1 and -2):
+"
+
+Given vimwiki (Heading with many bad caracters {{{1):
+  # One !@#@#(!%#&$^(!@
+  ## Two !!~!!:"@!>@!>?<
+
+Execute (Set syntax markdown && VimwikiTOC):
+  call SetSyntax('markdown')
+  set sw=8
+  VimwikiTOC
+
+Expect (Bad characters are removed):
+  # Contents
+  
+  - [One !@#@#(!%#&$^(!@](#one-)
+          - [Two !!~!!:"@!>@!>?<](#two-)
+  
+  # One !@#@#(!%#&$^(!@
+  ## Two !!~!!:"@!>@!>?<
+
+
+# Large previous tests {{{1
 
 Execute (Reset TOC header to default):
   call vimwiki#vars#set_wikilocal('toc_header', 'Contents')
@@ -28,11 +71,11 @@ Execute (VimwikiTOC):
 Expect (With a TOC sw=8):
   # Contents
 
-  - [Header 1](#Header 1)
+  - [Header 1](#header-1)
-          - [Header 1.1](#Header 1#Header 1.1)
+          - [Header 1.1](#header-11)
-                  - [Header 1.1.1](#Header 1#Header 1.1#Header 1.1.1)
+                  - [Header 1.1.1](#header-111)
-  - [Header 2](#Header 2)
+  - [Header 2](#header-2)
-                  - [Header 2.1.1](#Header 2#Header 2.1.1)
+                  - [Header 2.1.1](#header-211)
 
   # Header 1
   random text
@@ -51,11 +94,11 @@ Execute (Set sw=4  && VimwikiTOC):
 Expect (With a TOC sw=4):
   # Contents
 
-  - [Header 1](#Header 1)
+  - [Header 1](#header-1)
-      - [Header 1.1](#Header 1#Header 1.1)
+      - [Header 1.1](#header-11)
-          - [Header 1.1.1](#Header 1#Header 1.1#Header 1.1.1)
+          - [Header 1.1.1](#header-111)
-  - [Header 2](#Header 2)
+  - [Header 2](#header-2)
-          - [Header 2.1.1](#Header 2#Header 2.1.1)
+          - [Header 2.1.1](#header-211)
 
   # Header 1
   random text
@@ -79,11 +122,11 @@ Execute (VimwikiTOC):
 Expect (Brand new TOC):
   # Contents
 
-  - [Header 1](#Header 1)
+  - [Header 1](#header-1)
-      - [Header 1.1](#Header 1#Header 1.1)
+      - [Header 1.1](#header-11)
-          - [Header 1.1.1](#Header 1#Header 1.1#Header 1.1.1)
+          - [Header 1.1.1](#header-111)
-  - [Header 2](#Header 2)
+  - [Header 2](#header-2)
-          - [Header 2.1.1](#Header 2#Header 2.1.1)
+          - [Header 2.1.1](#header-211)
 
   # Header 1
   random text
@@ -103,11 +146,11 @@ Execute (Let toc_header = Sommaire && VimwikiTOC):
 Expect (Append a Sommaire && Leave Contents alone):
   # Sommaire
 
-  - [Header 1](#Header 1)
+  - [Header 1](#header-1)
-      - [Header 1.1](#Header 1#Header 1.1)
+      - [Header 1.1](#header-11)
-          - [Header 1.1.1](#Header 1#Header 1.1#Header 1.1.1)
+          - [Header 1.1.1](#header-111)
-  - [Header 2](#Header 2)
+  - [Header 2](#header-2)
-          - [Header 2.1.1](#Header 2#Header 2.1.1)
+          - [Header 2.1.1](#header-211)
 
   # Header 1
   random text
@@ -131,11 +174,11 @@ Execute (VimwikiTOC):
 Expect (Brand new TOC with sommaire):
   # Sommaire
 
-  - [Header 1](#Header 1)
+  - [Header 1](#header-1)
-      - [Header 1.1](#Header 1#Header 1.1)
+      - [Header 1.1](#header-11)
-          - [Header 1.1.1](#Header 1#Header 1.1#Header 1.1.1)
+          - [Header 1.1.1](#header-111)
-  - [Header 2](#Header 2)
+  - [Header 2](#header-2)
-          - [Header 2.1.1](#Header 2#Header 2.1.1)
+          - [Header 2.1.1](#header-211)
 
   # Header 1
   random text
@@ -157,11 +200,11 @@ Execute (call vimwiki#vars#set_global('toc_header_level', 6):
 Expect (Content prepended):
   ###### Sommaire
 
-  - [Header 1](#Header 1)
+  - [Header 1](#header-1)
-      - [Header 1.1](#Header 1#Header 1.1)
+      - [Header 1.1](#header-11)
-          - [Header 1.1.1](#Header 1#Header 1.1#Header 1.1.1)
+          - [Header 1.1.1](#header-111)
-  - [Header 2](#Header 2)
+  - [Header 2](#header-2)
-          - [Header 2.1.1](#Header 2#Header 2.1.1)
+          - [Header 2.1.1](#header-211)
 
   # Header 1
   random text
@@ -173,4 +216,4 @@ Expect (Content prepended):
   # Header 2
   ### Header 2.1.1
 
-" vim: sw=2 foldmethod=indent foldlevel=30 foldignore=#
+" vim: sw=2 foldmethod=marker foldlevel=30 foldignore=#