markup: Reimplement pygmentsCodefencesGuessSyntax

Fixes #6565

go.mod

@@ -54,8 +54,8 @@ require (
  github.com/spf13/viper v1.4.0
  github.com/tdewolff/minify/v2 v2.6.1
  github.com/yosssi/ace v0.0.5
- github.com/yuin/goldmark v1.1.10
- github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a
+ github.com/yuin/goldmark v1.1.11
+ github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5
  go.opencensus.io v0.22.0 // indirect
  gocloud.dev v0.15.0
  golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff

go.sum

@@ -359,8 +359,14 @@ github.com/yuin/goldmark v1.1.8 h1:d0m8Ac9JaetYjPZLC4P4W32ac7I0lpJpQbvxZtFqBoM=
 github.com/yuin/goldmark v1.1.8/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.10 h1:bg3TC1aj4DbjGdhvjSSffGfAgVUdBEIpccuCozwOYWo=
 github.com/yuin/goldmark v1.1.10/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.11 h1:OO08ilczi3F4swaYWPB99s08WRxP9DdLBemiLFQ6vCo=
+github.com/yuin/goldmark v1.1.11/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a h1:L7FTUnbc0WEBqGWgjbx4sPNAOX1/q5W/3KCD6g8XkKo=
 github.com/yuin/goldmark-highlighting v0.0.0-20191124122839-ede94e40cc3a/go.mod h1:1gshkGdH4gcrIH5MGSScGH42rOOCO+4Ks6acjAkA9C0=
+github.com/yuin/goldmark-highlighting v0.0.0-20191126180129-d7a4bf4d7ea4 h1:vI4Jv29V1cMPqetuLPMW1CMB9xNgxsHVBo8Mid6bwH8=
+github.com/yuin/goldmark-highlighting v0.0.0-20191126180129-d7a4bf4d7ea4/go.mod h1:4QGn5rJFOASBa2uK4Q2h3BRTyJqRfsAucPFIipSTcaM=
+github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5 h1:QbH7ca1qtgZHrzvcVAEoiJIwBqrXxMOfHYfwZIniIK0=
+github.com/yuin/goldmark-highlighting v0.0.0-20191202084645-78f32c8dd6d5/go.mod h1:4QGn5rJFOASBa2uK4Q2h3BRTyJqRfsAucPFIipSTcaM=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.mongodb.org/mongo-driver v1.0.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=

hugolib/hugo_sites_build_test.go

@@ -671,7 +671,7 @@ END
 
  b.CreateSites().Build(BuildCfg{})
 
- contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header99\">Another header</h2>", "<p>The End.</p>"}
+ contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header-99\">Another header</h2>", "<p>The End.</p>"}
 
  for i := 1; i <= numPages; i++ {
  if i%3 != 0 {
@@ -691,13 +691,13 @@ END
  checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), contentMatchers...)
  }
 
- checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033\n\nRender 1: View: 8033\n\nRender 2: View: 8033\n\nRender 3: View: 8033\n\nRender 4: View: 8033\n\nEND\n")
- checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8033", "Render 4: View: 8033\n\nEND")
- checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8033", "4: View: 8033\n\nEND")
+ checkContent(b, "public/s1/index.html", "P: s1/_index.md\nList: 10|List Content: 8132\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8132\n\nRender 1: View: 8132\n\nRender 2: View: 8132\n\nRender 3: View: 8132\n\nRender 4: View: 8132\n\nEND\n")
+ checkContent(b, "public/s2/index.html", "P: s2/_index.md\nList: 10|List Content: 8132", "Render 4: View: 8132\n\nEND")
+ checkContent(b, "public/index.html", "P: _index.md\nList: 10|List Content: 8132", "4: View: 8132\n\nEND")
 
  // Check paginated pages
  for i := 2; i <= 9; i++ {
- checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8033\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8033", "Render 4: View: 8033\n\nEND")
+ checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), fmt.Sprintf("Page: %d", i), "Content: 8132\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8132", "Render 4: View: 8132\n\nEND")
  }
 }
 

hugolib/page_test.go

@@ -1658,7 +1658,7 @@ $$$
 
  b.AssertFileContent("public/page/index.html",
  `<nav id="TableOfContents">`,
- `<li><a href="#shortcode-tshort-in-header">Shortcode T-SHORT in header</a></li>`,
+ `<li><a href="#shortcode-t-short-in-header">Shortcode T-SHORT in header</a></li>`,
  `<code class="language-bash" data-lang="bash"><span class="hl">SHORT`,
  `<code class="language-bash" data-lang="bash"><span class="hl">MARKDOWN`)
 }

markup/goldmark/convert.go

@@ -198,6 +198,7 @@ func newHighlighting(cfg highlight.Config) goldmark.Extender {
 
  e := hl.NewHighlighting(
  hl.WithStyle(cfg.Style),
+ hl.WithGuessLanguage(cfg.GuessSyntax),
  hl.WithCodeBlockOptions(highlight.GetCodeBlockOptions()),
  hl.WithFormatOptions(
  cfg.ToHTMLOptions()...,

markup/goldmark/convert_test.go

@@ -224,4 +224,25 @@ LINE5
  result = convertForConfig(c, cfg, lines, "bash {linenos=table}")
  c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
  })
+
+ c.Run("No language", func(c *qt.C) {
+ cfg := highlight.DefaultConfig
+ cfg.NoClasses = false
+ cfg.LineNos = true
+ cfg.LineNumbersInTable = false
+
+ result := convertForConfig(c, cfg, lines, "")
+ c.Assert(result, qt.Contains, "<pre><code>LINE1\n")
+ })
+
+ c.Run("No language, guess syntax", func(c *qt.C) {
+ cfg := highlight.DefaultConfig
+ cfg.NoClasses = false
+ cfg.GuessSyntax = true
+ cfg.LineNos = true
+ cfg.LineNumbersInTable = false
+
+ result := convertForConfig(c, cfg, lines, "")
+ c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+ })
 }

markup/highlight/config.go

@@ -58,6 +58,8 @@ type Config struct {
 
  // TabWidth sets the number of characters for a tab. Defaults to 4.
  TabWidth int
+
+ GuessSyntax bool
 }
 
 func (cfg Config) ToHTMLOptions() []html.Option {
@@ -104,6 +106,10 @@ func ApplyLegacyConfig(cfg config.Provider, conf *Config) error {
  conf.CodeFences = cfg.GetBool("pygmentsCodeFences")
  }
 
+ if conf.GuessSyntax == DefaultConfig.GuessSyntax && cfg.IsSet("pygmentsCodefencesGuessSyntax") {
+ conf.GuessSyntax = cfg.GetBool("pygmentsCodefencesGuessSyntax")
+ }
+
  if cfg.IsSet("pygmentsOptions") {
  if err := applyOptionsFromString(cfg.GetString("pygmentsOptions"), conf); err != nil {
  return err

markup/highlight/highlight.go

@@ -52,6 +52,14 @@ func highlight(code, lang string, cfg Config) (string, error) {
  lexer = lexers.Get(lang)
  }
 
+ if lexer == nil && cfg.GuessSyntax {
+ lexer = lexers.Analyse(code)
+ if lexer == nil {
+ lexer = lexers.Fallback
+ }
+ lang = strings.ToLower(lexer.Config().Name)
+ }
+
  if lexer == nil {
  wrapper := getPreWrapper(lang)
  fmt.Fprint(w, wrapper.Start(true, ""))

markup/highlight/highlight_test.go

@@ -84,4 +84,26 @@ LINE5
  result, _ = h.Highlight(lines, "bash", "linenos=table")
  c.Assert(result, qt.Contains, "<span class=\"lnt\">1\n</span>")
  })
+
+ c.Run("No language", func(c *qt.C) {
+ cfg := DefaultConfig
+ cfg.NoClasses = false
+ cfg.LineNos = true
+ h := New(cfg)
+
+ result, _ := h.Highlight(lines, "", "")
+ c.Assert(result, qt.Equals, "<pre><code>LINE1\nLINE2\nLINE3\nLINE4\nLINE5\n</code></pre>")
+ })
+
+ c.Run("No language, guess syntax", func(c *qt.C) {
+ cfg := DefaultConfig
+ cfg.NoClasses = false
+ cfg.GuessSyntax = true
+ cfg.LineNos = true
+ cfg.LineNumbersInTable = false
+ h := New(cfg)
+
+ result, _ := h.Highlight(lines, "", "")
+ c.Assert(result, qt.Contains, "<span class=\"ln\">2</span>LINE2\n<")
+ })
 }

markup/markup_config/config_test.go

@@ -55,7 +55,7 @@ func TestConfig(t *testing.T) {
  v.Set("footnoteAnchorPrefix", "myprefix")
  v.Set("footnoteReturnLinkContents", "myreturn")
  v.Set("pygmentsStyle", "hugo")
-
+ v.Set("pygmentsCodefencesGuessSyntax", true)
  conf, err := Decode(v)
 
  c.Assert(err, qt.IsNil)
@@ -64,6 +64,7 @@ func TestConfig(t *testing.T) {
  c.Assert(conf.BlackFriday.FootnoteReturnLinkContents, qt.Equals, "myreturn")
  c.Assert(conf.Highlight.Style, qt.Equals, "hugo")
  c.Assert(conf.Highlight.CodeFences, qt.Equals, true)
+ c.Assert(conf.Highlight.GuessSyntax, qt.Equals, true)
  })
 
 }