Adds ability to recursively evaluate nested shortcodes

src/core/Wyam.Common/Shortcodes/IShortcode.cs

@@ -11,6 +11,13 @@ namespace Wyam.Common.Shortcodes
  /// <summary>
  /// Contains the code for a given shortcode (see the <c>Shortcodes</c> module).
  /// </summary>
+ /// <remarks>
+ /// Shortcode instances are created once-per-document and reused for the life of that document.
+ /// An exception is that nested shortcodes are always processed by a new instance of the shortcode
+ /// implementation (which remains in use for that nested content). If a shortcode class also
+ /// implements <see cref="IDisposable"/>, the shortcode will be disposed at the processing conclusion
+ /// of the document or nested content.
+ /// </remarks>
  public interface IShortcode
  {
  /// <summary>

src/core/Wyam.Core/Modules/Contents/Shortcodes.cs

@@ -9,6 +9,7 @@
 using Wyam.Common.Shortcodes;
 using Wyam.Core.Meta;
 using Wyam.Core.Shortcodes;
+using Wyam.Core.Util;
 
 namespace Wyam.Core.Modules.Contents
 {
@@ -18,8 +19,8 @@ namespace Wyam.Core.Modules.Contents
  /// <category>Content</category>
  public class Shortcodes : IModule
  {
- private readonly string _startDelimiter = ShortcodeParser.DefaultPostRenderStartDelimiter;
- private readonly string _endDelimiter = ShortcodeParser.DefaultPostRenderEndDelimiter;
+ private readonly string _startDelimiter;
+ private readonly string _endDelimiter;
 
  /// <summary>
  /// Renders shortcodes in the input documents using the default start and end delimiters.
@@ -59,86 +60,129 @@ public IEnumerable<IDocument> Execute(IReadOnlyList<IDocument> inputs, IExecutio
  {
  return inputs.AsParallel().Select(context, input =>
  {
- // Parse the input stream looking for shortcodes
- ShortcodeParser parser = new ShortcodeParser(_startDelimiter, _endDelimiter, context.Shortcodes);
- List<ShortcodeLocation> locations = parser.Parse(input.GetStream()); // The input stream will get disposed in the parser
-
- // Return the original document if we didn't find any
- if (locations.Count == 0)
+ Stream stream = input.GetStream();
+ if (ProcessShortcodes(stream, input, context, out IDocument result))
  {
- return input;
+ return result;
  }
+ stream.Dispose();
+ return input;
+ });
+ }
 
- // Otherwise, create a shortcode instance for each named shortcode
- Dictionary<string, IShortcode> shortcodes =
- locations
- .Select(x => x.Name)
- .Distinct(StringComparer.OrdinalIgnoreCase)
- .ToDictionary(x => x, x => context.Shortcodes.CreateInstance(x), StringComparer.OrdinalIgnoreCase);
+ // The inputStream will be disposed if this returns <c>true</c> but will not otherwise
+ private bool ProcessShortcodes(Stream inputStream, IDocument input, IExecutionContext context, out IDocument result)
+ {
+ // Parse the input stream looking for shortcodes
+ ShortcodeParser parser = new ShortcodeParser(_startDelimiter, _endDelimiter, context.Shortcodes);
+ if (!inputStream.CanSeek)
+ {
+ inputStream = new SeekableStream(inputStream, true);
+ }
+ List<ShortcodeLocation> locations = parser.Parse(inputStream);
+
+ // Reset the position because we're going to use the stream again when we do replacements
+ inputStream.Position = 0;
 
- // Execute each of the shortcodes in order
- List<InsertingStreamLocation> insertingLocations = locations
- .Select(x =>
+ // Return the original document if we didn't find any
+ if (locations.Count == 0)
+ {
+ result = null;
+ return false;
+ }
+
+ // Otherwise, create a shortcode instance for each named shortcode
+ Dictionary<string, IShortcode> shortcodes =
+ locations
+ .Select(x => x.Name)
+ .Distinct(StringComparer.OrdinalIgnoreCase)
+ .ToDictionary(x => x, x => context.Shortcodes.CreateInstance(x), StringComparer.OrdinalIgnoreCase);
+
+ // Execute each of the shortcodes in order
+ List<InsertingStreamLocation> insertingLocations = locations
+ .Select(x =>
+ {
+ // Execute the shortcode
+ IShortcodeResult shortcodeResult = shortcodes[x.Name].Execute(x.Arguments, x.Content, input, context);
+
+ // Merge output metadata with the current input document
+ // Creating a new document is the easiest way to ensure all the metadata from shortcodes gets accumulated correctly
+ if (shortcodeResult?.Metadata != null)
  {
- IShortcodeResult result = shortcodes[x.Name].Execute(x.Arguments, x.Content, input, context);
- if (result?.Metadata != null)
+ input = context.GetDocument(input, shortcodeResult.Metadata);
+ }
+
+ // Recursively parse shortcodes
+ Stream shortcodeResultStream = shortcodeResult?.Stream;
+ if (shortcodeResultStream != null)
+ {
+ if (!shortcodeResultStream.CanSeek)
+ {
+ shortcodeResultStream = new SeekableStream(shortcodeResultStream, true);
+ }
+ if (ProcessShortcodes(shortcodeResultStream, input, context, out IDocument nestedResult))
+ {
+ input = nestedResult;
+ shortcodeResultStream = nestedResult.GetStream(); // Will get disposed in the replacement operation below
+ }
+ else
  {
- // Creating a new document is the easiest way to ensure all the metadata from shortcodes gets accumulated correctly
- input = context.GetDocument(input, result.Metadata);
+ shortcodeResultStream.Position = 0;
  }
- return new InsertingStreamLocation(x.FirstIndex, x.LastIndex, result?.Stream);
- })
- .ToList();
+ return new InsertingStreamLocation(x.FirstIndex, x.LastIndex, shortcodeResultStream);
+ }
 
- // Dispose any shortcodes that implement IDisposable
- foreach (IDisposable disposableShortcode
- in shortcodes.Values.Select(x => x as IDisposable).Where(x => x != null))
- {
- disposableShortcode.Dispose();
- }
+ return new InsertingStreamLocation(x.FirstIndex, x.LastIndex, null);
+ })
+ .ToList();
+
+ // Dispose any shortcodes that implement IDisposable
+ foreach (IDisposable disposableShortcode
+ in shortcodes.Values.Select(x => x as IDisposable).Where(x => x != null))
+ {
+ disposableShortcode.Dispose();
+ }
 
- // Construct a new stream with the shortcodes inserted
- // We have to use all TextWriter/TextReaders over the streams to ensure consistent encoding
- Stream resultStream = context.GetContentStream();
- char[] buffer = new char[4096];
- using (TextWriter writer = new StreamWriter(resultStream, Encoding.UTF8, 4096, true))
+ // Construct a new stream with the shortcode results inserted
+ // We have to use all TextWriter/TextReaders over the streams to ensure consistent encoding
+ Stream resultStream = context.GetContentStream();
+ char[] buffer = new char[4096];
+ using (TextWriter writer = new StreamWriter(resultStream, Encoding.UTF8, 4096, true))
+ {
+ // The input stream will get disposed when the reader is
+ using (TextReader reader = new StreamReader(inputStream))
  {
- // The input stream will get disposed when the reader is
- Stream inputStream = input.GetStream();
- using (TextReader reader = new StreamReader(inputStream))
+ int position = 0;
+ int length = 0;
+ foreach (InsertingStreamLocation insertingLocation in insertingLocations)
  {
- int position = 0;
- int length = 0;
- foreach (InsertingStreamLocation insertingLocation in insertingLocations)
- {
- // Copy up to the start of this shortcode
- length = insertingLocation.FirstIndex - position;
- Read(reader, writer, length, ref buffer);
- position += length;
+ // Copy up to the start of this shortcode
+ length = insertingLocation.FirstIndex - position;
+ Read(reader, writer, length, ref buffer);
+ position += length;
 
- // Copy the shortcode content to the result stream
- if (insertingLocation.Stream != null)
+ // Copy the shortcode content to the result stream
+ if (insertingLocation.Stream != null)
+ {
+ // This will dispose the shortcode content stream when done
+ using (TextReader insertingReader = new StreamReader(insertingLocation.Stream))
  {
- // This will dispose the shortcode content stream when done
- using (TextReader insertingReader = new StreamReader(insertingLocation.Stream))
- {
- Read(insertingReader, writer, null, ref buffer);
- }
+ Read(insertingReader, writer, null, ref buffer);
  }
-
- // Skip the shortcode text
- length = insertingLocation.LastIndex - insertingLocation.FirstIndex + 1;
- Read(reader, null, length, ref buffer);
- position += length;
  }
 
- // Copy remaining
- Read(reader, writer, null, ref buffer);
+ // Skip the shortcode text
+ length = insertingLocation.LastIndex - insertingLocation.FirstIndex + 1;
+ Read(reader, null, length, ref buffer);
+ position += length;
  }
- }
 
- return context.GetDocument(input, resultStream);
- });
+ // Copy remaining
+ Read(reader, writer, null, ref buffer);
+ }
+ }
+ result = context.GetDocument(input, resultStream);
+ return true;
  }
 
  // writer = null to just skip length in reader

src/core/Wyam.Core/Shortcodes/IO/Include.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Text;
 using Wyam.Common.Documents;
 using Wyam.Common.Execution;
 using Wyam.Common.IO;
+using Wyam.Common.Meta;
 using Wyam.Common.Shortcodes;
+using Wyam.Common.Tracing;
 
 namespace Wyam.Core.Shortcodes.IO
 {
@@ -18,11 +21,44 @@ namespace Wyam.Core.Shortcodes.IO
  /// <parameter>The path to the file to include.</parameter>
  public class Include : IShortcode
  {
+ private FilePath _sourcePath;
+
  /// <inheritdoc />
  public IShortcodeResult Execute(KeyValuePair<string, string>[] args, string content, IDocument document, IExecutionContext context)
  {
- IFile file = context.FileSystem.GetInputFile(new FilePath(args.SingleValue()));
- return context.GetShortcodeResult(file.Exists ? file.OpenRead() : null);
+ // Get the included path relative to the document
+ FilePath includedPath = new FilePath(args.SingleValue());
+ if (_sourcePath == null)
+ {
+ // Cache the source path for this shortcode instance since it'll be the same for all future shortcodes
+ _sourcePath = document.FilePath("IncludeShortcodeSource", document.Source);
+ }
+
+ // Try to find the file relative to the current document path
+ IFile includedFile = null;
+ if (includedPath.IsRelative && _sourcePath != null)
+ {
+ includedFile = context.FileSystem.GetFile(_sourcePath.Directory.CombineFile(includedPath));
+ }
+
+ // If that didn't work, try relative to the input folder
+ if (includedFile?.Exists != true)
+ {
+ includedFile = context.FileSystem.GetInputFile(includedPath);
+ }
+
+ // Get the included file
+ if (!includedFile.Exists)
+ {
+ Trace.Warning($"Included file {includedPath.FullPath} does not exist");
+ return context.GetShortcodeResult((Stream)null);
+ }
+
+ // Set the currently included shortcode source so nested includes can use it
+ return context.GetShortcodeResult(includedFile.OpenRead(), new MetadataItems
+ {
+ { "IncludeShortcodeSource", includedFile.Path.FullPath }
+ });
  }
  }
 }

src/core/Wyam.Core/Shortcodes/ShortcodeParser.cs

@@ -34,7 +34,7 @@ public ShortcodeParser(string startDelimiter, string endDelimiter, IReadOnlyShor
  /// <summary>
  /// Identifies shortcode locations in a stream.
  /// </summary>
- /// <param name="stream">The stream to parse. This method will dispose the passed-in stream.</param>
+ /// <param name="stream">The stream to parse. This method will not dispose the passed-in stream.</param>
  /// <returns>All of the shortcode locations in the stream.</returns>
  public List<ShortcodeLocation> Parse(Stream stream)
  {
@@ -44,7 +44,7 @@ public List<ShortcodeLocation> Parse(Stream stream)
  ShortcodeLocation shortcode = null;
  StringBuilder content = null;
 
- using (TextReader reader = new StreamReader(stream))
+ using (TextReader reader = new StreamReader(stream, Encoding.UTF8, true, 1024, true))
  {
  int r;
  int i = 0;

tests/core/Wyam.Core.Tests/Modules/Contents/ShortcodesFixture.cs

@@ -38,6 +38,23 @@ public void ProcessesShortcode()
  results.Single().Content.ShouldBe("123Foo456");
  }
 
+ [Test]
+ public void ProcessesNestedShortcode()
+ {
+ // Given
+ TestExecutionContext context = new TestExecutionContext();
+ context.Shortcodes.Add<TestShortcode>("Nested");
+ context.Shortcodes.Add<NestedShortcode>("Bar");
+ IDocument document = new TestDocument("123<?# Bar /?>456");
+ Core.Modules.Contents.Shortcodes module = new Core.Modules.Contents.Shortcodes();
+
+ // When
+ List<IDocument> results = module.Execute(new[] { document }, context).ToList();
+
+ // Then
+ results.Single().Content.ShouldBe("123ABCFooXYZ456");
+ }
+
  [Test]
  public void ShortcodeSupportsNullStreamResult()
  {
@@ -152,6 +169,12 @@ public IShortcodeResult Execute(KeyValuePair<string, string>[] args, string cont
  context.GetShortcodeResult(context.GetContentStream("Foo"));
  }
 
+ public class NestedShortcode : IShortcode
+ {
+ public IShortcodeResult Execute(KeyValuePair<string, string>[] args, string content, IDocument document, IExecutionContext context) =>
+ context.GetShortcodeResult(context.GetContentStream("ABC<?# Nested /?>XYZ"));
+ }
+
  public class NullStreamShortcode : IShortcode
  {
  public IShortcodeResult Execute(KeyValuePair<string, string>[] args, string content, IDocument document, IExecutionContext context) =>