re-implement shortcode parsing to/from mdast

package.json

@@ -171,6 +171,7 @@
     "textarea-caret-position": "^0.1.1",
     "unified": "^6.1.4",
     "unist-builder": "^1.0.2",
+    "unist-util-map": "^1.0.3",
     "unist-util-modify-children": "^1.1.1",
     "uuid": "^2.0.3",
     "whatwg-fetch": "^1.0.0"

src/components/Widgets/Markdown/MarkdownControl/VisualEditor/index.js

@@ -139,6 +139,7 @@ const NODE_COMPONENTS = {
   },
   'shortcode': props => {
     const { attributes, node, state: editorState } = props;
+    const { data } = node;
     const isSelected = editorState.selection.hasFocusIn(node);
     return (
       <div
@@ -146,7 +147,7 @@ const NODE_COMPONENTS = {
         {...attributes}
         draggable
       >
-        {getShortcodeId(props)}
+        {data.get('shortcode')}
       </div>
     );
   },

src/components/Widgets/Markdown/unified.js

@@ -1,4 +1,5 @@
-import { get, find, isEmpty } from 'lodash';
+import { get, has, find, isEmpty } from 'lodash';
+import { renderToString } from 'react-dom/server';
 import unified from 'unified';
 import u from 'unist-builder';
 import markdownToRemarkPlugin from 'remark-parse';
@@ -20,7 +21,6 @@ import { reduce, capitalize } from 'lodash';
 
 // Remove the yaml tokenizer, as the rich text editor doesn't support frontmatter
 delete markdownToRemarkPlugin.Parser.prototype.blockTokenizers.yamlFrontMatter;
-console.log(markdownToRemarkPlugin.Parser.prototype.blockTokenizers);
 
 const shortcodeAttributePrefix = 'ncp';
 
@@ -150,6 +150,90 @@ function remarkPrecompileShortcodes() {
   visitors.text = node => node.value;
 };
 
+const remarkShortcodes = ({ plugins }) => {
+  return transform;
+
+  function transform(node) {
+    if (node.children) {
+      node.children = node.children.reduce(reducer, []);
+    }
+    return node;
+
+    function reducer(newChildren, childNode) {
+      if (!['text', 'html'].includes(childNode.type)) {
+        const processedNode = childNode.children ? transform(childNode) : childNode;
+        newChildren.push(processedNode);
+        return newChildren;
+      }
+
+      const text = childNode.value;
+      let lastPlugin;
+      let match;
+      const plugin = plugins.find(p => {
+        match = text.match(p.pattern);
+        return match;
+      });
+      if (!plugin) {
+        newChildren.push(childNode);
+        return newChildren;
+      }
+      const matchValue = match[0];
+      const matchLength = matchValue.length;
+      const matchAll = matchLength === text.length;
+
+      if (matchAll) {
+        const shortcodeNode = createShortcodeNode(text, plugin, match);
+        newChildren.push(shortcodeNode);
+        return newChildren;
+      }
+
+      const tempChildren = [];
+      const matchAtStart = match.index === 0;
+      const matchAtEnd = match.index + matchLength === text.length;
+
+      if (!matchAtStart) {
+        const textBeforeMatch = text.slice(0, match.index);
+        const result = reducer([], { type: 'text', value: textBeforeMatch });
+        tempChildren.push(...result);
+      }
+
+      const matchNode = createShortcodeNode(matchValue, plugin, match);
+      tempChildren.push(matchNode);
+
+      if (!matchAtEnd) {
+        const textAfterMatch = text.slice(match.index + matchLength);
+        const result = reducer([], { type: 'text', value: textAfterMatch });
+        tempChildren.push(...result);
+      }
+
+      newChildren.push(...tempChildren);
+      return newChildren;
+    }
+
+    function createShortcodeNode(text, plugin, match) {
+      const shortcode = plugin.id;
+      const shortcodeData = plugin.fromBlock(match);
+      return { type: 'html', value: text, data: { shortcode, shortcodeData } };
+    }
+  }
+};
+
+const remarkToRehypeShortcodes = ({ plugins }) => {
+  return transform;
+
+  function transform(node) {
+    const children = node.children ? node.children.map(transform) : node.children;
+    if (!has(node, ['data', 'shortcode'])) {
+      return { ...node, children };
+    }
+    const { shortcode, shortcodeData } = node.data;
+    const plugin = plugins.get(shortcode);
+    const value = plugin.toPreview(shortcodeData);
+    const valueHtml = typeof value === 'string' ? value : renderToString(value);
+    return { ...node, value: valueHtml };
+  }
+};
+
 const parseShortcodesFromMarkdown = markdown => {
   const plugins = registry.getEditorComponents();
   const markdownLines = markdown.split('\n');
@@ -191,7 +275,7 @@ const remarkToSlatePlugin = () => {
     delete: 'strikethrough',
     inlineCode: 'code',
   };
-  const toTextNode = text => ({ kind: 'text', text });
+  const toTextNode = (text, data) => ({ kind: 'text', text, data });
   const wrapText = (node, index, parent) => {
     if (['text', 'html'].includes(node.type)) {
       parent.children.splice(index, 1, u('paragraph', [node]));
@@ -199,11 +283,13 @@ const remarkToSlatePlugin = () => {
   };
 
   let getDefinition;
-  const transform = node => {
+  const transform = (node, index, siblings, parent) => {
     let nodes;
 
     if (node.type === 'root') {
+      // Create definition getter for link and image references
       getDefinition = mdastDefinitions(node);
+      // Ensure top level text nodes are wrapped in paragraphs
       modifyChildren(wrapText)(node);
     }
 
@@ -213,8 +299,10 @@ const remarkToSlatePlugin = () => {
       // If a node returns a falsey value, exclude it. Some nodes do not
       // translate from MDAST to Slate, such as definitions for link/image
       // references or footnotes.
-      nodes = node.children.reduce((acc, childNode) => {
-        const transformed = transform(childNode);
+      //
+      // Consider using unist-util-remove instead for this.
+      nodes = node.children.reduce((acc, childNode, idx, sibs) => {
+        const transformed = transform(childNode, idx, sibs, node);
         if (transformed) {
           acc.push(transformed);
         }
@@ -228,7 +316,17 @@ const remarkToSlatePlugin = () => {
 
     // Process raw html as text, since it's valid markdown
     if (['text', 'html'].includes(node.type)) {
-      return toTextNode(node.value);
+      const { value, data } = node;
+      const shortcode = get(data, 'shortcode');
+      if (shortcode) {
+        const isBlock = parent.type === 'paragraph' && siblings.length === 1;
+        data.shortcodeValue = value;
+
+        if (isBlock) {
+          return { kind: 'block', type: 'shortcode', data, isVoid: true, nodes: [toTextNode('')] };
+        }
+      }
+      return toTextNode(value, data);
     }
 
     if (node.type === 'inlineCode') {
@@ -315,7 +413,10 @@ const remarkToSlatePlugin = () => {
       return { kind: 'block', type: typeMap['image'], data };
     }
   };
-  return transform;
+
+  // Since `transform` is used for recursive child mapping, ensure that only the
+  // first argument is supplied on the initial call.
+  return node => transform(node);
 };
 
 const slateToRemarkPlugin = () => {
@@ -327,9 +428,14 @@ const slateToRemarkPlugin = () => {
 };
 
 export const markdownToRemark = markdown => {
-  const result = unified()
+  const parsed = unified()
     .use(markdownToRemarkPlugin, { fences: true, pedantic: true, footnotes: true, commonmark: true })
     .parse(markdown);
+
+  const result = unified()
+    .use(remarkShortcodes, { plugins: registry.getEditorComponents() })
+    .runSync(parsed);
+
   return result;
 };
 
@@ -399,6 +505,7 @@ export const slateToRemark = raw => {
           }
         });
       } else {
+
         acc.push(u('html', childNode.text));
       }
       return acc;
@@ -408,6 +515,10 @@ export const slateToRemark = raw => {
       return u('root', children);
     }
 
+    if (node.type === 'shortcode') {
+      return u('html', { data: node.data }, node.data.shortcodeValue);
+    }
+
     if (node.type.startsWith('heading')) {
       const depths = { one: 1, two: 2, three: 3, four: 4, five: 5, six: 6 };
       const depth = node.type.split('-')[1];
@@ -448,19 +559,21 @@ export const slateToRemark = raw => {
     }
   }
   raw.type = 'root';
-  const result = transform(raw);
+  const mdast = transform(raw);
+
+  const result = unified()
+    .use(remarkShortcodes, { plugins: registry.getEditorComponents() })
+    .runSync(mdast);
+
   return result;
 };
 
 export const remarkToHtml = mdast => {
   const result = unified()
+    .use(remarkToRehypeShortcodes, { plugins: registry.getEditorComponents() })
     .use(remarkToRehype, { allowDangerousHTML: true })
-    .use(rehypeReparse)
     .use(rehypeRemoveEmpty)
     .use(rehypeMinifyWhitespace)
-    .use(() => node => {
-      return node;
-    })
     .runSync(mdast);
 
   const output = unified()

yarn.lock

@@ -5321,7 +5321,7 @@ mdast-util-compact@^1.0.0:
     unist-util-modify-children "^1.0.0"
     unist-util-visit "^1.1.0"
 
-mdast-util-definitions@^1.2.0:
+mdast-util-definitions@^1.2.0, mdast-util-definitions@^1.2.2:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/mdast-util-definitions/-/mdast-util-definitions-1.2.2.tgz#673f4377c3e23d3de7af7a4fe2214c0e221c5ac7"
   dependencies:
@@ -9010,7 +9010,13 @@ unist-util-is@^2.0.0, unist-util-is@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-2.1.1.tgz#0c312629e3f960c66e931e812d3d80e77010947b"
 
-unist-util-modify-children@^1.0.0:
+unist-util-map@^1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/unist-util-map/-/unist-util-map-1.0.3.tgz#26a913d7cddb3cd3e9a886d135d37a3d1f54e514"
+  dependencies:
+    object-assign "^4.0.1"
+
+unist-util-modify-children@^1.0.0, unist-util-modify-children@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/unist-util-modify-children/-/unist-util-modify-children-1.1.1.tgz#66d7e6a449e6f67220b976ab3cb8b5ebac39e51d"
   dependencies: