feat: Code Widget + Markdown Widget Internal Overhaul (#2828)

* wip - upgrade to slate 0.43

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* finish list handling logic

* add plugins directory

* tests wip

* setup testing

* wip

* add selection commands

* finish list testing

* stuff

* add codemirror

* abstract codemirror from slate

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* codemirror mostly working, some bugs

* upgrade to slate 46

* upgrade to slate 47

* wip

* wip

* progress

* wip

* mostly working links with surrounding marks

* wip

* tests passing

* add test

* fix formatting

* update snapshots

* close self closing tag in markdown html output

* wip - commonmark

* hold on commonmark work

* all tests passing

* fix e2e specs

* ignore tests in esm builds

* break/backspace plugins wip

* finish enter/backspace spec

* fix soft break handling

* wip - editor component deletion

* add insertion points

* make insertion points invisible

* fix empty mark nodes output to markdown

* fix pasting

* improve insertion points

* add static bottom insertion point

* improve click handling at insertion points

* restore current table functionality

* add paste support for Slate fragments

* support cut/copy markdown, paste between rich/raw editor

* fix copy paste

* wip - paste/select bug fixing

* fixed known slate issues

* split plugins

* fix editor toggles

* force text cursor in code widget

* wip - reorg plugins

* finish markdown control reorg

* configure plugin types

* quote block adjacent handling with tests

* wip

* finish quote logic and tests

* fix copy paste plugin migration regressions

* fix force insert before node

* fix trailing insertion point

* remove empty headers

* codemirror working properly in markdown widget

* return focus to codemirror on lang select enter

* fix state issues for widgets with local state

* wip - vim working, just need to work out distribution

* add settings pane

* wip - default modes

* fix deps

* add programming language data

* implement linguist langs in code widget

* everything built in

* remove old registration code, fix focus styling

* fix/update linting setup

* fix js lint errors

* remove stylelint from format script

* fix remaining linting errors

* fix reducer test failures

* chore: update commitlint for worktree support

* chore: fix remaining tests

* chore: drop unused monaco plugin

* chore: remove extraneous global styles rendering

* chore: fix failing tests

* fix: tests

* fix: quote/list nesting (tests still broken)

* fix: update quote tests

* chore: bring back code widget test config

* fix: autofocus

* fix: code blocks without the code widget

* fix: code editor component state issues

* fix: error

* fix: add code block test, few fixes

* chore: remove notes

* fix: [wip] update stateful shortcodes on undo/redo

* fix: support code styled links, handle unknown langs

* fix: few fixes

* fix: autofocus on insert, focus on all clicks

* fix: linting

* fix: autofocus

* fix: update code block fixture

* fix: remove unused cypress snapshot plugin

* fix: drop node 8 test, add node 12

* fix: use lodash.flatten instead of Array.flat

* fix: remove console logs

packages/netlify-cms-widget-code/README.md

@@ -0,0 +1,11 @@
+# Docs coming soon!
+
+Netlify CMS was recently converted from a single npm package to a "monorepo" of over 20 packages.
+That's over 20 Readme's! We haven't created one for this package yet, but we will soon.
+
+In the meantime, you can:
+
+1. Check out the [main readme](https://github.com/netlify/netlify-cms/#readme) or the [documentation
+   site](https://www.netlifycms.org) for more info.
+2. Reach out to the [community chat](https://gitter.im/netlify/netlifycms/) if you need help.
+3. Help out and [write the readme yourself](https://github.com/netlify/netlify-cms/edit/master/packages/netlify-cms-widget-code/README.md)!

packages/netlify-cms-widget-code/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "netlify-cms-widget-code",
+  "description": "Widget for editing code in Netlify CMS",
+  "version": "1.0.0",
+  "homepage": "https://www.netlifycms.org/docs/widgets/#code",
+  "repository": "https://github.com/netlify/netlify-cms/tree/master/packages/netlify-cms-widget-code",
+  "bugs": "https://github.com/netlify/netlify-cms/issues",
+  "module": "dist/esm/index.js",
+  "main": "dist/netlify-cms-widget-code.js",
+  "license": "MIT",
+  "keywords": [
+    "netlify",
+    "netlify-cms",
+    "widget",
+    "code",
+    "codemirror",
+    "editor",
+    "code editor"
+  ],
+  "sideEffects": false,
+  "scripts": {
+    "develop": "yarn build:esm --watch",
+    "build": "cross-env NODE_ENV=production webpack",
+    "build:esm": "cross-env NODE_ENV=esm babel src --out-dir dist/esm --ignore \"**/__tests__\" --root-mode upward",
+    "process:languages": "node ./scripts/process-languages"
+  },
+  "peerDependencies": {
+    "@emotion/core": "^10.0.9",
+    "lodash": "^4.17.11",
+    "codemirror": "^5.46.0",
+    "react": "^16.8.4",
+    "netlify-cms-ui-default": "^2.6.2"
+  },
+  "dependencies": {
+    "re-resizable": "^4.11.0",
+    "react-codemirror2": "^6.0.0",
+    "react-select": "^2.4.3"
+  }
+}

packages/netlify-cms-widget-code/scripts/process-languages.js

@@ -0,0 +1,45 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+const uniq = require('lodash/uniq');
+
+const rawDataPath = '../data/languages-raw.yml';
+const outputPath = '../data/languages.json';
+
+async function fetchData() {
+  const filePath = path.resolve(__dirname, rawDataPath);
+  const fileContent = await fs.readFile(filePath);
+  return yaml.safeLoad(fileContent);
+}
+
+function outputData(data) {
+  const filePath = path.resolve(__dirname, outputPath);
+  return fs.writeJson(filePath, data);
+}
+
+function transform(data) {
+  return Object.entries(data).reduce((acc, [label, lang]) => {
+    const { extensions = [], aliases = [], codemirror_mode, codemirror_mime_type } = lang;
+    if (codemirror_mode) {
+      const dotlessExtensions = extensions.map(ext => ext.slice(1));
+      const identifiers = uniq(
+        [label.toLowerCase(), ...aliases, ...dotlessExtensions].filter(alias => {
+          if (!alias) {
+            return;
+          }
+          return !/[^a-zA-Z]/.test(alias);
+        }),
+      );
+      acc.push({ label, identifiers, codemirror_mode, codemirror_mime_type });
+    }
+    return acc;
+  }, []);
+}
+
+async function process() {
+  const data = await fetchData();
+  const transformedData = transform(data);
+  return outputData(transformedData);
+}
+
+process();

packages/netlify-cms-widget-code/src/CodeControl.js

@@ -0,0 +1,315 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import { ClassNames } from '@emotion/core';
+import { Map } from 'immutable';
+import { uniq, isEqual, isEmpty } from 'lodash';
+import uuid from 'uuid/v4';
+import { UnControlled as ReactCodeMirror } from 'react-codemirror2';
+import CodeMirror from 'codemirror';
+import 'codemirror/keymap/vim';
+import 'codemirror/keymap/sublime';
+import 'codemirror/keymap/emacs';
+import codeMirrorStyles from 'codemirror/lib/codemirror.css';
+import materialTheme from 'codemirror/theme/material.css';
+import SettingsPane from './SettingsPane';
+import SettingsButton from './SettingsButton';
+import languageData from '../data/languages.json';
+
+// TODO: relocate as a utility function
+function getChangedProps(previous, next, keys) {
+  const propNames = keys || uniq(Object.keys(previous), Object.keys(next));
+  const changedProps = propNames.reduce((acc, prop) => {
+    if (previous[prop] !== next[prop]) {
+      acc[prop] = next[prop];
+    }
+    return acc;
+  }, {});
+  if (!isEmpty(changedProps)) {
+    return changedProps;
+  }
+}
+
+const languages = languageData.map(lang => ({
+  label: lang.label,
+  name: lang.identifiers[0],
+  mode: lang.codemirror_mode,
+  mimeType: lang.codemirror_mime_type,
+}));
+
+const styleString = `
+  padding: 0;
+`;
+
+const defaultLang = { name: '', mode: '', label: 'none' };
+
+function valueToOption(val) {
+  if (typeof val === 'string') {
+    return { value: val, label: val };
+  }
+  return { value: val.name, label: val.label || val.name };
+}
+
+const modes = languages.map(valueToOption);
+
+const themes = ['default', 'material'];
+
+const settingsPersistKeys = {
+  theme: 'cms.codemirror.theme',
+  keyMap: 'cms.codemirror.keymap',
+};
+
+export default class CodeControl extends React.Component {
+  static propTypes = {
+    field: ImmutablePropTypes.map.isRequired,
+    onChange: PropTypes.func.isRequired,
+    value: PropTypes.node,
+    forID: PropTypes.string.isRequired,
+    classNameWrapper: PropTypes.string.isRequired,
+    widget: PropTypes.object.isRequired,
+  };
+
+  keys = this.getKeys(this.props.field);
+
+  state = {
+    isActive: false,
+    unknownLang: null,
+    lang: '',
+    keyMap: localStorage.getItem(settingsPersistKeys['keyMap']) || 'default',
+    settingsVisible: false,
+    codeMirrorKey: uuid(),
+    theme: localStorage.getItem(settingsPersistKeys['theme']) || themes[themes.length - 1],
+    lastKnownValue: this.valueIsMap() ? this.props.value?.get(this.keys.code) : this.props.value,
+  };
+
+  shouldComponentUpdate(nextProps, nextState) {
+    return (
+      !isEqual(this.state, nextState) || this.props.classNameWrapper !== nextProps.classNameWrapper
+    );
+  }
+
+  componentDidMount() {
+    this.setState({
+      lang: this.getInitialLang() || '',
+    });
+  }
+
+  componentDidUpdate(prevProps, prevState) {
+    this.updateCodeMirrorProps(prevState);
+  }
+
+  updateCodeMirrorProps(prevState) {
+    const keys = ['lang', 'theme', 'keyMap'];
+    const changedProps = getChangedProps(prevState, this.state, keys);
+    if (changedProps) {
+      this.handleChangeCodeMirrorProps(changedProps);
+    }
+  }
+
+  getLanguageByName = name => {
+    return languages.find(lang => lang.name === name);
+  };
+
+  getKeyMapOptions = () => {
+    return Object.keys(CodeMirror.keyMap)
+      .sort()
+      .filter(keyMap => ['emacs', 'vim', 'sublime', 'default'].includes(keyMap))
+      .map(keyMap => ({ value: keyMap, label: keyMap }));
+  };
+
+  // This widget is not fully controlled, it only takes a value through props
+  // upon initialization.
+  getInitialLang = () => {
+    const { value, field } = this.props;
+    const lang =
+      (this.valueIsMap() && value && value.get(this.keys.lang)) || field.get('defaultLanguage');
+    const langInfo = this.getLanguageByName(lang);
+    if (lang && !langInfo) {
+      this.setState({ unknownLang: lang });
+    }
+    return lang;
+  };
+
+  // If `allow_language_selection` is not set, default to true. Otherwise, use
+  // its value.
+  allowLanguageSelection =
+    !this.props.field.has('allow_language_selection') ||
+    !!this.props.field.get('allow_language_selection');
+
+  toValue = this.valueIsMap()
+    ? (type, value) => (this.props.value || Map()).set(this.keys[type], value)
+    : (type, value) => (type === 'code' ? value : this.props.value);
+
+  // If the value is a map, keys can be customized via config.
+  getKeys(field) {
+    const defaults = {
+      code: 'code',
+      lang: 'lang',
+    };
+
+    // Force default keys if widget is an editor component code block.
+    if (this.props.isEditorComponent) {
+      return defaults;
+    }
+
+    const keys = field.get('keys', Map()).toJS();
+    return { ...defaults, ...keys };
+  }
+
+  // Determine if the persisted value is a map rather than a plain string. A map
+  // value allows both the code string and the language to be persisted.
+  valueIsMap() {
+    const { field, isEditorComponent } = this.props;
+    return !field.get('output_code_only') || isEditorComponent;
+  }
+
+  async handleChangeCodeMirrorProps(changedProps) {
+    const { onChange } = this.props;
+
+    if (changedProps.lang) {
+      const { mode } = this.getLanguageByName(changedProps.lang) || {};
+      if (mode) {
+        await import(`codemirror/mode/${mode}/${mode}.js`);
+      }
+    }
+
+    // Changing CodeMirror props requires re-initializing the
+    // detached/uncontrolled React CodeMirror component, so here we save and
+    // restore the selections and cursor position after the state change.
+    if (this.cm) {
+      const cursor = this.cm.doc.getCursor();
+      const selections = this.cm.doc.listSelections();
+      this.setState({ codeMirrorKey: uuid() }, () => {
+        this.cm.doc.setCursor(cursor);
+        this.cm.doc.setSelections(selections);
+      });
+    }
+
+    for (const key of ['theme', 'keyMap']) {
+      if (changedProps[key]) {
+        localStorage.setItem(settingsPersistKeys[key], changedProps[key]);
+      }
+    }
+
+    // Only persist the language change if supported - requires the value to be
+    // a map rather than just a code string.
+    if (changedProps.lang && this.valueIsMap()) {
+      onChange(this.toValue('lang', changedProps.lang));
+    }
+  }
+
+  handleChange(newValue) {
+    const cursor = this.cm.doc.getCursor();
+    const selections = this.cm.doc.listSelections();
+    this.setState({ lastKnownValue: newValue });
+    this.props.onChange(this.toValue('code', newValue), { cursor, selections });
+  }
+
+  showSettings = () => {
+    this.setState({ settingsVisible: true });
+  };
+
+  hideSettings = () => {
+    if (this.state.settingsVisible) {
+      this.setState({ settingsVisible: false });
+    }
+    this.cm.focus();
+  };
+
+  handleFocus = () => {
+    this.hideSettings();
+    this.props.setActiveStyle();
+    this.setActive();
+  };
+
+  handleBlur = () => {
+    this.setInactive();
+    this.props.setInactiveStyle();
+  };
+
+  setActive = () => this.setState({ isActive: true });
+  setInactive = () => this.setState({ isActive: false });
+
+  render() {
+    const { classNameWrapper, forID, widget, isNewEditorComponent } = this.props;
+    const { lang, settingsVisible, keyMap, codeMirrorKey, theme, lastKnownValue } = this.state;
+    const langInfo = this.getLanguageByName(lang);
+    const mode = langInfo?.mimeType || langInfo?.mode;
+
+    return (
+      <ClassNames>
+        {({ css, cx }) => (
+          <div
+            className={cx(
+              classNameWrapper,
+              css`
+                ${codeMirrorStyles};
+                ${materialTheme};
+                ${styleString};
+              `,
+            )}
+          >
+            {!settingsVisible && <SettingsButton onClick={this.showSettings} />}
+            {settingsVisible && (
+              <SettingsPane
+                hideSettings={this.hideSettings}
+                forID={forID}
+                modes={modes}
+                mode={valueToOption(langInfo || defaultLang)}
+                theme={themes.find(t => t === theme)}
+                themes={themes}
+                keyMap={{ value: keyMap, label: keyMap }}
+                keyMaps={this.getKeyMapOptions()}
+                allowLanguageSelection={this.allowLanguageSelection}
+                onChangeLang={newLang => this.setState({ lang: newLang })}
+                onChangeTheme={newTheme => this.setState({ theme: newTheme })}
+                onChangeKeyMap={newKeyMap => this.setState({ keyMap: newKeyMap })}
+              />
+            )}
+            <ReactCodeMirror
+              key={codeMirrorKey}
+              id={forID}
+              className={css`
+                height: 100%;
+
+                .CodeMirror {
+                  height: auto;
+                  cursor: text;
+                  min-height: 300px;
+                }
+
+                .CodeMirror-scroll {
+                  min-height: 300px;
+                }
+              `}
+              options={{
+                lineNumbers: true,
+                ...widget.codeMirrorConfig,
+                extraKeys: {
+                  'Shift-Tab': 'indentLess',
+                  Tab: 'indentMore',
+                  ...(widget.codeMirrorConfig.extraKeys || {}),
+                },
+                theme,
+                mode,
+                keyMap,
+                viewportMargin: Infinity,
+              }}
+              detach={true}
+              editorDidMount={cm => {
+                this.cm = cm;
+                if (isNewEditorComponent) {
+                  this.handleFocus();
+                }
+              }}
+              value={lastKnownValue}
+              onChange={(editor, data, newValue) => this.handleChange(newValue)}
+              onFocus={this.handleFocus}
+              onBlur={this.handleBlur}
+            />
+          </div>
+        )}
+      </ClassNames>
+    );
+  }
+}

packages/netlify-cms-widget-code/src/CodePreview.js

@@ -0,0 +1,29 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { Map } from 'immutable';
+import { isString } from 'lodash';
+import { WidgetPreviewContainer } from 'netlify-cms-ui-default';
+
+const toValue = (value, field) => {
+  if (isString(value)) {
+    return value;
+  }
+  if (Map.isMap(value)) {
+    return value.get(field.getIn(['keys', 'code'], 'code'), '');
+  }
+  return '';
+};
+
+const CodePreview = props => (
+  <WidgetPreviewContainer>
+    <pre>
+      <code>{toValue(props.value, props.field)}</code>
+    </pre>
+  </WidgetPreviewContainer>
+);
+
+CodePreview.propTypes = {
+  value: PropTypes.node,
+};
+
+export default CodePreview;

packages/netlify-cms-widget-code/src/SettingsButton.js

@@ -0,0 +1,31 @@
+import React from 'react';
+import styled from '@emotion/styled';
+import { Icon, buttons, shadows } from 'netlify-cms-ui-default';
+
+const StyledSettingsButton = styled.button`
+  ${buttons.button};
+  ${buttons.default};
+  ${shadows.drop};
+  display: block;
+  position: absolute;
+  z-index: 100;
+  right: 8px;
+  top: 8px;
+  opacity: 0.8;
+  padding: 2px 4px;
+  line-height: 1;
+  height: auto;
+
+  ${Icon} {
+    position: relative;
+    top: 1px;
+  }
+`;
+
+const SettingsButton = ({ showClose, onClick }) => (
+  <StyledSettingsButton onClick={onClick}>
+    <Icon type={showClose ? 'close' : 'settings'} size="small" />
+  </StyledSettingsButton>
+);
+
+export default SettingsButton;

packages/netlify-cms-widget-code/src/SettingsPane.js

@@ -0,0 +1,109 @@
+import React from 'react';
+import styled from '@emotion/styled';
+import Select from 'react-select';
+import isHotkey from 'is-hotkey';
+import { text, shadows } from 'netlify-cms-ui-default';
+import SettingsButton from './SettingsButton';
+import languageSelectStyles from './languageSelectStyles';
+
+const SettingsPaneContainer = styled.div`
+  position: absolute;
+  right: 0;
+  width: 200px;
+  z-index: 10;
+  height: 100%;
+  background-color: #fff;
+  overflow-y: scroll;
+  padding: 12px;
+  ${shadows.drop};
+`;
+
+const SettingsFieldLabel = styled.label`
+  ${text.fieldLabel};
+  font-size: 11px;
+  display: block;
+  margin-top: 8px;
+  margin-bottom: 2px;
+`;
+
+const SettingsSectionTitle = styled.h3`
+  font-size: 14px;
+  margin-top: 14px;
+  margin-bottom: 0;
+
+  &:first-of-type {
+    margin-top: 4px;
+  }
+`;
+
+const SettingsSelect = ({ value, options, onChange, forID, type, autoFocus }) => (
+  <Select
+    inputId={`${forID}-select-${type}`}
+    styles={languageSelectStyles}
+    value={value}
+    options={options}
+    onChange={opt => onChange(opt.value)}
+    menuPlacement="auto"
+    captureMenuScroll={false}
+    autoFocus={autoFocus}
+  />
+);
+
+const SettingsPane = ({
+  hideSettings,
+  forID,
+  modes,
+  mode,
+  theme,
+  themes,
+  keyMap,
+  keyMaps,
+  allowLanguageSelection,
+  onChangeLang,
+  onChangeTheme,
+  onChangeKeyMap,
+}) => (
+  <SettingsPaneContainer onKeyDown={e => isHotkey('esc', e) && hideSettings()}>
+    <SettingsButton onClick={hideSettings} showClose={true} />
+    {allowLanguageSelection && (
+      <>
+        <SettingsSectionTitle>Field Settings</SettingsSectionTitle>
+        <SettingsFieldLabel htmlFor={`${forID}-select-mode`}>Mode</SettingsFieldLabel>
+        <SettingsSelect
+          type="mode"
+          forID={forID}
+          value={mode}
+          options={modes}
+          onChange={onChangeLang}
+          autoFocus
+        />
+      </>
+    )}
+    <>
+      <SettingsSectionTitle>Global Settings</SettingsSectionTitle>
+      {themes && (
+        <>
+          <SettingsFieldLabel htmlFor={`${forID}-select-theme`}>Theme</SettingsFieldLabel>
+          <SettingsSelect
+            type="theme"
+            forID={forID}
+            value={{ value: theme, label: theme }}
+            options={themes.map(t => ({ value: t, label: t }))}
+            onChange={onChangeTheme}
+            autoFocus={!allowLanguageSelection}
+          />
+        </>
+      )}
+      <SettingsFieldLabel htmlFor={`${forID}-select-keymap`}>KeyMap</SettingsFieldLabel>
+      <SettingsSelect
+        type="keymap"
+        forID={forID}
+        value={keyMap}
+        options={keyMaps}
+        onChange={onChangeKeyMap}
+      />
+    </>
+  </SettingsPaneContainer>
+);
+
+export default SettingsPane;

packages/netlify-cms-widget-code/src/index.js

@@ -0,0 +1,14 @@
+import controlComponent from './CodeControl';
+import previewComponent from './CodePreview';
+
+const Widget = (opts = {}) => ({
+  name: 'code',
+  controlComponent,
+  previewComponent,
+  allowMapValue: true,
+  codeMirrorConfig: {},
+  ...opts,
+});
+
+export const NetlifyCmsWidgetCode = { Widget, controlComponent, previewComponent };
+export default NetlifyCmsWidgetCode;

packages/netlify-cms-widget-code/src/languageSelectStyles.js

@@ -0,0 +1,35 @@
+import { reactSelectStyles, borders } from 'netlify-cms-ui-default';
+
+const languageSelectStyles = {
+  ...reactSelectStyles,
+  container: provided => ({
+    ...reactSelectStyles.container(provided),
+    'margin-top': '2px',
+  }),
+  control: provided => ({
+    ...reactSelectStyles.control(provided),
+    border: borders.textField,
+    padding: 0,
+    fontSize: '13px',
+    minHeight: 'auto',
+  }),
+  dropdownIndicator: provided => ({
+    ...reactSelectStyles.dropdownIndicator(provided),
+    padding: '4px',
+  }),
+  option: (provided, state) => ({
+    ...reactSelectStyles.option(provided, state),
+    padding: 0,
+    paddingLeft: '8px',
+  }),
+  menu: provided => ({
+    ...reactSelectStyles.menu(provided),
+    margin: '2px 0',
+  }),
+  menuList: provided => ({
+    ...provided,
+    'max-height': '200px',
+  }),
+};
+
+export default languageSelectStyles;

packages/netlify-cms-widget-code/webpack.config.js

@@ -0,0 +1,3 @@
+const { getConfig } = require('../../scripts/webpack.js');
+
+module.exports = getConfig();