Rich Text Editor (WIP) x2

Gemfile

@@ -94,6 +94,8 @@ gem 'json-ld', '~> 3.0'
 gem 'json-ld-preloaded', '~> 3.0'
 gem 'rdf-normalize', '~> 0.3'
 
+gem 'redcarpet', '~> 3.4'
+
 group :development, :test do
   gem 'fabrication', '~> 2.20'
   gem 'fuubar', '~> 2.3'

Gemfile.lock

@@ -479,6 +479,7 @@ GEM
       link_header (~> 0.0, >= 0.0.8)
     rdf-normalize (0.3.3)
       rdf (>= 2.2, < 4.0)
+    redcarpet (3.4.0)
     redis (4.1.2)
     redis-actionpack (5.0.2)
       actionpack (>= 4.0, < 6)
@@ -740,6 +741,7 @@ DEPENDENCIES
   rails-i18n (~> 5.1)
   rails-settings-cached (~> 0.6)
   rdf-normalize (~> 0.3)
+  redcarpet (~> 3.4)
   redis (~> 4.1)
   redis-namespace (~> 1.5)
   redis-rails (~> 5.0)

app/javascript/gabsocial/actions/importer/normalizer.js

@@ -1,7 +1,4 @@
 import escapeTextContentForBrowser from 'escape-html'
-import { markdownToDraft } from 'markdown-draft-js'
-import { Remarkable } from 'remarkable'
-import * as entities from 'entities'
 import emojify from '../../components/emoji/emoji'
 import { unescapeHTML } from '../../utils/html'
 import { expandSpoilers } from '../../initial_state'
@@ -66,39 +63,7 @@ export function normalizeStatus(status, normalOldStatus) {
     const spoilerText   = normalStatus.spoiler_text || '';
     const searchContent = [spoilerText, status.content].join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
     const emojiMap      = makeEmojiMap(normalStatus);
-    
+    const theContent = !!normalStatus.rich_content ? normalStatus.rich_content : normalStatus.content;
-    let theContent
-    if (!!normalStatus.rich_content) {
-      theContent = normalStatus.rich_content
-      // let rawObject = markdownToDraft(theContent, {
-      //   preserveNewlines: true,
-      //   remarkablePreset: 'commonmark',
-      //   remarkableOptions: {
-      //     enable: {
-      //       inline: ['del', 'ins'],
-      //     }
-      //   }
-      // });
-
-      const md = new Remarkable({
-        html: false,
-        breaks: true,
-      })
-      let html = md.render(theContent)
-      html = entities.decodeHTML(html)
-      
-      theContent = html
-
-      console.log("html:", html)
-      console.log("theContent:", theContent)
-      console.log("status:", status)
-      console.log("normalStatus:", normalStatus)
-      // console.log("rawObject:", rawObject)
-    } else {
-      theContent = normalStatus.content
-    }
-    // let theContent = !!normalStatus.rich_content ? normalStatus.rich_content : normalStatus.content;
-
 
     normalStatus.search_index = domParser.parseFromString(searchContent, 'text/html').documentElement.textContent;
     normalStatus.contentHtml  = emojify(theContent, emojiMap, false, true);

app/javascript/gabsocial/components/composer.js

@@ -4,10 +4,8 @@ import {
   CompositeDecorator,
   RichUtils,
   convertToRaw,
-  convertFromRaw,
-  ContentState,
 } from 'draft-js'
-import { draftToMarkdown } from 'markdown-draft-js'
+import draftToMarkdown from '../features/ui/util/draft-to-markdown'
 import { urlRegex } from '../features/ui/util/url_regex'
 import classNames from 'classnames/bind'
 import RichTextEditorBar from './rich_text_editor_bar'
@@ -134,7 +132,8 @@ class Composer extends PureComponent {
 
     const rawObject = convertToRaw(content);
     const markdownString = draftToMarkdown(rawObject, {
-      preserveNewlines: true,
+      escapeMarkdownCharacters: false,
+      preserveNewlines: false,
       remarkablePreset: 'commonmark',
       remarkableOptions: {
         disable: {
@@ -147,6 +146,7 @@ class Composer extends PureComponent {
     });
 
     console.log("text:", markdownString)
+    // console.log("html:", html)
 
     this.props.onChange(null, text, markdownString, selectionStart)
   }
@@ -154,11 +154,12 @@ class Composer extends PureComponent {
   // **bold**
   // *italic*
   // __underline__
-  // ~strikethrough~
+  // ~~strike~~
-  // # title
+  // # header
   // > quote
-  // `code`
+  // ```
-  // ```code```
+  // code
+  // ```
 
   focus = () => {
     this.textbox.editor.focus()

app/javascript/gabsocial/components/rich_text_editor_bar.js

@@ -1,5 +1,4 @@
 import { RichUtils } from 'draft-js'
-import { defineMessages, injectIntl } from 'react-intl'
 import classNames from 'classnames/bind'
 import { me } from '../initial_state'
 import { makeGetAccount } from '../selectors'
@@ -70,10 +69,6 @@ const RTE_ITEMS = [
   },
 ]
 
-const messages = defineMessages({
-  follow: { id: 'follow', defaultMessage: 'Follow' },
-})
-
 const mapStateToProps = (state) => {
   const getAccount = makeGetAccount()
   const account = getAccount(state, me)
@@ -86,13 +81,11 @@ const mapStateToProps = (state) => {
 }
 
 export default
-@injectIntl
 @connect(mapStateToProps)
 class RichTextEditorBar extends PureComponent {
 
   static propTypes = {
     editorState: PropTypes.object.isRequired,
-    intl: PropTypes.object.isRequired,
     isPro: PropTypes.bool.isRequired,
     rteControlsVisible: PropTypes.bool.isRequired,
     onChange: PropTypes.func.isRequired,
@@ -127,7 +120,7 @@ class RichTextEditorBar extends PureComponent {
             />
           ))
         }
-        <Button
+        {/*<Button
           backgroundColor='none'
           color='secondary'
           onClick={this.handleOnTogglePopoutEditor}
@@ -137,7 +130,7 @@ class RichTextEditorBar extends PureComponent {
           iconClassName={_s.inheritFill}
           iconSize='12px'
           radiusSmall
-        />
+        />*/}
       </div>
     )
   }

app/javascript/gabsocial/components/status_list.js

@@ -3,7 +3,7 @@ import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import { createSelector } from 'reselect';
-import sample from 'lodash.sample';
+// import sample from 'lodash.sample';
 import debounce from 'lodash.debounce'
 import { me, promotions } from '../initial_state';
 import { dequeueTimeline } from '../actions/timelines';
@@ -44,7 +44,7 @@ const mapStateToProps = (state, { timelineId }) => {
   if (!timelineId) return {}
 
   const getStatusIds = makeGetStatusIds();
-  const promotion = promotions.length > 0 && sample(promotions.filter(p => p.timeline_id === timelineId));
+  const promotion = false//promotions.length > 0 && sample(promotions.filter(p => p.timeline_id === timelineId));
 
   const statusIds = getStatusIds(state, {
     type: timelineId.substring(0, 5) === 'group' ? 'group' : timelineId,

app/javascript/gabsocial/features/ui/util/draft-to-markdown.js

@@ -0,0 +1,456 @@
+// https://raw.githubusercontent.com/Rosey/markdown-draft-js/main/src/draft-to-markdown.js
+const TRAILING_WHITESPACE = /[ \u0020\t\n]*$/;
+
+// This escapes some markdown but there's a few cases that are TODO -
+// - List items
+// - Back tics  (see https://github.com/Rosey/markdown-draft-js/issues/52#issuecomment-388458017)
+// - Complex markdown, like links or images. Not sure it's even worth it, because if you're typing
+// that into draft chances are you know its markdown and maybe expect it convert? :/
+const MARKDOWN_STYLE_CHARACTERS = ['*', '_', '~', '`'];
+const MARKDOWN_STYLE_CHARACTER_REGXP = /(\*|_|~|\\|`)/g;
+
+// I hate this a bit, being outside of the function’s scope
+// but can’t think of a better way to keep track of how many ordered list
+// items were are on, as draft doesn’t explicitly tell us in the raw object 😢.
+// This is a hash that will be assigned values based on depth, so like
+// orderedListNumber[0] = 1 would mean that ordered list at depth 0 is on number 1.
+// orderedListNumber[0] = 2 would mean that ordered list at depth 0 is on number 2.
+// This is so we have the right number of numbers when doing a list, eg
+// 1. Item One
+// 2. Item two
+// 3. Item three
+// And so on.
+var orderedListNumber = {},
+    previousOrderedListDepth = 0;
+
+// A map of draftjs block types -> markdown open and close characters
+// Both the open and close methods must exist, even if they simply return an empty string.
+// They should always return a string.
+const StyleItems = {
+  // BLOCK LEVEL
+  'unordered-list-item': {
+    open: function () {
+      return '- ';
+    },
+
+    close: function () {
+      return '';
+    }
+  },
+
+  'ordered-list-item': {
+    open: function (block, number = 1) {
+      return `${number}. `;
+    },
+
+    close: function () {
+      return '';
+    }
+  },
+
+  'blockquote': {
+    open: function () {
+      return '> ';
+    },
+
+    close: function () {
+      return '';
+    }
+  },
+
+  'header-one': {
+    open: function () {
+      return '# ';
+    },
+
+    close: function () {
+      return '';
+    }
+  },
+
+  'code-block': {
+    open: function (block) {
+      return '```' + (block.data.language || '') + '\n';
+    },
+
+    close: function () {
+      return '\n```';
+    }
+  },
+
+  // INLINE LEVEL
+  'BOLD': {
+    open: function () {
+      return '**';
+    },
+
+    close: function () {
+      return '**';
+    }
+  },
+
+  'ITALIC': {
+    open: function () {
+      return '*';
+    },
+
+    close: function () {
+      return '*';
+    }
+  },
+
+  'UNDERLINE': {
+    open: function () {
+      return '_';
+    },
+
+    close: function () {
+      return '_';
+    }
+  },
+
+  'STRIKETHROUGH': {
+    open: function () {
+      return '~~';
+    },
+
+    close: function () {
+      return '~~';
+    }
+  },
+
+  'CODE': {
+    open: function () {
+      return '`';
+    },
+
+    close: function () {
+      return '`';
+    }
+  }
+};
+
+// A map of draftjs entity types -> markdown open and close characters
+// entities are different from block types because they have additional data attached to them.
+// an entity object is passed in to both open and close, in case it's needed for string generation.
+//
+// Both the open and close methods must exist, even if they simply return an empty string.
+// They should always return a string.
+const EntityItems = {
+  //
+}
+
+// Bit of a hack - we normally want a double newline after a block,
+// but for list items we just want one (unless it's the _last_ list item in a group.)
+const SingleNewlineAfterBlock = [
+  'unordered-list-item',
+  'ordered-list-item'
+];
+
+function isEmptyBlock(block) {
+  return block.text.length === 0 && block.entityRanges.length === 0 && Object.keys(block.data || {}).length === 0;
+}
+
+/**
+ * Generate markdown for a single block javascript object
+ * DraftJS raw object contains an array of blocks, which is the main "structure"
+ * of the text. Each block = a new line.
+ *
+ * @param {Object} block - block to generate markdown for
+ * @param {Number} index - index of the block in the blocks array
+ * @param {Object} rawDraftObject - entire raw draft object (needed for accessing the entityMap)
+ * @param {Object} options - additional options passed in by the user calling this method.
+ *
+ * @return {String} markdown string
+**/
+function renderBlock(block, index, rawDraftObject, options) {
+  var openInlineStyles = [],
+      markdownToAdd = [];
+  var markdownString = '',
+      customStyleItems = options.styleItems || {},
+      customEntityItems = options.entityItems || {},
+      escapeMarkdownCharacters = options.hasOwnProperty('escapeMarkdownCharacters') ? options.escapeMarkdownCharacters : true;
+
+  var type = block.type;
+
+  var markdownStyleCharactersToEscape = [];
+
+  // draft-js emits empty blocks that have type set… don’t style them unless the user wants to preserve new lines
+  // (if newlines are preserved each empty line should be "styled" eg in case of blockquote we want to see a blockquote.)
+  // but if newlines aren’t preserved then we'd end up having double or triple or etc markdown characters, which is a bug.
+  if (isEmptyBlock(block) && !options.preserveNewlines) {
+    type = 'unstyled';
+  }
+
+  // Render main block wrapping element
+  if (customStyleItems[type] || StyleItems[type]) {
+    if (type === 'unordered-list-item' || type === 'ordered-list-item') {
+      markdownString += ' '.repeat(block.depth * 4);
+    }
+
+    if (type === 'ordered-list-item') {
+      orderedListNumber[block.depth] = orderedListNumber[block.depth] || 1;
+      markdownString += (customStyleItems[type] || StyleItems[type]).open(block, orderedListNumber[block.depth]);
+      orderedListNumber[block.depth]++;
+
+      // Have to reset the number for orderedListNumber if we are breaking out of a list so that if
+      // there's another nested list at the same level further down, it starts at 1 again.
+      // COMPLICATED 😭
+      if (previousOrderedListDepth > block.depth) {
+        orderedListNumber[previousOrderedListDepth] = 1;
+      }
+
+      previousOrderedListDepth = block.depth;
+    } else {
+      orderedListNumber = {};
+      markdownString += (customStyleItems[type] || StyleItems[type]).open(block);
+    }
+  }
+
+  // Render text within content, along with any inline styles/entities
+  Array.from(block.text).some(function (character, characterIndex) {
+    // Close any entity tags that need closing
+    block.entityRanges.forEach(function (range, rangeIndex) {
+      if (range.offset + range.length === characterIndex) {
+        var entity = rawDraftObject.entityMap[range.key];
+        if (customEntityItems[entity.type] || EntityItems[entity.type]) {
+          markdownString += (customEntityItems[entity.type] || EntityItems[entity.type]).close(entity);
+        }
+      }
+    });
+
+    // Close any inline tags that need closing
+    openInlineStyles.forEach(function (style, styleIndex) {
+      if (style.offset + style.length === characterIndex) {
+        if ((customStyleItems[style.style] || StyleItems[style.style])) {
+          var styleIndex = openInlineStyles.indexOf(style);
+          // Handle nested case - close any open inline styles before closing the parent
+          if (styleIndex > -1 && styleIndex !== openInlineStyles.length - 1) {
+            for (var i = openInlineStyles.length - 1; i !== styleIndex; i--) {
+              var styleItem = (customStyleItems[openInlineStyles[i].style] || StyleItems[openInlineStyles[i].style]);
+              if (styleItem) {
+                var trailingWhitespace = TRAILING_WHITESPACE.exec(markdownString);
+                markdownString = markdownString.slice(0, markdownString.length - trailingWhitespace[0].length);
+                markdownString += styleItem.close();
+                markdownString += trailingWhitespace[0];
+              }
+            }
+          }
+
+          // Close the actual inline style being closed
+          // Have to trim whitespace first and then re-add after because markdown can't handle leading/trailing whitespace
+          var trailingWhitespace = TRAILING_WHITESPACE.exec(markdownString);
+          markdownString = markdownString.slice(0, markdownString.length - trailingWhitespace[0].length);
+
+          markdownString += (customStyleItems[style.style] || StyleItems[style.style]).close();
+          markdownString += trailingWhitespace[0];
+
+          // Handle nested case - reopen any inline styles after closing the parent
+          if (styleIndex > -1 && styleIndex !== openInlineStyles.length - 1) {
+            for (var i = openInlineStyles.length - 1; i !== styleIndex; i--) {
+              var styleItem = (customStyleItems[openInlineStyles[i].style] || StyleItems[openInlineStyles[i].style]);
+              if (styleItem && openInlineStyles[i].offset + openInlineStyles[i].length > characterIndex) {
+                markdownString += styleItem.open();
+              } else {
+                openInlineStyles.splice(i, 1);
+              }
+            }
+          }
+
+          openInlineStyles.splice(styleIndex, 1);
+        }
+      }
+    });
+
+    // Open any inline tags that need opening
+    block.inlineStyleRanges.forEach(function (style, styleIndex) {
+      if (style.offset === characterIndex) {
+        if ((customStyleItems[style.style] || StyleItems[style.style])) {
+          var styleToAdd = (customStyleItems[style.style] || StyleItems[style.style]).open();
+          markdownToAdd.push({
+            type: 'style',
+            style: style,
+            value: styleToAdd
+          });
+        }
+      }
+    });
+
+    // Open any entity tags that need opening
+    block.entityRanges.forEach(function (range, rangeIndex) {
+      if (range.offset === characterIndex) {
+        var entity = rawDraftObject.entityMap[range.key];
+        if (customEntityItems[entity.type] || EntityItems[entity.type]) {
+          var entityToAdd = (customEntityItems[entity.type] || EntityItems[entity.type]).open(entity);
+          markdownToAdd.push({
+            type: 'entity',
+            value: entityToAdd
+          });
+        }
+      }
+    });
+
+    // These are all the opening entity and style types being added to the markdown string for this loop
+    // we store in an array and add here because if the character is WS character, we want to hang onto it and not apply it until the next non-whitespace
+    // character before adding the markdown, since markdown doesn’t play nice with leading whitespace (eg '** bold**' is no  good, whereas ' **bold**' is good.)
+    if (character !== ' ' && markdownToAdd.length) {
+      markdownString += markdownToAdd.map(function (item) {
+        return item.value;
+      }).join('');
+
+      markdownToAdd.forEach(function (item) {
+        if (item.type === 'style') {
+          // We hang on to this because we may need to close it early and then re-open if there are nested styles being opened and closed.
+          openInlineStyles.push(item.style);
+        }
+      });
+
+      markdownToAdd = [];
+    }
+
+    if (block.type !== 'code-block' && escapeMarkdownCharacters) {
+      let insideInlineCodeStyle = openInlineStyles.find((style) => style.style === 'CODE');
+
+      if (insideInlineCodeStyle) {
+        // Todo - The syntax to escape backtics when inside backtic code already is to use MORE backtics wrapping.
+        // So we need to see how many backtics in a row we have and then when converting to markdown, use that # + 1
+
+        // EG  ``Test ` Hllo ``
+        // OR   ```Test `` Hello```
+        // OR ````Test ``` Hello ````
+        // Similar work has to be done for codeblocks.
+      } else {
+        // Special escape logic for blockquotes and heading characters
+        if (characterIndex === 0 && character === '#' && block.text[1] && block.text[1] === ' ') {
+          character = character.replace('#', '\\#');
+        } else if (characterIndex === 0 && character === '>') {
+          character = character.replace('>', '\\>');
+        }
+
+        // Escaping inline markdown characters
+        // 🧹 If someone can think of a more elegant solution, I would love that.
+        // orginally this was just a little char replace using a simple regular expression, but there’s lots of cases where
+        // a markdown character does not actually get converted to markdown, like this case: http://google.com/i_am_a_link
+        // so this code now tries to be smart and keeps track of potential “opening” characters as well as potential “closing”
+        // characters, and only escapes if both opening and closing exist, and they have the correct whitepace-before-open, whitespace-or-end-of-string-after-close pattern
+        if (MARKDOWN_STYLE_CHARACTERS.includes(character)) {
+          let openingStyle = markdownStyleCharactersToEscape.find(function (item) {
+            return item.character === character;
+          });
+
+          if (!openingStyle && block.text[characterIndex - 1] === ' ' && block.text[characterIndex + 1] !== ' ') {
+            markdownStyleCharactersToEscape.push({
+              character: character,
+              index: characterIndex,
+              markdownStringIndexStart: markdownString.length + character.length - 1,
+              markdownStringIndexEnd: markdownString.length + character.length
+            });
+          } else if (openingStyle && block.text[characterIndex - 1] === character && characterIndex === openingStyle.index + 1) {
+            openingStyle.markdownStringIndexEnd += 1;
+          } else if (openingStyle) {
+            let openingStyleLength = openingStyle.markdownStringIndexEnd - openingStyle.markdownStringIndexStart;
+            let escapeCharacter = false;
+            let popOpeningStyle = false;
+            if (openingStyleLength === 1 && (block.text[characterIndex + 1] === ' ' || !block.text[characterIndex + 1])) {
+              popOpeningStyle = true;
+              escapeCharacter = true;
+            }
+
+            if (openingStyleLength === 2 && block.text[characterIndex + 1] === character) {
+              escapeCharacter = true;
+            }
+
+            if (openingStyleLength === 2 && block.text[characterIndex - 1] === character && (block.text[characterIndex + 1] === ' ' || !block.text[characterIndex + 1])) {
+              popOpeningStyle = true;
+              escapeCharacter = true;
+            }
+
+            if (popOpeningStyle) {
+              markdownStyleCharactersToEscape.splice(markdownStyleCharactersToEscape.indexOf(openingStyle), 1);
+              let replacementString = markdownString.slice(openingStyle.markdownStringIndexStart, openingStyle.markdownStringIndexEnd);
+              replacementString = replacementString.replace(MARKDOWN_STYLE_CHARACTER_REGXP, '\\$1');
+              markdownString = (markdownString.slice(0, openingStyle.markdownStringIndexStart) + replacementString + markdownString.slice(openingStyle.markdownStringIndexEnd));
+            }
+
+            if (escapeCharacter) {
+              character = `\\${character}`;
+            }
+          }
+        }
+      }
+    }
+
+    if (character === '\n' && type === 'blockquote') {
+      markdownString += '\n> ';
+    } else {
+      markdownString += character;
+    }
+  });
+
+  // Close any remaining entity tags
+  block.entityRanges.forEach(function (range, rangeIndex) {
+    if (range.offset + range.length === Array.from(block.text).length) {
+      var entity = rawDraftObject.entityMap[range.key];
+      if (customEntityItems[entity.type] || EntityItems[entity.type]) {
+        markdownString += (customEntityItems[entity.type] || EntityItems[entity.type]).close(entity);
+      }
+    }
+  });
+
+  // Close any remaining inline tags (if an inline tag ends at the very last char, we won't catch it inside the loop)
+  openInlineStyles.reverse().forEach(function (style) {
+    var trailingWhitespace = TRAILING_WHITESPACE.exec(markdownString);
+    markdownString = markdownString.slice(0, markdownString.length - trailingWhitespace[0].length);
+    markdownString += (customStyleItems[style.style] || StyleItems[style.style]).close();
+    markdownString += trailingWhitespace[0];
+  });
+
+  // Close block level item
+  if (customStyleItems[type] || StyleItems[type]) {
+    markdownString += (customStyleItems[type] || StyleItems[type]).close(block);
+  }
+
+  // Determine how many newlines to add - generally we want 2, but for list items we just want one when they are succeeded by another list item.
+  if (SingleNewlineAfterBlock.indexOf(type) !== -1 && rawDraftObject.blocks[index + 1] && SingleNewlineAfterBlock.indexOf(rawDraftObject.blocks[index + 1].type) !== -1) {
+    markdownString += '\n';
+  } else if (rawDraftObject.blocks[index + 1]) {
+    if (rawDraftObject.blocks[index].text) {
+      if (SingleNewlineAfterBlock.indexOf(type) !== -1
+          && SingleNewlineAfterBlock.indexOf(rawDraftObject.blocks[index + 1].type) === -1) {
+        markdownString += '\n\n';
+      } else if (!options.preserveNewlines) {
+        // 2 newlines if not preserving
+        markdownString += '\n\n';
+      } else {
+        markdownString += '\n';
+      }
+    } else if (options.preserveNewlines) {
+      markdownString += '\n';
+    }
+  }
+
+  return markdownString;
+}
+
+/**
+ * Generate markdown for a raw draftjs object
+ * DraftJS raw object contains an array of blocks, which is the main "structure"
+ * of the text. Each block = a new line.
+ *
+ * @param {Object} rawDraftObject - draftjs object to generate markdown for
+ * @param {Object} options - optional additional data, see readme for what options can be passed in.
+ *
+ * @return {String} markdown string
+**/
+function draftToMarkdown(rawDraftObject, options) {
+  options = options || {};
+  var markdownString = '';
+  rawDraftObject.blocks.forEach(function (block, index) {
+    markdownString += renderBlock(block, index, rawDraftObject, options);
+  });
+
+  orderedListNumber = {}; // See variable definitions at the top of the page to see why we have to do this sad hack.
+  return markdownString;
+}
+
+export default draftToMarkdown;

app/javascript/styles/global.css

@@ -169,6 +169,10 @@ pre {
   margin-bottom: 0.5rem;
 }
 
+.statusContent h1 * {
+  font-size: var(--fs_xl) !important;
+}
+
 .statusContent ul,
 .statusContent ol {
   padding-left: 40px;
@@ -183,6 +187,12 @@ pre {
   /* list-style-type: disc; */
 }
 
+.statusContent code {
+  background-color: rgba(0,0,0,.05);
+  padding-left: 0.5em;
+  padding-right: 0.5em;
+}
+
 .dangerousContent,
 .dangerousContent * {
   margin-top: 0;

app/lib/formatter.rb

@@ -3,6 +3,73 @@
 require 'singleton'
 require_relative './sanitize_config'
 
+class HTMLRenderer < Redcarpet::Render::HTML
+  def block_code(code, language)
+    "<pre><code>#{encode(code).gsub("\n", "<br/>")}</code></pre>"
+  end
+
+  def block_quote(quote)
+    "<blockquote>#{quote}</blockquote>"
+  end
+
+  def codespan(code)
+    "<code>#{code}</code>"
+  end
+
+  def double_emphasis(text)
+    "<strong>#{text}</strong>"
+  end
+  
+  def emphasis(text)
+    "<em>#{text}</em>"
+  end
+  
+  def header(text, header_level)
+    "<h1>#{text}</h1>"
+  end
+  
+  def triple_emphasis(text)
+    "<b><em>#{text}</em></b>"
+  end
+  
+  def strikethrough(text)
+    "<del>#{text}</del>"
+  end
+  
+  def underline(text)
+    "<u>#{text}</u>"
+  end
+
+  def list(contents, list_type)
+    if list_type == :ordered
+      "<ol>#{contents}</ol>"
+    elsif list_type == :unordered
+      "<ul>#{contents}</ul>"
+    else
+      content
+    end
+  end
+  
+  def list_item(text, list_type)
+    "<li>#{text}</li>"
+  end
+
+  def autolink(link, link_type)
+    return link if link_type == :email
+    Formatter.instance.link_url(link)
+  end
+
+  private
+
+  def html_entities
+    @html_entities ||= HTMLEntities.new
+  end
+
+  def encode(html)
+    html_entities.encode(html)
+  end
+end
+
 class Formatter
   include Singleton
   include RoutingHelper
@@ -39,20 +106,39 @@ class Formatter
     linkable_accounts << status.account
 
     html = raw_content
-
+    puts "TELLY FORMAT-1: " + html.to_s
-    html = encode_and_link_urls(html, linkable_accounts)
+    html = format_markdown(html) if options[:use_markdown]
-
+    puts "TELLY FORMAT-2: " + html.to_s
+    html = encode_and_link_urls(html, linkable_accounts, keep_html: options[:use_markdown])
+    puts "TELLY FORMAT-3: " + html.to_s
+    html = reformat(html, true) unless options[:use_markdown]
+    puts "TELLY FORMAT-4: " + html.to_s
     html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify]
 
-    html = simple_format(html, {}, sanitize: false)
+    puts "TELLY FORMAT-5: " + html.to_s
 
+    unless options[:use_markdown]
+      puts "TELLY FORMAT-4: " + html
+      html = html.gsub(/(?:\n\r?|\r\n?)/, '<br />')
       html = html.delete("\n")
-
-    html.html_safe # rubocop:disable Rails/OutputSafety
     end
 
-  def reformat(html)
+    puts "TELLY FORMAT-6: " + html.to_s
-    sanitize(html, Sanitize::Config::GABSOCIAL_STRICT)
+
+    html.html_safe # rubocop:disable Rails/OutputSafety
+
+    puts "telly-html: " + html.to_s
+
+    html
+  end
+
+  def format_markdown(html)
+    html = markdown_formatter.render(html)
+    html.delete("\r").delete("\n")
+  end
+
+  def reformat(html, outgoing = false)
+    sanitize(html, Sanitize::Config::GABSOCIAL_STRICT.merge(outgoing: outgoing))
   rescue ArgumentError
     ''
   end
@@ -93,8 +179,7 @@ class Formatter
   end
 
   def format_field(account, str, **options)
-    return reformat(str).html_safe unless account.local? # rubocop:disable Rails/OutputSafety
+    html = account.local? ? encode_and_link_urls(str, me: true) : reformat(str)
-    html = encode_and_link_urls(str, me: true)
     html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify]
     html.html_safe # rubocop:disable Rails/OutputSafety
   end
@@ -107,8 +192,43 @@ class Formatter
     html.html_safe # rubocop:disable Rails/OutputSafety
   end
 
+  def link_url(url)
+    "<a href=\"#{encode(url)}\" target=\"blank\" rel=\"nofollow noopener noreferrer\">#{link_html(url)}</a>"
+  end
+
   private
 
+  def markdown_formatter
+    extensions = {
+      autolink: false,
+      no_intra_emphasis: true,
+      fenced_code_blocks: true,
+      disable_indented_code_blocks: true,
+      strikethrough: true,
+      lax_spacing: true,
+      space_after_headers: true,
+      superscript: false,
+      underline: true,
+      highlight: false,
+      footnotes: false,
+    }
+
+    renderer = HTMLRenderer.new({
+      filter_html: false,
+      escape_html: false,
+      no_images: true,
+      no_styles: true,
+      safe_links_only: false,
+      hard_wrap: false,
+      no_links: true,
+      with_toc_data: false,
+      prettify: false,
+      link_attributes: nil
+    })
+
+    Redcarpet::Markdown.new(renderer, extensions)
+  end
+
   def html_entities
     @html_entities ||= HTMLEntities.new
   end
@@ -125,7 +245,7 @@ class Formatter
       accounts = nil
     end
 
-    rewrite(html.dup, entities) do |entity|
+    rewrite(html.dup, entities, options[:keep_html]) do |entity|
       if entity[:url]
         link_to_url(entity, options)
       elsif entity[:hashtag]
@@ -195,7 +315,7 @@ class Formatter
     html
   end
 
-  def rewrite(text, entities)
+  def rewrite(text, entities, keep_html = false)
     text = text.to_s
 
     # Sort by start index
@@ -208,12 +328,12 @@ class Formatter
 
     last_index = entities.reduce(0) do |index, entity|
       indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices]
-      result << encode(text[index...indices.first])
+      result << (keep_html ? text[index...indices.first] : encode(text[index...indices.first]))
       result << yield(entity)
       indices.last
     end
 
-    result << encode(text[last_index..-1])
+    result << (keep_html ? text[last_index..-1] : encode(text[last_index..-1]))
 
     result.flatten.join
   end
@@ -256,6 +376,29 @@ class Formatter
     Extractor.remove_overlapping_entities(special + standard)
   end
 
+  def html_friendly_extractor(html, options = {})
+    gaps = []
+    total_offset = 0
+
+    escaped = html.gsub(/<[^>]*>|&#[0-9]+;/) do |match|
+      total_offset += match.length - 1
+      end_offset = Regexp.last_match.end(0)
+      gaps << [end_offset - total_offset, total_offset]
+      "\u200b"
+    end
+
+    entities = Extractor.extract_hashtags_with_indices(escaped, :check_url_overlap => false) +
+               Extractor.extract_mentions_or_lists_with_indices(escaped)
+    Extractor.remove_overlapping_entities(entities).map do |extract|
+      pos = extract[:indices].first
+      offset_idx = gaps.rindex { |gap| gap.first <= pos }
+      offset = offset_idx.nil? ? 0 : gaps[offset_idx].last
+      next extract.merge(
+        :indices => [extract[:indices].first + offset, extract[:indices].last + offset]
+      )
+    end
+  end
+
   def link_to_url(entity, options = {})
     url        = Addressable::URI.parse(entity[:url])
     html_attrs = { target: '_blank', rel: 'nofollow noopener noreferrer' }

app/serializers/rest/status_serializer.rb

@@ -74,9 +74,6 @@ class REST::StatusSerializer < ActiveModel::Serializer
 
   def rich_content
     Formatter.instance.format(object, use_markdown: true).strip
-    # raw_content = object.markdown
-    # return '' if raw_content.blank?
-    # raw_content.strip
   end
 
   def url

package.json

@@ -92,7 +92,6 @@
     "draft-js": "^0.11.4",
     "draftjs-to-markdown": "^0.6.0",
     "emoji-mart": "Gargron/emoji-mart#build",
-    "entities": "^2.0.3",
     "es6-symbol": "^3.1.1",
     "escape-html": "^1.0.3",
     "exif-js": "^2.3.0",
@@ -116,7 +115,6 @@
     "lodash.throttle": "^4.1.1",
     "lodash.unescape": "^4.0.1",
     "mark-loader": "^0.1.6",
-    "markdown-draft-js": "^2.2.0",
     "mini-css-extract-plugin": "^0.5.0",
     "mkdirp": "^0.5.1",
     "moment-mini": "^2.24.0",
@@ -154,7 +152,6 @@
     "redux": "^4.0.1",
     "redux-immutable": "^4.0.0",
     "redux-thunk": "^2.2.0",
-    "remarkable": "^2.0.1",
     "requestidlecallback": "^0.3.0",
     "reselect": "^4.0.0",
     "rimraf": "^2.6.3",

yarn.lock

@@ -1472,7 +1472,7 @@ are-we-there-yet@~1.1.2:
     delegates "^1.0.0"
     readable-stream "^2.0.6"
 
-argparse@^1.0.10, argparse@^1.0.7:
+argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
   integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==
@@ -1640,13 +1640,6 @@ atob@^2.1.2:
   resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9"
   integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==
 
-autolinker@^3.11.0:
-  version "3.14.1"
-  resolved "https://registry.yarnpkg.com/autolinker/-/autolinker-3.14.1.tgz#6ae4b812b6eaf42d4d68138b9e67757cbf2bc1e4"
-  integrity sha512-yvsRHIaY51EYDml6MGlbqyJGfl4n7zezGYf+R7gvM8c5LNpRGc4SISkvgAswSS8SWxk/OrGCylKV9mJyVstz7w==
-  dependencies:
-    tslib "^1.9.3"
-
 autoprefixer@^9.5.1:
   version "9.7.6"
   resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.7.6.tgz#63ac5bbc0ce7934e6997207d5bb00d68fa8293a4"
@@ -3394,11 +3387,6 @@ entities@^2.0.0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4"
   integrity sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw==
 
-entities@^2.0.3:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.3.tgz#5c487e5742ab93c15abb5da22759b8590ec03b7f"
-  integrity sha512-MyoZ0jgnLvB2X3Lg5HqpFmn1kybDiIfEQmKzTb5apr51Rb+T3KdmMiqa70T+bhGnyv7bQ6WMj2QMHpGMmlrUYQ==
-
 enzyme-adapter-react-16@^1.7.1:
   version "1.15.2"
   resolved "https://registry.yarnpkg.com/enzyme-adapter-react-16/-/enzyme-adapter-react-16-1.15.2.tgz#b16db2f0ea424d58a808f9df86ab6212895a4501"
@@ -6099,13 +6087,6 @@ mark-loader@^0.1.6:
   resolved "https://registry.yarnpkg.com/mark-loader/-/mark-loader-0.1.6.tgz#0abb477dca7421d70e20128ff6489f5cae8676d5"
   integrity sha1-CrtHfcp0IdcOIBKP9kifXK6GdtU=
 
-markdown-draft-js@^2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/markdown-draft-js/-/markdown-draft-js-2.2.0.tgz#1a4796c4d7907761ae417e2176ab858dfaa1166a"
-  integrity sha512-OW53nd5m7KrnuXjH8jNKB3SQ5KidKD/+pHS+3daVKTsyjsiozghjUxzNH+5JLxzaBECHmJ7itE39RzMgYm5EFQ==
-  dependencies:
-    remarkable "2.0.0"
-
 md5.js@^1.3.4:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f"
@@ -8230,22 +8211,6 @@ regjsparser@^0.6.4:
   dependencies:
     jsesc "~0.5.0"
 
-remarkable@2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/remarkable/-/remarkable-2.0.0.tgz#795f965bede8300362ce51a716edc322d9e7a4ca"
-  integrity sha512-3gvKFAgL4xmmVRKAMNm6UzDo/rO2gPVkZrWagp6AXEA4JvCcMcRx9aapYbb7AJAmLLvi/u06+EhzqoS7ha9qOg==
-  dependencies:
-    argparse "^1.0.10"
-    autolinker "^3.11.0"
-
-remarkable@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/remarkable/-/remarkable-2.0.1.tgz#280ae6627384dfb13d98ee3995627ca550a12f31"
-  integrity sha512-YJyMcOH5lrR+kZdmB0aJJ4+93bEojRZ1HGDn9Eagu6ibg7aVZhc3OWbbShRid+Q5eAfsEqWxpe+g5W5nYNfNiA==
-  dependencies:
-    argparse "^1.0.10"
-    autolinker "^3.11.0"
-
 remove-trailing-separator@^1.0.1:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef"
@@ -9455,7 +9420,7 @@ tryer@^1.0.1:
   resolved "https://registry.yarnpkg.com/tryer/-/tryer-1.0.1.tgz#f2c85406800b9b0f74c9f7465b81eaad241252f8"
   integrity sha512-c3zayb8/kWWpycWYg87P71E1S1ZL6b6IJxfb5fvsUgsf0S2MVGaDhDXXjDMpdCpfWXqptc+4mXwmiy1ypXqRAA==
 
-tslib@^1.9.0, tslib@^1.9.3:
+tslib@^1.9.0:
   version "1.11.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.11.1.tgz#eb15d128827fbee2841549e171f45ed338ac7e35"
   integrity sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==