feat: change to markdown and map html to markdown

package.json

@@ -54,6 +54,9 @@
     "jsonwebtoken": "^9.0.2",
     "libsodium-wrappers": "^0.7.11",
     "lint-staged": "^13.1.0",
+    "mdast-util-from-markdown": "^0.8.5",
+    "mdast-util-gfm": "^0.1.2",
+    "micromark-extension-gfm": "^0.3.3",
     "lodash": "^4.17.21",
     "ms": "^2.1.3",
     "node-html-parser": "^6.1.5",

src/bindings/config.ts

@@ -65,9 +65,7 @@ export const loadConfig = async (context: Context): Promise<BotConfig> => {
       permitBaseUrl: process.env.PERMIT_BASE_URL || permitBaseUrl,
     },
     unassign: {
-      timeRangeForMaxIssue: process.env.DEFAULT_TIME_RANGE_FOR_MAX_ISSUE
-        ? Number(process.env.DEFAULT_TIME_RANGE_FOR_MAX_ISSUE)
-        : timeRangeForMaxIssue,
+      timeRangeForMaxIssue: process.env.DEFAULT_TIME_RANGE_FOR_MAX_ISSUE ? Number(process.env.DEFAULT_TIME_RANGE_FOR_MAX_ISSUE) : timeRangeForMaxIssue,
       timeRangeForMaxIssueEnabled: process.env.DEFAULT_TIME_RANGE_FOR_MAX_ISSUE_ENABLED
         ? process.env.DEFAULT_TIME_RANGE_FOR_MAX_ISSUE_ENABLED == "true"
         : timeRangeForMaxIssueEnabled,

src/configs/ubiquibot-config-default.ts

@@ -87,7 +87,20 @@ export const DefaultConfig: MergedConfig = {
   ],
   incentives: {
     comment: {
-      elements: {},
+      issue: {
+        assignee: false,
+        creator: false,
+        default: true,
+      },
+      pullRequest: {
+        assignee: false,
+        creator: false,
+        reviewer: true,
+        default: true,
+      },
+      elements: {
+        p: 0.1,
+      },
       totals: {
         word: 0,
       },

src/decs.d.ts

@@ -0,0 +1 @@
+declare module "mdast-util-gfm";

src/handlers/payout/post.ts

@@ -2,12 +2,13 @@ import { getWalletAddress } from "../../adapters/supabase";
 import { getBotContext, getLogger } from "../../bindings";
 import { getAllIssueComments, getAllPullRequestReviews, getIssueDescription, parseComments } from "../../helpers";
 import { getLatestPullRequest, gitLinkedPrParser } from "../../helpers/parser";
-import { Incentives, MarkdownItem, Payload, UserType } from "../../types";
+import { Incentives, MarkdownItem, MarkdownItems, Payload, UserType } from "../../types";
 import { RewardsResponse, commentParser } from "../comment";
 import Decimal from "decimal.js";
 import { bountyInfo } from "../wildcard";
 import { IncentivesCalculationResult } from "./action";
 import { BigNumber } from "ethers";
+import { HTMLItem } from "../../types/html";
 
 export interface CreatorCommentResult {
   title: string;
@@ -50,16 +51,16 @@ export const calculateIssueConversationReward = async (calculateIncentives: Ince
       logger.info(`Skipping to parse the comment because it contains commands. comment: ${JSON.stringify(issueComment)}`);
       continue;
     }
-    if (!issueComment.body_html) {
-      logger.info(`Skipping to parse the comment because body_html is undefined. comment: ${JSON.stringify(issueComment)}`);
+    if (!issueComment.body) {
+      logger.info(`Skipping to parse the comment because body is undefined. comment: ${JSON.stringify(issueComment)}`);
       continue;
     }
 
     // Store the comment along with user's login and node_id
     if (!issueCommentsByUser[user.login]) {
       issueCommentsByUser[user.login] = { id: user.id, comments: [] };
     }
-    issueCommentsByUser[user.login].comments.push(issueComment.body_html);
+    issueCommentsByUser[user.login].comments.push(issueComment.body);
   }
   logger.info(`Filtering by the user type done. commentsByUser: ${JSON.stringify(issueCommentsByUser)}`);
 
@@ -298,45 +299,80 @@ const generatePermitForComments = async (
  * @returns - The reward value
  */
 const calculateRewardValue = (
-  comments: Record<string, string[]>,
+  comments: Record<MarkdownItem, string[]>,
   incentives: Incentives
 ): { sum: Decimal; sumByType: Record<string, { count: number; reward: Decimal }> } => {
   let sum = new Decimal(0);
   const sumByType: Record<string, { count: number; reward: Decimal }> = {};
 
-  for (const key of Object.keys(comments)) {
-    const value = comments[key];
+  for (const item of MarkdownItems) {
+    const value = comments[item];
 
     // Initialize the sum for this key if it doesn't exist
-    if (!sumByType[key]) {
-      sumByType[key] = {
+    if (!sumByType[item]) {
+      sumByType[item] = {
         count: 0,
         reward: new Decimal(0),
       };
     }
 
     // if it's a text node calculate word count and multiply with the reward value
-    if (key == "#text") {
+    if (item === MarkdownItem.Text) {
       if (!incentives.comment.totals.word) {
         continue;
       }
       const wordReward = new Decimal(incentives.comment.totals.word);
       const wordCount = value.map((str) => str.trim().split(" ").length).reduce((totalWords, wordCount) => totalWords + wordCount, 0);
       const reward = wordReward.mul(wordCount);
-      sumByType[key].count += wordCount;
-      sumByType[key].reward = wordReward;
+      sumByType[item].count += wordCount;
+      sumByType[item].reward = wordReward;
       sum = sum.add(reward);
     } else {
-      if (!incentives.comment.elements[key]) {
+      const htmlTag = MarkdownItemToHTMLTag[item];
+      if (!htmlTag || !incentives.comment.elements[htmlTag]) {
         continue;
       }
-      const rewardValue = new Decimal(incentives.comment.elements[key]);
+      const rewardValue = new Decimal(incentives.comment.elements[htmlTag]);
       const reward = rewardValue.mul(value.length);
-      sumByType[key].count += value.length;
-      sumByType[key].reward = rewardValue;
+      sumByType[item].count += value.length;
+      sumByType[item].reward = rewardValue;
       sum = sum.add(reward);
     }
   }
 
   return { sum, sumByType };
 };
+
+const MarkdownItemToHTMLTag: Record<MarkdownItem, HTMLItem> = {
+  [MarkdownItem.Text]: HTMLItem.P,
+  [MarkdownItem.Paragraph]: HTMLItem.P,
+  [MarkdownItem.Heading]: HTMLItem.H1,
+  [MarkdownItem.Heading1]: HTMLItem.H1,
+  [MarkdownItem.Heading2]: HTMLItem.H2,
+  [MarkdownItem.Heading3]: HTMLItem.H3,
+  [MarkdownItem.Heading4]: HTMLItem.H4,
+  [MarkdownItem.Heading5]: HTMLItem.H5,
+  [MarkdownItem.Heading6]: HTMLItem.H6,
+  [MarkdownItem.ListItem]: HTMLItem.LI,
+  [MarkdownItem.List]: HTMLItem.UL,
+  [MarkdownItem.Link]: HTMLItem.A,
+  [MarkdownItem.Image]: HTMLItem.IMG,
+  [MarkdownItem.BlockQuote]: HTMLItem.BLOCKQUOTE,
+  [MarkdownItem.Code]: HTMLItem.PRE,
+  [MarkdownItem.Emphasis]: HTMLItem.EM,
+  [MarkdownItem.Strong]: HTMLItem.STRONG,
+  [MarkdownItem.Delete]: HTMLItem.DEL,
+  [MarkdownItem.HTML]: HTMLItem.HTML,
+  [MarkdownItem.InlineCode]: HTMLItem.CODE,
+  [MarkdownItem.LinkReference]: HTMLItem.A,
+  [MarkdownItem.ImageReference]: HTMLItem.IMG,
+  [MarkdownItem.FootnoteReference]: HTMLItem.SUP,
+  [MarkdownItem.FootnoteDefinition]: HTMLItem.P,
+  [MarkdownItem.Table]: HTMLItem.TABLE,
+  [MarkdownItem.TableCell]: HTMLItem.TD,
+  [MarkdownItem.TableRow]: HTMLItem.TR,
+  [MarkdownItem.ThematicBreak]: HTMLItem.HR,
+  [MarkdownItem.Break]: HTMLItem.BR,
+  [MarkdownItem.Root]: HTMLItem.HTML,
+  [MarkdownItem.Definition]: HTMLItem.DL,
+};

src/helpers/comment.ts

@@ -1,45 +1,92 @@
+import { MarkdownItem } from "../types";
+import fromMarkdown from "mdast-util-from-markdown";
+import gfmFromMarkdown from "mdast-util-gfm";
+import gfm from "micromark-extension-gfm";
 import Decimal from "decimal.js";
 import { isEmpty } from "lodash";
-import * as parse5 from "parse5";
 
 type Node = {
-  nodeName: string;
-  tagName?: string;
-  value?: string;
-  childNodes?: Node[];
+  type: MarkdownItem;
+  value: string;
+  depth?: number;
+  children: Node[];
 };
 
-const traverse = (result: Record<string, string[]>, node: Node, itemsToExclude: string[]): Record<string, string[]> => {
-  if (itemsToExclude.includes(node.nodeName)) {
-    return result;
+const traverse = (result: Record<MarkdownItem, string[]>, node: Node, itemsToExclude: string[]): Record<MarkdownItem, string[]> => {
+  if (!result[node.type]) {
+    result[node.type] = [];
   }
 
-  if (!result[node.nodeName]) {
-    result[node.nodeName] = [];
+  if (node.type === MarkdownItem.Heading) {
+    node.type = `heading${node.depth}` as MarkdownItem;
   }
 
-  result[node.nodeName].push(node.value?.trim() ?? "");
+  result[node.type].push(node.value?.trim() ?? "");
+
+  if (itemsToExclude.includes(node.type)) {
+    return result;
+  }
 
-  if (node.childNodes && node.childNodes.length > 0) {
-    node.childNodes.forEach((child) => traverse(result, child, itemsToExclude));
+  if (node.children && node.children.length > 0) {
+    node.children.forEach((child) => traverse(result, child, itemsToExclude));
   }
 
   return result;
 };
 
-export const parseComments = (comments: string[], itemsToExclude: string[]): Record<string, string[]> => {
-  const result: Record<string, string[]> = {};
+export const parseComments = async (comments: string[], itemsToExclude: string[]): Promise<Record<MarkdownItem, string[]>> => {
+  const result: Record<MarkdownItem, string[]> = {
+    [MarkdownItem.Text]: [],
+    [MarkdownItem.Paragraph]: [],
+    [MarkdownItem.Heading]: [],
+    [MarkdownItem.Heading1]: [],
+    [MarkdownItem.Heading2]: [],
+    [MarkdownItem.Heading3]: [],
+    [MarkdownItem.Heading4]: [],
+    [MarkdownItem.Heading5]: [],
+    [MarkdownItem.Heading6]: [],
+    [MarkdownItem.ListItem]: [],
+    [MarkdownItem.List]: [],
+    [MarkdownItem.Link]: [],
+    [MarkdownItem.Image]: [],
+    [MarkdownItem.BlockQuote]: [],
+    [MarkdownItem.Code]: [],
+    [MarkdownItem.Emphasis]: [],
+    [MarkdownItem.Strong]: [],
+    [MarkdownItem.Delete]: [],
+    [MarkdownItem.HTML]: [],
+    [MarkdownItem.InlineCode]: [],
+    [MarkdownItem.LinkReference]: [],
+    [MarkdownItem.ImageReference]: [],
+    [MarkdownItem.FootnoteReference]: [],
+    [MarkdownItem.FootnoteDefinition]: [],
+    [MarkdownItem.Table]: [],
+    [MarkdownItem.TableCell]: [],
+    [MarkdownItem.TableRow]: [],
+    [MarkdownItem.ThematicBreak]: [],
+    [MarkdownItem.Break]: [],
+    [MarkdownItem.Root]: [],
+    [MarkdownItem.Definition]: [],
+  };
 
   for (const comment of comments) {
-    const fragment = parse5.parseFragment(comment);
-    traverse(result, fragment as Node, itemsToExclude);
+    const tree = fromMarkdown(comment, {
+      extensions: [gfm()],
+      mdastExtensions: [gfmFromMarkdown.fromMarkdown],
+    });
+    console.log(`Comment Mdast Tree: ${JSON.stringify(tree, null, 2)}`);
+    traverse(result, tree as Node, itemsToExclude);
   }
 
+  console.log(`Comment Parsed: ${JSON.stringify(result, null, 2)}`);
+
   // remove empty values
-  if (result["#text"]) {
-    result["#text"] = result["#text"].filter((str) => str.length > 0);
+  if (result[MarkdownItem.Text]) {
+    result[MarkdownItem.Text] = result[MarkdownItem.Text].filter((str) => str.length > 0);
   }
 
+  console.log(`Comment Parsed Cleaned: ${JSON.stringify(result, null, 2)}`);
+
   return result;
 };
 
@@ -99,7 +146,7 @@
 
   if (!isEmpty(debug)) {
     const data = Object.entries(debug)
       .filter(([_, value]) => value.count > 0)
       .map(([key, value]) => {
         const element = key === "#text" ? "words" : key;
         const units = value.count;

src/types/config.ts

@@ -13,6 +13,23 @@ export type LabelItem = Static<typeof LabelItemSchema>;
 
 const CommentIncentivesSchema = Type.Object(
   {
+    issue: Type.Object(
+      {
+        assignee: Type.Boolean(),
+        creator: Type.Boolean(),
+        default: Type.Boolean(),
+      },
+      { additionalProperties: false }
+    ),
+    pullRequest: Type.Object(
+      {
+        assignee: Type.Boolean(),
+        creator: Type.Boolean(),
+        reviewer: Type.Boolean(),
+        default: Type.Boolean(),
+      },
+      { additionalProperties: false }
+    ),
     elements: Type.Record(Type.String(), Type.Number()),
     totals: Type.Object(
       {

src/types/html.ts

@@ -0,0 +1,27 @@
+export enum HTMLItem {
+  P = "p",
+  H1 = "h1",
+  H2 = "h2",
+  H3 = "h3",
+  H4 = "h4",
+  H5 = "h5",
+  H6 = "h6",
+  UL = "ul",
+  A = "a",
+  IMG = "img",
+  BLOCKQUOTE = "blockquote",
+  CODE = "code",
+  EM = "em",
+  STRONG = "strong",
+  DEL = "del",
+  HTML = "html",
+  SUP = "sup",
+  LI = "li",
+  TABLE = "table",
+  TR = "tr",
+  TD = "td",
+  HR = "hr",
+  BR = "br",
+  DL = "dl",
+  PRE = "pre",
+}