chore: Update npm dependencies and website parser

Author: n4ze3m

server/package.json

@@ -74,6 +74,7 @@
     "html-to-text": "^9.0.5",
     "ignore": "^5.2.4",
     "ioredis": "^5.4.1",
+    "jsdom": "^24.1.1",
     "jsonwebtoken": "^9.0.2",
     "langchain": "^0.1.25",
     "mammoth": "^1.6.0",
@@ -96,6 +97,7 @@
   },
   "devDependencies": {
     "@types/bcryptjs": "^2.4.2",
+    "@types/jsdom": "^21.1.7",
     "@types/jsonwebtoken": "^9.0.6",
     "@types/node": "20.4.4",
     "@types/pubsub-js": "^1.8.3",

server/src/utils/website-parser.ts

@@ -1,10 +1,47 @@
-import * as cheerio from "cheerio";
-import TurndownService from "turndown";
-let turndownService = new TurndownService();
+import * as cheerio from "cheerio"
+import TurndownService from "turndown"
+import { Readability, isProbablyReaderable } from "@mozilla/readability"
+import { JSDOM } from "jsdom"
 
 export const websiteParser = (html: string) => {
-  const $ = cheerio.load(html);
-  const mainContent = $('[role="main"]').html() || $("main").html() || $.html();
-  const markdown = turndownService.turndown(mainContent);
-  return markdown;
+  const $ = cheerio.load(html)
+
+  $('script, style, link, svg, [src^="data:image/"]').remove()
+  
+  const jsdom = new JSDOM($.html())
+
+  const doc = jsdom.window.document
+
+  if (isProbablyReaderable(doc)) {
+    const reader = new Readability(doc)
+    const article = reader.parse()
+    const turndownService = new TurndownService({
+      headingStyle: 'atx',
+      codeBlockStyle: 'fenced'
+    })
+    return turndownService.turndown(article.content).trim()
+  }
+
+
+
+  $('*').each((_, element) => {
+    if ('attribs' in element) {
+      const attributes = element.attribs
+      for (const attr in attributes) {
+        if (attr !== 'href' && attr !== 'src') {
+          $(element).removeAttr(attr)
+        }
+      }
+    }
+  })
+
+  const mainContent = $('[role="main"]').html() || $("main").html() || $("body").html() || ""
+
+  const turndownService = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced'
+  })
+  const markdown = turndownService.turndown(mainContent)
+
+  return markdown.trim()
 };