diff --git a/index.html b/index.html
index 63bc6cf..9f172e5 100644
--- a/index.html
+++ b/index.html
@@ -11,6 +11,8 @@
+
+
diff --git a/package-lock.json b/package-lock.json
index 7a1f347..09de7b1 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,6 +10,7 @@
"dependencies": {
"bootstrap": "^5.3.3",
"diff-match-patch": "^1.0.5",
+ "htmlarkdown": "^1.0.2",
"markdown-it": "^14.1.0",
"openai": "^4.36.0",
"react": "^18.2.0",
@@ -3107,6 +3108,28 @@
"react-is": "^16.7.0"
}
},
+ "node_modules/htmlarkdown": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/htmlarkdown/-/htmlarkdown-1.0.2.tgz",
+ "integrity": "sha512-OwS8myXDtTdxmJUX59Yhe05I+e9/iUnCBM4o1q8J0nI70XK+HZaoc/2+Yeh5iZyEo/duAgku7tUw0HA1BXUfnw==",
+ "dependencies": {
+ "escape-string-regexp": "^5.0.0",
+ "lodash": "^4.17.21",
+ "predicate-hof": "^2.0.1",
+ "relateurl": "^0.2.7"
+ }
+ },
+ "node_modules/htmlarkdown/node_modules/escape-string-regexp": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz",
+ "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==",
+ "engines": {
+ "node": ">=12"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/humanize-ms": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
@@ -3681,6 +3704,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/lodash": {
+ "version": "4.17.21",
+ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+ },
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -4145,6 +4173,11 @@
"integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
"dev": true
},
+ "node_modules/predicate-hof": {
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/predicate-hof/-/predicate-hof-2.0.1.tgz",
+ "integrity": "sha512-/V8uGCZiqYlvimWeHYQkChkDO9xzO4rf62ovn88XffuDOYlCH9UREAz0HEikEzjVlOKORZY20tuipqNb4Aybjw=="
+ },
"node_modules/prelude-ls": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -4364,6 +4397,14 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/relateurl": {
+ "version": "0.2.7",
+ "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz",
+ "integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==",
+ "engines": {
+ "node": ">= 0.10"
+ }
+ },
"node_modules/resolve": {
"version": "2.0.0-next.5",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz",
diff --git a/package.json b/package.json
index 9013730..2a12aa0 100644
--- a/package.json
+++ b/package.json
@@ -12,6 +12,7 @@
"dependencies": {
"bootstrap": "^5.3.3",
"diff-match-patch": "^1.0.5",
+ "htmlarkdown": "^1.0.2",
"markdown-it": "^14.1.0",
"openai": "^4.36.0",
"react": "^18.2.0",
diff --git a/src/helpers/rag.js b/src/helpers/rag.js
index a51d309..d54fe39 100644
--- a/src/helpers/rag.js
+++ b/src/helpers/rag.js
@@ -1,6 +1,7 @@
import sendOpenAIMessage from "./openaiMessage";
import bingWebSearch from "./bingSearch";
import { escapeHtml } from "markdown-it/lib/common/utils.mjs";
+import { HTMLarkdown } from 'htmlarkdown'
const defaultModel = 'gpt-3.5-turbo'
@@ -79,6 +80,46 @@ function rank(query, r1, r2, r3, model = defaultModel) {
return r
}
+async function scrape_whole_docs(docs) {
+ // could also use webarchives if needed
+ const htmlarkdown = new HTMLarkdown()
+ let promises = docs.map(async (doc) => {
+ const url = 'https://corsproxy.io/?' + encodeURIComponent(doc.url);
+ // const url = 'https://archive.today/?run=1&url=' + encodeURIComponent(doc.url)
+ // https://webcache.googleusercontent.com/search?q=cache:https%3A%2F%2Fmedium.com%2F%40cybersphere%2Ffetch-api-the-ultimate-guide-to-cors-and-no-cors-cbcef88d371e
+ // https://archive.today/?run=1&url=https%3A%2F%2Fmedium.com%2F%40cybersphere%2Ffetch-api-the-ultimate-guide-to-cors-and-no-cors-cbcef88d371e
+ // https://archive.today/?run=1&url=https%3A%2F%2Fmedium.com%2F%40cybersphere%2Ffetch-api-the-ultimate-guide-to-cors-and-no-cors-cbcef88d371e
+ let md
+
+ // a = fetch(url, { mode: 'no-cors', redirect: 'follow', referrer: '', window: window, headers: { Referer: url } }); a
+
+ try {
+ let response = await fetch(
+ url, { mode: 'no-cors', redirect: 'follow', referrer: '', window: window }
+ )
+ if (!response.ok) {
+ throw new Error(`HTTP error! status: ${response.status}`);
+ }
+ let body = await response.text()
+
+ md = htmlarkdown.convert(body)
+ } catch (e) {
+ console.log(e)
+ md = null
+ }
+ return {
+ url: url,
+ content: md,
+ source: doc.source + '[web]',
+ name: doc.name,
+ query: doc.query
+ }
+ })
+ return await Promise.all(promises)
+}
+
+
+
function answer_with_docs(query, rankedDocs, model = defaultModel) {
let history_summary = ''
let messages = [
@@ -124,6 +165,16 @@ function search(q, model = defaultModel, setReply, setStatus, setDocs) {
let answer = rankedDocsP.then((rankedDocs) => {
setDocs(rankedDocs)
+ setStatus({ status: "Scraping" })
+ // Step 4: scrape whole docs, for top 10
+ let top10 = rankedDocs.slice(0, 10)
+ return scrape_whole_docs(top10).then(fullDocs => {
+ fullDocs = fullDocs.filter(d => d.content != null)
+ return [...fullDocs, ...rankedDocs.slice(0, 50)]
+
+ })
+
+ }).then((rankedDocs) => {
setStatus({ status: "Answering" })
return answer_with_docs(q, rankedDocs, model)
}).catch((err) => {