wassname/rag_search_cite
1
0
Fork 0
mirror of https://github.com/wassname/rag_search_cite.git synced 2026-08-18 12:21:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

tries scraping, failed due to cors

Browse Source
This commit is contained in:
wassname
2024-06-29 20:18:49 +08:00
parent 9208ccf4ca
commit bc0da235b4
4 changed files with 95 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
index.html
+2
View File
@@ -11,6 +11,8 @@
<meta
http-equiv="Content-Security-Policy"
content="*' 'unsafe-inline' 'unsafe-eval';" />
<meta http-equiv="Access-Control-Allow-Origin" content="*" />
</head>
<body>
package-lock.json
Generated
+41
View File
@@ -10,6 +10,7 @@
"dependencies": {
"bootstrap": "^5.3.3",
"diff-match-patch": "^1.0.5",
"htmlarkdown": "^1.0.2",
"markdown-it": "^14.1.0",
"openai": "^4.36.0",
"react": "^18.2.0",
@@ -3107,6 +3108,28 @@
"react-is": "^16.7.0"
}
},
"node_modules/htmlarkdown": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/htmlarkdown/-/htmlarkdown-1.0.2.tgz",
"integrity": "sha512-OwS8myXDtTdxmJUX59Yhe05I+e9/iUnCBM4o1q8J0nI70XK+HZaoc/2+Yeh5iZyEo/duAgku7tUw0HA1BXUfnw==",
"dependencies": {
"escape-string-regexp": "^5.0.0",
"lodash": "^4.17.21",
"predicate-hof": "^2.0.1",
"relateurl": "^0.2.7"
}
},
"node_modules/htmlarkdown/node_modules/escape-string-regexp": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz",
"integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/humanize-ms": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
@@ -3681,6 +3704,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/lodash": {
"version": "4.17.21",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
},
"node_modules/lodash.merge": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
@@ -4145,6 +4173,11 @@
"integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==",
"dev": true
},
"node_modules/predicate-hof": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/predicate-hof/-/predicate-hof-2.0.1.tgz",
"integrity": "sha512-/V8uGCZiqYlvimWeHYQkChkDO9xzO4rf62ovn88XffuDOYlCH9UREAz0HEikEzjVlOKORZY20tuipqNb4Aybjw=="
},
"node_modules/prelude-ls": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -4364,6 +4397,14 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/relateurl": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz",
"integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==",
"engines": {
"node": ">= 0.10"
}
},
"node_modules/resolve": {
"version": "2.0.0-next.5",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-2.0.0-next.5.tgz",
package.json
+1
View File
@@ -12,6 +12,7 @@
"dependencies": {
"bootstrap": "^5.3.3",
"diff-match-patch": "^1.0.5",
"htmlarkdown": "^1.0.2",
"markdown-it": "^14.1.0",
"openai": "^4.36.0",
"react": "^18.2.0",
src/helpers/rag.js
+51
View File
@@ -1,6 +1,7 @@
import sendOpenAIMessage from "./openaiMessage";
import bingWebSearch from "./bingSearch";
import { escapeHtml } from "markdown-it/lib/common/utils.mjs";
import { HTMLarkdown } from 'htmlarkdown'
const defaultModel = 'gpt-3.5-turbo'
@@ -79,6 +80,46 @@ function rank(query, r1, r2, r3, model = defaultModel) {
return r
}
async function scrape_whole_docs(docs) {
// could also use webarchives if needed
const htmlarkdown = new HTMLarkdown()
let promises = docs.map(async (doc) => {
const url = 'https://corsproxy.io/?' + encodeURIComponent(doc.url);
// const url = 'https://archive.today/?run=1&url=' + encodeURIComponent(doc.url)
// https://webcache.googleusercontent.com/search?q=cache:https%3A%2F%2Fmedium.com%2F%40cybersphere%2Ffetch-api-the-ultimate-guide-to-cors-and-no-cors-cbcef88d371e
// https://archive.today/?run=1&url=https%3A%2F%2Fmedium.com%2F%40cybersphere%2Ffetch-api-the-ultimate-guide-to-cors-and-no-cors-cbcef88d371e
// https://archive.today/?run=1&url=https%3A%2F%2Fmedium.com%2F%40cybersphere%2Ffetch-api-the-ultimate-guide-to-cors-and-no-cors-cbcef88d371e
let md
// a = fetch(url, { mode: 'no-cors', redirect: 'follow', referrer: '', window: window, headers: { Referer: url } }); a
try {
let response = await fetch(
url, { mode: 'no-cors', redirect: 'follow', referrer: '', window: window }
)
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
let body = await response.text()
md = htmlarkdown.convert(body)
} catch (e) {
console.log(e)
md = null
}
return {
url: url,
content: md,
source: doc.source + '[web]',
name: doc.name,
query: doc.query
}
})
return await Promise.all(promises)
}
function answer_with_docs(query, rankedDocs, model = defaultModel) {
let history_summary = ''
let messages = [
@@ -124,6 +165,16 @@ function search(q, model = defaultModel, setReply, setStatus, setDocs) {
let answer = rankedDocsP.then((rankedDocs) => {
setDocs(rankedDocs)
setStatus({ status: "Scraping" })
// Step 4: scrape whole docs, for top 10
let top10 = rankedDocs.slice(0, 10)
return scrape_whole_docs(top10).then(fullDocs => {
fullDocs = fullDocs.filter(d => d.content != null)
return [...fullDocs, ...rankedDocs.slice(0, 50)]
})
}).then((rankedDocs) => {
setStatus({ status: "Answering" })
return answer_with_docs(q, rankedDocs, model)
}).catch((err) => {