From 7677a59e049d24502a88e02c2a6f62b0d3a44432 Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Fri, 13 Jan 2023 10:36:13 +0100 Subject: [PATCH 01/40] installed sharp for next standalone deployment --- website/package-lock.json | 645 ++++++++++++++++++++++++++++++++++++-- website/package.json | 1 + 2 files changed, 617 insertions(+), 29 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 672ddb05..b215f733 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -41,6 +41,7 @@ "react-dom": "18.2.0", "react-feature-flags": "^1.0.0", "react-icons": "^4.7.1", + "sharp": "^0.31.3", "swr": "^2.0.0", "tailwindcss": "^3.2.4", "use-debounce": "^9.0.2" @@ -14993,7 +14994,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -15083,6 +15083,16 @@ "file-uri-to-path": "1.0.0" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/blob-util": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz", @@ -15439,7 +15449,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -15859,8 +15868,7 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "node_modules/chrome-trace-event": { "version": "1.0.3", @@ -16162,6 +16170,18 @@ "node": ">=0.10.0" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -16175,6 +16195,15 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -16184,6 +16213,22 @@ "color-support": "bin.js" } }, + "node_modules/color/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "node_modules/color2k": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/color2k/-/color2k-2.0.0.tgz", @@ -17524,6 +17569,20 @@ "node": ">=0.10" } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -17556,6 +17615,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -17701,6 +17768,14 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -18123,7 +18198,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "dependencies": { "once": "^1.4.0" } @@ -19255,6 +19329,14 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, "node_modules/expect": { "version": "29.3.1", "resolved": "https://registry.npmjs.org/expect/-/expect-29.3.1.tgz", @@ -20400,6 +20482,11 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fs-extra": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", @@ -20665,6 +20752,11 @@ "assert-plus": "^1.0.0" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "node_modules/github-slugger": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.5.0.tgz", @@ -21454,7 +21546,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -27077,6 +27168,17 @@ "node": ">=6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/min-document": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", @@ -27253,6 +27355,11 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "node_modules/move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -27342,6 +27449,11 @@ "node": ">=0.10.0" } }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -27494,6 +27606,52 @@ "tslib": "^2.0.3" } }, + "node_modules/node-abi": { + "version": "3.31.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.31.0.tgz", + "integrity": "sha512-eSKV6s+APenqVh8ubJyiu/YhZgxQpGP66ntzUb3lY1xB9ukSRaGnx0AIxI+IM+1+IVYC1oWobgG5L3Lt9ARykQ==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abi/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abi/node_modules/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-abi/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/node-addon-api": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.0.0.tgz", + "integrity": "sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA==" + }, "node_modules/node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", @@ -31758,6 +31916,31 @@ "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-3.8.0.tgz", "integrity": "sha512-WuxUnVtlWL1OfZFQFuqvnvs6MiAGk9UNsBostyBOB0Is9wb5uRESevA6rnl/rkksXaGX3GzZhPup5d6Vp1nFew==" }, + "node_modules/prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -32009,7 +32192,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dev": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -32196,6 +32378,33 @@ "webpack": "^4.0.0 || ^5.0.0" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", @@ -32530,7 +32739,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -33159,8 +33367,7 @@ "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "node_modules/safe-regex": { "version": "1.1.0", @@ -33712,6 +33919,58 @@ "node": ">=8" } }, + "node_modules/sharp": { + "version": "0.31.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz", + "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.1", + "node-addon-api": "^5.0.0", + "prebuild-install": "^7.1.1", + "semver": "^7.3.8", + "simple-get": "^4.0.1", + "tar-fs": "^2.1.1", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/sharp/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp/node_modules/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -33750,6 +34009,62 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -34434,7 +34749,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -34443,7 +34757,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, "funding": [ { "type": "github", @@ -34877,6 +35190,32 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/tar/node_modules/chownr": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", @@ -35439,7 +35778,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "dev": true, "dependencies": { "safe-buffer": "^5.0.1" }, @@ -48424,8 +48762,7 @@ "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -48485,6 +48822,16 @@ "file-uri-to-path": "1.0.0" } }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "blob-util": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz", @@ -48774,7 +49121,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "requires": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -49079,8 +49425,7 @@ "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "dev": true + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "chrome-trace-event": { "version": "1.0.3", @@ -49314,6 +49659,30 @@ "object-visit": "^1.0.0" } }, + "color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "requires": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "dependencies": { + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + } + } + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -49327,6 +49696,15 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, + "color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "requires": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -50417,6 +50795,14 @@ "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true }, + "decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "requires": { + "mimic-response": "^3.1.0" + } + }, "dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -50446,6 +50832,11 @@ "which-typed-array": "^1.1.8" } }, + "deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -50552,6 +50943,11 @@ "repeat-string": "^1.5.4" } }, + "detect-libc": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", + "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==" + }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -50898,7 +51294,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "requires": { "once": "^1.4.0" } @@ -51761,6 +52156,11 @@ } } }, + "expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" + }, "expect": { "version": "29.3.1", "resolved": "https://registry.npmjs.org/expect/-/expect-29.3.1.tgz", @@ -52669,6 +53069,11 @@ } } }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "fs-extra": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", @@ -52875,6 +53280,11 @@ "assert-plus": "^1.0.0" } }, + "github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "github-slugger": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.5.0.tgz", @@ -53468,8 +53878,7 @@ "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, "iferr": { "version": "0.1.5", @@ -57731,6 +58140,11 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" + }, "min-document": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", @@ -57872,6 +58286,11 @@ "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", "dev": true }, + "mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -57948,6 +58367,11 @@ "to-regex": "^3.0.1" } }, + "napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -58046,6 +58470,42 @@ "tslib": "^2.0.3" } }, + "node-abi": { + "version": "3.31.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.31.0.tgz", + "integrity": "sha512-eSKV6s+APenqVh8ubJyiu/YhZgxQpGP66ntzUb3lY1xB9ukSRaGnx0AIxI+IM+1+IVYC1oWobgG5L3Lt9ARykQ==", + "requires": { + "semver": "^7.3.5" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, + "node-addon-api": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.0.0.tgz", + "integrity": "sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA==" + }, "node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", @@ -61022,6 +61482,25 @@ } } }, + "prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "requires": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + } + }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -61214,7 +61693,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "dev": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -61346,6 +61824,29 @@ "schema-utils": "^3.0.0" } }, + "rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "requires": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "dependencies": { + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" + } + } + }, "react": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", @@ -61580,7 +62081,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dev": true, "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -62062,8 +62562,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "safe-regex": { "version": "1.1.0", @@ -62524,6 +63023,44 @@ "kind-of": "^6.0.2" } }, + "sharp": { + "version": "0.31.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.3.tgz", + "integrity": "sha512-XcR4+FCLBFKw1bdB+GEhnUNXNXvnt0tDo4WsBsraKymuo/IAuPuCBVAL2wIkUw2r/dwFW5Q5+g66Kwl2dgDFVg==", + "requires": { + "color": "^4.2.3", + "detect-libc": "^2.0.1", + "node-addon-api": "^5.0.0", + "prebuild-install": "^7.1.1", + "semver": "^7.3.8", + "simple-get": "^4.0.1", + "tar-fs": "^2.1.1", + "tunnel-agent": "^0.6.0" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } + } + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -62553,6 +63090,36 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" + }, + "simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "requires": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "requires": { + "is-arrayish": "^0.3.1" + }, + "dependencies": { + "is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + } + } + }, "sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -63139,7 +63706,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "requires": { "safe-buffer": "~5.2.0" }, @@ -63147,8 +63713,7 @@ "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" } } }, @@ -63474,6 +64039,29 @@ } } }, + "tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "requires": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + }, "telejson": { "version": "6.0.8", "resolved": "https://registry.npmjs.org/telejson/-/telejson-6.0.8.tgz", @@ -63885,7 +64473,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "dev": true, "requires": { "safe-buffer": "^5.0.1" } diff --git a/website/package.json b/website/package.json index 917c819b..dd0ca93a 100644 --- a/website/package.json +++ b/website/package.json @@ -58,6 +58,7 @@ "react-dom": "18.2.0", "react-feature-flags": "^1.0.0", "react-icons": "^4.7.1", + "sharp": "^0.31.3", "swr": "^2.0.0", "tailwindcss": "^3.2.4", "use-debounce": "^9.0.2" From 9451aff6ccce98f64e706d511664f1d29f9fc7f2 Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Sat, 14 Jan 2023 03:49:19 +0000 Subject: [PATCH 02/40] [fix] @ekurtulus major logic bug in summarization --- .../custom_datasets/summarization.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 model/supervised_finetuning/custom_datasets/summarization.py diff --git a/model/supervised_finetuning/custom_datasets/summarization.py b/model/supervised_finetuning/custom_datasets/summarization.py new file mode 100644 index 00000000..76147928 --- /dev/null +++ b/model/supervised_finetuning/custom_datasets/summarization.py @@ -0,0 +1,57 @@ +import random +from datasets import load_dataset +from torch.utils.data import Dataset + +SUMMARIZATION_SPECIAL_TOKENS = {"Text": "", "Summary": ["TL;DR:", "Summarize this", "Give me the summary"]} + + + +summarization_config_mapping = { + "cnn_dailymail": ("3.0.0",), + "samsum": (), + "xsum": (), + "multi_news": (), + "scitldr": ("AIC",), + "billsum": (), + "reddit": (), +} + +summarization_name_mapping = { + "cnn_dailymail": ("article", "highlights"), + "samsum": ("dialogue", "summary"), + "xsum": ("document", "summary"), + "multi_news": ("document", "summary"), + "scitldr": ("source", "target"), + "billsum": ("text", "summary"), + "reddit": ("content", "summary"), +} + + +def index_summary_default(text, summary): + return text.replace('\n\n', '\n'), summary + + +def index_summary_merge(text, summary): + return " ".join(text), " ".join(summary) + + +class SummarizationDataset(Dataset): + def __init__(self, dataset, cache_dir, split): + self.dataset = load_dataset(dataset, *summarization_config_mapping[dataset], cache_dir=cache_dir, split=split) + self.text_column, self.summary_column = summarization_name_mapping[dataset] + self.preprocess_fn = index_summary_merge if dataset == "scitdlr" else index_summary_default + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + data = self.dataset[idx] + text, summary = data[self.text_column], data[self.summary_column] + text, summary = self.preprocess_fn(text, summary) + prompt = random.choice(SUMMARIZATION_SPECIAL_TOKENS["Summary"]) + + return ( + "".join([SUMMARIZATION_SPECIAL_TOKENS["Text"], ' '.join(text.split(' ')[:256]), prompt]), + summary + ) + From 3966024871697bd47de52f9f4ec846216722f669 Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Sat, 14 Jan 2023 05:49:22 +0000 Subject: [PATCH 03/40] [fix] Fix summarizer bug and QA typo issue --- model/supervised_finetuning/README.md | 2 + .../supervised_finetuning/configs/config.yaml | 14 +- .../custom_datasets/__init__.py | 152 ++------------- .../custom_datasets/dialogue_collator.py | 5 +- .../custom_datasets/prompt_dialogue.py | 21 +-- .../custom_datasets/qa_datasets.py | 174 ++++++++++++++++++ .../custom_datasets/summarization.py | 22 ++- .../tests/test_datasets.py | 41 +++++ .../supervised_finetuning/tests/test_utils.py | 9 + 9 files changed, 267 insertions(+), 173 deletions(-) create mode 100644 model/supervised_finetuning/custom_datasets/qa_datasets.py create mode 100644 model/supervised_finetuning/tests/test_datasets.py create mode 100644 model/supervised_finetuning/tests/test_utils.py diff --git a/model/supervised_finetuning/README.md b/model/supervised_finetuning/README.md index bd202397..822121d8 100644 --- a/model/supervised_finetuning/README.md +++ b/model/supervised_finetuning/README.md @@ -58,6 +58,8 @@ the end to trigger deepspeed python trainer.py --configs defaults your-model-name --deepspeed ``` +## Dataset choices + ## Results Experimental results in wandb diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index bd35f168..b0684bda 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -6,7 +6,7 @@ defaults: per_device_eval_batch_size: 2 weight_decay: 0.00 warmup_steps: 600 - eval_steps: 100 + eval_steps: 500 save_steps: 500 max_length: 512 num_train_epochs: 3 @@ -18,7 +18,7 @@ defaults: datasets: - webgpt - prompt_dialogue - cache_dir: ~/.cache + cache_dir: .cache loss_fn: CrossEntropyLoss eval_size: log_dir: "base" @@ -48,14 +48,14 @@ gpt-jt: per_device_eval_batch_size: 4 codegen: - learning_rate: 2e-6 + learning_rate: 8e-6 model_name: Salesforce/codegen-2B-multi weight_decay: 0.01 - max_length: 812 - warmup_steps: 600 + max_length: 512 + warmup_steps: 1000 gradient_checkpointing: false - gradient_accumulation_steps: 5 - per_device_train_batch_size: 4 + gradient_accumulation_steps: 10 + per_device_train_batch_size: 2 per_device_eval_batch_size: 4 debug: diff --git a/model/supervised_finetuning/custom_datasets/__init__.py b/model/supervised_finetuning/custom_datasets/__init__.py index c0cd424b..6b7906f7 100644 --- a/model/supervised_finetuning/custom_datasets/__init__.py +++ b/model/supervised_finetuning/custom_datasets/__init__.py @@ -1,136 +1,11 @@ -import numpy as np -from datasets import load_dataset +from custom_datasets.prompt_dialogue import PromptGeneratedDataset +from custom_datasets.qa_datasets import SODA, JokeExplaination, QADataset, WebGPT +from custom_datasets.summarization import SummarizationDataset from sklearn.model_selection import train_test_split -from torch.utils.data import Dataset, Subset +from torch.utils.data import Subset -from .prompt_dialogue import PromptGeneratedDataset - -QA_SPECIAL_TOKENS = {"Question": "", "Answer": ""} -SUMMARIZATION_SPECIAL_TOKENS = {"Text": "", "Summary": "TL;DR:"} - -summarization_name_mapping = { - "cnn_dailymail": ("article", "highlights"), - "samsum": ("dialogue", "summary"), - "xsum": ("document", "summary"), - "multi_news": ("document", "summary"), - "scitldr": ("source", "target"), - "billsum": ("text", "summary"), - "reddit": ("content", "summary"), -} -summarization_config_mapping = { - "cnn_dailymail": ("3.0.0",), - "samsum": (), - "xsum": (), - "multi_news": (), - "scitldr": ("AIC",), - "billsum": (), - "reddit": (), -} - -QA_DATASETS = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_noconext"] -SUMMARIZATION_DATASETS = ["xsum", "cnn_dailymail", "samsum", "multi_news"] - - -def index_squad_v2(example): - return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0] - - -def index_trivia_qa_nocontext(example): - # dummy return one randomly - return example["question"], example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))] - - -def index_trivia_qa_context(example): - question = example["question"] - title = example["title"][np.random.randint(len(example["title"]))] - context = example["search_context"][np.random.randint(len(example["search_context"]))] - answer = example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))] - - return title + ". " + context + " " + question, answer - - -def index_adversarial_qa(example): - return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0] - - -class QADataset(Dataset): - def __init__(self, dataset, cache_dir, split): - if dataset == "squad_v2": - self.index_fn = index_squad_v2 - self.dataset = load_dataset("squad_v2", cache_dir=cache_dir, split=split) - elif dataset == "trivia_qa_nocontext": - self.index_fn = index_trivia_qa_nocontext - self.dataset = load_dataset("trivia_qa", "rc.nocontext") - elif dataset == "trivia_qa_context": - self.index_fn = index_trivia_qa_context - self.dataset = load_dataset("trivia_qa", "rc") - elif dataset == "adversarial_qa": - self.index_fn = index_adversarial_qa - self.dataset = load_dataset("adversarial_qa", "adversarialQA") - else: - raise ValueError("Unknown dataset : " + dataset) - - def __len__(self): - return len(self.dataset) - - def __getitem__(self, idx): - data = self.dataset[idx] - return self.index_fn(data) - - -def index_summary_default(text, summary): - return text, summary - - -def index_summary_merge(text, summary): - return " ".join(text), " ".join(summary) - - -class SummarizationDataset(Dataset): - def __init__(self, dataset, cache_dir, split): - self.dataset = load_dataset(dataset, *summarization_config_mapping[dataset], cache_dir=cache_dir, split=split) - self.summary_column, self.text_column = summarization_name_mapping[dataset] - self.preprocess_fn = index_summary_merge if dataset == "scitdlr" else index_summary_merge - - def __len__(self): - return len(self.dataset) - - def __getitem__(self, idx): - data = self.dataset[idx] - text, summary = data[self.text_column], data[self.summary_column] - text, summary = self.preprocess_fn(text, summary) - - return "".join( - SUMMARIZATION_SPECIAL_TOKENS["Text"], text, " ", SUMMARIZATION_SPECIAL_TOKENS["Summary"], summary - ) - - -class WebGPT(Dataset): - def __init__(self) -> None: - super().__init__() - - dataset = load_dataset("openai/webgpt_comparisons") - questions = {} - # using prompt as our index will allows us - # to add additional generated prompt later - self.index2question = {} - for row in dataset["train"]: - question = row["question"]["full_text"] - if question not in self.index2question: - self.index2question[len(self.index2question)] = question - - # only keep the best answer - questions[question] = row["answer_0" if row["score_0"] > row["score_1"] else "answer_1"] - - self.questions = questions - - def __len__(self): - return len(self.index2question) - - def __getitem__(self, index): - question = self.index2question[index] - answer = self.questions[question] - return [question, answer] +QA_DATASETS = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_nocontext"] +SUMMARIZATION_DATASETS = ["xsum", "cnn_dailymail", "samsum", "multi_news", "scitldr", "billsum", "reddit"] def train_val_dataset(dataset, val_split=0.2): @@ -143,19 +18,24 @@ def train_val_dataset(dataset, val_split=0.2): def get_one_dataset(conf, dataset_name): dataset_name = dataset_name.lower() - if dataset_name in ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_noconext"]: + if dataset_name in QA_DATASETS: train = QADataset(dataset_name, conf.cache_dir, "train") eval = QADataset(dataset_name, conf.cache_dir, "validation") - elif dataset_name in ["xsum", "cnn_dailymail", "samsum", "multi_news", "scitldr", "billsum", "reddit"]: + elif dataset_name in SUMMARIZATION_DATASETS: train = SummarizationDataset(dataset_name, conf.cache_dir, "train") - eval = SummarizationDataset(dataset_name, conf.cache_dir, "validation") - + eval = SummarizationDataset(dataset_name, conf.cache_dir, "validation" if dataset_name != "billsum" else "test") elif dataset_name == "webgpt": dataset = WebGPT() train, eval = train_val_dataset(dataset, val_split=0.2) elif dataset_name == "prompt_dialogue": - dataset = PromptGeneratedDataset() + dataset = PromptGeneratedDataset(conf.cache_dir) + train, eval = train_val_dataset(dataset, val_split=0.2) + elif dataset_name == "soda": + dataset = SODA(conf.cache_dir) + train, eval = train_val_dataset(dataset, val_split=0.1) + elif dataset_name == "joke": + dataset = JokeExplaination(conf.cache_dir) train, eval = train_val_dataset(dataset, val_split=0.2) else: raise ValueError(f"Unknown dataset {dataset_name}") diff --git a/model/supervised_finetuning/custom_datasets/dialogue_collator.py b/model/supervised_finetuning/custom_datasets/dialogue_collator.py index 479931f6..2efe160f 100644 --- a/model/supervised_finetuning/custom_datasets/dialogue_collator.py +++ b/model/supervised_finetuning/custom_datasets/dialogue_collator.py @@ -3,11 +3,10 @@ from typing import Optional, Union import numpy as np import torch +from custom_datasets.qa_datasets import QA_SPECIAL_TOKENS from torch.nn import functional as F from transformers.tokenization_utils_base import PaddingStrategy, PreTrainedTokenizerBase -from . import QA_SPECIAL_TOKENS - @dataclass class DialogueDataCollator: @@ -35,7 +34,7 @@ class DialogueDataCollator: # Add a way for the model to terminate generation # When we predict the start of a new expected question, we want to be able to stop generation - messages.append(QA_SPECIAL_TOKENS["Question"]) + messages.append(self.tokenizer.eos_token) flatten_message = self.tokenizer( "".join(messages), diff --git a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py index 17911141..372ea27f 100644 --- a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py +++ b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py @@ -16,10 +16,10 @@ class PromptGeneratedDataset(Dataset): url = "https://github.com/Rallio67/language-model-agents/raw/main/chat_dialogue_v2_c.txt" - def __init__(self) -> None: + def __init__(self, cache_dir) -> None: super().__init__() - os.makedirs("datasets", exist_ok=True) - chat_dialogue = os.path.join("datasets", "chat_dialogue_v2_c.txt") + os.makedirs(cache_dir, exist_ok=True) + chat_dialogue = os.path.join(cache_dir, "chat_dialogue_v2_c.txt") if not os.path.exists(chat_dialogue): with urlopen(self.url) as file: content = file.read().decode() @@ -49,18 +49,3 @@ class PromptGeneratedDataset(Dataset): def __getitem__(self, index): question, answer = self.pairs[index] return question, answer - - -if __name__ == "__main__": - from torch.utils.data import DataLoader - from transformers import AutoTokenizer - - from .dialogue_collator import DialogueDataCollator - - tokenizer = AutoTokenizer.from_pretrained("Salesforce/codegen-2B-multi") - tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"}) - dataset = PromptGeneratedDataset() - collate_fn = DialogueDataCollator(tokenizer, padding=True, max_length=128) - dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=5) - for batch in dataloader: - print(batch["input_ids"].shape) diff --git a/model/supervised_finetuning/custom_datasets/qa_datasets.py b/model/supervised_finetuning/custom_datasets/qa_datasets.py new file mode 100644 index 00000000..d0fcffda --- /dev/null +++ b/model/supervised_finetuning/custom_datasets/qa_datasets.py @@ -0,0 +1,174 @@ +import json +import os +from urllib.request import urlopen + +import numpy as np +from datasets import load_dataset +from torch.utils.data import Dataset + +QA_SPECIAL_TOKENS = {"Question": "", "Answer": "", "StartPrefix": "", "EndPrefix": ""} + + +def index_squad_v2(example): + if len(example["answers"]["text"]): + answer = example["answers"]["text"][0] + else: + answer = "I do not have answer for that" + return example["context"] + " " + example["question"], answer + + +def index_trivia_qa_nocontext(example): + # dummy return one randomly + return example["question"], example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))] + + +def index_trivia_qa_context(example): + question = example["question"] + if len(example["search_results"]["search_context"]): + context = example["search_results"]["search_context"][ + np.random.randint(len(example["search_results"]["search_context"])) + ] + else: + context = "" + answer = example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))] + + return context + " " + question, answer + + +def index_adversarial_qa(example): + return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0] + + +class QADataset(Dataset): + def __init__(self, dataset, cache_dir, split): + if dataset == "squad_v2": + self.index_fn = index_squad_v2 + self.dataset = load_dataset("squad_v2", cache_dir=cache_dir, split=split) + elif dataset == "trivia_qa_nocontext": + self.index_fn = index_trivia_qa_nocontext + self.dataset = load_dataset("trivia_qa", "rc.nocontext", split=split) + elif dataset == "trivia_qa_context": + self.index_fn = index_trivia_qa_context + self.dataset = load_dataset("trivia_qa", "rc", split=split) + elif dataset == "adversarial_qa": + self.index_fn = index_adversarial_qa + self.dataset = load_dataset("adversarial_qa", "adversarialQA", split=split) + else: + raise ValueError("Unknown dataset : " + dataset) + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, idx): + data = self.dataset[idx] + return self.index_fn(data) + + +class WebGPT(Dataset): + def __init__(self) -> None: + super().__init__() + + dataset = load_dataset("openai/webgpt_comparisons") + questions = {} + # using prompt as our index will allows us + # to add additional generated prompt later + self.index2question = {} + for row in dataset["train"]: + question = row["question"]["full_text"] + if question not in self.index2question: + self.index2question[len(self.index2question)] = question + + # only keep the best answer + questions[question] = row["answer_0" if row["score_0"] > row["score_1"] else "answer_1"] + + self.questions = questions + + def __len__(self): + return len(self.index2question) + + def __getitem__(self, index): + question = self.index2question[index] + answer = self.questions[question] + return [question, answer] + + +class SODA(Dataset): + def process_soda_convo(self, data): + pairs = [] + play_as = data["speakers"][1] + prefix = "{}. {}".format(data["narrative"], "your name {}".format(play_as)) + question, answer = "", "" + prefix, postfix = "", "" + previous_chat = [] + + for idx, convo in enumerate(data["dialogue"]): + if idx % 2 == 0: + question = convo + prefix = data["speakers"][idx] + else: + answer = convo + postfix = data["speakers"][idx] + if len(question) and len(answer) and prefix != postfix and postfix == play_as: + history = "".join(["{}{}".format(*p) for p in previous_chat]) + if len(history): + history += "" + pairs.append((prefix + history + question, answer)) + previous_chat.append((question, answer)) + return pairs + + def __init__(self, cache_dir, max_sample_size=10000, input_max_length=1024) -> None: + super().__init__() + + self.pairs = [] + dataset = load_dataset("allenai/soda", cache_dir=cache_dir)["train"] + for data in dataset: + data_pair = self.process_soda_convo(data) + for (prompt, answer) in data_pair: + if len(prompt) < input_max_length: + self.pairs.append((prompt, answer)) + + if len(self.pairs) > max_sample_size: + break + + def __len__(self): + return len(self.pairs) + + def __getitem__(self, index): + question, answer = self.pairs[index] + return question, answer + + +class JokeExplaination(Dataset): + """ """ + + url = "https://gist.github.com/theblackcat102/42b697e24a13fdb499e20edfbf618361/raw/1834dca207898c15f93b809d1195f6f6e47c9e1e/joke_explained.jsonl" + + def __init__(self, cache_dir) -> None: + super().__init__() + os.makedirs(cache_dir, exist_ok=True) + joke_explain_filename = os.path.join(cache_dir, "joke_explaination.jsonl") + if not os.path.exists(joke_explain_filename): + with urlopen(self.url) as file: + content = file.read().decode() + with open(joke_explain_filename, "w") as fout: + fout.write(content) + + question = "" + answer = "" + self.pairs = [] + with open(joke_explain_filename, "r") as f: + for line in f: + data = json.loads(line) + joke = data["joke"] + explanation = data["explaination"] + self.pairs.append((joke, explanation)) + + if len(question) > 0 and len(answer) > 0: + self.pairs.append((question, answer)) + + def __len__(self): + return len(self.pairs) + + def __getitem__(self, index): + question, answer = self.pairs[index] + return question, answer diff --git a/model/supervised_finetuning/custom_datasets/summarization.py b/model/supervised_finetuning/custom_datasets/summarization.py index 76147928..41fa6dc0 100644 --- a/model/supervised_finetuning/custom_datasets/summarization.py +++ b/model/supervised_finetuning/custom_datasets/summarization.py @@ -1,10 +1,14 @@ import random + from datasets import load_dataset from torch.utils.data import Dataset SUMMARIZATION_SPECIAL_TOKENS = {"Text": "", "Summary": ["TL;DR:", "Summarize this", "Give me the summary"]} - +SUMMARY_SPECIAL_PROMPT = { + "multi_news": ["Summarize in bullet points", "Generate summary in list of points"], + "xsum": ["Give me summary in one sentence", "Short TLDR", "Give me a concise summary"], +} summarization_config_mapping = { "cnn_dailymail": ("3.0.0",), @@ -28,7 +32,7 @@ summarization_name_mapping = { def index_summary_default(text, summary): - return text.replace('\n\n', '\n'), summary + return text.replace("\n\n", "\n"), summary def index_summary_merge(text, summary): @@ -37,9 +41,10 @@ def index_summary_merge(text, summary): class SummarizationDataset(Dataset): def __init__(self, dataset, cache_dir, split): + self.name = dataset self.dataset = load_dataset(dataset, *summarization_config_mapping[dataset], cache_dir=cache_dir, split=split) self.text_column, self.summary_column = summarization_name_mapping[dataset] - self.preprocess_fn = index_summary_merge if dataset == "scitdlr" else index_summary_default + self.preprocess_fn = index_summary_merge if dataset == "scitldr" else index_summary_default def __len__(self): return len(self.dataset) @@ -48,10 +53,9 @@ class SummarizationDataset(Dataset): data = self.dataset[idx] text, summary = data[self.text_column], data[self.summary_column] text, summary = self.preprocess_fn(text, summary) - prompt = random.choice(SUMMARIZATION_SPECIAL_TOKENS["Summary"]) - - return ( - "".join([SUMMARIZATION_SPECIAL_TOKENS["Text"], ' '.join(text.split(' ')[:256]), prompt]), - summary - ) + if self.name in SUMMARY_SPECIAL_PROMPT: + prompt = random.choice(SUMMARIZATION_SPECIAL_TOKENS["Summary"]) + else: + prompt = random.choice(SUMMARIZATION_SPECIAL_TOKENS["Summary"]) + return ("".join([SUMMARIZATION_SPECIAL_TOKENS["Text"], " ".join(text.split(" ")[:256]), prompt]), summary) diff --git a/model/supervised_finetuning/tests/test_datasets.py b/model/supervised_finetuning/tests/test_datasets.py new file mode 100644 index 00000000..721fdfd3 --- /dev/null +++ b/model/supervised_finetuning/tests/test_datasets.py @@ -0,0 +1,41 @@ +from argparse import Namespace + +from custom_datasets import get_one_dataset +from custom_datasets.dialogue_collator import DialogueDataCollator + + +def test_all_datasets(): + qa_base = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_nocontext"] + summarize_base = ["scitldr", "xsum", "cnn_dailymail", "samsum", "multi_news", "billsum"] + others = ["prompt_dialogue", "webgpt", "soda", "joke"] + + config = Namespace(cache_dir=".cache") + for dataset_name in others + qa_base + summarize_base: + print(dataset_name) + train, eval = get_one_dataset(config, dataset_name) + # sanity check + for idx in range(min(len(train), 1000)): + train[idx] + for idx in range(min(len(eval), 1000)): + eval[idx] + + +def test_collate_fn(): + from torch.utils.data import DataLoader + from utils import get_tokenizer + + config = Namespace(cache_dir=".cache", model_name="Salesforce/codegen-2B-multi") + tokenizer = get_tokenizer(config) + collate_fn = DialogueDataCollator(tokenizer, max_length=512) + train, eval = get_one_dataset(config, "multi_news") + dataloader = DataLoader(train, collate_fn=collate_fn, batch_size=128) + for batch in dataloader: + # print(batch.keys()) + # print(tokenizer.decode(batch['input_ids'][0])) + # print('-----') + # print(tokenizer.decode(batch['targets'][0][batch['label_masks'][0]])) + assert batch["targets"].shape[1] <= 512 + + +if __name__ == "__main__": + test_all_datasets() diff --git a/model/supervised_finetuning/tests/test_utils.py b/model/supervised_finetuning/tests/test_utils.py new file mode 100644 index 00000000..ad40e534 --- /dev/null +++ b/model/supervised_finetuning/tests/test_utils.py @@ -0,0 +1,9 @@ +from argparse import Namespace + +from utils import get_tokenizer + + +def test_tokenizer(): + get_tokenizer(Namespace(model_name="Salesforce/codegen-2B-multi", cache_dir=".cache")) + get_tokenizer(Namespace(model_name="facebook/galactica-1.3b", cache_dir=".cache")) + get_tokenizer(Namespace(model_name="", cache_dir=".cache")) From 154611109465e3ca1c22efd08ce8bb15e52f519d Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Sat, 14 Jan 2023 06:24:47 +0000 Subject: [PATCH 04/40] [feature] added GSM8k and code refactoring --- .../supervised_finetuning/configs/config.yaml | 12 +++++++++ .../custom_datasets/__init__.py | 10 ++++--- .../custom_datasets/qa_datasets.py | 16 ++++++++--- .../tests/test_datasets.py | 27 ++++++++++++++----- model/supervised_finetuning/utils.py | 3 ++- 5 files changed, 53 insertions(+), 15 deletions(-) diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index b0684bda..0440201a 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -18,6 +18,18 @@ defaults: datasets: - webgpt - prompt_dialogue + - squad_v2 + - adversarial_qa + - trivia_qa_nocontext + - xsum + - cnn_dailymail + - prompt_dialogue + - multi_news + - scitldr + - soda + - joke + - + - joke cache_dir: .cache loss_fn: CrossEntropyLoss eval_size: diff --git a/model/supervised_finetuning/custom_datasets/__init__.py b/model/supervised_finetuning/custom_datasets/__init__.py index 6b7906f7..e293af3d 100644 --- a/model/supervised_finetuning/custom_datasets/__init__.py +++ b/model/supervised_finetuning/custom_datasets/__init__.py @@ -4,8 +4,8 @@ from custom_datasets.summarization import SummarizationDataset from sklearn.model_selection import train_test_split from torch.utils.data import Subset -QA_DATASETS = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_nocontext"] -SUMMARIZATION_DATASETS = ["xsum", "cnn_dailymail", "samsum", "multi_news", "scitldr", "billsum", "reddit"] +QA_DATASETS = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_nocontext", "gsm8k"] +SUMMARIZATION_DATASETS = ["xsum", "cnn_dailymail", "samsum", "multi_news", "scitldr", "billsum"] def train_val_dataset(dataset, val_split=0.2): @@ -20,11 +20,13 @@ def get_one_dataset(conf, dataset_name): if dataset_name in QA_DATASETS: train = QADataset(dataset_name, conf.cache_dir, "train") - eval = QADataset(dataset_name, conf.cache_dir, "validation") + val_name = "validation" if dataset_name not in ["gsm8k"] else "test" + eval = QADataset(dataset_name, conf.cache_dir, val_name) elif dataset_name in SUMMARIZATION_DATASETS: train = SummarizationDataset(dataset_name, conf.cache_dir, "train") - eval = SummarizationDataset(dataset_name, conf.cache_dir, "validation" if dataset_name != "billsum" else "test") + val_name = "validation" if dataset_name not in ["billsum"] else "test" + eval = SummarizationDataset(dataset_name, conf.cache_dir, val_name) elif dataset_name == "webgpt": dataset = WebGPT() train, eval = train_val_dataset(dataset, val_split=0.2) diff --git a/model/supervised_finetuning/custom_datasets/qa_datasets.py b/model/supervised_finetuning/custom_datasets/qa_datasets.py index d0fcffda..eed9c644 100644 --- a/model/supervised_finetuning/custom_datasets/qa_datasets.py +++ b/model/supervised_finetuning/custom_datasets/qa_datasets.py @@ -39,6 +39,10 @@ def index_adversarial_qa(example): return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0] +def index_gsm8k(example): + return example["question"], example["answer"] + + class QADataset(Dataset): def __init__(self, dataset, cache_dir, split): if dataset == "squad_v2": @@ -46,13 +50,19 @@ class QADataset(Dataset): self.dataset = load_dataset("squad_v2", cache_dir=cache_dir, split=split) elif dataset == "trivia_qa_nocontext": self.index_fn = index_trivia_qa_nocontext - self.dataset = load_dataset("trivia_qa", "rc.nocontext", split=split) + self.dataset = load_dataset("trivia_qa", "rc.nocontext", split=split, cache_dir=cache_dir) elif dataset == "trivia_qa_context": self.index_fn = index_trivia_qa_context - self.dataset = load_dataset("trivia_qa", "rc", split=split) + self.dataset = load_dataset("trivia_qa", "rc", split=split, cache_dir=cache_dir) elif dataset == "adversarial_qa": self.index_fn = index_adversarial_qa - self.dataset = load_dataset("adversarial_qa", "adversarialQA", split=split) + self.dataset = load_dataset("adversarial_qa", "adversarialQA", split=split, cache_dir=cache_dir) + elif dataset == "gsm8k": + self.index_fn = index_gsm8k + self.dataset = load_dataset("gsm8k", "main", split=split, cache_dir=cache_dir) + elif dataset == "adversarial_qa": + self.index_fn = index_adversarial_qa + self.dataset = load_dataset("adversarial_qa", "adversarialQA", split=split, cache_dir=cache_dir) else: raise ValueError("Unknown dataset : " + dataset) diff --git a/model/supervised_finetuning/tests/test_datasets.py b/model/supervised_finetuning/tests/test_datasets.py index 721fdfd3..c9363303 100644 --- a/model/supervised_finetuning/tests/test_datasets.py +++ b/model/supervised_finetuning/tests/test_datasets.py @@ -1,12 +1,12 @@ from argparse import Namespace -from custom_datasets import get_one_dataset +from custom_datasets import QA_DATASETS, SUMMARIZATION_DATASETS, get_one_dataset from custom_datasets.dialogue_collator import DialogueDataCollator def test_all_datasets(): - qa_base = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_nocontext"] - summarize_base = ["scitldr", "xsum", "cnn_dailymail", "samsum", "multi_news", "billsum"] + qa_base = QA_DATASETS + summarize_base = SUMMARIZATION_DATASETS others = ["prompt_dialogue", "webgpt", "soda", "joke"] config = Namespace(cache_dir=".cache") @@ -21,21 +21,34 @@ def test_all_datasets(): def test_collate_fn(): - from torch.utils.data import DataLoader + from torch.utils.data import ConcatDataset, DataLoader from utils import get_tokenizer config = Namespace(cache_dir=".cache", model_name="Salesforce/codegen-2B-multi") tokenizer = get_tokenizer(config) collate_fn = DialogueDataCollator(tokenizer, max_length=512) - train, eval = get_one_dataset(config, "multi_news") - dataloader = DataLoader(train, collate_fn=collate_fn, batch_size=128) + qa_base = QA_DATASETS + summarize_base = SUMMARIZATION_DATASETS + others = ["prompt_dialogue", "webgpt", "soda", "joke", "gsm8k"] + + trains, evals = [], [] + for dataset_name in others + qa_base + summarize_base: + print(dataset_name) + train, eval = get_one_dataset(config, dataset_name) + trains.append(train) + evals.append(eval) + + dataloader = DataLoader(ConcatDataset(trains), collate_fn=collate_fn, batch_size=128) for batch in dataloader: # print(batch.keys()) # print(tokenizer.decode(batch['input_ids'][0])) # print('-----') # print(tokenizer.decode(batch['targets'][0][batch['label_masks'][0]])) assert batch["targets"].shape[1] <= 512 + dataloader = DataLoader(ConcatDataset(evals), collate_fn=collate_fn, batch_size=128) + for batch in dataloader: + assert batch["targets"].shape[1] <= 512 if __name__ == "__main__": - test_all_datasets() + test_collate_fn() diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index f598dde1..85fb86db 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -6,8 +6,9 @@ import nltk import numpy as np import transformers import yaml -from custom_datasets import QA_DATASETS, QA_SPECIAL_TOKENS, SUMMARIZATION_DATASETS, get_one_dataset +from custom_datasets import QA_DATASETS, SUMMARIZATION_DATASETS, get_one_dataset from custom_datasets.dialogue_collator import DialogueDataCollator +from custom_datasets.qa_datasets import QA_SPECIAL_TOKENS from losses import CrossEntropyLoss, PolyLoss from models import freeze_top_n_layers, get_specific_model from sklearn.model_selection import train_test_split From 6f6c590e5798b6aa0c37df5be1655bfd19b3eeca Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Sat, 14 Jan 2023 06:47:21 +0000 Subject: [PATCH 05/40] [fix] Disable task specific evaluation --- .../custom_datasets/summarization.py | 1 + model/supervised_finetuning/utils.py | 57 ++++++++++--------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/model/supervised_finetuning/custom_datasets/summarization.py b/model/supervised_finetuning/custom_datasets/summarization.py index 41fa6dc0..69e4b51d 100644 --- a/model/supervised_finetuning/custom_datasets/summarization.py +++ b/model/supervised_finetuning/custom_datasets/summarization.py @@ -8,6 +8,7 @@ SUMMARIZATION_SPECIAL_TOKENS = {"Text": "", "Summary": ["TL;DR:", "Summarize thi SUMMARY_SPECIAL_PROMPT = { "multi_news": ["Summarize in bullet points", "Generate summary in list of points"], "xsum": ["Give me summary in one sentence", "Short TLDR", "Give me a concise summary"], + "samsum": ["TLDR;", "Summarize this dialogue", "Summarize dialogue"], } summarization_config_mapping = { diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index 85fb86db..7b6e03b6 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -1,12 +1,13 @@ -from functools import partial +# from functools import partial from pathlib import Path import evaluate -import nltk -import numpy as np + +# import nltk +# import numpy as np import transformers import yaml -from custom_datasets import QA_DATASETS, SUMMARIZATION_DATASETS, get_one_dataset +from custom_datasets import get_one_dataset from custom_datasets.dialogue_collator import DialogueDataCollator from custom_datasets.qa_datasets import QA_SPECIAL_TOKENS from losses import CrossEntropyLoss, PolyLoss @@ -52,25 +53,25 @@ def preprocess_qa(eval_pred): return (eval_pred.predictions, eval_pred.label_ids) -def postprocess_summarization(preds, labels): - preds = [pred.strip() for pred in preds] - labels = [label.strip() for label in labels] +# def postprocess_summarization(preds, labels): +# preds = [pred.strip() for pred in preds] +# labels = [label.strip() for label in labels] - preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds] - labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels] +# preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds] +# labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels] - return preds, labels +# return preds, labels -def preprocess_summarization(eval_pred, tokenizer, ignore_pad_token_for_loss=True): - preds, labels = eval_pred - decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True) - if ignore_pad_token_for_loss: - labels = np.where(labels != -100, labels, tokenizer.pad_token_id) - decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) +# def preprocess_summarization(eval_pred, tokenizer, ignore_pad_token_for_loss=True): +# preds, labels = eval_pred +# decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True) +# if ignore_pad_token_for_loss: +# labels = np.where(labels != -100, labels, tokenizer.pad_token_id) +# decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) - decoded_preds, decoded_labels = postprocess_summarization(decoded_preds, decoded_labels) - return decoded_preds, decoded_labels +# decoded_preds, decoded_labels = postprocess_summarization(decoded_preds, decoded_labels) +# return decoded_preds, decoded_labels def get_metrics(conf, tokenizer): @@ -78,16 +79,16 @@ def get_metrics(conf, tokenizer): # metrics in the future for more thorough evaluation metrics, preprocess_fns = [evaluate.load("accuracy")], [default_preprocess] - if any(dataset in QA_DATASETS for dataset in conf.datasets): - raise ValueError("TODO") - metrics.append(evaluate.load("squad_v2")) - preprocess_fns.append(preprocess_qa) - if any(dataset in SUMMARIZATION_DATASETS for dataset in conf.datasets): - raise ValueError("TODO") - metrics.append(evaluate.load("rouge")) - preprocess_fns.append( - partial(preprocess_summarization, tokenizer, ignore_pad_token_for_loss=conf.ignore_pad_token_for_loss) - ) + # if any(dataset in QA_DATASETS for dataset in conf.datasets): + # raise ValueError("TODO") + # metrics.append(evaluate.load("squad_v2")) + # preprocess_fns.append(preprocess_qa) + # if any(dataset in SUMMARIZATION_DATASETS for dataset in conf.datasets): + # raise ValueError("TODO") + # metrics.append(evaluate.load("rouge")) + # preprocess_fns.append( + # partial(preprocess_summarization, tokenizer, ignore_pad_token_for_loss=conf.ignore_pad_token_for_loss) + # ) return metrics, preprocess_fns From f9c8d1dd81922e5b59e86204169c2027dfe7949c Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 14 Jan 2023 16:55:14 +0900 Subject: [PATCH 06/40] Combining the admin API routes with the backends user routes --- backend/oasst_backend/api/v1/users.py | 4 +- website/src/components/UsersCell.tsx | 21 +++-- website/src/lib/oasst_api_client.ts | 86 ++++++++++++++++++++ website/src/pages/admin/manage_user/[id].tsx | 38 +++++---- website/src/pages/api/admin/update_user.ts | 26 +++--- website/src/pages/api/admin/user_messages.ts | 13 ++- website/src/pages/api/admin/users.ts | 34 ++++++-- website/src/types/Users.ts | 51 ++++++++++++ 8 files changed, 220 insertions(+), 53 deletions(-) create mode 100644 website/src/types/Users.ts diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py index 5bc0195e..565499a7 100644 --- a/backend/oasst_backend/api/v1/users.py +++ b/backend/oasst_backend/api/v1/users.py @@ -15,7 +15,7 @@ from starlette.status import HTTP_204_NO_CONTENT router = APIRouter() -@router.get("/users/{user_id}", response_model=protocol.User) +@router.get("/users/{user_id}", response_model=protocol.FrontEndUser) def get_user( user_id: UUID, api_client_id: UUID = None, @@ -27,7 +27,7 @@ def get_user( """ ur = UserRepository(db, api_client) user: User = ur.get_user(user_id, api_client_id) - return protocol.User(user.username, user.display_name, user.auth_method) + return user.to_protocol_frontend_user() @router.put("/users/{user_id}", status_code=HTTP_204_NO_CONTENT) diff --git a/website/src/components/UsersCell.tsx b/website/src/components/UsersCell.tsx index 63c923a7..f5545be1 100644 --- a/website/src/components/UsersCell.tsx +++ b/website/src/components/UsersCell.tsx @@ -15,6 +15,7 @@ import { import Link from "next/link"; import { useState } from "react"; import { get } from "src/lib/api"; +import type { User } from "src/types/Users"; import useSWR from "swr"; /** @@ -22,7 +23,7 @@ import useSWR from "swr"; */ const UsersCell = () => { const [pageIndex, setPageIndex] = useState(0); - const [users, setUsers] = useState([]); + const [users, setUsers] = useState([]); // Fetch and save the users. // This follows useSWR's recommendation for simple pagination: @@ -53,21 +54,23 @@ const UsersCell = () => { Id - Email + Auth Id + Auth Method Name Role Update - {users.map((user, index) => ( - - {user.id} - {user.email} - {user.name} - {user.role} + {users.map(({ id, user_id, auth_method, display_name, role }) => ( + + {user_id} + {id} + {auth_method} + {display_name} + {role} - Manage + Manage ))} diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 381e6e0f..08cd0b56 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -1,4 +1,6 @@ import { JWT } from "next-auth/jwt"; +import type { Message } from "src/types/Conversation"; +import type { BackendUser } from "src/types/Users"; export class OasstError { message: string; @@ -12,6 +14,48 @@ export class OasstError { } } +/** + * Reports the Backend's knowledge of a user. + */ +export interface BackendUser { + /** + * The user's unique ID according to the `auth_method`. + */ + id: string; + + /** + * The user's set name + */ + display_name: string; + + /** + * The authorization method. One of: + * - discord + * - local + */ + auth_method: string; + + /** + * The backend's UUID for this user. + */ + user_id: string; + + /** + * Arbitrary notes about the user. + */ + notes: string; + + /** + * True when the user is able to access the platform. False otherwise. + */ + enabled: boolean; + + /** + * True when the user is marked for deletion. False otherwise. + */ + deleted: boolean; +} + export class OasstApiClient { constructor(private readonly oasstApiUrl: string, private readonly oasstApiKey: string) {} @@ -43,6 +87,32 @@ export class OasstApiClient { return await resp.json(); } + private async put(path: string, body: any): Promise { + const resp = await fetch(`${this.oasstApiUrl}${path}`, { + method: "PUT", + headers: { + "X-API-Key": this.oasstApiKey, + }, + }); + + if (resp.status === 204) { + return null; + } + + if (resp.status >= 300) { + const errorText = await resp.text(); + let error: any; + try { + error = JSON.parse(errorText); + } catch (e) { + throw new OasstError(errorText, 0, resp.status); + } + throw new OasstError(error.message ?? error, error.error_code, resp.status); + } + + return await resp.json(); + } + private async get(path: string): Promise { const resp = await fetch(`${this.oasstApiUrl}${path}`, { method: "GET", @@ -121,6 +191,22 @@ export class OasstApiClient { }); } + async fetch_user(user_id: string): Promise { + return this.get(`/api/v1/users/users/${user_id}`); + } + + async fetch_users(max_count: number): Promise { + return this.get(`/api/v1/frontend_users/?max_count=${max_count}`); + } + + async fetch_user_messages(user_id: string): Promise { + return this.get(`/api/v1/users/${user_id}/messages`); + } + + async set_user_status(user_id: string, is_enabled: boolean, notes): Promise { + return this.put(`/api/v1/users/users/${user_id}?enabled=${is_enabled}¬es=${notes}`); + } + //Fetch valid labels. This is called every task. though the call may be redundant //keeping this for future where the valid labels may change per task async fetch_valid_text(): Promise { diff --git a/website/src/pages/admin/manage_user/[id].tsx b/website/src/pages/admin/manage_user/[id].tsx index 6386c155..90698f4a 100644 --- a/website/src/pages/admin/manage_user/[id].tsx +++ b/website/src/pages/admin/manage_user/[id].tsx @@ -7,6 +7,7 @@ import { useEffect } from "react"; import { getAdminLayout } from "src/components/Layout"; import { UserMessagesCell } from "src/components/UserMessagesCell"; import { post } from "src/lib/api"; +import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; import useSWRMutation from "swr/mutation"; @@ -68,24 +69,17 @@ const ManageUser = ({ user }) => { }} >
+ - + + {({ field }) => ( - Username + Display Name )} - - {({ field }) => ( - - Email - - - )} - - {({ field }) => ( @@ -98,13 +92,21 @@ const ManageUser = ({ user }) => { )} + + {({ field }) => ( + + Notes + + + )} + - + ); @@ -114,15 +116,17 @@ const ManageUser = ({ user }) => { * Fetch the user's data on the server side when rendering. */ export async function getServerSideProps({ query }) { - const user = await prisma.user.findUnique({ - where: { id: query.id }, + const backend_user = await oasstApiClient.fetch_user(query.id); + const local_user = await prisma.user.findUnique({ + where: { id: backend_user.id }, select: { - id: true, - name: true, - email: true, role: true, }, }); + const user = { + ...backend_user, + role: local_user?.role || "general", + }; return { props: { user, diff --git a/website/src/pages/api/admin/update_user.ts b/website/src/pages/api/admin/update_user.ts index 95ddff4b..341ec736 100644 --- a/website/src/pages/api/admin/update_user.ts +++ b/website/src/pages/api/admin/update_user.ts @@ -1,22 +1,28 @@ import { withRole } from "src/lib/auth"; +import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; /** * Update's the user's data in the database. Accessible only to admins. */ const handler = withRole("admin", async (req, res) => { - const { id, role } = req.body; + const { id, auth_method, user_id, notes, role } = req.body; - await prisma.user.update({ - where: { - id, - }, - data: { - role, - }, - }); + // If the user is authorized by the web, update their role. + if (auth_method === "local") { + await prisma.user.update({ + where: { + id, + }, + data: { + role, + }, + }); + } + // Tell the backend the user's enabled or not enabled status. + await oasstApiClient.set_user_status(user_id, role !== "banned", notes); - res.status(200).end(); + res.status(200).json({}); }); export default handler; diff --git a/website/src/pages/api/admin/user_messages.ts b/website/src/pages/api/admin/user_messages.ts index 254bf9c6..67f72a4e 100644 --- a/website/src/pages/api/admin/user_messages.ts +++ b/website/src/pages/api/admin/user_messages.ts @@ -1,15 +1,12 @@ +import { oasstApiClient } from "src/lib/oasst_api_client"; import { withRole } from "src/lib/auth"; +/** + * Returns the messages recorded by the backend for a user. + */ const handler = withRole("admin", async (req, res) => { const { user } = req.query; - - const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/frontend_users/local/${user}/messages`, { - method: "GET", - headers: { - "X-API-Key": process.env.FASTAPI_KEY, - }, - }); - const messages = await messagesRes.json(); + const messages = await oasstApiClient.fetch_user_messages(user); res.status(200).json(messages); }); diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index 7c71b667..af1479f2 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -1,4 +1,5 @@ import { withRole } from "src/lib/auth"; +import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; // The number of users to fetch in any request. @@ -14,18 +15,37 @@ const handler = withRole("admin", async (req, res) => { // Note: with Prisma this isn't the most efficient but it's the only possible // option with cuid based User IDs. const { pageIndex } = req.query; - const skip = parseInt(pageIndex as string) * PAGE_SIZE || 0; - // Fetch 20 users. - const users = await prisma.user.findMany({ + // First, get all the users according to the backend. + const all_users = await oasstApiClient.fetch_users(20); + + // Next, get all the users stored in the web's auth datbase to fetch their role. + const local_user_ids = all_users.map(({ id }) => id); + const local_users = await prisma.user.findMany({ + where: { + id: { + in: local_user_ids, + }, + }, select: { id: true, role: true, - name: true, - email: true, }, - skip, - take: PAGE_SIZE, + }); + + // Combine the information by updating the set of full users with their role. + // Default any users without a role set locally as "general". + const local_user_map = local_users.reduce((result, user) => { + result.set(user.id, user.role); + return result; + }, new Map()); + + const users = all_users.map((user) => { + const role = local_user_map.get(user.id) || "general"; + return { + ...user, + role, + }; }); res.status(200).json(users); diff --git a/website/src/types/Users.ts b/website/src/types/Users.ts new file mode 100644 index 00000000..eeb1903a --- /dev/null +++ b/website/src/types/Users.ts @@ -0,0 +1,51 @@ +/** + * Reports the Backend's knowledge of a user. + */ +export interface BackendUser { + /** + * The user's unique ID according to the `auth_method`. + */ + id: string; + + /** + * The user's set name + */ + display_name: string; + + /** + * The authorization method. One of: + * - discord + * - local + */ + auth_method: string; + + /** + * The backend's UUID for this user. + */ + user_id: string; + + /** + * Arbitrary notes about the user. + */ + notes: string; + + /** + * True when the user is able to access the platform. False otherwise. + */ + enabled: boolean; + + /** + * True when the user is marked for deletion. False otherwise. + */ + deleted: boolean; +} + +/** + * An expanded User for the web. + */ +export interface User extends BackendUser { + /** + * The user's roles within the webapp. + */ + role: string; +} From ddfb4977646d8082a3724792dfa5842f74041f52 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 14 Jan 2023 16:57:06 +0900 Subject: [PATCH 07/40] Fixing some lint errors and adding a TODO --- website/src/lib/oasst_api_client.ts | 2 +- website/src/pages/api/admin/user_messages.ts | 2 +- website/src/pages/api/admin/users.ts | 9 +-------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 08cd0b56..95e2fb58 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -87,7 +87,7 @@ export class OasstApiClient { return await resp.json(); } - private async put(path: string, body: any): Promise { + private async put(path: string): Promise { const resp = await fetch(`${this.oasstApiUrl}${path}`, { method: "PUT", headers: { diff --git a/website/src/pages/api/admin/user_messages.ts b/website/src/pages/api/admin/user_messages.ts index 67f72a4e..f862bfc0 100644 --- a/website/src/pages/api/admin/user_messages.ts +++ b/website/src/pages/api/admin/user_messages.ts @@ -1,5 +1,5 @@ -import { oasstApiClient } from "src/lib/oasst_api_client"; import { withRole } from "src/lib/auth"; +import { oasstApiClient } from "src/lib/oasst_api_client"; /** * Returns the messages recorded by the backend for a user. diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index af1479f2..04bcacf0 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -2,19 +2,12 @@ import { withRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; -// The number of users to fetch in any request. -const PAGE_SIZE = 20; - /** * Returns a list of user results from the database when the requesting user is * a logged in admin. */ const handler = withRole("admin", async (req, res) => { - // Figure out the pagination index and skip that number of users. - // - // Note: with Prisma this isn't the most efficient but it's the only possible - // option with cuid based User IDs. - const { pageIndex } = req.query; + // TODO(#673): Update this to support pagination. // First, get all the users according to the backend. const all_users = await oasstApiClient.fetch_users(20); From e1309b1d5aa05756d16af0c5fc1cdb66ef23ef9e Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 14 Jan 2023 17:05:24 +0900 Subject: [PATCH 08/40] Adding some function comments --- website/src/lib/oasst_api_client.ts | 54 +++++++---------------------- 1 file changed, 12 insertions(+), 42 deletions(-) diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index a26b421d..43ca6c49 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -14,48 +14,6 @@ export class OasstError { } } -/** - * Reports the Backend's knowledge of a user. - */ -export interface BackendUser { - /** - * The user's unique ID according to the `auth_method`. - */ - id: string; - - /** - * The user's set name - */ - display_name: string; - - /** - * The authorization method. One of: - * - discord - * - local - */ - auth_method: string; - - /** - * The backend's UUID for this user. - */ - user_id: string; - - /** - * Arbitrary notes about the user. - */ - notes: string; - - /** - * True when the user is able to access the platform. False otherwise. - */ - enabled: boolean; - - /** - * True when the user is marked for deletion. False otherwise. - */ - deleted: boolean; -} - export class OasstApiClient { constructor(private readonly oasstApiUrl: string, private readonly oasstApiKey: string) {} @@ -191,18 +149,30 @@ export class OasstApiClient { }); } + /** + * Returns the `BackendUser` associated with `user_id` + */ async fetch_user(user_id: string): Promise { return this.get(`/api/v1/users/users/${user_id}`); } + /** + * Returns the `max_count` `BackendUser`s stored by the backend. + */ async fetch_users(max_count: number): Promise { return this.get(`/api/v1/frontend_users/?max_count=${max_count}`); } + /** + * Returns the `Message`s associated with `user_id` in the backend. + */ async fetch_user_messages(user_id: string): Promise { return this.get(`/api/v1/users/${user_id}/messages`); } + /** + * Updates the backend's knowledge about the `user_id`. + */ async set_user_status(user_id: string, is_enabled: boolean, notes): Promise { return this.put(`/api/v1/users/users/${user_id}?enabled=${is_enabled}¬es=${notes}`); } From 7de25f299c790ae9701e7ebcd636ebe1697dd42e Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 14 Jan 2023 17:07:37 +0900 Subject: [PATCH 09/40] fixing a type error --- website/src/pages/api/admin/user_messages.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/pages/api/admin/user_messages.ts b/website/src/pages/api/admin/user_messages.ts index f862bfc0..91e5f45e 100644 --- a/website/src/pages/api/admin/user_messages.ts +++ b/website/src/pages/api/admin/user_messages.ts @@ -6,7 +6,7 @@ import { oasstApiClient } from "src/lib/oasst_api_client"; */ const handler = withRole("admin", async (req, res) => { const { user } = req.query; - const messages = await oasstApiClient.fetch_user_messages(user); + const messages = await oasstApiClient.fetch_user_messages(user as string); res.status(200).json(messages); }); From 5282c1182725cc3045ab1fd5dbf080bf0a88ea87 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 14 Jan 2023 18:14:29 +0900 Subject: [PATCH 10/40] Adding an explicit type --- website/src/pages/api/admin/user_messages.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/src/pages/api/admin/user_messages.ts b/website/src/pages/api/admin/user_messages.ts index 91e5f45e..236afa2d 100644 --- a/website/src/pages/api/admin/user_messages.ts +++ b/website/src/pages/api/admin/user_messages.ts @@ -1,12 +1,13 @@ import { withRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; +import type { Message } from "src/types/Conversation"; /** * Returns the messages recorded by the backend for a user. */ const handler = withRole("admin", async (req, res) => { const { user } = req.query; - const messages = await oasstApiClient.fetch_user_messages(user as string); + const messages: Message[] = await oasstApiClient.fetch_user_messages(user as string); res.status(200).json(messages); }); From b02429c1f7f9699e10c036fb002abc5b14ce6cfc Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 14 Jan 2023 18:55:28 +0900 Subject: [PATCH 11/40] Tag in the user table when they are new and include it in the user session --- website/prisma/schema.prisma | 1 + website/src/pages/api/auth/[...nextauth].ts | 6 ++++-- website/src/pages/api/update_task.ts | 3 +++ website/src/pages/dashboard.tsx | 6 ++++++ website/types/next-auth.d.ts | 4 ++++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/website/prisma/schema.prisma b/website/prisma/schema.prisma index f9eab3b7..dc6a700d 100644 --- a/website/prisma/schema.prisma +++ b/website/prisma/schema.prisma @@ -41,6 +41,7 @@ model User { email String? @unique emailVerified DateTime? image String? + isNew Boolean @default(true) role String @default("general") accounts Account[] diff --git a/website/src/pages/api/auth/[...nextauth].ts b/website/src/pages/api/auth/[...nextauth].ts index 363c1404..254e4f9d 100644 --- a/website/src/pages/api/auth/[...nextauth].ts +++ b/website/src/pages/api/auth/[...nextauth].ts @@ -86,6 +86,7 @@ export const authOptions: AuthOptions = { */ async session({ session, token }) { session.user.role = token.role; + session.user.isNew = token.isNew; return session; }, /** @@ -93,11 +94,12 @@ export const authOptions: AuthOptions = { * This let's use forward the role to the session object. */ async jwt({ token }) { - const { role } = await prisma.user.findUnique({ + const { isNew, role } = await prisma.user.findUnique({ where: { id: token.sub }, - select: { role: true }, + select: { role: true, isNew: true }, }); token.role = role; + token.isNew = isNew; return token; }, }, diff --git a/website/src/pages/api/update_task.ts b/website/src/pages/api/update_task.ts index f3aa6fb1..02982daa 100644 --- a/website/src/pages/api/update_task.ts +++ b/website/src/pages/api/update_task.ts @@ -17,6 +17,9 @@ const handler = withoutRole("banned", async (req, res, token) => { // Parse out the local task ID and the interaction contents. const { id: frontendId, content, update_type } = req.body; + // Record that the user has done meaningful work and is no longer new. + await prisma.user.update({ where: { id: token.sub }, data: { isNew: false } }); + // Accept the task so that we can complete it, this will probably go away soon. const registeredTask = await prisma.registeredTask.findUniqueOrThrow({ where: { id: frontendId } }); const task = registeredTask.task as Prisma.JsonObject; diff --git a/website/src/pages/dashboard.tsx b/website/src/pages/dashboard.tsx index d296592d..42481f1e 100644 --- a/website/src/pages/dashboard.tsx +++ b/website/src/pages/dashboard.tsx @@ -1,9 +1,15 @@ import Head from "next/head"; +import { useSession } from "next-auth/react"; import { LeaderboardTable, TaskOption } from "src/components/Dashboard"; import { getDashboardLayout } from "src/components/Layout"; import { TaskCategory } from "src/components/Tasks/TaskTypes"; const Dashboard = () => { + const { data: session } = useSession(); + + // TODO(#670): Do something more meaningful when the user is new. + console.log(session?.user?.isNew); + return ( <> diff --git a/website/types/next-auth.d.ts b/website/types/next-auth.d.ts index 25027600..738ff8f2 100644 --- a/website/types/next-auth.d.ts +++ b/website/types/next-auth.d.ts @@ -6,6 +6,8 @@ declare module "next-auth" { user: { /** The user's role. */ role: string; + /** True when the user is new. */ + isNew: boolean; } & DefaultSession["user"]; } } @@ -14,5 +16,7 @@ declare module "next-auth/jwt" { interface JWT { /** The user's role. */ role?: string; + /** True when the user is new. */ + isNew?: boolean; } } From 670be60ca80e084c116e13847085ec604068c280 Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Sat, 14 Jan 2023 12:17:58 +0000 Subject: [PATCH 12/40] [fix] Fix config typo --- model/supervised_finetuning/configs/config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index 0440201a..2eaa6686 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -28,8 +28,8 @@ defaults: - scitldr - soda - joke - - - - joke + - gsm8k + - samsum cache_dir: .cache loss_fn: CrossEntropyLoss eval_size: @@ -63,10 +63,10 @@ codegen: learning_rate: 8e-6 model_name: Salesforce/codegen-2B-multi weight_decay: 0.01 - max_length: 512 + max_length: 520 warmup_steps: 1000 gradient_checkpointing: false - gradient_accumulation_steps: 10 + gradient_accumulation_steps: 9 per_device_train_batch_size: 2 per_device_eval_batch_size: 4 From a902c600faa20af2847a95aedd3ffde3abf9752e Mon Sep 17 00:00:00 2001 From: Nil Andreu <65730003+Nil-Andreu@users.noreply.github.com> Date: Sat, 14 Jan 2023 13:22:55 +0100 Subject: [PATCH 13/40] Store Message Toxicity in database (#553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [NEW] MessageToxicity table * [NEW] Alembic message Toxicity * [NEW] Model name enum * [NEW] Refactor Enum HF * [NEW] Settings: DEBUT_SKIP_TOXICITY_CALCULATION * [NEW] Store toxicity values * [FIX] Merge conflict * [FIX] Documentation * [NEW] save_toxicity: function * [FIX] Formatted string * [NEW] DEBUG_SKIP_TOXICITY_CALCULATION=True * [FIX] HfClassificationModel * [FIX] Alembic merge heads * [NEW] Refactor save_toxicity * [NEW] Separating score/label * [NEW] Store score and label * [FIX] Cleaning Alembic * [NEW] Clean HF names * [NEW] Not type hinting * [NEW] Update alembic versions * [NEW] Revert the changes * [NEW] Type hinting label & score * Updated down_revision in migration script Co-authored-by: Andreas Köpf --- ansible/dev.yaml | 1 + ...01_08_2200-bcc2fe18d214_messagetoxicity.py | 40 +++++++++++++++++++ backend/oasst_backend/api/v1/hugging_face.py | 5 ++- backend/oasst_backend/config.py | 1 + backend/oasst_backend/models/__init__.py | 2 + .../oasst_backend/models/message_toxicity.py | 24 +++++++++++ backend/oasst_backend/prompt_repository.py | 23 +++++++++-- backend/oasst_backend/tree_manager.py | 23 ++++++++++- backend/oasst_backend/utils/hugging_face.py | 6 ++- docker-compose.yaml | 1 + scripts/backend-development/run-local.sh | 1 + 11 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py create mode 100644 backend/oasst_backend/models/message_toxicity.py diff --git a/ansible/dev.yaml b/ansible/dev.yaml index 3cf061a5..2bf67b01 100644 --- a/ansible/dev.yaml +++ b/ansible/dev.yaml @@ -83,6 +83,7 @@ MAX_WORKERS: "1" RATE_LIMIT: "false" DEBUG_SKIP_EMBEDDING_COMPUTATION: "true" + DEBUG_SKIP_TOXICITY_CALCULATION: "true" ports: - 8080:8080 diff --git a/backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py b/backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py new file mode 100644 index 00000000..1d17b9d2 --- /dev/null +++ b/backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py @@ -0,0 +1,40 @@ +"""MessageToxicity + +Revision ID: bcc2fe18d214 +Revises: 20cd871f4ec7 +Create Date: 2023-01-08 22:00:43.297719 + +""" +import sqlalchemy as sa +import sqlmodel +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "bcc2fe18d214" +down_revision = "846cc08ac79f" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "message_toxicity", + sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("toxicity", sa.Float(), nullable=True), + sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), + sa.Column("model", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False), + sa.ForeignKeyConstraint( + ["message_id"], + ["message.id"], + ), + sa.PrimaryKeyConstraint("message_id", "model"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("message_toxicity") + # ### end Alembic commands ### diff --git a/backend/oasst_backend/api/v1/hugging_face.py b/backend/oasst_backend/api/v1/hugging_face.py index 62d2ea6b..a6715574 100644 --- a/backend/oasst_backend/api/v1/hugging_face.py +++ b/backend/oasst_backend/api/v1/hugging_face.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, Depends from oasst_backend.api import deps from oasst_backend.models import ApiClient from oasst_backend.schemas.hugging_face import ToxicityClassification -from oasst_backend.utils.hugging_face import HfUrl, HuggingFaceAPI +from oasst_backend.utils.hugging_face import HfClassificationModel, HfUrl, HuggingFaceAPI router = APIRouter() @@ -25,7 +25,8 @@ async def get_text_toxicity( ToxicityClassification: the score of toxicity of the message. """ - api_url: str = HfUrl.HUGGINGFACE_TOXIC_ROBERTA.value + api_url: str = HfUrl.HUGGINGFACE_TOXIC_CLASSIFICATION.value + "/" + HfClassificationModel.TOXIC_ROBERTA.value + hugging_face_api = HuggingFaceAPI(api_url) response = await hugging_face_api.post(msg) diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index c18bd4c2..2967884c 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -79,6 +79,7 @@ class Settings(BaseSettings): ) DEBUG_ALLOW_SELF_LABELING: bool = False # allow users to label their own messages DEBUG_SKIP_EMBEDDING_COMPUTATION: bool = False + DEBUG_SKIP_TOXICITY_CALCULATION: bool = False HUGGING_FACE_API_KEY: str = "" diff --git a/backend/oasst_backend/models/__init__.py b/backend/oasst_backend/models/__init__.py index 0873381c..9dc052d7 100644 --- a/backend/oasst_backend/models/__init__.py +++ b/backend/oasst_backend/models/__init__.py @@ -3,6 +3,7 @@ from .journal import Journal, JournalIntegration from .message import Message from .message_embedding import MessageEmbedding from .message_reaction import MessageReaction +from .message_toxicity import MessageToxicity from .message_tree_state import MessageTreeState from .task import Task from .text_labels import TextLabels @@ -17,6 +18,7 @@ __all__ = [ "MessageEmbedding", "MessageReaction", "MessageTreeState", + "MessageToxicity", "Task", "TextLabels", "Journal", diff --git a/backend/oasst_backend/models/message_toxicity.py b/backend/oasst_backend/models/message_toxicity.py new file mode 100644 index 00000000..8a78e2dc --- /dev/null +++ b/backend/oasst_backend/models/message_toxicity.py @@ -0,0 +1,24 @@ +from datetime import datetime +from typing import Optional +from uuid import UUID + +import sqlalchemy as sa +import sqlalchemy.dialects.postgresql as pg +from sqlmodel import Field, Float, SQLModel + + +class MessageToxicity(SQLModel, table=True): + __tablename__ = "message_toxicity" + __table_args__ = (sa.PrimaryKeyConstraint("message_id", "model"),) + + message_id: UUID = Field(sa_column=sa.Column(pg.UUID(as_uuid=True), sa.ForeignKey("message.id"), nullable=False)) + model: str = Field(max_length=256, nullable=False) + + # Storing the score and the label of the message + score: float = Field(sa_column=sa.Column(Float), nullable=False) + label: str = Field(max_length=256, nullable=False) + + # In the case that the Message Embedding is created afterwards + created_date: Optional[datetime] = Field( + sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()) + ) diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index cb6e70f7..0c40daa0 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -14,6 +14,7 @@ from oasst_backend.models import ( Message, MessageEmbedding, MessageReaction, + MessageToxicity, MessageTreeState, Task, TextLabels, @@ -293,6 +294,25 @@ class PromptRepository: return reaction, task + def insert_toxicity(self, message_id: UUID, model: str, score: float, label: str) -> MessageToxicity: + """Save the toxicity score of a new message in the database. + Args: + message_id (UUID): the identifier of the message we want to save its toxicity score + model (str): the model used for creating the toxicity score + score (float): the toxicity score that we obtained from the model + label (str): the final classification in toxicity of the model + Raises: + OasstError: if misses some of the before params + Returns: + MessageToxicity: the instance in the database of the score saved for that message + """ + + message_toxicity = MessageToxicity(message_id=message_id, model=model, score=score, label=label) + self.db.add(message_toxicity) + self.db.commit() + self.db.refresh(message_toxicity) + return message_toxicity + def insert_message_embedding(self, message_id: UUID, model: str, embedding: List[float]) -> MessageEmbedding: """Insert the embedding of a new message in the database. @@ -308,9 +328,6 @@ class PromptRepository: MessageEmbedding: the instance in the database of the embedding saved for that message """ - if None in (message_id, model, embedding): - raise OasstError("Paramters missing to add embedding", OasstErrorCode.GENERIC_ERROR) - message_embedding = MessageEmbedding(message_id=message_id, model=model, embedding=embedding) self.db.add(message_embedding) self.db.commit() diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index d7abd9f8..fe45ca0e 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -1,7 +1,7 @@ import random from enum import Enum from http import HTTPStatus -from typing import Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from uuid import UUID import numpy as np @@ -11,7 +11,7 @@ from oasst_backend.api.v1.utils import prepare_conversation, prepare_conversatio from oasst_backend.config import TreeManagerConfiguration, settings from oasst_backend.models import Message, MessageReaction, MessageTreeState, TextLabels, message_tree_state from oasst_backend.prompt_repository import PromptRepository -from oasst_backend.utils.hugging_face import HfEmbeddingModel, HfUrl, HuggingFaceAPI +from oasst_backend.utils.hugging_face import HfClassificationModel, HfEmbeddingModel, HfUrl, HuggingFaceAPI from oasst_shared.exceptions.oasst_api_error import OasstError, OasstErrorCode from oasst_shared.schemas import protocol as protocol_schema from sqlalchemy.sql import text @@ -363,6 +363,25 @@ class TreeManager: f"Could not fetch embbeddings for text reply to {interaction.message_id=} with {interaction.text=} by {interaction.user=}." ) + if not settings.DEBUG_SKIP_TOXICITY_CALCULATION: + try: + model_name: str = HfClassificationModel.TOXIC_ROBERTA.value + hugging_face_api: HuggingFaceAPI = HuggingFaceAPI( + f"{HfUrl.HUGGINGFACE_FEATURE_EXTRACTION.value}/{model_name}" + ) + + toxicity: List[List[Dict[str, Any]]] = await hugging_face_api.post(interaction.text) + toxicity = toxicity[0][0] + + pr.insert_toxicity( + message_id=message.id, model=model_name, score=toxicity["score"], label=toxicity["label"] + ) + + except OasstError: + logger.error( + f"Could not compute toxicity for text reply to {interaction.message_id=} with {interaction.text=} by {interaction.user=}." + ) + case protocol_schema.MessageRating: logger.info( f"Frontend reports rating of {interaction.message_id=} with {interaction.rating=} by {interaction.user=}." diff --git a/backend/oasst_backend/utils/hugging_face.py b/backend/oasst_backend/utils/hugging_face.py index 099bc51f..1aef23ce 100644 --- a/backend/oasst_backend/utils/hugging_face.py +++ b/backend/oasst_backend/utils/hugging_face.py @@ -8,10 +8,14 @@ from oasst_shared.exceptions import OasstError, OasstErrorCode class HfUrl(str, Enum): - HUGGINGFACE_TOXIC_ROBERTA = ("https://api-inference.huggingface.co/models/unitary/multilingual-toxic-xlm-roberta",) + HUGGINGFACE_TOXIC_CLASSIFICATION = "https://api-inference.huggingface.co/models" HUGGINGFACE_FEATURE_EXTRACTION = "https://api-inference.huggingface.co/pipeline/feature-extraction" +class HfClassificationModel(str, Enum): + TOXIC_ROBERTA = "unitary/multilingual-toxic-xlm-roberta" + + class HfEmbeddingModel(str, Enum): MINILM = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2" diff --git a/docker-compose.yaml b/docker-compose.yaml index 858acb68..cde65166 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -101,6 +101,7 @@ services: - DEBUG_USE_SEED_DATA=True - DEBUG_ALLOW_SELF_LABELING=True - MAX_WORKERS=1 + - DEBUG_SKIP_TOXICITY_CALCULATION=True - DEBUG_SKIP_EMBEDDING_COMPUTATION=True depends_on: db: diff --git a/scripts/backend-development/run-local.sh b/scripts/backend-development/run-local.sh index 2433c67e..22701ace 100755 --- a/scripts/backend-development/run-local.sh +++ b/scripts/backend-development/run-local.sh @@ -6,6 +6,7 @@ pushd "$parent_path/../../backend" export DEBUG_SKIP_API_KEY_CHECK=True export DEBUG_USE_SEED_DATA=True +export DEBUG_SKIP_TOXICITY_CALCULATION=True export DEBUG_ALLOW_SELF_LABELING=True export DEBUG_SKIP_EMBEDDING_COMPUTATION=True From feba2de20cacf9b9b7e93cc747b68aa3d0b0801a Mon Sep 17 00:00:00 2001 From: klotske Date: Sat, 14 Jan 2023 15:49:33 +0300 Subject: [PATCH 14/40] Bring back the NextLink --- website/src/components/Header/UserMenu.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/src/components/Header/UserMenu.tsx b/website/src/components/Header/UserMenu.tsx index 8835c7c2..c5efec52 100644 --- a/website/src/components/Header/UserMenu.tsx +++ b/website/src/components/Header/UserMenu.tsx @@ -12,6 +12,7 @@ import { useColorModeValue, } from "@chakra-ui/react"; import { signOut, useSession } from "next-auth/react"; +import NextLink from "next/link"; import React from "react"; import { FiAlertTriangle, FiLayout, FiLogOut, FiSettings, FiShield } from "react-icons/fi"; @@ -77,7 +78,7 @@ export function UserMenu() { {accountOptions.map((item) => ( - +