wassname/prob_jsonformer
1
0
Fork 0
mirror of https://github.com/wassname/prob_jsonformer.git synced 2026-09-10 12:37:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix deps

Browse Source
This commit is contained in:
rahulgs12
2023-05-01 16:30:55 -04:00
parent 38a700abfe
commit e065a127c7
5 changed files with 84 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,3 +1,4 @@
jsonllm/__pycache__
.venv
workspace.ipynb
dist
license.txt
+17
View File
@@ -0,0 +1,17 @@
MIT License
Copyright (c) 2018 Rahul Sengottuvelu
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
poetry.lock
Generated
+44 -133
View File
@@ -1,10 +1,10 @@
# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
# This file is automatically @generated by Poetry and should not be changed by hand.
[[package]]
name = "accelerate"
version = "0.18.0"
description = "Accelerate"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7.0"
files = [
@@ -59,25 +59,6 @@ six = "*"
[package.extras]
test = ["astroid", "pytest"]
[[package]]
name = "attrs"
version = "23.1.0"
description = "Classes Without Boilerplate"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"},
{file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"},
]
[package.extras]
cov = ["attrs[tests]", "coverage[toml] (>=5.3)"]
dev = ["attrs[docs,tests]", "pre-commit"]
docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"]
tests = ["attrs[tests-no-zope]", "zope-interface"]
tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
[[package]]
name = "backcall"
version = "0.2.0"
@@ -94,7 +75,7 @@ files = [
name = "bitsandbytes"
version = "0.38.1"
description = "8-bit optimizers and matrix multiplication routines."
category = "main"
category = "dev"
optional = false
python-versions = "*"
files = [
@@ -106,7 +87,7 @@ files = [
name = "certifi"
version = "2022.12.7"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
category = "dev"
optional = false
python-versions = ">=3.6"
files = [
@@ -195,7 +176,7 @@ pycparser = "*"
name = "charset-normalizer"
version = "3.1.0"
description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7.0"
files = [
@@ -280,7 +261,7 @@ files = [
name = "cmake"
version = "3.26.3"
description = "CMake is an open-source, cross-platform family of tools designed to build, test and package software"
category = "main"
category = "dev"
optional = false
python-versions = "*"
files = [
@@ -310,7 +291,7 @@ test = ["codecov (>=2.0.5)", "coverage (>=4.2)", "flake8 (>=3.0.4)", "path.py (>
name = "colorama"
version = "0.4.6"
description = "Cross-platform colored terminal text."
category = "main"
category = "dev"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
files = [
@@ -397,7 +378,7 @@ tests = ["asttokens", "littleutils", "pytest", "rich"]
name = "filelock"
version = "3.12.0"
description = "A platform independent file lock."
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -413,7 +394,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.3)", "diff-cover (>=7.5)", "p
name = "fsspec"
version = "2023.4.0"
description = "File-system specification"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.8"
files = [
@@ -449,7 +430,7 @@ tqdm = ["tqdm"]
name = "huggingface-hub"
version = "0.14.1"
description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7.0"
files = [
@@ -481,7 +462,7 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t
name = "idna"
version = "3.4"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.5"
files = [
@@ -586,7 +567,7 @@ testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"]
name = "jinja2"
version = "3.1.2"
description = "A very fast and expressive template engine."
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -600,26 +581,6 @@ MarkupSafe = ">=2.0"
[package.extras]
i18n = ["Babel (>=2.7)"]
[[package]]
name = "jsonschema"
version = "4.17.3"
description = "An implementation of JSON Schema validation for Python"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "jsonschema-4.17.3-py3-none-any.whl", hash = "sha256:a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6"},
{file = "jsonschema-4.17.3.tar.gz", hash = "sha256:0f864437ab8b6076ba6707453ef8f98a6a0d512a80e93f8abdb676f737ecb60d"},
]
[package.dependencies]
attrs = ">=17.4.0"
pyrsistent = ">=0.14.0,<0.17.0 || >0.17.0,<0.17.1 || >0.17.1,<0.17.2 || >0.17.2"
[package.extras]
format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"]
format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"]
[[package]]
name = "jupyter-client"
version = "8.2.0"
@@ -668,7 +629,7 @@ test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"]
name = "lit"
version = "16.0.2"
description = "A Software Testing Tool"
category = "main"
category = "dev"
optional = false
python-versions = "*"
files = [
@@ -679,7 +640,7 @@ files = [
name = "markupsafe"
version = "2.1.2"
description = "Safely add untrusted strings to HTML/XML markup."
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -754,7 +715,7 @@ traitlets = "*"
name = "mpmath"
version = "1.3.0"
description = "Python library for arbitrary-precision floating-point arithmetic"
category = "main"
category = "dev"
optional = false
python-versions = "*"
files = [
@@ -784,7 +745,7 @@ files = [
name = "networkx"
version = "3.1"
description = "Python package for creating and manipulating graphs and networks"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.8"
files = [
@@ -803,7 +764,7 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"]
name = "numpy"
version = "1.24.3"
description = "Fundamental package for array computing in Python"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.8"
files = [
@@ -841,7 +802,7 @@ files = [
name = "nvidia-cublas-cu11"
version = "11.10.3.66"
description = "CUBLAS native runtime libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -857,7 +818,7 @@ wheel = "*"
name = "nvidia-cuda-cupti-cu11"
version = "11.7.101"
description = "CUDA profiling tools runtime libs."
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -873,7 +834,7 @@ wheel = "*"
name = "nvidia-cuda-nvrtc-cu11"
version = "11.7.99"
description = "NVRTC native runtime libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -890,7 +851,7 @@ wheel = "*"
name = "nvidia-cuda-runtime-cu11"
version = "11.7.99"
description = "CUDA Runtime native Libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -906,7 +867,7 @@ wheel = "*"
name = "nvidia-cudnn-cu11"
version = "8.5.0.96"
description = "cuDNN runtime libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -922,7 +883,7 @@ wheel = "*"
name = "nvidia-cufft-cu11"
version = "10.9.0.58"
description = "CUFFT native runtime libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -934,7 +895,7 @@ files = [
name = "nvidia-curand-cu11"
version = "10.2.10.91"
description = "CURAND native runtime libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -950,7 +911,7 @@ wheel = "*"
name = "nvidia-cusolver-cu11"
version = "11.4.0.1"
description = "CUDA solver native runtime libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -967,7 +928,7 @@ wheel = "*"
name = "nvidia-cusparse-cu11"
version = "11.7.4.91"
description = "CUSPARSE native runtime libraries"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -983,7 +944,7 @@ wheel = "*"
name = "nvidia-nccl-cu11"
version = "2.14.3"
description = "NVIDIA Collective Communication Library (NCCL) Runtime"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -994,7 +955,7 @@ files = [
name = "nvidia-nvtx-cu11"
version = "11.7.91"
description = "NVIDIA Tools Extension"
category = "main"
category = "dev"
optional = false
python-versions = ">=3"
files = [
@@ -1010,7 +971,7 @@ wheel = "*"
name = "packaging"
version = "23.1"
description = "Core utilities for Python packages"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -1096,7 +1057,7 @@ wcwidth = "*"
name = "psutil"
version = "5.9.5"
description = "Cross-platform lib for process and system monitoring in Python."
category = "main"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
files = [
@@ -1173,43 +1134,6 @@ files = [
[package.extras]
plugins = ["importlib-metadata"]
[[package]]
name = "pyrsistent"
version = "0.19.3"
description = "Persistent/Functional/Immutable data structures"
category = "main"
optional = false
python-versions = ">=3.7"
files = [
{file = "pyrsistent-0.19.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:20460ac0ea439a3e79caa1dbd560344b64ed75e85d8703943e0b66c2a6150e4a"},
{file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c18264cb84b5e68e7085a43723f9e4c1fd1d935ab240ce02c0324a8e01ccb64"},
{file = "pyrsistent-0.19.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4b774f9288dda8d425adb6544e5903f1fb6c273ab3128a355c6b972b7df39dcf"},
{file = "pyrsistent-0.19.3-cp310-cp310-win32.whl", hash = "sha256:5a474fb80f5e0d6c9394d8db0fc19e90fa540b82ee52dba7d246a7791712f74a"},
{file = "pyrsistent-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:49c32f216c17148695ca0e02a5c521e28a4ee6c5089f97e34fe24163113722da"},
{file = "pyrsistent-0.19.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9"},
{file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ab2204234c0ecd8b9368dbd6a53e83c3d4f3cab10ecaf6d0e772f456c442393"},
{file = "pyrsistent-0.19.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e42296a09e83028b3476f7073fcb69ffebac0e66dbbfd1bd847d61f74db30f19"},
{file = "pyrsistent-0.19.3-cp311-cp311-win32.whl", hash = "sha256:64220c429e42a7150f4bfd280f6f4bb2850f95956bde93c6fda1b70507af6ef3"},
{file = "pyrsistent-0.19.3-cp311-cp311-win_amd64.whl", hash = "sha256:016ad1afadf318eb7911baa24b049909f7f3bb2c5b1ed7b6a8f21db21ea3faa8"},
{file = "pyrsistent-0.19.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c4db1bd596fefd66b296a3d5d943c94f4fac5bcd13e99bffe2ba6a759d959a28"},
{file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeda827381f5e5d65cced3024126529ddc4289d944f75e090572c77ceb19adbf"},
{file = "pyrsistent-0.19.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42ac0b2f44607eb92ae88609eda931a4f0dfa03038c44c772e07f43e738bcac9"},
{file = "pyrsistent-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:e8f2b814a3dc6225964fa03d8582c6e0b6650d68a232df41e3cc1b66a5d2f8d1"},
{file = "pyrsistent-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9bb60a40a0ab9aba40a59f68214eed5a29c6274c83b2cc206a359c4a89fa41b"},
{file = "pyrsistent-0.19.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a2471f3f8693101975b1ff85ffd19bb7ca7dd7c38f8a81701f67d6b4f97b87d8"},
{file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc5d149f31706762c1f8bda2e8c4f8fead6e80312e3692619a75301d3dbb819a"},
{file = "pyrsistent-0.19.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3311cb4237a341aa52ab8448c27e3a9931e2ee09561ad150ba94e4cfd3fc888c"},
{file = "pyrsistent-0.19.3-cp38-cp38-win32.whl", hash = "sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c"},
{file = "pyrsistent-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:c147257a92374fde8498491f53ffa8f4822cd70c0d85037e09028e478cababb7"},
{file = "pyrsistent-0.19.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b735e538f74ec31378f5a1e3886a26d2ca6351106b4dfde376a26fc32a044edc"},
{file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99abb85579e2165bd8522f0c0138864da97847875ecbd45f3e7e2af569bfc6f2"},
{file = "pyrsistent-0.19.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a8cb235fa6d3fd7aae6a4f1429bbb1fec1577d978098da1252f0489937786f3"},
{file = "pyrsistent-0.19.3-cp39-cp39-win32.whl", hash = "sha256:c74bed51f9b41c48366a286395c67f4e894374306b197e62810e0fdaf2364da2"},
{file = "pyrsistent-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:878433581fc23e906d947a6814336eee031a00e6defba224234169ae3d3d6a98"},
{file = "pyrsistent-0.19.3-py3-none-any.whl", hash = "sha256:ccf0d6bd208f8111179f0c26fdf84ed7c3891982f2edaeae7422575f47e66b64"},
{file = "pyrsistent-0.19.3.tar.gz", hash = "sha256:1a2994773706bbb4995c31a97bc94f1418314923bd1048c6d964837040376440"},
]
[[package]]
name = "python-dateutil"
version = "2.8.2"
@@ -1253,7 +1177,7 @@ files = [
name = "pyyaml"
version = "6.0"
description = "YAML parser and emitter for Python"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.6"
files = [
@@ -1393,7 +1317,7 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""}
name = "regex"
version = "2023.3.23"
description = "Alternative regular expression module, to replace re."
category = "main"
category = "dev"
optional = false
python-versions = ">=3.8"
files = [
@@ -1463,7 +1387,7 @@ files = [
name = "requests"
version = "2.29.0"
description = "Python HTTP for Humans."
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -1485,7 +1409,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
name = "setuptools"
version = "67.7.2"
description = "Easily download, build, install, upgrade, and uninstall Python packages"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -1534,7 +1458,7 @@ tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"]
name = "sympy"
version = "1.11.1"
description = "Computer algebra system (CAS) in Python"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.8"
files = [
@@ -1564,7 +1488,7 @@ tests = ["pytest", "pytest-cov"]
name = "tokenizers"
version = "0.13.3"
description = "Fast and Customizable Tokenizers"
category = "main"
category = "dev"
optional = false
python-versions = "*"
files = [
@@ -1619,14 +1543,10 @@ testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests"]
name = "torch"
version = "2.0.0"
description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.8.0"
files = [
{file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"},
{file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"},
{file = "torch-2.0.0-1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9"},
{file = "torch-2.0.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203"},
{file = "torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca"},
{file = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4"},
{file = "torch-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a"},
@@ -1696,7 +1616,7 @@ files = [
name = "tqdm"
version = "4.65.0"
description = "Fast, Extensible Progress Meter"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -1733,7 +1653,7 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"]
name = "transformers"
version = "4.28.1"
description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7.0"
files = [
@@ -1801,19 +1721,10 @@ vision = ["Pillow"]
name = "triton"
version = "2.0.0"
description = "A language and compiler for custom Deep Learning operations"
category = "main"
category = "dev"
optional = false
python-versions = "*"
files = [
{file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"},
{file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"},
{file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"},
{file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"},
{file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"},
{file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"},
{file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"},
{file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"},
{file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"},
{file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"},
{file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"},
{file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"},
@@ -1839,7 +1750,7 @@ tutorials = ["matplotlib", "pandas", "tabulate"]
name = "typing-extensions"
version = "4.5.0"
description = "Backported and Experimental Type Hints for Python 3.7+"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -1851,7 +1762,7 @@ files = [
name = "urllib3"
version = "1.26.15"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
files = [
@@ -1880,7 +1791,7 @@ files = [
name = "wheel"
version = "0.40.0"
description = "A built-package format for Python"
category = "main"
category = "dev"
optional = false
python-versions = ">=3.7"
files = [
@@ -1894,4 +1805,4 @@ test = ["pytest (>=6.0.0)"]
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
content-hash = "b57850a2d5b4340321c5c7f0c5cfffd843b22d84e71c6c560743a761d91da612"
content-hash = "97f0806a8bd15b25baacb0d2a6fef273dba9f620795ad927f55c46c802306fa4"
pyproject.toml
+9 -6
View File
@@ -1,22 +1,25 @@
[tool.poetry]
name = "jsonformer"
version = "0.1.0"
version = "0.3.0"
description = ""
authors = ["1rgs <rgsduke@gmail.com>"]
readme = "README.md"
[tool.poetry.dependencies]
python = "^3.10"
transformers = "^4.27.4"
jsonschema = "^4.17.3"
torch = "^2.0.0"
accelerate = "^0.18.0"
bitsandbytes = "^0.38.1"
termcolor = "^2.3.0"
[tool.poetry.group.dev.dependencies]
ipykernel = "^6.22.0"
torch = "^2.0.0"
accelerate = "^0.18.0"
bitsandbytes = "^0.38.1"
termcolor = "^2.3.0"
transformers = "^4.27.4"
[build-system]
requires = ["poetry-core"]
readme.md
+13 -20
View File
@@ -12,6 +12,16 @@ that specifies the structure of the JSON.
Current approaches to this problem are brittle and error-prone. They rely on prompt engineering, fine-tuning, and post-processing, but they still fail to generate syntactically correct JSON in many cases.
Jsonformer is a new approach to this problem. In structured data, many tokens are fixed and predictable. Jsonformer is a wrapper around HuggingFace models that fills in the fixed tokens during the generation process, and only delegates the generation of content tokens to the language model. This makes it more efficient and bulletproof than existing approaches.
This currently supports a subset of JSON Schema. Below is a list of the supported schema types:
- number
- boolean
- string
- array
- object
## Example
```python
@@ -40,33 +50,16 @@ generated_data = jsonformer()
print(generated_data)
```
Jsonformer is a new approach to this problem. In structured data, many tokens are fixed and predictable. Jsonformer is a wrapper around HuggingFace models that fills in the fixed tokens during the generation process, and only delegates the generation of content tokens to the language model. This makes it more efficient and bulletproof than existing approaches.
This currently supports a subset of JSON Schema. Below is a list of the supported schema types:
- number
- boolean
- string
- array
- object
## Features
- Bulletproof JSON generation: Jsonformer ensures that the generated JSON is always syntactically correct and conforms to the specified schema.
- Efficiency: By generating only the content tokens and filling in the fixed tokens, Jsonformer is more efficient than generating a full JSON string and parsing it.
- Flexible and extendable: Jsonformer is built on top of the HuggingFace transformers library, making it compatible with any model that supports the HuggingFace interface.
## Usage
## Installation
To use Jsonformer, you need to provide a language model, a tokenizer, a JSON schema, and a prompt. Optionally, you can enable debugging, and set the maximum array length for generated arrays.
```python
from transformers import PreTrainedModel, PreTrainedTokenizer
from typing import Dict, Any
from jsonformer import Jsonformer
jsonformer = Jsonformer(model, tokenizer, json_schema, prompt, debug=True, max_array_length=10)
generated_data = jsonformer()
```bash
pip install jsonformer
```
## License