mirror of
https://github.com/wassname/prob_jsonformer.git
synced 2026-09-09 11:29:57 +08:00
fix, update readme and example
This commit is contained in:
wassname
+130
-145
@@ -111,12 +111,36 @@
|
||||
" name: \u001b[32m\"Snowboard X-15\"\u001b[0m,\n",
|
||||
" description: \u001b[32m\"Snowboard for all levels\"\u001b[0m,\n",
|
||||
" category: \u001b[32m\"Snowboards\"\u001b[0m,\n",
|
||||
" price: \u001b[32m20.0375\u001b[0m,\n",
|
||||
" price: \u001b[32m20.0\u001b[0m,\n",
|
||||
" inStock: \u001b[32mTrue\u001b[0m,\n",
|
||||
" rating: \u001b[32m5.0\u001b[0m,\n",
|
||||
" images: [\n",
|
||||
" \u001b[32m\"https://s3.amazonaws.com/mikesskisport/images/Snow\"\u001b[0m\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" productId: \u001b[32m\"2\"\u001b[0m,\n",
|
||||
" name: \u001b[32m\"Mike's Ski Shop Exclusive\"\u001b[0m,\n",
|
||||
" description: \u001b[32m\"Mike's Ski Shop Exclusive\"\u001b[0m,\n",
|
||||
" category: \u001b[32m\"Ski Shops\"\u001b[0m,\n",
|
||||
" price: \u001b[32m20.0\u001b[0m,\n",
|
||||
" inStock: \u001b[32mTrue\u001b[0m,\n",
|
||||
" rating: \u001b[32m5.0\u001b[0m,\n",
|
||||
" images: [\n",
|
||||
" \u001b[32m\"https://s3.amazonaws.com/mikesskisport/images/Mike\"\u001b[0m\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" productId: \u001b[32m\"3\"\u001b[0m,\n",
|
||||
" name: \u001b[32m\"Mike's Ski Shop Exclusive\"\u001b[0m,\n",
|
||||
" description: \u001b[32m\"Mike's Ski Shop Exclusive\"\u001b[0m,\n",
|
||||
" category: \u001b[32m\"Ski Shops\"\u001b[0m,\n",
|
||||
" price: \u001b[32m20.0\u001b[0m,\n",
|
||||
" inStock: \u001b[32mTrue\u001b[0m,\n",
|
||||
" rating: \u001b[32m5.0\u001b[0m,\n",
|
||||
" images: [\n",
|
||||
" \u001b[32m\"https://s3.amazonaws.com/mikesskisport/images/Mike\"\u001b[0m\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
@@ -175,62 +199,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Generating...\n",
|
||||
"{\n",
|
||||
" make: \u001b[32m\"Mazda\"\u001b[0m,\n",
|
||||
" model: [\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.8154296875\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"Kea\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.184814453125\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"Mazda\"\u001b[0m\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" new: [\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.90185546875\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"true\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.09808349609375\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"false\"\u001b[0m\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" rating: [\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.221435546875\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"1\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.394775390625\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"2\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.382568359375\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"3\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.0013370513916015625\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"4\"\u001b[0m\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" year: \u001b[32m2016.0\u001b[0m,\n",
|
||||
" colors_available: [\n",
|
||||
" \u001b[32m\"red\"\u001b[0m\n",
|
||||
" ]\n",
|
||||
"}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"car = {\n",
|
||||
" \"type\": \"object\",\n",
|
||||
@@ -262,96 +233,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Generating...\n",
|
||||
"{\n",
|
||||
" car: {\n",
|
||||
" make: \u001b[32m\"Rolls Royce\"\u001b[0m,\n",
|
||||
" model: \u001b[32m\"Phantom\"\u001b[0m,\n",
|
||||
" year: \u001b[32m2014.0\u001b[0m,\n",
|
||||
" colors: [\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.001560211181640625\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"white\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.833984375\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"red\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.0865478515625\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"black\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.048553466796875\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"blue\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.0294342041015625\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"green\"\u001b[0m\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" as_new: [\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.96533203125\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"true\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.03460693359375\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"false\"\u001b[0m\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" rating: [\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.05462646484375\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"1\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.233642578125\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"2\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.2093505859375\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"3\"\u001b[0m\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" prob: \u001b[32m0.50244140625\u001b[0m,\n",
|
||||
" choice: \u001b[32m\"4\"\u001b[0m\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" features: {\n",
|
||||
" audio: {\n",
|
||||
" brand: \u001b[32m\"Mercedes-Benz\"\u001b[0m,\n",
|
||||
" speakers: \u001b[32m2.09999\u001b[0m,\n",
|
||||
" hasBluetooth: \u001b[32mTrue\u001b[0m\n",
|
||||
" },\n",
|
||||
" safety: {\n",
|
||||
" airbags: \u001b[32m2.09999\u001b[0m,\n",
|
||||
" parkingSensors: \u001b[32mTrue\u001b[0m,\n",
|
||||
" laneAssist: \u001b[32mTrue\u001b[0m\n",
|
||||
" },\n",
|
||||
" performance: {\n",
|
||||
" engine: \u001b[32m\"Mercedes-Benz 6.2 L\"\u001b[0m,\n",
|
||||
" horsepower: \u001b[32m423.09999\u001b[0m,\n",
|
||||
" topSpeed: \u001b[32m220.09999\u001b[0m\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" },\n",
|
||||
" owner: {\n",
|
||||
" firstName: \u001b[32m\"John\"\u001b[0m,\n",
|
||||
" lastName: \u001b[32m\"Doe\"\u001b[0m,\n",
|
||||
" age: \u001b[32m40.09999\u001b[0m\n",
|
||||
" }\n",
|
||||
"}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"complex_car = {\n",
|
||||
" \"type\": \"object\",\n",
|
||||
@@ -418,6 +302,107 @@
|
||||
"\n",
|
||||
"highlight_values(output)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Readme example"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'name': 'John Doe',\n",
|
||||
" 'age': 20,\n",
|
||||
" 'age_probs': [{'prob': 0.794921875, 'choice': '12'},\n",
|
||||
" {'prob': 0.068359375, 'choice': '10'},\n",
|
||||
" {'prob': 0.04345703125, 'choice': '16'},\n",
|
||||
" {'prob': 0.03228759765625, 'choice': '14'},\n",
|
||||
" {'prob': 0.0175628662109375, 'choice': '11'},\n",
|
||||
" {'prob': 0.0157318115234375, 'choice': '15'},\n",
|
||||
" {'prob': 0.013671875, 'choice': '20'},\n",
|
||||
" {'prob': 0.006664276123046875, 'choice': '18'},\n",
|
||||
" {'prob': 0.0046539306640625, 'choice': '13'},\n",
|
||||
" {'prob': 0.0018215179443359375, 'choice': '21'},\n",
|
||||
" {'prob': 0.00041294097900390625, 'choice': '17'},\n",
|
||||
" {'prob': 0.00028824806213378906, 'choice': '19'},\n",
|
||||
" {'prob': 0.00014495849609375, 'choice': '22'},\n",
|
||||
" {'prob': 6.955862045288086e-05, 'choice': '23'},\n",
|
||||
" {'prob': 2.968311309814453e-05, 'choice': '25'},\n",
|
||||
" {'prob': 2.8789043426513672e-05, 'choice': '26'},\n",
|
||||
" {'prob': 1.901388168334961e-05, 'choice': '24'},\n",
|
||||
" {'prob': 1.1742115020751953e-05, 'choice': '28'},\n",
|
||||
" {'prob': 1.1920928955078125e-06, 'choice': '27'},\n",
|
||||
" {'prob': 7.748603820800781e-07, 'choice': '29'}],\n",
|
||||
" 'unit_time': 0.01,\n",
|
||||
" 'is_student': True,\n",
|
||||
" 'is_student_probs': [{'prob': 0.8310546875, 'choice': 'true'},\n",
|
||||
" {'prob': 0.1688232421875, 'choice': 'false'}],\n",
|
||||
" 'courses': ['C1'],\n",
|
||||
" 'trim': None,\n",
|
||||
" 'color': 'white'}"
|
||||
]
|
||||
},
|
||||
"execution_count": 22,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from prob_jsonformer import Jsonformer\n",
|
||||
"from transformers import AutoModelForCausalLM, AutoTokenizer\n",
|
||||
"\n",
|
||||
"model_name = \"databricks/dolly-v2-3b\"\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(model_name)\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
|
||||
"\n",
|
||||
"json_schema = {\n",
|
||||
" \"type\": \"object\",\n",
|
||||
" \"properties\": {\n",
|
||||
" \"name\": {\"type\": \"string\"},\n",
|
||||
" \"age\": {\"type\": \"integer\"},\n",
|
||||
" \"age_probs\": {\"type\": \"choice_probs\", \"enum\": [str(s) for s in range(10, 30)]},\n",
|
||||
" \"unit_time\": {\"type\": \"number\"},\n",
|
||||
" \"is_student\": {\"type\": \"boolean\"},\n",
|
||||
" \"is_student_probs\": {\"type\": \"choice_probs\", \"enum\": [\"true\", \"false\"]},\n",
|
||||
" \"courses\": {\n",
|
||||
" \"type\": \"array\",\n",
|
||||
" \"items\": {\"type\": \"string\"}\n",
|
||||
" },\n",
|
||||
" \"trim\": {\"type\": [\"string\", \"null\"]},\n",
|
||||
" \"color\": {\n",
|
||||
" \"type\": \"enum\",\n",
|
||||
" \"values\": [\"red\", \"green\", \"blue\", \"brown\", \"white\", \"black\"],\n",
|
||||
" },\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"prompt = \"Generate a young person's information based on the following schema:\"\n",
|
||||
"jsonformer = Jsonformer(model, tokenizer, json_schema, prompt, temperature=0)\n",
|
||||
"generated_data = jsonformer()\n",
|
||||
"\n",
|
||||
"generated_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
Reference in New Issue
Block a user