From 7b2c4b2b2420438d5419df39dd87a2f9cdac1f9c Mon Sep 17 00:00:00 2001 From: rahul Date: Thu, 4 May 2023 14:35:41 -0400 Subject: [PATCH] Created using Colaboratory --- Jsonformer_example.ipynb | 2018 +++----------------------------------- 1 file changed, 118 insertions(+), 1900 deletions(-) diff --git a/Jsonformer_example.ipynb b/Jsonformer_example.ipynb index 9c7cb70..d2d3f9a 100644 --- a/Jsonformer_example.ipynb +++ b/Jsonformer_example.ipynb @@ -1,11 +1,10 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": { - "colab_type": "text", - "id": "view-in-github" + "id": "view-in-github", + "colab_type": "text" }, "source": [ "\"Open" @@ -19,46 +18,48 @@ "base_uri": "https://localhost:8080/" }, "id": "p-kogdhHO3PC", - "outputId": "2b5204a9-9bd2-41ee-a0c6-1e73d9e9bd3f" + "outputId": "d41737a8-998e-4451-cce3-6ac6c98cc69b" }, "outputs": [ { - "name": "stdout", "output_type": "stream", + "name": "stdout", "text": [ "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", - "Collecting transformers\n", - " Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m47.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting jsonformer==0.7.0\n", - " Downloading jsonformer-0.7.0-py3-none-any.whl (6.0 kB)\n", - "Requirement already satisfied: termcolor<3.0.0,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from jsonformer==0.7.0) (2.3.0)\n", + "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.28.1)\n", + "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.18.0)\n", + "Requirement already satisfied: jsonformer in /usr/local/lib/python3.10/dist-packages (0.9.0)\n", + "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.14.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n", - "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n", - "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n", - " Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m59.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n", + "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n", - "Collecting huggingface-hub<1.0,>=0.11.0\n", - " Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n", + "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.13.3)\n", + "Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.0+cu118)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n", + "Requirement already satisfied: termcolor<3.0.0,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from jsonformer) (2.3.0)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (2023.4.0)\n", - "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->accelerate) (1.11.1)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->accelerate) (3.1)\n", + "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->accelerate) (2.0.0)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->accelerate) (3.1.2)\n", + "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.4.0->accelerate) (16.0.2)\n", + "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.4.0->accelerate) (3.25.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n", - "Installing collected packages: tokenizers, jsonformer, huggingface-hub, transformers\n", - "Successfully installed huggingface-hub-0.14.1 jsonformer-0.7.0 tokenizers-0.13.3 transformers-4.28.1\n" + "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n", + "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.4.0->accelerate) (2.1.2)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.4.0->accelerate) (1.3.0)\n" ] } ], "source": [ - "!pip install transformers jsonformer accelerate" + "!pip install transformers accelerate jsonformer" ] }, { @@ -66,151 +67,17 @@ "execution_count": 2, "metadata": { "colab": { - "base_uri": "https://localhost:8080/", - "height": 213, - "referenced_widgets": [ - "1418e875d6584870a0d4a7611debf932", - "e7abe3ccedf246f197160945cd26b545", - "086a6440bd9d4a3390bed968a4d4d3d6", - "02ad93e956474727ad642c9aaeb8aad8", - "f19b249748974079a996170be29105d3", - "ca421d7484db4edcac62db3237d0d67c", - "45eed3a78d584bdfa88583e5810ab164", - "ef111124d856465d9d8f01771d1e0b40", - "042b3a59f0524ee78c871d6b08308e0a", - "944a28e252bb42c5a4d2ca7a6ff99042", - "f4bef42fd47747d1a0961884377fd1f9", - "15d1160a1a0a496da2a89c9cf2e99665", - "4983e8390b66460a90e83edd04de3e7f", - "182ef356260a4c88abcba5ceb34496cb", - "9ec59d2b85304523a56672bb36285d34", - "5c37eb60432b464996eba00d416744fa", - "9913415740e5423db050290cc08aff7b", - "399adac27db848debc7c7bc501d254b4", - "fcaa8f487b6f4c86902045ac3fde6c21", - "4abc7b557ef74fa39bc94d2b907fc696", - "ddb49badd8ad4f18a25584de6db3a2b3", - "f5ed0b885ea043dbb6b99cfb1bda4507", - "273101883ae34535975eed607a790cc9", - "c6d4c42ff4bc41b880ee9f16e94b10e4", - "00230a356acc4f61aae15a908e05b78a", - "8b44e3366078474888d0c63220fa48c7", - "35a57225fb85450b9724eaeb7430f163", - "6be9f39c48f14e6da245fcb4f444be85", - "826b1f1be9db46d69cdba91d9ecc6192", - "9e2b4ea7851d4a06834b094dbbe66d68", - "e2404160c47146fab50115b8ca1767dd", - "b4712966b3f547d0aeaaab981d4749bd", - "28c9cdc32acc4ae2962688be3a15dbbd", - "4a94b28e25d24e26b64fa6505bc57b8a", - "60c8788a69134829b2f3b1b170c0a46e", - "7e9b2ac126aa4813ad750ef2187f38eb", - "9e5ee727aaaa4528a2c84ae71a68a955", - "d85c645681c640608877688387dea4b4", - "77bd5864b9f542a390de2343c04bbf34", - "193146bfbbb64f4985a172b83cf1cdae", - "79ee8cdc1bec4d76b95b3d7b9c1f5cca", - "ecee68f18c204e6ba89839312418c29b", - "e30b343221d748d5b200c9cf37d26217", - "15eb2a6ad12a4845a6ba16b993adabe3", - "39c845eb44494bafb6831988f8f175cc", - "af45e3b7ab2a46188adb40cf36a5006b", - "14e1309603cc49bfb8a77d9a1227b682", - "74cb4236ca5f420fb133ea3941280b7d", - "3854cc6cbbe94913b7eb071c37b5de16", - "86f51c48d0ac4293bc2cef377d90f712", - "762ea404c1264e88bb6a1bd5fd3df083", - "5e606fa6538745cdafa1fdcffa860b0e", - "8ec28c348727451793423fbb8e460d67", - "855b92bde94f4b838a0ef4f88e27b8b5", - "137c0112348646b19ab2522486aea6e8" - ] + "base_uri": "https://localhost:8080/" }, "id": "9y0Jjj0sOtor", - "outputId": "545d45d5-45ad-4791-fde7-e76ad4546ee8" + "outputId": "6c5ea465-384f-4aa1-8304-6fa721534f73" }, "outputs": [ { - "name": "stdout", "output_type": "stream", - "text": [ - "Loading model and tokenizer...\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1418e875d6584870a0d4a7611debf932", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Downloading (…)lve/main/config.json: 0%| | 0.00/819 [00:00