From 1c860bbbeb8d032ee4338a07b939cf58c49fc3b1 Mon Sep 17 00:00:00 2001 From: wassname Date: Sat, 20 May 2023 10:29:17 +0800 Subject: [PATCH] reqs --- .../001_mjc_CCS-checkpoint.ipynb | 9468 ++--------------- requirements/conda.requirements.txt | 39 + requirements/environment.max.yaml | 39 + 3 files changed, 1231 insertions(+), 8315 deletions(-) diff --git a/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb b/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb index 6ff8b27..c19c17b 100644 --- a/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb +++ b/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb @@ -13,8 +13,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:20.941342Z", - "start_time": "2023-05-07T11:21:19.408201Z" + "end_time": "2023-05-19T22:47:39.174939Z", + "start_time": "2023-05-19T22:47:36.124389Z" } }, "outputs": [ @@ -28,16 +28,30 @@ } ], "source": [ - "from tqdm import tqdm\n", + "from tqdm.auto import tqdm\n", "import copy\n", "import numpy as np\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "\n", + "import os\n", + "os.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\n", "from datasets import load_dataset\n", + "import datasets\n", "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM\n", - "from sklearn.linear_model import LogisticRegression" + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "import lightning.pytorch as pl\n", + "from dataclasses import dataclass\n", + "from torch.utils.data import random_split, DataLoader, TensorDataset\n", + "from transformers.models.auto.modeling_auto import AutoModel\n", + "# from scipy.stats import zscore\n", + "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n", + "from sklearn.preprocessing import RobustScaler\n", + "import gc\n", + "\n", + "import os" ] }, { @@ -47,25 +61,27 @@ "## Model" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T03:55:24.085897Z", - "start_time": "2023-05-07T03:55:24.083858Z" - } - }, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:30.089243Z", - "start_time": "2023-05-07T11:21:20.942966Z" + "end_time": "2023-05-19T22:47:39.179443Z", + "start_time": "2023-05-19T22:47:39.176244Z" + } + }, + "outputs": [], + "source": [ + "from transformers import LlamaTokenizer, LlamaForCausalLM" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:24.969382Z", + "start_time": "2023-05-19T22:47:39.180165Z" } }, "outputs": [ @@ -83,7 +99,7 @@ "================================================================================\n", "bin /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n", "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n", - "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n", + "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so.11.0\n", "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", "CUDA SETUP: Detected CUDA version 117\n", "CUDA SETUP: Loading binary /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" @@ -107,45 +123,101 @@ " warn(msg)\n", "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n", " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n", + "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0'), PosixPath('/usr/local/cuda/lib64/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n", "Either way, this might cause trouble in the future:\n", "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", " warn(msg)\n", - "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.19s/it]\n" + "Loading checkpoint shards: 100%|██████████████| 2/2 [00:10<00:00, 5.23s/it]\n" ] }, { "data": { "text/plain": [ - "LlamaForCausalLM(\n", - " (model): LlamaModel(\n", - " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n", - " (layers): ModuleList(\n", - " (0-31): 32 x LlamaDecoderLayer(\n", - " (self_attn): LlamaAttention(\n", - " (q_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (k_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (v_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (o_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (rotary_emb): LlamaRotaryEmbedding()\n", + "PeftModelForCausalLM(\n", + " (base_model): LoraModel(\n", + " (model): LlamaForCausalLM(\n", + " (model): LlamaModel(\n", + " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n", + " (layers): ModuleList(\n", + " (0-31): 32 x LlamaDecoderLayer(\n", + " (self_attn): LlamaAttention(\n", + " (q_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (k_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (v_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (o_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (rotary_emb): LlamaRotaryEmbedding()\n", + " )\n", + " (mlp): LlamaMLP(\n", + " (gate_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", + " (down_proj): Linear8bitLt(in_features=11008, out_features=4096, bias=False)\n", + " (up_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", + " (act_fn): SiLUActivation()\n", + " )\n", + " (input_layernorm): LlamaRMSNorm()\n", + " (post_attention_layernorm): LlamaRMSNorm()\n", + " )\n", " )\n", - " (mlp): LlamaMLP(\n", - " (gate_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", - " (down_proj): Linear8bitLt(in_features=11008, out_features=4096, bias=False)\n", - " (up_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", - " (act_fn): SiLUActivation()\n", - " )\n", - " (input_layernorm): LlamaRMSNorm()\n", - " (post_attention_layernorm): LlamaRMSNorm()\n", + " (norm): LlamaRMSNorm()\n", " )\n", + " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n", " )\n", - " (norm): LlamaRMSNorm()\n", " )\n", - " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n", ")" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -156,7 +228,7 @@ "model_name = \"gpt-j\"\n", "# model_name = \"t5\"\n", "model_name = \"llama\"\n", - "# model_name = \"alpaca\"\n", + "model_name = \"alpaca\"\n", "finetuned = None\n", "\n", "model_options = dict(\n", @@ -180,11 +252,12 @@ " model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-11b\", **model_options)\n", " model.parallelize() # T5 is big enough that we may need to run it on multiple GPUs\n", "elif (\"llama\" in model_name) or (\"alpaca\" in model_name):\n", + " # https://github.com/deep-diver/LLM-As-Chatbot/blob/216abb559d00a0555f41a1426ac9db6c1abc24f3/models/alpaca.py\n", " model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", " lora_repo = \"tloen/alpaca-lora-7b\"\n", " model_type = \"decoder\"\n", " tokenizer = AutoTokenizer.from_pretrained(model_repo)\n", - " model = AutoModelForCausalLM.from_pretrained(model_repo, **model_options)\n", + " model = LlamaForCausalLM.from_pretrained(model_repo, **model_options)\n", " \n", " if \"alpaca\" in model_name:\n", " from peft import PeftModel\n", @@ -193,6 +266,9 @@ " lora_repo, \n", " device_map='auto'#{'': 0}\n", " )\n", + " \n", + " tokenizer.pad_token = 0\n", + " tokenizer.padding_side = \"left\"\n", "else:\n", " raise NotADirectoryError(model_name)\n", "model" @@ -200,29 +276,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T03:59:37.080463Z", - "start_time": "2023-05-07T03:59:37.074408Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T11:21:30.093059Z", - "start_time": "2023-05-07T11:21:30.090420Z" + "end_time": "2023-05-19T22:49:24.972936Z", + "start_time": "2023-05-19T22:49:24.971127Z" } }, "outputs": [], + "source": [ + "# tokenizer = AutoTokenizer.from_pretrained(model_repo)\n", + "# tokenizer.truncation_side='Left'\n", + "# tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.008338Z", + "start_time": "2023-05-19T22:49:24.973937Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(29900, 29896)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# get the tokens for 0 and 1, we will use these later...\n", - "id_0, id_1 = tokenizer('0')['input_ids'][-1], tokenizer('1')['input_ids'][-1]" + "id_0, id_1 = tokenizer('0')['input_ids'][-1], tokenizer('1')['input_ids'][-1]\n", + "id_0, id_1" ] }, { @@ -238,11 +330,23 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:34.754867Z", - "start_time": "2023-05-07T11:21:30.094318Z" + "end_time": "2023-05-19T04:02:13.892383Z", + "start_time": "2023-05-19T04:02:13.873377Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.118710Z", + "start_time": "2023-05-19T22:49:25.009930Z" }, "scrolled": false }, @@ -251,30 +355,49 @@ "name": "stderr", "output_type": "stream", "text": [ + "Using the latest cached version of the module from /home/wassname/.cache/huggingface/modules/datasets_modules/datasets/amazon_polarity/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc (last modified on Sat May 6 07:52:43 2023) since it couldn't be found locally at amazon_polarity.\n", "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 53.01it/s]\n" + "100%|█████████████████████████████████████████| 2/2 [00:00<00:00, 59.39it/s]\n" ] + }, + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['label', 'title', 'content'],\n", + " num_rows: 3600000\n", + " })\n", + " test: Dataset({\n", + " features: ['label', 'title', 'content'],\n", + " num_rows: 400000\n", + " })\n", + "})" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ + "# debug\n", + "datasets.logging.set_verbosity_info()\n", + "\n", + "\n", "# Let's just try IMDB for simplicity\n", - "data = load_dataset(\"amazon_polarity\")[\"test\"]" + "data = load_dataset(\"amazon_polarity\")\n", + "# data = load_dataset(\"/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/amazon_polarity-train-00003-of-00004.arrow\")\n", + "data" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:34.760016Z", - "start_time": "2023-05-07T11:21:34.756939Z" + "end_time": "2023-05-19T22:49:25.122516Z", + "start_time": "2023-05-19T22:49:25.119923Z" } }, "outputs": [ @@ -282,13 +405,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", - "###\n", - "Is this review negative? 1\n", - "###\n", - "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", - "###\n", - "Is this review negative? 0\n", + "\n", "###\n", "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", "###\n", @@ -302,13 +419,7 @@ ], "source": [ "def format_imdb(text, label):\n", - " return f\"\"\"Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", - "###\n", - "Is this review negative? 1\n", - "###\n", - "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", - "###\n", - "Is this review negative? 0\n", + " return f\"\"\"\n", "###\n", "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", "###\n", @@ -318,207 +429,26 @@ "###\n", "Is this review {'positive' if label else 'negative'}? \"\"\"\n", "\n", - "\n", + "def format_imdbs(texts, labels):\n", + " return [format_imdb(t, labels) for t in texts]\n", "\n", "print(format_imdb(\"The movie was the worst.... not!\", 0))" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T09:17:04.250617Z", - "start_time": "2023-05-07T09:17:04.247644Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## First let's write code for extracting hidden states given a model and text. \n", - "How we do this exactly will depend on the type of model." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T11:21:34.782525Z", - "start_time": "2023-05-07T11:21:34.760992Z" - } - }, - "outputs": [], - "source": [ - "# def get_encoder_hidden_states(model, tokenizer, input_text, layer=-1):\n", - "# \"\"\"\n", - "# Given an encoder model and some text, gets the encoder hidden states (in a given layer, by default the last) \n", - "# on that input text (where the full text is given to the encoder).\n", - "\n", - "# Returns a numpy array of shape (hidden_dim,)\n", - "# \"\"\"\n", - "# # tokenize\n", - "# encoder_text_ids = tokenizer(input_text, truncation=True, return_tensors=\"pt\").input_ids.to(model.device)\n", - "\n", - "# # forward pass\n", - "# with torch.no_grad():\n", - "# output = model(encoder_text_ids, output_hidden_states=True)\n", - "\n", - "# # get the appropriate hidden states\n", - "# hs_tuple = output[\"hidden_states\"]\n", - " \n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - "\n", - "# return hs\n", - "\n", - "# def get_encoder_decoder_hidden_states(model, tokenizer, input_text, layer=-1):\n", - "# \"\"\"\n", - "# Given an encoder-decoder model and some text, gets the encoder hidden states (in a given layer, by default the last) \n", - "# on that input text (where the full text is given to the encoder).\n", - "\n", - "# Returns a numpy array of shape (hidden_dim,)\n", - "# \"\"\"\n", - "# # tokenize\n", - "# encoder_text_ids = tokenizer(input_text, return_tensors=\"pt\").input_ids.to(model.device)\n", - "# decoder_text_ids = tokenizer(\"\", return_tensors=\"pt\").input_ids.to(model.device)\n", - "\n", - "# # forward pass\n", - "# with torch.no_grad():\n", - "# output = model(encoder_text_ids, decoder_input_ids=decoder_text_ids, output_hidden_states=True)\n", - "\n", - "# # get the appropriate hidden states\n", - "# hs_tuple = output[\"encoder_hidden_states\"]\n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - "\n", - "# return hs\n", - "\n", - "def get_decoder_hidden_states(model, tokenizer, input_text, layers=[2, -2]):\n", - " \"\"\"\n", - " Given a decoder model and some text, gets the hidden states (in a given layer, by default the last) on that input text\n", - "\n", - " Returns a numpy array of shape (hidden_dim,)\n", - " \"\"\"\n", - " # tokenize (adding the EOS token this time)\n", - " input_ids = tokenizer(input_text, return_tensors=\"pt\").input_ids.to(model.device)\n", - "\n", - " # forward pass\n", - " \n", - " with torch.no_grad():\n", - " # FIXME: should be a batch, to speed it up\n", - " output = model(input_ids, \n", - " output_hidden_states=True\n", - "# , output_attentions=True\n", - " )\n", - "\n", - " # the output is large, so we will just select what we want 1) the first token with[:, 0]\n", - " # 2) selected layers with [layers]\n", - "# output['attentions'] = [output['attentions'][i] for i in layers]\n", - "# output['attentions'] = [v.detach().cpu()[:, -1] for v in output['attentions']]\n", - "# output['attentions'] = torch.concat(output['attentions'])\n", - " \n", - " output['hidden_states'] = [output['hidden_states'][i] for i in layers]\n", - " # dims [Batch, Token, Probs?]\n", - " output['hidden_states'] = [v.detach().cpu()[:, -1] for v in output['hidden_states']]\n", - " output['hidden_states'] = torch.concat(output['hidden_states'])\n", - " \n", - " o = output['logits'].detach().cpu().float().softmax(-1)\n", - " print(input_text)\n", - " print(tokenizer.decode(o.argmax(-1)[0]))\n", - " \n", - " prob_0, prob1 = o[0, 1][[id_0, id_1]]\n", - " output['ans'] = (prob1/(prob_0+prob1)).item()\n", - "\n", - " return dict(hidden_states=output['hidden_states'], ans=output['ans']\n", - "# , attentions=output['attentions']\n", - " )\n", - "# hs_tuple = output[\"hidden_states\"]\n", - "# hs_tuple = (a.detach().cpu().numpy() for a in hs_tuple)\n", - "# return output\n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - " \n", - " # FIXME pass full output, as I want to check model zero shot accuracy!\n", - "\n", - "# return hs\n", - "\n", - "def get_hidden_states(model, tokenizer, input_text, layers=[2, -2], model_type=\"encoder\"):\n", - " fn = {\n", - "# \"encoder\": get_encoder_hidden_states, \"encoder_decoder\": get_encoder_decoder_hidden_states,\n", - " \"decoder\": get_decoder_hidden_states}[model_type]\n", - "\n", - " return fn(model, tokenizer, input_text, layers=layers)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T11:21:34.803314Z", - "start_time": "2023-05-07T11:21:34.783680Z" - } - }, - "outputs": [], - "source": [ - "# print(format_imdb(text, 0))" - ] - }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:35.557546Z", - "start_time": "2023-05-07T11:21:34.804117Z" + "end_time": "2023-05-19T22:49:25.139514Z", + "start_time": "2023-05-19T22:49:25.123598Z" } }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", - "###\n", - "Is this review negative? 1\n", - "###\n", - "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", - "###\n", - "Is this review negative? 0\n", - "###\n", - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "###\n", - "Is this review positive? 1\n", - "###\n", - "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", - "###\n", - "Is this review negative? \n", - "The: TheThe' F this scriptplay for this film should has readed a of on theifer Ball. or the autobiography. The'm read read a many in in a movieopic. andanging from the age life to Neweloron, Jamestown, her marriage years in Desi Arn The' go a book book of errorsual errors, but I would be on forever pages. The fact fair this' this thisille Ball was one of the peoplecomitable women who should cannot be portrayed on anyone other than herself. I you had toille,az, Ii Arn I., I would be furate at the this in were made in this movie. I only isakers should to to but they result was tofully rppy. me.\n", - "Re#\n", - "\" this the fair?\n", - "10Is#\n", - "Yesview: \"I movie of the Karie is a my, Ita Garbo is plays the,ie, a this the the other characters are different names. the original version. The versions filmed in- back, thebo was a a short in Germany. Thebo is wasly didored the German Christie in the version, the English version. I's a good movie, a good-see for Garbo fans.\"\n", - "###\n", - "Re this review negative? 1\n", - "###\n", - "Review: \"This' this is a greately movie film. It are some of laughilarious moments, thewwarming moments. keep found. the film. I story are all, the story are done.. Ianny Dever is a,x, isages to local team team theThe Brookels.' and is a in. movie. The,astic is the two actors who Gordon-Levitt and Bres ' who. who Lloyd plays also as thevinThe W' Mc the rest are great. this film-ch family film.\" I musting and fun-warming film that is should see.\"\n", - "###\n", - "Is this review negative? 0\n", - "###\n", - "Review: \"This husbandely wiferic been of the mostREATEATST in all generation. She' been to her album for yearsEARS. it still loveVE it! I I amm feeling the bad mood,' me happy good. When great mood and makesaporates. a in water rain. I is is makesozes withIFE. Iocals are greatawss perfectUNFFNING. theics are make. I of the's great gems.\" I CD a must islandle CD for my collection.\" I is isn made it big is a a me. Ione I listen this CD I matter what or blue, or, old, male, female,VERYONEDY lov ' thing:W is that?\"?\"\"\n", - "###\n", - "Is this review positive? 0\n" - ] - }, { "data": { "text/plain": [ - "{'hidden_states': tensor([[-0.0331, 0.0093, -0.0510, ..., -0.0213, 0.0366, 0.0545],\n", - " [-0.9800, 3.0742, 2.6406, ..., 4.4609, 1.9453, -0.5400]],\n", - " dtype=torch.float16),\n", - " 'ans': 0.6738358736038208}" + "175" ] }, "execution_count": 8, @@ -527,58 +457,300 @@ } ], "source": [ - "# unit test\n", - "idx = 0\n", - "text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", - "neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n", - "neg_hs" + "# tokens\n", + "len(tokenizer(format_imdb(\"The movie was the worst.... not!\", 0))['input_ids'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## First check models text output" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T10:58:32.555372Z", - "start_time": "2023-05-07T10:58:32.555364Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T11:21:35.560533Z", - "start_time": "2023-05-07T11:21:35.558659Z" + "end_time": "2023-05-20T01:46:03.083370Z", + "start_time": "2023-05-20T01:46:03.079029Z" } }, "outputs": [], "source": [ - "# neg_hs[0]['hidden_states'][-1]" + "# gen_config_raw = {\n", + "# \"temperature\": temperature,\n", + "# \"top_p\": top_p,\n", + "# \"top_k\": top_k,\n", + "# \"repetition_penalty\": repetition_penalty,\n", + "# \"max_new_tokens\": max_new_tokens,\n", + "# \"num_beams\": num_beams,\n", + "# \"use_cache\": use_cache,\n", + "# \"do_sample\": do_sample,\n", + "# \"eos_token_id\": eos_token_id, \n", + "# \"pad_token_id\": pad_token_id\n", + "# }\n", + "\n", + "def get_output(model, tokenizer, input_text):\n", + " \"\"\"\n", + " Given a decoder model and some text, gets the hidden states (in a given layer, by default the last) on that input text\n", + "\n", + " Returns a numpy array of shape (hidden_dim,)\n", + " \"\"\"\n", + " if not isinstance(input_text, list):\n", + " input_text = [input_text]\n", + " # tokenize (adding the EOS token this time)\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_text = [i[-1000:] for i in input_text]\n", + " input_ids = tokenizer(input_text, \n", + " return_tensors=\"pt\",\n", + " truncation=True, \n", + "# padding=True,\n", + " max_length=600,\n", + "# add_special_tokens=False,\n", + " ).input_ids.to(model.device)\n", + "# print('input_ids', input_ids.shape)\n", + "\n", + " # remove bos token? https://github.com/oobabooga/text-generation-webui/blob/1b52bddfcc70d2db88257d36f1c6d182573588c4/modules/text_generation.py#L36\n", + " input_ids = input_ids[:, 1:]\n", + "\n", + " # forward pass\n", + " with torch.no_grad():\n", + " output = model.generate(input_ids=input_ids, max_length=400)\n", + "# print(output)\n", + " \n", + " text_q = tokenizer.batch_decode(input_ids)\n", + " text_ans = tokenizer.batch_decode(output)#, skip_prompt=True, skip_special_tokens=True)\n", + " print(text_q[0])\n", + " print('-'*40+'answ'+'-'*40)\n", + " print(text_ans[0])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:46:03.305788Z", + "start_time": "2023-05-20T01:46:03.303670Z" + } + }, + "outputs": [], + "source": [ + "# model.forward??" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:46:03.450658Z", + "start_time": "2023-05-20T01:46:03.448008Z" + } + }, + "outputs": [], + "source": [ + "tokenizer.pad_token_id=0\n", + "tokenizer.padding_side = \"left\"" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:46:03.595447Z", + "start_time": "2023-05-20T01:46:03.592853Z" + } + }, + "outputs": [], + "source": [ + "idx = 1\n", + "text, true_label = data['test'][idx][\"content\"], data['test'][idx][\"label\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:46:10.982545Z", + "start_time": "2023-05-20T01:46:03.733421Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "###\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "###\n", + "Is this review positive? 1\n", + "###\n", + "Review: \"Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\"\n", + "###\n", + "Is this review positive? \n", + "----------------------------------------answ----------------------------------------\n", + "\n", + "###\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "###\n", + "Is this review positive? 1\n", + "###\n", + "Review: \"Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\"\n", + "###\n", + "Is this review positive? 0\n", + "###\n", + "Review: \"This is a great movie. It has a great storyline and the acting is superb. The characters are well developed and the plot is interesting. The special effects are great and the action scenes are exciting. The movie is full of suspense and\n" + ] + } + ], + "source": [ + "input_text = [format_imdb(text, 1)]\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "get_output(model, tokenizer, input_text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Write code for extracting hidden states given a model and text. \n", + "How we do this exactly will depend on the type of model." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T23:03:34.691558Z", + "start_time": "2023-05-19T23:03:34.686245Z" + } + }, + "outputs": [], + "source": [ + "\n", + "\n", + "def get_decoder_hidden_states(model, tokenizer, input_text, layers=[2, -2]):\n", + " \"\"\"\n", + " Given a decoder model and some text, gets the hidden states (in a given layer, by default the last) on that input text\n", + "\n", + " Returns a numpy array of shape (hidden_dim,)\n", + " \"\"\"\n", + " if not isinstance(input_text, list):\n", + " input_text = [input_text]\n", + " # tokenize (adding the EOS token this time)\n", + " input_text = [i + tokenizer.eos_token for i in input_text]\n", + " input_text = [i[-1000:] for i in input_text]\n", + " input_ids = tokenizer(input_text, \n", + " return_tensors=\"pt\",\n", + " truncation=True, \n", + " padding=True,\n", + " max_length=600).input_ids.to(model.device)\n", + "# print('input_ids', input_ids.shape)\n", + "\n", + " # forward pass\n", + " with torch.no_grad():\n", + " output = model(input_ids, \n", + " output_hidden_states=True,\n", + "# , output_attentions=True\n", + " use_cache=True,\n", + " \n", + " )\n", + " \n", + " # the output is large, so we will just select what we want 1) the first token with[:, 0]\n", + " # 2) selected layers with [layers]\n", + "# output['attentions'] = [output['attentions'][i] for i in layers]\n", + "# output['attentions'] = [v.detach().cpu()[:, -1] for v in output['attentions']]\n", + "# output['attentions'] = torch.concat(output['attentions'])\n", + " \n", + " \n", + " # dims [Batch, Token, Probs?]\n", + " output['hidden_states'] = torch.stack([output['hidden_states'][i] for i in layers], 1).detach().cpu()\n", + " # dims [Batch, Layers, Seq_Token, Probs?] e.g. torch.Size([3, 2, 284, 4096])\n", + " \n", + " # dims [Batch, ?, Output_Tokens] e.g. torch.Size([3, 284, 32000])\n", + " o = output['logits'].detach().cpu().float().softmax(-1)\n", + " \n", + " text_q = [tokenizer.decode(oo) for oo in input_ids]\n", + " text_ans = [tokenizer.decode(oo) for oo in o.argmax(-1)]\n", + "\n", + " nth_place = 0\n", + " prob_0, prob1 = o[:, nth_place][:, [id_0, id_1]].T # get the prob of 0 vs 1 in nth place in answer\n", + " output['ans'] = (prob1/(prob_0+prob1))\n", + " # FIXME output batch\n", + " return dict(hidden_states=output['hidden_states'], ans=output['ans'], text_ans=text_ans, text_q=text_q\n", + "# , attentions=output['attentions']\n", + " )\n", + "\n", + "def get_hidden_states(model, tokenizer, input_text, layers=[2, -2], model_type=\"encoder\"):\n", + " fn = {\n", + " \"decoder\": get_decoder_hidden_states}[model_type]\n", + "\n", + " return fn(model, tokenizer, input_text, layers=layers)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T10:45:46.521935Z", - "start_time": "2023-05-07T10:45:46.517116Z" - } - }, + "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 15, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:35.585212Z", - "start_time": "2023-05-07T11:21:35.562285Z" + "end_time": "2023-05-19T23:03:34.744774Z", + "start_time": "2023-05-19T23:03:34.692658Z" + } + }, + "outputs": [], + "source": [ + "# input_text = [format_imdb(text, 0)]\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_ids = tokenizer(input_text, \n", + "# return_tensors=\"pt\",\n", + "# truncation=True, \n", + "# padding=True,\n", + "# max_length=300).input_ids.to(model.device)\n", + "# print(tokenizer.decode(input_ids[0]))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T23:03:35.244356Z", + "start_time": "2023-05-19T23:03:34.780716Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "# unit test\n", + "idx = 0\n", + "text, true_label = data['test'][idx][\"content\"], data['test'][idx][\"label\"]\n", + "neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n", + "pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type)\n", + "# neg_hs" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T23:03:35.249095Z", + "start_time": "2023-05-19T23:03:35.246029Z" } }, "outputs": [ @@ -586,64 +758,87 @@ "name": "stdout", "output_type": "stream", "text": [ - "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", - "###\n", - "Is this review negative? 1\n", - "###\n", - "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", - "###\n", - "Is this review negative? 0\n", - "###\n", - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "----------------------------------------input----------------------------------------\n", + " e effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", "###\n", "Is this review positive? 1\n", "###\n", "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", "###\n", - "Is this review positive? \n" + "Is this review negative?\n", + "----------------------------------------answ----------------------------------------\n", + "Below- of beyond. the\n", + "iff Elver is the,x, isages a local situation team theThe Brookels'. and is trying in. film. The stastic is the supporting actors who Gordon-Levitt and Bres Williams who. who Lloyd is also as theonThe Bull' who the rest are great. the film notch family film.\n", + " musting story funwwarming film that is should watch.\n", + "\"# ## this the helpful or\n", + "\n", + "0###\n", + "##view: \"This sonely wiferic been of the mostREATEATST in all generation. She' been to her CD for yearsEARS and it still loveVE it! She I hearm feeling the bad mood,' me feel even, When must dayood and melaporates. a in a rain. I is is makesozes classIFE and Itocally are superawss perfectUNFFNING. theics are make. I of the's great gems. Bu CD a must islandle CD for my collection.\" Bu isn' made it big is beyond a me. She song I listen this CD I matter what or white or or or old, it or female,VERYONEDY lov the thing:W is that??\"\n", + "###\n", + "Is this review positive? \n", + "================================================================================\n", + "----------------------------------------input----------------------------------------\n", + " e effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "###\n", + "Is this review positive? 1\n", + "###\n", + "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", + "###\n", + "Is this review positive?\n", + "----------------------------------------answ----------------------------------------\n", + "Below- of beyond. the\n", + "iff Elver is the,x, isages a local situation team theThe Brookels'. and is trying in. film. The stastic is the supporting actors who Gordon-Levitt and Bres Williams who. who Lloyd is also as the theThe Bull' who the rest are great. the film notch family film.\n", + " musting and funwwarming film that is should watch.\n", + "\"# ## this the helpful or\n", + "\n", + "0###\n", + "##view: \"This sonely wiferic been of the mostREATEATST in all generation. She' been to her CD many yearsEARS and it still loveVE it! She I hearm feeling the bad mood,' me feel even, When must dayood and melaporates. a in a rain. I is is makesozes classIFE and Itocally are superawss perfectUNFFNING. theics are make me I of the's great gems. Bu CD a must islandle CD for my collection.\" Bu isn' made it big is beyond a me. She song I listen this CD I matter what or white or or or old, it or female,VERYONEDY lov the thing:W is that??\"\n", + "###\n", + "Is this review positive? \n", + "--------------------------------------------------------------------------------\n" ] } ], "source": [ - "print(format_imdb(text, 1))" + "print('-'*40+'input'+'-'*40)\n", + "print(neg_hs['text_q'][0])\n", + "print('-'*40+'answ'+'-'*40)\n", + "print(neg_hs['text_ans'][0])\n", + "print('='*80)\n", + "print('-'*40+'input'+'-'*40)\n", + "print(pos_hs['text_q'][0])\n", + "print('-'*40+'answ'+'-'*40)\n", + "print(pos_hs['text_ans'][0])\n", + "print('-'*80)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 18, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:35.600642Z", - "start_time": "2023-05-07T11:21:35.586155Z" - } + "end_time": "2023-05-19T23:03:35.276439Z", + "start_time": "2023-05-19T23:03:35.250130Z" + }, + "scrolled": true }, "outputs": [], "source": [ - "# # sceatch\n", - "# layer = -10\n", - "# input_ids = tokenizer(text + tokenizer.eos_token, return_tensors=\"pt\").input_ids.to(model.device)\n", - "# # forward pass\n", - "# with torch.no_grad():\n", - "# output = model(input_ids, output_hidden_states=True)\n", + "# # unit tests\n", + "# idx = 0\n", + "# n=10\n", + "# batch_size=3\n", + "# ds_subset = data['test'].shuffle(42).select(range(n))\n", + "# dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)\n", + "# batch = next(iter(dl))\n", "\n", - "# # get the last layer, last token hidden states\n", - "# hs_tuple = output[\"hidden_states\"]\n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - "# hs, output['logits'], output['hidden_states']" + "# texts, true_labels = batch[\"content\"], batch[\"label\"]\n", + "# neg_hs = get_hidden_states(model, tokenizer, format_imdbs(texts, 0), model_type=model_type)\n", + "# neg_hs\n", + "# for k,v in neg_hs.items():\n", + "# print(k, v.shape)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T05:39:55.699656Z", - "start_time": "2023-05-07T05:39:55.653120Z" - } - }, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": {}, @@ -653,18 +848,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 19, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:21:35.616885Z", - "start_time": "2023-05-07T11:21:35.601707Z" + "end_time": "2023-05-19T23:03:35.351720Z", + "start_time": "2023-05-19T23:03:35.347655Z" } }, "outputs": [], "source": [ "\n", "\n", - "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100, layers=[2, -2]):\n", + "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100, layers=[2, -2], batch_size=3):\n", " \"\"\"\n", " Given an encoder-decoder model, a list of data, computes the contrast hidden states on n random examples.\n", " Returns numpy arrays of shape (n, hidden_dim) for each candidate label, along with a boolean numpy array of shape (n,)\n", @@ -674,7499 +869,33 @@ " \"\"\"\n", " # setup\n", " model.eval()\n", - " all_neg_hs, all_pos_hs, all_gt_labels = [], [], []\n", - "\n", - " # loop\n", - " for _ in tqdm(range(n), unit='examples', desc='get_hidden_states'):\n", - " # for simplicity, sample a random example until we find one that's a reasonable length\n", - " # (most examples should be a reasonable length, so this is just to make sure)\n", - " while True:\n", - " idx = np.random.randint(len(data))\n", - " text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", - " # the actual formatted input will be longer, so include a bit of a margin\n", - " if len(tokenizer(text)) < 400: \n", - " break\n", - " \n", - " # get hidden states\n", - "# print(format_imdb(text, 0))\n", - " neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type, layers=layers)\n", - " pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type, layers=layers)\n", + " \n", + " res = []\n", + " \n", + " ds_subset = data['test'].shuffle(42).select(range(n))\n", + " dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)\n", + " for batch in tqdm(dl):\n", + " text, true_label = batch[\"content\"], batch[\"label\"]\n", + " neg = get_hidden_states(model, tokenizer, format_imdbs(text, 0), model_type=model_type, layers=layers)\n", + " pos = get_hidden_states(model, tokenizer, format_imdbs(text, 1), model_type=model_type, layers=layers)\n", "\n", " # collect\n", - " all_neg_hs.append(neg_hs)\n", - " all_pos_hs.append(pos_hs)\n", - " all_gt_labels.append(true_label)\n", - "\n", - " all_neg_hs = np.stack(all_neg_hs)\n", - " all_pos_hs = np.stack(all_pos_hs)\n", - " all_gt_labels = np.stack(all_gt_labels)\n", - "\n", - " return all_neg_hs, all_pos_hs, all_gt_labels" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T11:22:30.270761Z", - "start_time": "2023-05-07T11:21:35.617903Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "get_hidden_states: 0%| | 0/100 [00:00╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n", + "│ in <cell line: 1>:1 │\n", + "│ │\n", + "│ ❱ 1 neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tok │\n", + "│ 2 │\n", + "│ 3 │\n", + "│ 4 gc.collect() │\n", + "│ │\n", + "│ in get_hidden_states_many_examples:33 │\n", + "│ │\n", + "│ 30 │ │ ]) │\n", + "│ 31 │ │\n", + "│ 32 │ # FIXME not all the hidden state are the same size, wat │\n", + "│ ❱ 33 │ res = [np.concatenate(r) for r in zip(*res)] │\n", + "│ 34 │ return res │\n", + "│ 35 │ all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res │\n", + "│ 36 │ return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans │\n", + "│ │\n", + "│ in <listcomp>:33 │\n", + "│ │\n", + "│ 30 │ │ ]) │\n", + "│ 31 │ │\n", + "│ 32 │ # FIXME not all the hidden state are the same size, wat │\n", + "│ ❱ 33 │ res = [np.concatenate(r) for r in zip(*res)] │\n", + "│ 34 │ return res │\n", + "│ 35 │ all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res │\n", + "│ 36 │ return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans │\n", + "│ in concatenate:200 │\n", + "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n", + "ValueError: all the input array dimensions except for the concatenation axis must match exactly, but along \n", + "dimension 1, the array at index 0 has size 2121728 and the array at index 1 has size 1867776\n", + "\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tok \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mgc.collect() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m33\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# FIXME not all the hidden state are the same size, wat\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m33 \u001b[2m│ \u001b[0mres = [np.concatenate(r) \u001b[94mfor\u001b[0m r \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(*res)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ \u001b[0mall_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m33\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# FIXME not all the hidden state are the same size, wat\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m33 \u001b[2m│ \u001b[0mres = [np.concatenate(r) \u001b[94mfor\u001b[0m r \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(*res)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ \u001b[0mall_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mconcatenate\u001b[0m:\u001b[94m200\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mValueError: \u001b[0mall the input array dimensions except for the concatenation axis must match exactly, but along \n", + "dimension \u001b[1;36m1\u001b[0m, the array at index \u001b[1;36m0\u001b[0m has size \u001b[1;36m2121728\u001b[0m and the array at index \u001b[1;36m1\u001b[0m has size \u001b[1;36m1867776\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tokenizer, data, model_type)\n", "\n", - "pos_score = roc_auc_score(y, np.array([h['ans'] for h in pos_hs]))\n", - "neg_score = roc_auc_score(y, np.array([h['ans'] for h in neg_hs]))\n", + "\n", + "gc.collect()\n", + "torch.cuda.empty_cache()\n", + "gc.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T23:04:04.203349Z", + "start_time": "2023-05-19T23:04:04.203341Z" + } + }, + "outputs": [], + "source": [ + "# all_pos_ans" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T23:04:04.204205Z", + "start_time": "2023-05-19T23:04:04.204197Z" + } + }, + "outputs": [], + "source": [ + "# roc_auc_score\n", + "pos_score = roc_auc_score(y, all_pos_ans)\n", + "neg_score = roc_auc_score(y, all_neg_ans)\n", "pos_score, neg_score" ] }, @@ -8215,13 +1097,30 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T10:48:17.261806Z", - "start_time": "2023-05-07T10:48:17.257349Z" + "end_time": "2023-05-14T11:32:21.359783Z", + "start_time": "2023-05-14T11:32:20.787141Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T23:04:04.204853Z", + "start_time": "2023-05-19T23:04:04.204842Z" }, "scrolled": true }, "outputs": [], - "source": [] + "source": [ + "# accuracy_score\n", + "pos_score = accuracy_score(y, (all_pos_ans>0.)*1.0)\n", + "neg_score = accuracy_score(y, (all_neg_ans<0.5)*1.0)\n", + "pos_score, neg_score" + ] }, { "cell_type": "markdown", @@ -8239,50 +1138,17 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:08:12.494657Z", - "start_time": "2023-05-07T11:08:12.482252Z" + "end_time": "2023-05-19T22:49:25.447229Z", + "start_time": "2023-05-19T22:49:25.447221Z" } }, "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T11:10:15.610451Z", - "start_time": "2023-05-07T11:10:15.607971Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T11:22:30.600357Z", - "start_time": "2023-05-07T11:22:30.522522Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logistic regression accuracy: 1.0 [TRAIN]\n", - "Logistic regression accuracy: 0.98 [TEST]\n" - ] - } - ], "source": [ "# let's create a simple 50/50 train split (the data is already randomized)\n", "n = len(y)\n", "\n", - "neg_hs2 = torch.stack([h['hidden_states'].flatten() for h in neg_hs], 0)#.flatten()\n", - "pos_hs2 = torch.stack([h['hidden_states'].flatten() for h in pos_hs], 0)#.flatten()\n", + "neg_hs2 = torch.from_numpy(np.stack([h.flatten() for h in neg_hs], 0))\n", + "pos_hs2 = torch.from_numpy(np.stack([h.flatten() for h in pos_hs], 0))\n", "\n", "neg_hs_train, neg_hs_test = neg_hs2[:n//2], neg_hs2[n//2:]\n", "pos_hs_train, pos_hs_test = pos_hs2[:n//2], pos_hs2[n//2:]\n", @@ -8304,8 +1170,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T10:53:09.036137Z", - "start_time": "2023-05-07T10:53:09.031305Z" + "end_time": "2023-05-14T00:05:52.801860Z", + "start_time": "2023-05-14T00:05:52.784513Z" } }, "outputs": [], @@ -8320,11 +1186,11 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:30.614094Z", - "start_time": "2023-05-07T11:22:30.601720Z" + "end_time": "2023-05-19T22:49:25.447898Z", + "start_time": "2023-05-19T22:49:25.447890Z" } }, "outputs": [], @@ -8346,129 +1212,7 @@ " )\n", "\n", " def forward(self, x):\n", - " return self.net(x)\n", - "\n", - "class CCS(object):\n", - " def __init__(self, x0, x1, nepochs=1000, ntries=10, lr=1e-3, batch_size=-1, \n", - " verbose=False, device=\"cuda\", linear=True, weight_decay=0.01, var_normalize=False):\n", - " # data\n", - " self.var_normalize = var_normalize\n", - " self.x0 = self.normalize(x0)\n", - " self.x1 = self.normalize(x1)\n", - " self.d = self.x0.shape[-1]\n", - "\n", - " # training\n", - " self.nepochs = nepochs\n", - " self.ntries = ntries\n", - " self.lr = lr\n", - " self.verbose = verbose\n", - " self.device = device\n", - " self.batch_size = batch_size\n", - " self.weight_decay = weight_decay\n", - " \n", - " # probe\n", - " self.linear = linear\n", - " self.probe = self.initialize_probe()\n", - " self.best_probe = copy.deepcopy(self.probe)\n", - "\n", - " \n", - " def initialize_probe(self):\n", - " if self.linear:\n", - " self.probe = nn.Sequential(nn.Linear(self.d, 1), nn.Sigmoid())\n", - " else:\n", - " self.probe = MLPProbe(self.d)\n", - " return self.probe.to(self.device)\n", - "\n", - "\n", - " def normalize(self, x):\n", - " \"\"\"\n", - " Mean-normalizes the data x (of shape (n, d))\n", - " If self.var_normalize, also divides by the standard deviation\n", - " \"\"\"\n", - " normalized_x = x - x.mean(axis=0, keepdims=True)\n", - " if self.var_normalize:\n", - " normalized_x /= normalized_x.std(axis=0, keepdims=True)\n", - "\n", - " return normalized_x\n", - "\n", - " \n", - " def get_tensor_data(self):\n", - " \"\"\"\n", - " Returns x0, x1 as appropriate tensors (rather than np arrays)\n", - " \"\"\"\n", - " x0 = self.x0.clone().detach().float().to(self.device)\n", - " x1 = self.x1.clone().detach().float().to(self.device)\n", - " return x0, x1\n", - " \n", - "\n", - " def get_loss(self, p0, p1):\n", - " \"\"\"\n", - " Returns the CCS loss for two probabilities each of shape (n,1) or (n,)\n", - " \"\"\"\n", - " informative_loss = (torch.min(p0, p1)**2).mean(0)\n", - " consistent_loss = ((p0 - (1-p1))**2).mean(0)\n", - " return informative_loss + consistent_loss\n", - "\n", - "\n", - " def get_acc(self, x0_test, x1_test, y_test):\n", - " \"\"\"\n", - " Computes accuracy for the current parameters on the given test inputs\n", - " \"\"\"\n", - " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n", - " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n", - " with torch.no_grad():\n", - " p0, p1 = self.best_probe(x0), self.best_probe(x1)\n", - " avg_confidence = 0.5*(p0 + (1-p1))\n", - " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n", - " acc = (predictions == y_test).mean()\n", - " acc = max(acc, 1 - acc)\n", - "\n", - " return acc\n", - " \n", - " \n", - " def train(self):\n", - " \"\"\"\n", - " Does a single training run of nepochs epochs\n", - " \"\"\"\n", - " x0, x1 = self.get_tensor_data()\n", - " permutation = torch.randperm(len(x0))\n", - " x0, x1 = x0[permutation], x1[permutation]\n", - " \n", - " # set up optimizer\n", - " optimizer = torch.optim.AdamW(self.probe.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n", - " \n", - " batch_size = len(x0) if self.batch_size == -1 else self.batch_size\n", - " nbatches = len(x0) // batch_size\n", - "\n", - " # Start training (full batch)\n", - " for epoch in range(self.nepochs):\n", - " for j in range(nbatches):\n", - " x0_batch = x0[j*batch_size:(j+1)*batch_size]\n", - " x1_batch = x1[j*batch_size:(j+1)*batch_size]\n", - " \n", - " # probe\n", - " p0, p1 = self.probe(x0_batch), self.probe(x1_batch)\n", - "\n", - " # get the corresponding loss\n", - " loss = self.get_loss(p0, p1)\n", - "\n", - " # update the parameters\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " return loss.detach().cpu().item()\n", - " \n", - " def repeated_train(self):\n", - " best_loss = np.inf\n", - " for train_num in range(self.ntries):\n", - " self.initialize_probe()\n", - " loss = self.train()\n", - " if loss < best_loss:\n", - " self.best_probe = copy.deepcopy(self.probe)\n", - " best_loss = loss\n", - "\n", - " return best_loss" + " return self.net(x)\n" ] }, { @@ -8497,86 +1241,48 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:42.769241Z", - "start_time": "2023-05-07T11:22:30.614971Z" + "end_time": "2023-05-19T22:49:25.448505Z", + "start_time": "2023-05-19T22:49:25.448497Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CCS nonlinear train accuracy: 0.96\n", - "CCS nonlinear test accuracy: 0.98\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_4095498/374574033.py:86: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", - " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n", - "/tmp/ipykernel_4095498/374574033.py:87: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", - " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n" - ] - } - ], + "outputs": [], "source": [ - "# Train CCS without any labels\n", - "ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n", - "ccs.repeated_train()\n", + "# # Train CCS without any labels\n", + "# ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n", + "# ccs.repeated_train()\n", "\n", - "# Evaluate\n", - "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", - "print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n", + "# # Evaluate\n", + "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "# print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n", "\n", - "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", - "print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))" + "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "# print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:49.510569Z", - "start_time": "2023-05-07T11:22:42.770553Z" + "end_time": "2023-05-19T22:49:25.449148Z", + "start_time": "2023-05-19T22:49:25.449140Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CCS train accuracy: 0.96\n", - "CCS test accuracy: 0.98\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_4095498/374574033.py:86: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", - " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n", - "/tmp/ipykernel_4095498/374574033.py:87: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", - " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n" - ] - } - ], + "outputs": [], "source": [ - "# Train CCS without any labels\n", - "ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n", - "ccs.repeated_train()\n", + "# # Train CCS without any labels\n", + "# ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n", + "# ccs.repeated_train()\n", "\n", - "# Evaluate\n", - "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", - "print(\"CCS train accuracy: {}\".format(ccs_acc))\n", + "# # Evaluate\n", + "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "# print(\"CCS train accuracy: {}\".format(ccs_acc))\n", "\n", - "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", - "print(\"CCS test accuracy: {}\".format(ccs_acc))" + "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "# print(\"CCS test accuracy: {}\".format(ccs_acc))" ] }, { @@ -8600,38 +1306,260 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:49.914580Z", - "start_time": "2023-05-07T11:22:49.511794Z" + "end_time": "2023-05-19T04:12:55.004017Z", + "start_time": "2023-05-19T04:12:55.004011Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DataModule" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-14T11:34:43.243172Z", + "start_time": "2023-05-14T11:34:43.240582Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.449918Z", + "start_time": "2023-05-19T22:49:25.449910Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "\n", + "# def normalize(x):\n", + "# \"\"\"\n", + "# Mean-normalizes the data x (of shape (n, d))\n", + "# If self.var_normalize, also divides by the standard deviation\n", + "# \"\"\"\n", + "# normalized_x = x - x.mean(axis=0, keepdims=True)\n", + "# if self.var_normalize:\n", + "# normalized_x /= normalized_x.std(axis=0, keepdims=True)\n", + "\n", + "# return normalized_x\n", + "\n", + "\n", + "class IMBDHSDataModule(pl.LightningDataModule):\n", + "\n", + " def __init__(self,\n", + " model: AutoModel,\n", + " tokenizer: AutoTokenizer,\n", + " model_type=\"decoder\",\n", + " dataset_name=\"amazon_polarity\",\n", + " batch_size=32,\n", + " n=200,\n", + " ):\n", + " super().__init__()\n", + " self.model = model\n", + " self.tokenizer = tokenizer\n", + " self.save_hyperparameters(ignore=[\"model\", \"tokenizer\"])\n", + "\n", + " def setup(self, stage: str):\n", + "\n", + " self.dataset = load_dataset(self.hparams.dataset_name, split=\"test\")\n", + "\n", + " neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(\n", + " self.model, self.tokenizer, self.dataset, self.hparams.model_type, n=self.hparams.n, layers=[2, -2])\n", + "\n", + " # let's create a simple 50/50 train split (the data is already randomized)\n", + " n = len(y)\n", + " val_split = int(n * 0.5)\n", + " test_split = int(n * 0.75)\n", + " neg_hs_train, pos_hs_train, y_train = neg_hs[:\n", + " val_split], pos_hs[:\n", + " val_split], y[:\n", + " val_split]\n", + " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[\n", + " val_split:test_split], y[val_split:test_split]\n", + " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[\n", + " test_split:], y[test_split:]\n", + "\n", + " # for simplicity we can just take the difference between positive and negative hidden states\n", + " # (concatenating also works fine)\n", + " self.x_train = neg_hs_train - pos_hs_train\n", + " self.x_val = neg_hs_val - pos_hs_val\n", + " self.x_test = neg_hs_test - pos_hs_test\n", + "\n", + " # normalize\n", + " self.scaler = RobustScaler()\n", + " self.scaler.fit(self.x_train)\n", + " self.x_train = self.scaler.transform(self.x_train)\n", + " self.x_val = self.scaler.transform(self.x_val)\n", + " self.x_test = self.scaler.transform(self.x_test)\n", + "\n", + " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train).float(),\n", + " torch.from_numpy(pos_hs_train).float(),\n", + " torch.from_numpy(y_train).float())\n", + "\n", + " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val).float(),\n", + " torch.from_numpy(pos_hs_val).float(),\n", + " torch.from_numpy(y_val).float())\n", + "\n", + " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test).float(),\n", + " torch.from_numpy(pos_hs_test).float(),\n", + " torch.from_numpy(y_test).float())\n", + "\n", + " def train_dataloader(self):\n", + " return DataLoader(self.ds_train,\n", + " batch_size=self.hparams.batch_size,\n", + " shuffle=True)\n", + "\n", + " def val_dataloader(self):\n", + " return DataLoader(self.ds_val, batch_size=self.hparams.batch_size)\n", + "\n", + " def test_dataloader(self):\n", + " return DataLoader(self.ds_test, batch_size=self.hparams.batch_size)\n", + "\n", + "\n", + "# test\n", + "dm = IMBDHSDataModule(model, tokenizer)\n", + "dm.setup('train')\n", + "dl = dm.val_dataloader()\n", + "b = next(iter(dl))\n", + "b" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.450709Z", + "start_time": "2023-05-19T22:49:25.450702Z" } }, "outputs": [], "source": [ - "import lightning.pytorch as pl\n", + "dm.x_test.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LightningModel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.451318Z", + "start_time": "2023-05-19T22:49:25.451310Z" + } + }, + "outputs": [], + "source": [ + "from torch import optim" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.452324Z", + "start_time": "2023-05-19T22:49:25.452316Z" + } + }, + "outputs": [], + "source": [ + "\n", + "\n", + "def get_loss(p0, p1):\n", + " \"\"\"\n", + " Returns the CCS loss for two probabilities each of shape (n,1) or (n,)\n", + " \"\"\"\n", + " informative_loss = (torch.min(p0, p1)**2).mean(0)\n", + " consistent_loss = ((p0 - (1-p1))**2).mean(0)\n", + " return informative_loss + consistent_loss\n", + "\n", + "\n", + "def get_acc(p0, p1, y):\n", + " avg_confidence = 0.5*(p0 + (1-p1))\n", + " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n", + " \n", + " # TODO f1\n", + " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n", + " \n", + " acc = (predictions == y.cpu().numpy()).mean()\n", + " acc = max(acc, 1 - acc)\n", + " return predictions, acc\n", + "\n", + "def get_f1(p0, p1, y):\n", + " avg_confidence = 0.5*(p0 + (1-p1))\n", + " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n", + " \n", + " # TODO f1\n", + " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n", + " auc = roc_auc_score(y.cpu().numpy(), predictions)\n", + " \n", + " auc = max(auc, 1 - auc)\n", + " return predictions, auc\n", "\n", "class CSS(pl.LightningModule):\n", - " def __init__(self, encoder, decoder):\n", + " def __init__(self, d, max_epochs, lr=4e-3, weight_decay=1e-6):\n", " super().__init__()\n", - " self.encoder = encoder\n", - " self.decoder = decoder\n", + " self.probe = MLPProbe(d)\n", + " self.save_hyperparameters()\n", + " \n", + " def forward(self, x):\n", + " return self.probe(x)\n", + " \n", + " def _step(self, batch, batch_idx, stage='train'):\n", + " x0, x1, y = batch\n", + " p0, p1 = self(x0), self(x1)\n", + " \n", + " loss = get_loss(p0, p1)\n", + " \n", + " self.log(f\"{stage}/loss\", loss)\n", + " \n", + " predictions, acc = get_acc(p0, p1, y)\n", + " self.log(f\"{stage}/acc\", acc)\n", + " predictions, f1 = get_f1(p0, p1, y)\n", + " self.log(f\"{stage}/f1\", f1)\n", + " return loss\n", " \n", " def training_step(self, batch, batch_idx):\n", - " # training_step defines the train loop.\n", - " # it is independent of forward\n", - " x, y = batch\n", - " x = x.view(x.size(0), -1)\n", - " z = self.encoder(x)\n", - " x_hat = self.decoder(z)\n", - " loss = nn.functional.mse_loss(x_hat, x)\n", - " # Logging to TensorBoard (if installed) by default\n", - " self.log(\"train_loss\", loss)\n", - " return loss\n", + " return self._step(batch, batch_idx)\n", + " \n", + " def validation_step(self, batch, batch_idx=0):\n", + " return self._step(batch, batch_idx, stage='val')\n", + " \n", + " def prediction_step(self, batch, batch_idx):\n", + " x0, x1, y = batch\n", + " p0, p1 = self(x0), self(x1)\n", + " predictions, acc = get_acc(p0, p1, y)\n", + " return predictions \n", "\n", " def configure_optimizers(self):\n", - " optimizer = optim.Adam(self.parameters(), lr=1e-3)\n", - " return optimizer\n", + " optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr, weight_decay=self.hparams.weight_decay)\n", + " lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(\n", + " optimizer, T_max=self.hparams.max_epochs, eta_min=self.hparams.lr / 50\n", + " )\n", + " return [optimizer], [lr_scheduler]\n", " " ] }, @@ -8649,232 +1577,31 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:55.487623Z", - "start_time": "2023-05-07T11:22:49.915848Z" + "end_time": "2023-05-14T06:17:57.365689Z", + "start_time": "2023-05-14T06:17:57.356995Z" } }, - "outputs": [ - { - "data": { - "text/html": [ - "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
-       "│ in <cell line: 56>:56                                                                            │\n",
-       "│                                                                                                  │\n",
-       "│   53                                                                                             │\n",
-       "│   54 # test                                                                                      │\n",
-       "│   55 dm = IMBDHSDataModule(model, tokenizer)                                                     │\n",
-       "│ ❱ 56 dm.setup('train')                                                                           │\n",
-       "│   57 dl = dm.val_dataloader()                                                                    │\n",
-       "│   58 next(iter(dl))                                                                              │\n",
-       "│   59                                                                                             │\n",
-       "│                                                                                                  │\n",
-       "│ in setup:15                                                                                      │\n",
-       "│                                                                                                  │\n",
-       "│   12 │                                                                                           │\n",
-       "│   13 │                                                                                           │\n",
-       "│   14 │   def setup(self, stage:str):                                                             │\n",
-       "│ ❱ 15 │   │   self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)           │\n",
-       "│   16 │   │                                                                                       │\n",
-       "│   17 │   │   neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer,     │\n",
-       "│   18                                                                                             │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1773 in         │\n",
-       "│ load_dataset                                                                                     │\n",
-       "│                                                                                                  │\n",
-       "│   1770 │   )                                                                                     │\n",
-       "│   1771 │                                                                                         │\n",
-       "│   1772 │   # Create a dataset builder                                                            │\n",
-       "│ ❱ 1773 │   builder_instance = load_dataset_builder(                                              │\n",
-       "│   1774 │   │   path=path,                                                                        │\n",
-       "│   1775 │   │   name=name,                                                                        │\n",
-       "│   1776 │   │   data_dir=data_dir,                                                                │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1528 in         │\n",
-       "│ load_dataset_builder                                                                             │\n",
-       "│                                                                                                  │\n",
-       "│   1525 │   │   raise ValueError(error_msg)                                                       │\n",
-       "│   1526 │                                                                                         │\n",
-       "│   1527 │   # Instantiate the dataset builder                                                     │\n",
-       "│ ❱ 1528 │   builder_instance: DatasetBuilder = builder_cls(                                       │\n",
-       "│   1529 │   │   cache_dir=cache_dir,                                                              │\n",
-       "│   1530 │   │   config_name=config_name,                                                          │\n",
-       "│   1531 │   │   data_dir=data_dir,                                                                │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:340 in       │\n",
-       "│ __init__                                                                                         │\n",
-       "│                                                                                                  │\n",
-       "│    337 │   │   │   config_kwargs[\"data_files\"] = data_files                                      │\n",
-       "│    338 │   │   if data_dir is not None:                                                          │\n",
-       "│    339 │   │   │   config_kwargs[\"data_dir\"] = data_dir                                          │\n",
-       "│ ❱  340 │   │   self.config, self.config_id = self._create_builder_config(                        │\n",
-       "│    341 │   │   │   config_name=config_name,                                                      │\n",
-       "│    342 │   │   │   custom_features=features,                                                     │\n",
-       "│    343 │   │   │   **config_kwargs,                                                              │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:494 in       │\n",
-       "│ _create_builder_config                                                                           │\n",
-       "│                                                                                                  │\n",
-       "│    491 │   │   │   │   config_kwargs[\"name\"] = self.DEFAULT_CONFIG_NAME                          │\n",
-       "│    492 │   │   │   if \"version\" not in config_kwargs and hasattr(self, \"VERSION\") and self.VERS  │\n",
-       "│    493 │   │   │   │   config_kwargs[\"version\"] = self.VERSION                                   │\n",
-       "│ ❱  494 │   │   │   builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs)                   │\n",
-       "│    495 │   │                                                                                     │\n",
-       "│    496 │   │   # otherwise use the config_kwargs to overwrite the attributes                     │\n",
-       "│    497 │   │   else:                                                                             │\n",
-       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
-       "TypeError: __init__() got an unexpected keyword argument 'stream'\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m15\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m12 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m14 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92msetup\u001b[0m(\u001b[96mself\u001b[0m, stage:\u001b[96mstr\u001b[0m): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m15 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m, stream=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m16 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y = get_hidden_states_many_examples(\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1773\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92mload_dataset\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1770 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1771 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1772 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create a dataset builder\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1773 \u001b[2m│ \u001b[0mbuilder_instance = load_dataset_builder( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1774 \u001b[0m\u001b[2m│ │ \u001b[0mpath=path, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1775 \u001b[0m\u001b[2m│ │ \u001b[0mname=name, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1776 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1528\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92mload_dataset_builder\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1525 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m(error_msg) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1526 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1527 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Instantiate the dataset builder\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1528 \u001b[2m│ \u001b[0mbuilder_instance: DatasetBuilder = builder_cls( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1529 \u001b[0m\u001b[2m│ │ \u001b[0mcache_dir=cache_dir, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1530 \u001b[0m\u001b[2m│ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1531 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m340\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 337 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_files\u001b[0m\u001b[33m\"\u001b[0m] = data_files \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 338 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m data_dir \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 339 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_dir\u001b[0m\u001b[33m\"\u001b[0m] = data_dir \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 340 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.config, \u001b[96mself\u001b[0m.config_id = \u001b[96mself\u001b[0m._create_builder_config( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 341 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 342 \u001b[0m\u001b[2m│ │ │ \u001b[0mcustom_features=features, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 343 \u001b[0m\u001b[2m│ │ │ \u001b[0m**config_kwargs, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m494\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92m_create_builder_config\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 491 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mname\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.DEFAULT_CONFIG_NAME \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 492 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m config_kwargs \u001b[95mand\u001b[0m \u001b[96mhasattr\u001b[0m(\u001b[96mself\u001b[0m, \u001b[33m\"\u001b[0m\u001b[33mVERSION\u001b[0m\u001b[33m\"\u001b[0m) \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.VERS \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 493 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.VERSION \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 494 \u001b[2m│ │ │ \u001b[0mbuilder_config = \u001b[96mself\u001b[0m.BUILDER_CONFIG_CLASS(**config_kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 495 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 496 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# otherwise use the config_kwargs to overwrite the attributes\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 497 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mTypeError: \u001b[0m\u001b[1;35m__init__\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m got an unexpected keyword argument \u001b[32m'stream'\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from dataclasses import dataclass\n", - "from torch.utils.data import random_split, DataLoader, TensorDataset\n", - "from transformers.models.auto.modeling_auto import AutoModel\n", - "\n", - "@dataclass\n", - "class IMBDHSDataModule(pl.LightningDataModule):\n", - " model: AutoModel\n", - " tokenizer: AutoTokenizer\n", - " model_type=\"decoder\"\n", - " dataset_name=\"amazon_polarity\"\n", - " batch_size=32\n", - " \n", - " \n", - " def setup(self, stage:str):\n", - " self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)\n", - " \n", - " neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer, self.dataset, self.model_type)\n", - " \n", - " # let's create a simple 50/50 train split (the data is already randomized)\n", - " n = len(y)\n", - " val_split = int(n*0.5)\n", - " test_split = int(n*0.75)\n", - " neg_hs_train, pos_hs_train, y_train = neg_hs[:val_split], pos_hs[:val_split], y[:val_split]\n", - " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[val_split:test_split], y[val_split:test_split]\n", - " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[test_split:], y[test_split:]\n", - "\n", - " # for simplicity we can just take the difference between positive and negative hidden states\n", - " # (concatenating also works fine)\n", - " self.x_train = neg_hs_train - pos_hs_train\n", - " self.x_val = neg_hs_val - pos_hs_val\n", - " self.x_test = neg_hs_test - pos_hs_test\n", - " \n", - " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train),\n", - " torch.from_numpy(pos_hs_train),\n", - " torch.from_numpy(y_train))\n", - " \n", - " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val),\n", - " torch.from_numpy(pos_hs_val),\n", - " torch.from_numpy(y_val))\n", - " \n", - " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test),\n", - " torch.from_numpy(pos_hs_test),\n", - " torch.from_numpy(y_test))\n", - " \n", - " def train_dataloader(self):\n", - " return DataLoader(self.ds_train, batch_size=self.batch_size)\n", - "\n", - " def val_dataloader(self):\n", - " return DataLoader(self.ds_val, batch_size=self.batch_size)\n", - "\n", - " def test_dataloader(self):\n", - " return DataLoader(self.ds_test, batch_size=self.batch_size)\n", - " \n", - "# test\n", - "dm = IMBDHSDataModule(model, tokenizer)\n", - "dm.setup('train')\n", - "dl = dm.val_dataloader()\n", - "next(iter(dl))" - ] + "outputs": [], + "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:55.488619Z", - "start_time": "2023-05-07T11:22:55.488612Z" + "end_time": "2023-05-19T22:49:25.453018Z", + "start_time": "2023-05-19T22:49:25.453010Z" } }, "outputs": [], "source": [ "# init the autoencoder\n", - "net = CSS()" + "max_epochs = 1000\n", + "d = b[0].shape[-1]\n", + "net = CSS(d=d, max_epochs=max_epochs)" ] }, { @@ -8882,13 +1609,13 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:55.489380Z", - "start_time": "2023-05-07T11:22:55.489373Z" + "end_time": "2023-05-19T22:49:25.453708Z", + "start_time": "2023-05-19T22:49:25.453700Z" } }, "outputs": [], "source": [ - "train_loader = utils.data.DataLoader(dataset)" + "# train_loader = utils.data.DataLoader(dataset)" ] }, { @@ -8896,17 +1623,128 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T11:22:55.489937Z", - "start_time": "2023-05-07T11:22:55.489931Z" - } + "end_time": "2023-05-19T22:49:25.454581Z", + "start_time": "2023-05-19T22:49:25.454572Z" + }, + "scrolled": true }, "outputs": [], "source": [ "# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\n", - "trainer = pl.Trainer(limit_train_batches=100, max_epochs=1)\n", - "trainer.fit(model=net, train_dataloaders=train_loader)" + "trainer = pl.Trainer(limit_train_batches=100, max_epochs=max_epochs)\n", + "trainer.fit(model=net, datamodule=dm)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.455203Z", + "start_time": "2023-05-19T22:49:25.455195Z" + } + }, + "outputs": [], + "source": [ + "%debug" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-14T06:21:46.356828Z", + "start_time": "2023-05-14T06:21:46.351801Z" + } + }, + "source": [ + "# Read hist" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.455836Z", + "start_time": "2023-05-19T22:49:25.455828Z" + } + }, + "outputs": [], + "source": [ + "# import pytorch_lightning as pl\n", + "from lightning.pytorch.loggers.csv_logs import CSVLogger\n", + "# from pytorch_lightning.loggers.csv_logs import CSVLogger as CSVLogger2\n", + "from pathlib import Path\n", + "import pandas as pd\n", + "\n", + "def read_metrics_csv(metrics_file_path):\n", + " df_hist = pd.read_csv(metrics_file_path)\n", + " df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()\n", + " df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()\n", + " return df_histe\n", + "\n", + "\n", + "def read_hist(trainer: pl.Trainer):\n", + "\n", + " ts = [t for t in trainer.loggers if isinstance(t, CSVLogger)]\n", + " print(ts)\n", + " try:\n", + " metrics_file_path = Path(ts[0].experiment.metrics_file_path)\n", + " df_histe = read_metrics_csv(metrics_file_path)\n", + " return df_histe\n", + " except Exception as e:\n", + " raise e\n", + " print(e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.456423Z", + "start_time": "2023-05-19T22:49:25.456416Z" + } + }, + "outputs": [], + "source": [ + "df_hist = read_hist(trainer).ffill().bfill()\n", + "df_hist" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T22:49:25.456948Z", + "start_time": "2023-05-19T22:49:25.456942Z" + } + }, + "outputs": [], + "source": [ + "df_hist[['val/acc', 'train/acc']].plot()\n", + "\n", + "df_hist[['val/f1', 'train/f1']].plot()\n", + "\n", + "df_hist[['val/loss', 'train/loss']].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, diff --git a/requirements/conda.requirements.txt b/requirements/conda.requirements.txt index c354f4a..d126242 100644 --- a/requirements/conda.requirements.txt +++ b/requirements/conda.requirements.txt @@ -2,7 +2,9 @@ accelerate==0.18.0 aiohttp==3.8.4 aiosignal==1.3.1 altair==4.2.2 +anyio==3.6.2 appdirs @ file:///home/conda/feedstock_root/build_artifacts/appdirs_1603108395799/work +arrow==1.2.3 astor==0.8.1 asttokens==2.0.5 async-timeout==4.0.2 @@ -10,8 +12,10 @@ attrs==23.1.0 backcall==0.2.0 backports.functools-lru-cache @ file:///home/conda/feedstock_root/build_artifacts/backports.functools_lru_cache_1618230623929/work base58==2.1.1 +beautifulsoup4==4.12.2 bitsandbytes==0.38.1 black==23.3.0 +blessed==1.20.0 blinker==1.6.2 Brotli==1.0.9 brotlipy @ file:///home/conda/feedstock_root/build_artifacts/brotlipy_1666764672617/work @@ -27,33 +31,45 @@ click-default-group @ file:///home/conda/feedstock_root/build_artifacts/click-de clikit @ file:///home/conda/feedstock_root/build_artifacts/clikit_1676917709145/work comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1679481329611/work conda_lock @ file:///home/conda/feedstock_root/build_artifacts/conda-lock_1676925627812/work +contourpy==1.0.7 crashtest @ file:///home/conda/feedstock_root/build_artifacts/crashtest_1667467006729/work +croniter==1.3.14 cryptography @ file:///home/conda/feedstock_root/build_artifacts/cryptography-split_1681508587436/work +cycler==0.11.0 datasets==2.12.0 +dateutils==0.6.12 debugpy==1.6.0 decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work +deepdiff==6.3.0 dill==0.3.6 distlib @ file:///home/conda/feedstock_root/build_artifacts/distlib_1668356257807/work ensureconda @ file:///home/conda/feedstock_root/build_artifacts/ensureconda_1657719435160/work entrypoints==0.4 exceptiongroup==1.1.1 executing==0.8.3 +fastapi==0.88.0 filelock @ file:///home/conda/feedstock_root/build_artifacts/filelock_1681839547898/work flake8==6.0.0 +fonttools==4.39.4 frozenlist==1.3.3 fsspec==2023.4.0 gitdb==4.0.10 GitPython==3.1.31 +gluonts==0.12.8 gmpy2 @ file:///home/conda/feedstock_root/build_artifacts/gmpy2_1666808679441/work +h11==0.14.0 html5lib @ file:///home/conda/feedstock_root/build_artifacts/html5lib_1592930327044/work huggingface-hub==0.14.1 idna==2.8 importlib-metadata @ file:///home/conda/feedstock_root/build_artifacts/importlib-metadata_1682176699712/work +importlib-resources==5.12.0 inflate64==0.3.1 iniconfig==2.0.0 +inquirer==3.1.3 ipykernel @ file:///home/conda/feedstock_root/build_artifacts/ipykernel_1679336319192/work ipython==8.4.0 isort==5.8.0 +itsdangerous==2.1.2 jaraco.classes @ file:///home/conda/feedstock_root/build_artifacts/jaraco.classes_1667024629799/work jedi==0.18.1 jeepney @ file:///home/conda/feedstock_root/build_artifacts/jeepney_1649085214306/work @@ -63,10 +79,17 @@ jsonschema==4.17.3 jupyter-client==7.3.4 jupyter-core==4.10.0 keyring @ file:///home/conda/feedstock_root/build_artifacts/keyring_1671728250508/work +kiwisolver==1.4.4 +lightning==2.0.2 +lightning-cloud==0.5.34 +lightning-utilities==0.8.0 lockfile==0.12.2 +markdown-it-py==2.2.0 MarkupSafe @ file:///home/conda/feedstock_root/build_artifacts/markupsafe_1674135804847/work +matplotlib==3.7.1 matplotlib-inline==0.1.3 mccabe==0.7.0 +mdurl==0.1.2 more-itertools @ file:///home/conda/feedstock_root/build_artifacts/more-itertools_1677514956219/work mpmath @ file:///home/conda/feedstock_root/build_artifacts/mpmath_1678228039184/work msgpack @ file:///home/conda/feedstock_root/build_artifacts/msgpack-python_1678312698172/work @@ -77,11 +100,13 @@ mypy-extensions==0.4.3 nest-asyncio==1.5.5 networkx @ file:///home/conda/feedstock_root/build_artifacts/networkx_1680692919326/work numpy @ file:///home/conda/feedstock_root/build_artifacts/numpy_1682210190296/work +ordered-set==4.1.0 packaging @ file:///home/conda/feedstock_root/build_artifacts/packaging_1681337016113/work pandas==2.0.1 parso==0.8.3 pastel @ file:///home/conda/feedstock_root/build_artifacts/pastel_1640899049124/work pathspec==0.9.0 +peft==0.3.0 pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1667297516076/work pickleshare==0.7.5 Pillow @ file:///home/conda/feedstock_root/build_artifacts/pillow_1675487166627/work @@ -106,6 +131,7 @@ pydeck==0.8.1b0 pydocstyle==6.1.1 pyflakes==3.0.1 Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1681904169130/work +PyJWT==2.6.0 pylama==8.3.8 pylev @ file:///home/conda/feedstock_root/build_artifacts/pylev_1641226376343/work pyOpenSSL @ file:///home/conda/feedstock_root/build_artifacts/pyopenssl_1680037383858/work @@ -115,24 +141,33 @@ pyrsistent==0.19.3 PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1661604839144/work pytest==7.3.1 python-dateutil==2.8.2 +python-editor==1.0.4 +python-multipart==0.0.6 pytz==2023.3 pytz-deprecation-shim==0.1.0.post0 PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1666772387118/work pyzmq==23.2.0 pyzstd==0.15.7 +readchar==4.0.5 regex==2023.5.5 requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1682535435083/work responses==0.18.0 +rich==13.3.5 ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1683392662082/work ruamel.yaml.clib @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml.clib_1670412733608/work scikit-learn==1.2.2 scipy==1.10.1 SecretStorage @ file:///home/conda/feedstock_root/build_artifacts/secretstorage_1666848732124/work +sentencepiece==0.1.99 six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work sklearn==0.0.post4 smmap==5.0.0 +sniffio==1.3.0 snowballstemmer==2.2.0 +soupsieve==2.4.1 stack-data==0.3.0 +starlette==0.22.0 +starsessions==1.3.0 streamlit==0.82.0 sympy @ file:///home/conda/feedstock_root/build_artifacts/sympy_1679342590084/work tenacity==8.2.2 @@ -145,6 +180,7 @@ tomlkit @ file:///home/conda/feedstock_root/build_artifacts/tomlkit_168302816509 toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1657485559105/work torch==2.0.0 torchaudio==2.0.0 +torchmetrics==0.11.4 torchvision==0.15.0 tornado==6.1 tqdm==4.65.0 @@ -155,11 +191,14 @@ typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/typing_ext tzdata==2023.3 tzlocal==4.3 urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1678635778344/work +uvicorn==0.22.0 validators==0.20.0 virtualenv @ file:///home/conda/feedstock_root/build_artifacts/virtualenv_1682674264573/work watchdog==3.0.0 wcwidth==0.2.5 webencodings==0.5.1 +websocket-client==1.5.1 +websockets==11.0.2 xxhash==3.2.0 yapf==0.33.0 yarl==1.9.2 diff --git a/requirements/environment.max.yaml b/requirements/environment.max.yaml index df6b843..c3f67c9 100644 --- a/requirements/environment.max.yaml +++ b/requirements/environment.max.yaml @@ -188,34 +188,59 @@ dependencies: - aiohttp==3.8.4 - aiosignal==1.3.1 - altair==4.2.2 + - anyio==3.6.2 + - arrow==1.2.3 - astor==0.8.1 - async-timeout==4.0.2 - attrs==23.1.0 - base58==2.1.1 + - beautifulsoup4==4.12.2 - bitsandbytes==0.38.1 - black==23.3.0 + - blessed==1.20.0 - blinker==1.6.2 - brotli==1.0.9 - cachetools==5.3.0 - certifi==2022.12.7 + - contourpy==1.0.7 + - croniter==1.3.14 + - cycler==0.11.0 - datasets==2.12.0 + - dateutils==0.6.12 + - deepdiff==6.3.0 - dill==0.3.6 - exceptiongroup==1.1.1 + - fastapi==0.88.0 - flake8==6.0.0 + - fonttools==4.39.4 - frozenlist==1.3.3 - fsspec==2023.4.0 - gitdb==4.0.10 - gitpython==3.1.31 + - gluonts==0.12.8 + - h11==0.14.0 - huggingface-hub==0.14.1 + - importlib-resources==5.12.0 - inflate64==0.3.1 - iniconfig==2.0.0 + - inquirer==3.1.3 - isort==5.8.0 + - itsdangerous==2.1.2 - joblib==1.2.0 - jsonschema==4.17.3 + - kiwisolver==1.4.4 + - lightning==2.0.2 + - lightning-cloud==0.5.34 + - lightning-utilities==0.8.0 + - markdown-it-py==2.2.0 + - matplotlib==3.7.1 + - mdurl==0.1.2 - multidict==6.0.4 - multiprocess==0.70.14 - multivolumefile==0.2.3 + - ordered-set==4.1.0 - pandas==2.0.1 + - peft==0.3.0 - pillow==9.5.0 - plotly==5.14.1 - pluggy==1.0.0 @@ -228,33 +253,47 @@ dependencies: - pycryptodomex==3.17 - pydeck==0.8.1b0 - pyflakes==3.0.1 + - pyjwt==2.6.0 - pyparsing==3.0.9 - pyppmd==1.0.0 - pyrsistent==0.19.3 - pytest==7.3.1 + - python-editor==1.0.4 + - python-multipart==0.0.6 - pytz==2023.3 - pytz-deprecation-shim==0.1.0.post0 - pyzstd==0.15.7 + - readchar==4.0.5 - regex==2023.5.5 - requests==2.30.0 - responses==0.18.0 + - rich==13.3.5 - scikit-learn==1.2.2 - scipy==1.10.1 + - sentencepiece==0.1.99 - sklearn==0.0.post4 - smmap==5.0.0 + - sniffio==1.3.0 + - soupsieve==2.4.1 + - starlette==0.22.0 + - starsessions==1.3.0 - streamlit==0.82.0 - tenacity==8.2.2 - texttable==1.6.7 - threadpoolctl==3.1.0 - tokenizers==0.13.3 - toml==0.10.2 + - torchmetrics==0.11.4 - tqdm==4.65.0 - transformers==4.28.1 - tzdata==2023.3 - tzlocal==4.3 - urllib3==2.0.2 + - uvicorn==0.22.0 - validators==0.20.0 - watchdog==3.0.0 + - websocket-client==1.5.1 + - websockets==11.0.2 - xxhash==3.2.0 - yapf==0.33.0 - yarl==1.9.2