diff --git a/001_mjc_CCS.ipynb b/001_mjc_CCS.ipynb index 464b69b..f3d130c 100644 --- a/001_mjc_CCS.ipynb +++ b/001_mjc_CCS.ipynb @@ -13,8 +13,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:26:52.360728Z", - "start_time": "2023-05-19T00:26:50.894539Z" + "end_time": "2023-05-20T01:54:44.191549Z", + "start_time": "2023-05-20T01:54:41.824251Z" } }, "outputs": [ @@ -35,9 +35,23 @@ "import torch.nn as nn\n", "import torch.nn.functional as F\n", "\n", + "import os\n", + "os.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\n", "from datasets import load_dataset\n", + "import datasets\n", "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM\n", - "from sklearn.linear_model import LogisticRegression" + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "import lightning.pytorch as pl\n", + "from dataclasses import dataclass\n", + "from torch.utils.data import random_split, DataLoader, TensorDataset\n", + "from transformers.models.auto.modeling_auto import AutoModel\n", + "# from scipy.stats import zscore\n", + "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n", + "from sklearn.preprocessing import RobustScaler\n", + "import gc\n", + "\n", + "import os" ] }, { @@ -47,13 +61,28 @@ "## Model" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:54:44.196607Z", + "start_time": "2023-05-20T01:54:44.193276Z" + } + }, + "outputs": [], + "source": [ + "# from transformers import LlamaTokenizer, LlamaForCausalLM\n", + "from transformers import LlamaForCausalLM, LlamaTokenizer" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T03:55:24.085897Z", - "start_time": "2023-05-07T03:55:24.083858Z" + "end_time": "2023-05-20T01:54:27.257168Z", + "start_time": "2023-05-20T01:54:27.254862Z" } }, "outputs": [], @@ -61,11 +90,11 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:22.701155Z", - "start_time": "2023-05-19T00:26:52.362168Z" + "end_time": "2023-05-20T01:56:26.440636Z", + "start_time": "2023-05-20T01:54:44.197666Z" } }, "outputs": [ @@ -83,7 +112,7 @@ "================================================================================\n", "bin /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n", "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n", - "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so.11.0\n", + "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n", "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", "CUDA SETUP: Detected CUDA version 117\n", "CUDA SETUP: Loading binary /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" @@ -107,45 +136,101 @@ " warn(msg)\n", "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//matplotlib_inline.backend_inline'), PosixPath('module')}\n", " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0'), PosixPath('/usr/local/cuda/lib64/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n", + "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n", "Either way, this might cause trouble in the future:\n", "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", " warn(msg)\n", - "Loading checkpoint shards: 100%|███████████████████████████████████████████| 2/2 [00:06<00:00, 3.49s/it]\n" + "Loading checkpoint shards: 100%|██████████████| 2/2 [00:07<00:00, 3.78s/it]\n" ] }, { "data": { "text/plain": [ - "LlamaForCausalLM(\n", - " (model): LlamaModel(\n", - " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n", - " (layers): ModuleList(\n", - " (0-31): 32 x LlamaDecoderLayer(\n", - " (self_attn): LlamaAttention(\n", - " (q_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (k_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (v_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (o_proj): Linear8bitLt(in_features=4096, out_features=4096, bias=False)\n", - " (rotary_emb): LlamaRotaryEmbedding()\n", + "PeftModelForCausalLM(\n", + " (base_model): LoraModel(\n", + " (model): LlamaForCausalLM(\n", + " (model): LlamaModel(\n", + " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n", + " (layers): ModuleList(\n", + " (0-31): 32 x LlamaDecoderLayer(\n", + " (self_attn): LlamaAttention(\n", + " (q_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (k_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (v_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (o_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " (lora_embedding_A): ParameterDict()\n", + " (lora_embedding_B): ParameterDict()\n", + " )\n", + " (rotary_emb): LlamaRotaryEmbedding()\n", + " )\n", + " (mlp): LlamaMLP(\n", + " (gate_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", + " (down_proj): Linear8bitLt(in_features=11008, out_features=4096, bias=False)\n", + " (up_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", + " (act_fn): SiLUActivation()\n", + " )\n", + " (input_layernorm): LlamaRMSNorm()\n", + " (post_attention_layernorm): LlamaRMSNorm()\n", + " )\n", " )\n", - " (mlp): LlamaMLP(\n", - " (gate_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", - " (down_proj): Linear8bitLt(in_features=11008, out_features=4096, bias=False)\n", - " (up_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", - " (act_fn): SiLUActivation()\n", - " )\n", - " (input_layernorm): LlamaRMSNorm()\n", - " (post_attention_layernorm): LlamaRMSNorm()\n", + " (norm): LlamaRMSNorm()\n", " )\n", + " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n", " )\n", - " (norm): LlamaRMSNorm()\n", " )\n", - " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n", ")" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -156,7 +241,7 @@ "model_name = \"gpt-j\"\n", "# model_name = \"t5\"\n", "model_name = \"llama\"\n", - "# model_name = \"alpaca\"\n", + "model_name = \"alpaca\"\n", "finetuned = None\n", "\n", "model_options = dict(\n", @@ -180,11 +265,12 @@ " model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-11b\", **model_options)\n", " model.parallelize() # T5 is big enough that we may need to run it on multiple GPUs\n", "elif (\"llama\" in model_name) or (\"alpaca\" in model_name):\n", + " # https://github.com/deep-diver/LLM-As-Chatbot/blob/216abb559d00a0555f41a1426ac9db6c1abc24f3/models/alpaca.py\n", " model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", " lora_repo = \"tloen/alpaca-lora-7b\"\n", " model_type = \"decoder\"\n", - " tokenizer = AutoTokenizer.from_pretrained(model_repo)\n", - " model = AutoModelForCausalLM.from_pretrained(model_repo, **model_options)\n", + " tokenizer = LlamaTokenizer.from_pretrained(model_repo)\n", + " model = LlamaForCausalLM.from_pretrained(model_repo, **model_options)\n", " \n", " if \"alpaca\" in model_name:\n", " from peft import PeftModel\n", @@ -193,6 +279,9 @@ " lora_repo, \n", " device_map='auto'#{'': 0}\n", " )\n", + " \n", + " tokenizer.pad_token = 0\n", + " tokenizer.padding_side = \"left\"\n", "else:\n", " raise NotADirectoryError(model_name)\n", "model" @@ -200,29 +289,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T03:59:37.080463Z", - "start_time": "2023-05-07T03:59:37.074408Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-19T00:28:22.705285Z", - "start_time": "2023-05-19T00:28:22.702264Z" + "end_time": "2023-05-20T01:56:26.444015Z", + "start_time": "2023-05-20T01:56:26.442164Z" } }, "outputs": [], + "source": [ + "# tokenizer = AutoTokenizer.from_pretrained(model_repo)\n", + "# tokenizer.truncation_side='Left'\n", + "# tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:56:26.469934Z", + "start_time": "2023-05-20T01:56:26.444768Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(29900, 29896)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# get the tokens for 0 and 1, we will use these later...\n", - "id_0, id_1 = tokenizer('0')['input_ids'][-1], tokenizer('1')['input_ids'][-1]" + "id_0, id_1 = tokenizer('0')['input_ids'][-1], tokenizer('1')['input_ids'][-1]\n", + "id_0, id_1" ] }, { @@ -238,11 +343,23 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:26.441627Z", - "start_time": "2023-05-19T00:28:22.706863Z" + "end_time": "2023-05-19T04:02:13.892383Z", + "start_time": "2023-05-19T04:02:13.873377Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:56:27.020627Z", + "start_time": "2023-05-20T01:56:26.470949Z" }, "scrolled": false }, @@ -251,30 +368,49 @@ "name": "stderr", "output_type": "stream", "text": [ + "Using the latest cached version of the module from /home/wassname/.cache/huggingface/modules/datasets_modules/datasets/amazon_polarity/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc (last modified on Sat May 6 07:52:43 2023) since it couldn't be found locally at amazon_polarity.\n", "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "100%|██████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00, 1.87it/s]\n" + "100%|█████████████████████████████████████████| 2/2 [00:00<00:00, 3.96it/s]\n" ] + }, + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['label', 'title', 'content'],\n", + " num_rows: 3600000\n", + " })\n", + " test: Dataset({\n", + " features: ['label', 'title', 'content'],\n", + " num_rows: 400000\n", + " })\n", + "})" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ + "# debug\n", + "datasets.logging.set_verbosity_info()\n", + "\n", + "\n", "# Let's just try IMDB for simplicity\n", - "data = load_dataset(\"amazon_polarity\")[\"test\"]" + "data = load_dataset(\"amazon_polarity\")\n", + "# data = load_dataset(\"/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/amazon_polarity-train-00003-of-00004.arrow\")\n", + "data" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 5, + "execution_count": 22, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:26.446222Z", - "start_time": "2023-05-19T00:28:26.443481Z" + "end_time": "2023-05-20T02:25:05.461369Z", + "start_time": "2023-05-20T02:25:05.458241Z" } }, "outputs": [ @@ -282,171 +418,314 @@ "name": "stdout", "output_type": "stream", "text": [ - "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", - "###\n", - "Is this review negative? 1\n", - "###\n", - "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", - "###\n", - "Is this review negative? 0\n", - "###\n", "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "###\n", - "Is this review positive? 1\n", - "###\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", "Review: \"The movie was the worst.... not!\"\n", - "###\n", - "Is this review negative? \n" + "Question: Is this review negative?\n", + "Answer: \n", + "\n" ] } ], "source": [ "def format_imdb(text, label):\n", - " return f\"\"\"Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", - "###\n", - "Is this review negative? 1\n", - "###\n", - "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", - "###\n", - "Is this review negative? 0\n", - "###\n", - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "###\n", - "Is this review positive? 1\n", - "###\n", + " return f\"\"\"Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", "Review: \"{text}\"\n", - "###\n", - "Is this review {'positive' if label else 'negative'}? \"\"\"\n", - "\n", + "Question: Is this review {'positive' if label else 'negative'}?\n", + "Answer: \n", + "\"\"\"\n", "\n", + "def format_imdbs(texts, labels):\n", + " return [format_imdb(t, labels) for t in texts]\n", "\n", "print(format_imdb(\"The movie was the worst.... not!\", 0))" ] }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:05.902898Z", + "start_time": "2023-05-20T02:25:05.898745Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "174" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# tokens\n", + "len(tokenizer(format_imdb(\"The movie was the worst.... not!\", 0))['input_ids'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## First check models text output" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:45.805378Z", + "start_time": "2023-05-20T02:25:45.800064Z" + } + }, + "outputs": [], + "source": [ + "# gen_config_raw = {\n", + "# \"temperature\": temperature,\n", + "# \"top_p\": top_p,\n", + "# \"top_k\": top_k,\n", + "# \"repetition_penalty\": repetition_penalty,\n", + "# \"max_new_tokens\": max_new_tokens,\n", + "# \"num_beams\": num_beams,\n", + "# \"use_cache\": use_cache,\n", + "# \"do_sample\": do_sample,\n", + "# \"eos_token_id\": eos_token_id, \n", + "# \"pad_token_id\": pad_token_id\n", + "# }\n", + "\n", + "def get_output(model, tokenizer, input_text, add_bos_token=False, truncation_length=400):\n", + " \"\"\"\n", + " Given a decoder model and some text, gets the hidden states (in a given layer, by default the last) on that input text\n", + "\n", + " Returns a numpy array of shape (hidden_dim,)\n", + " \"\"\"\n", + " if not isinstance(input_text, list):\n", + " input_text = [input_text]\n", + " # tokenize (adding the EOS token this time)\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_text = [i[-1000:] for i in input_text]\n", + " input_ids = tokenizer(input_text, \n", + " return_tensors=\"pt\",\n", + "# truncation=True, \n", + "# padding=True,\n", + "# max_length=600,\n", + "# add_special_tokens=False,\n", + " ).input_ids.to(model.device)\n", + "# print('input_ids', input_ids.shape)\n", + "\n", + " # remove bos token? https://github.com/oobabooga/text-generation-webui/blob/1b52bddfcc70d2db88257d36f1c6d182573588c4/modules/text_generation.py#L36\n", + " if not add_bos_token and input_ids[0][0] == tokenizer.bos_token_id:\n", + " input_ids = input_ids[:, 1:]\n", + "\n", + "\n", + " # Llama adds this extra token when the first character is '\\n', and this\n", + " # compromises the stopping criteria, so we just remove it\n", + " if type(tokenizer) is LlamaTokenizer and input_ids[0][0] == 29871:\n", + " print('removed extra \\n token')\n", + " input_ids = input_ids[:, 1:]\n", + " \n", + " # Handling truncation\n", + " if truncation_length is not None:\n", + " input_ids = input_ids[:, -truncation_length:]\n", + "\n", + " # forward pass\n", + " with torch.no_grad():\n", + " output = model.generate(input_ids=input_ids, max_length=400)\n", + "# print(output)\n", + " \n", + " text_q = tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n", + " text_ans = tokenizer.batch_decode(output, skip_special_tokens=False)#, skip_prompt=True, skip_special_tokens=True)\n", + " print(text_q[0])\n", + " print('-'*40+'answ'+'-'*40)\n", + " print(text_ans[0])\n" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T09:17:04.250617Z", - "start_time": "2023-05-07T09:17:04.247644Z" + "end_time": "2023-05-20T01:54:09.323908Z", + "start_time": "2023-05-20T01:54:09.321888Z" } }, "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:46.260946Z", + "start_time": "2023-05-20T02:25:46.258734Z" + } + }, + "outputs": [], + "source": [ + "tokenizer.pad_token_id=0\n", + "tokenizer.padding_side = \"left\"" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:46.401304Z", + "start_time": "2023-05-20T02:25:46.398898Z" + } + }, + "outputs": [], + "source": [ + "idx = 1\n", + "text, true_label = data['test'][idx][\"content\"], data['test'][idx][\"label\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:53.587709Z", + "start_time": "2023-05-20T02:25:46.528753Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", + "Review: \"Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\"\n", + "Question: Is this review positive?\n", + "Answer: \n", + "\n", + "----------------------------------------answ----------------------------------------\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", + "Review: \"Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\"\n", + "Question: Is this review positive?\n", + "Answer: \n", + "---\n", + "Review: \"This is a great movie. It has a great storyline and the acting is superb. The characters are well developed and the plot is interesting. The special effects are great and the cinematography is beautiful. The music is also very good and fits the mood of the\n" + ] + } + ], + "source": [ + "input_text = [format_imdb(text, 1)]\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "get_output(model, tokenizer, input_text)" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## First let's write code for extracting hidden states given a model and text. \n", + "## Write code for extracting hidden states given a model and text. \n", "How we do this exactly will depend on the type of model." ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 41, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:26.468372Z", - "start_time": "2023-05-19T00:28:26.447139Z" + "end_time": "2023-05-20T02:27:33.628985Z", + "start_time": "2023-05-20T02:27:33.623501Z" } }, "outputs": [], "source": [ - "# def get_encoder_hidden_states(model, tokenizer, input_text, layer=-1):\n", - "# \"\"\"\n", - "# Given an encoder model and some text, gets the encoder hidden states (in a given layer, by default the last) \n", - "# on that input text (where the full text is given to the encoder).\n", "\n", - "# Returns a numpy array of shape (hidden_dim,)\n", - "# \"\"\"\n", - "# # tokenize\n", - "# encoder_text_ids = tokenizer(input_text, truncation=True, return_tensors=\"pt\").input_ids.to(model.device)\n", "\n", - "# # forward pass\n", - "# with torch.no_grad():\n", - "# output = model(encoder_text_ids, output_hidden_states=True)\n", - "\n", - "# # get the appropriate hidden states\n", - "# hs_tuple = output[\"hidden_states\"]\n", - " \n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - "\n", - "# return hs\n", - "\n", - "# def get_encoder_decoder_hidden_states(model, tokenizer, input_text, layer=-1):\n", - "# \"\"\"\n", - "# Given an encoder-decoder model and some text, gets the encoder hidden states (in a given layer, by default the last) \n", - "# on that input text (where the full text is given to the encoder).\n", - "\n", - "# Returns a numpy array of shape (hidden_dim,)\n", - "# \"\"\"\n", - "# # tokenize\n", - "# encoder_text_ids = tokenizer(input_text, return_tensors=\"pt\").input_ids.to(model.device)\n", - "# decoder_text_ids = tokenizer(\"\", return_tensors=\"pt\").input_ids.to(model.device)\n", - "\n", - "# # forward pass\n", - "# with torch.no_grad():\n", - "# output = model(encoder_text_ids, decoder_input_ids=decoder_text_ids, output_hidden_states=True)\n", - "\n", - "# # get the appropriate hidden states\n", - "# hs_tuple = output[\"encoder_hidden_states\"]\n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - "\n", - "# return hs\n", - "\n", - "def get_decoder_hidden_states(model, tokenizer, input_text, layers=[2, -2]):\n", + "def get_decoder_hidden_states(model, tokenizer, input_text, layers=[2, -2], add_bos_token=False, truncation_length=400):\n", " \"\"\"\n", " Given a decoder model and some text, gets the hidden states (in a given layer, by default the last) on that input text\n", "\n", " Returns a numpy array of shape (hidden_dim,)\n", " \"\"\"\n", + " if not isinstance(input_text, list):\n", + " input_text = [input_text]\n", " # tokenize (adding the EOS token this time)\n", - " input_ids = tokenizer(input_text, return_tensors=\"pt\").input_ids.to(model.device)\n", + " input_text = [i + tokenizer.eos_token for i in input_text]\n", + " input_text = [i[-1000:] for i in input_text]\n", + " input_ids = tokenizer(input_text, \n", + " return_tensors=\"pt\",\n", + "# truncation=True, \n", + " padding=True,\n", + "# max_length=600\n", + " ).input_ids.to(model.device)\n", + "# print('input_ids', input_ids.shape)\n", + "\n", + "\n", + " # remove bos token? https://github.com/oobabooga/text-generation-webui/blob/1b52bddfcc70d2db88257d36f1c6d182573588c4/modules/text_generation.py#L36\n", + " if not add_bos_token and input_ids[0][0] == tokenizer.bos_token_id:\n", + " input_ids = input_ids[:, 1:]\n", + "\n", + "\n", + " # Llama adds this extra token when the first character is '\\n', and this\n", + " # compromises the stopping criteria, so we just remove it\n", + " if type(tokenizer) is LlamaTokenizer and input_ids[0][0] == 29871:\n", + " print('removed extra \\n token')\n", + " input_ids = input_ids[:, 1:]\n", + " \n", + " # Handling truncation\n", + " if truncation_length is not None:\n", + " input_ids = input_ids[:, -truncation_length:]\n", "\n", " # forward pass\n", - " \n", " with torch.no_grad():\n", - " # FIXME: should be a batch, to speed it up\n", " output = model(input_ids, \n", - " output_hidden_states=True\n", + " output_hidden_states=True,\n", "# , output_attentions=True\n", + " use_cache=True,\n", + " \n", " )\n", - "\n", + " \n", " # the output is large, so we will just select what we want 1) the first token with[:, 0]\n", " # 2) selected layers with [layers]\n", "# output['attentions'] = [output['attentions'][i] for i in layers]\n", "# output['attentions'] = [v.detach().cpu()[:, -1] for v in output['attentions']]\n", "# output['attentions'] = torch.concat(output['attentions'])\n", " \n", - " output['hidden_states'] = [output['hidden_states'][i] for i in layers]\n", - " # dims [Batch, Token, Probs?]\n", - " output['hidden_states'] = [v.detach().cpu()[:, -1] for v in output['hidden_states']]\n", - " output['hidden_states'] = torch.concat(output['hidden_states'])\n", " \n", + " # dims [Batch, Token, Probs?]\n", + " output['hidden_states'] = torch.stack([output['hidden_states'][i] for i in layers], 1).detach().cpu()\n", + " # dims [Batch, Layers, Seq_Token, Probs?] e.g. torch.Size([3, 2, 284, 4096])\n", + " \n", + " # dims [Batch, ?, Output_Tokens] e.g. torch.Size([3, 284, 32000])\n", " o = output['logits'].detach().cpu().float().softmax(-1)\n", - "# print(input_text)\n", - "# print(tokenizer.decode(o.argmax(-1)[0]))\n", - " \n", - " prob_0, prob1 = o[0, 1][[id_0, id_1]]\n", - " output['ans'] = (prob1/(prob_0+prob1)).item()\n", + " \n", + " text_q = [tokenizer.decode(oo) for oo in input_ids]\n", + " text_ans = [tokenizer.decode(oo) for oo in o.argmax(-1)]\n", "\n", - " return dict(hidden_states=output['hidden_states'], ans=output['ans']\n", + " nth_place = 0\n", + " prob_0, prob1 = o[:, nth_place][:, [id_0, id_1]].T # get the prob of 0 vs 1 in nth place in answer\n", + " output['ans'] = (prob1/(prob_0+prob1))\n", + " # FIXME output batch\n", + " return dict(hidden_states=output['hidden_states'], ans=output['ans'], text_ans=text_ans, text_q=text_q\n", "# , attentions=output['attentions']\n", " )\n", - "# hs_tuple = output[\"hidden_states\"]\n", - "# hs_tuple = (a.detach().cpu().numpy() for a in hs_tuple)\n", - "# return output\n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - " \n", - " # FIXME pass full output, as I want to check model zero shot accuracy!\n", - "\n", - "# return hs\n", "\n", "def get_hidden_states(model, tokenizer, input_text, layers=[2, -2], model_type=\"encoder\"):\n", " fn = {\n", - "# \"encoder\": get_encoder_hidden_states, \"encoder_decoder\": get_encoder_decoder_hidden_states,\n", " \"decoder\": get_decoder_hidden_states}[model_type]\n", "\n", " return fn(model, tokenizer, input_text, layers=layers)" @@ -454,95 +733,59 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 42, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:26.489421Z", - "start_time": "2023-05-19T00:28:26.469251Z" + "end_time": "2023-05-20T02:27:33.946048Z", + "start_time": "2023-05-20T02:27:33.944125Z" } }, "outputs": [], "source": [ - "# print(format_imdb(text, 0))" + "# input_text = [format_imdb(text, 0)]\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_ids = tokenizer(input_text, \n", + "# return_tensors=\"pt\",\n", + "# truncation=True, \n", + "# padding=True,\n", + "# max_length=300).input_ids.to(model.device)\n", + "# print(tokenizer.decode(input_ids[0]))" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 43, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:27.461762Z", - "start_time": "2023-05-19T00:28:26.490379Z" - } + "end_time": "2023-05-20T02:27:34.584342Z", + "start_time": "2023-05-20T02:27:34.092856Z" + }, + "scrolled": true }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'hidden_states': tensor([[-0.0331, 0.0093, -0.0510, ..., -0.0213, 0.0366, 0.0545],\n", - " [-0.9800, 3.0742, 2.6406, ..., 4.4609, 1.9453, -0.5400]],\n", - " dtype=torch.float16),\n", - " 'ans': 0.6738358736038208}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# unit test\n", "idx = 0\n", - "text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", + "text, true_label = data['test'][idx][\"content\"], data['test'][idx][\"label\"]\n", "neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n", - "neg_hs" + "pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type)\n", + "# neg_hs" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T10:58:32.555372Z", - "start_time": "2023-05-07T10:58:32.555364Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-19T00:28:27.465171Z", - "start_time": "2023-05-19T00:28:27.463324Z" - } - }, - "outputs": [], - "source": [ - "# neg_hs[0]['hidden_states'][-1]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T10:45:46.521935Z", - "start_time": "2023-05-07T10:45:46.517116Z" - } - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-19T00:28:27.481113Z", - "start_time": "2023-05-19T00:28:27.467528Z" + "end_time": "2023-05-20T02:27:34.588474Z", + "start_time": "2023-05-20T02:27:34.585702Z" } }, "outputs": [ @@ -550,64 +793,93 @@ "name": "stdout", "output_type": "stream", "text": [ - "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", - "###\n", - "Is this review negative? 1\n", - "###\n", - "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", - "###\n", - "Is this review negative? 0\n", - "###\n", - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "###\n", - "Is this review positive? 1\n", - "###\n", + "----------------------------------------input----------------------------------------\n", + " hout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", - "###\n", - "Is this review positive? \n" + "Question: Is this review negative?\n", + "Answer: \n", + "\n", + "----------------------------------------answ----------------------------------------\n", + " Belowen\n", + "us hadver, a,x, isages a team situation team.The Brookels'. and is trying at. film.\n", + " stastic is the supporting actors who Gordon-Levitt and Ches Williams who. who Lloyd plays also as thevinThe Bull' who the rest are great. the film notch family production.\n", + " musting and funwwarming film that is should watch.\n", + "\": What this movie based or\n", + " AnswerAnswer: Yes\n", + "0\n", + "\n", + "Questionview: \"This reviewely wiferic been of the mostREATEATE in all generation. She' been to her CD many monthsEARS and it still loveVE it! She I hearm feeling the bad mood,' me smile even, When must dayood and makesaporates. a in a rain. I is is makesozes classIFE and IERYally are superawss perfectUNFFNING. theics are make me I of the's great gems. Bu CD a must islandle CD for my collection. I isn isn made it big is beyond a me. Sheone I listen this CD I matter what or white or or or old, it or female,VERYONEDY lov the thing:W is that?\"?\"\n", + "Question: Is this review positive?\n", + "Answer: 0---\n", + "================================================================================\n", + "----------------------------------------input----------------------------------------\n", + " hout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", + "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", + "Question: Is this review positive?\n", + "Answer: \n", + "\n", + "----------------------------------------answ----------------------------------------\n", + " Belowen\n", + "us hadver, a,x, isages a team situation team.The Brookels'. and is trying at. film.\n", + " stastic is the supporting actors who Gordon-Levitt and Ches Williams who. who Lloyd plays also as thevinThe Bull' who the rest are great. the film notch family production.\n", + " musting and funwwarming film that is should watch.\n", + "\": What this movie based or\n", + " AnswerAnswer: Yes\n", + "0\n", + "\n", + "Questionview: \"This reviewely wiferic been of the mostREATEATE in all generation. She' been to her CD many monthsEARS and it still loveVE it! She I hearm feeling the bad mood,' me feel even, When must mood and makesaporates. a in a rain. I is is makesozes classIFE and IERYally are superawss perfectUNPENING. theics are make me I of the's great gems. Bu CD a must islandle CD for my collection. I isn' made it big is beyond a me. She song I listen this CD I matter what or white or or or old, it or female,VERYONEDY lov the thing:W is that?\"?\"\n", + "Question: Is this review positive?\n", + "Answer: 1---\n", + "--------------------------------------------------------------------------------\n" ] } ], "source": [ - "print(format_imdb(text, 1))" + "print('-'*40+'input'+'-'*40)\n", + "print(neg_hs['text_q'][0])\n", + "print('-'*40+'answ'+'-'*40)\n", + "print(neg_hs['text_ans'][0])\n", + "print('='*80)\n", + "print('-'*40+'input'+'-'*40)\n", + "print(pos_hs['text_q'][0])\n", + "print('-'*40+'answ'+'-'*40)\n", + "print(pos_hs['text_ans'][0])\n", + "print('-'*80)" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 45, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:27.496331Z", - "start_time": "2023-05-19T00:28:27.482257Z" - } + "end_time": "2023-05-20T02:27:34.617886Z", + "start_time": "2023-05-20T02:27:34.589316Z" + }, + "scrolled": true }, "outputs": [], "source": [ - "# # sceatch\n", - "# layer = -10\n", - "# input_ids = tokenizer(text + tokenizer.eos_token, return_tensors=\"pt\").input_ids.to(model.device)\n", - "# # forward pass\n", - "# with torch.no_grad():\n", - "# output = model(input_ids, output_hidden_states=True)\n", + "# # unit tests\n", + "# idx = 0\n", + "# n=10\n", + "# batch_size=3\n", + "# ds_subset = data['test'].shuffle(42).select(range(n))\n", + "# dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)\n", + "# batch = next(iter(dl))\n", "\n", - "# # get the last layer, last token hidden states\n", - "# hs_tuple = output[\"hidden_states\"]\n", - "# hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", - "# hs, output['logits'], output['hidden_states']" + "# texts, true_labels = batch[\"content\"], batch[\"label\"]\n", + "# neg_hs = get_hidden_states(model, tokenizer, format_imdbs(texts, 0), model_type=model_type)\n", + "# neg_hs\n", + "# for k,v in neg_hs.items():\n", + "# print(k, v.shape)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T05:39:55.699656Z", - "start_time": "2023-05-07T05:39:55.653120Z" - } - }, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": {}, @@ -617,18 +889,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 18, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:28:27.512822Z", - "start_time": "2023-05-19T00:28:27.497303Z" + "end_time": "2023-05-20T01:56:35.161371Z", + "start_time": "2023-05-20T01:56:35.145412Z" } }, "outputs": [], "source": [ "\n", "\n", - "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100, layers=[2, -2]):\n", + "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100, layers=[2, -2], batch_size=3):\n", " \"\"\"\n", " Given an encoder-decoder model, a list of data, computes the contrast hidden states on n random examples.\n", " Returns numpy arrays of shape (n, hidden_dim) for each candidate label, along with a boolean numpy array of shape (n,)\n", @@ -638,51 +910,35 @@ " \"\"\"\n", " # setup\n", " model.eval()\n", - " all_neg_hs, all_pos_hs, all_gt_labels = [], [], []\n", - " all_neg_ans, all_pos_ans = [], []\n", - "\n", - " # loop\n", - " for _ in tqdm(range(n), unit='examples', desc='get_hidden_states'):\n", - " # for simplicity, sample a random example until we find one that's a reasonable length\n", - " # (most examples should be a reasonable length, so this is just to make sure)\n", - " while True:\n", - " idx = np.random.randint(len(data))\n", - " text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", - " # the actual formatted input will be longer, so include a bit of a margin\n", - " if len(tokenizer(text)) < 400: \n", - " break\n", - " \n", - " # get hidden states\n", - "# print(format_imdb(text, 0))\n", - " neg = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type, layers=layers)\n", - " pos = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type, layers=layers)\n", + " \n", + " res = []\n", + " \n", + " ds_subset = data['test'].shuffle(42).select(range(n))\n", + " dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)\n", + " for batch in tqdm(dl):\n", + " text, true_label = batch[\"content\"], batch[\"label\"]\n", + " neg = get_hidden_states(model, tokenizer, format_imdbs(text, 0), model_type=model_type, layers=layers)\n", + " pos = get_hidden_states(model, tokenizer, format_imdbs(text, 1), model_type=model_type, layers=layers)\n", "\n", " # collect\n", - " all_neg_hs.append(neg['hidden_states'].flatten())\n", - " all_pos_hs.append(pos['hidden_states'].flatten())\n", - " all_pos_ans.append(pos['ans'])\n", - " all_neg_ans.append(neg['ans'])\n", - " all_gt_labels.append(true_label)\n", - "\n", - " all_neg_hs = np.stack(all_neg_hs)\n", - " all_pos_hs = np.stack(all_pos_hs)\n", - " all_gt_labels = np.stack(all_gt_labels)\n", - "\n", - " return all_neg_hs, all_pos_hs, all_gt_labels, np.array(all_neg_ans), np.array(all_pos_ans)" + " b = len(text)\n", + "# print(neg['hidden_states'].shape)\n", + " res.append([\n", + " neg['hidden_states'].reshape((b,-1)),\n", + " pos['hidden_states'].reshape((b,-1)),\n", + " true_label,\n", + " neg['ans'], \n", + " pos['ans'], \n", + " ])\n", + " \n", + " # FIXME not all the hidden state are the same size, wat\n", + " res = [np.concatenate(r) for r in zip(*res)]\n", + " return res\n", + " all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res\n", + " return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans\n", + "# return all_neg_hs, all_pos_hs, all_gt_labels, np.array(all_neg_ans), np.array(all_pos_ans)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-13T11:50:54.694584Z", - "start_time": "2023-05-13T11:50:54.692039Z" - } - }, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": { @@ -697,11 +953,61 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 19, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:24.222213Z", - "start_time": "2023-05-19T00:28:27.513749Z" + "end_time": "2023-05-20T01:56:35.183039Z", + "start_time": "2023-05-20T01:56:35.162312Z" + } + }, + "outputs": [], + "source": [ + "# neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tokenizer, data, model_type, n=10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Speed\n", + "\n", + "- 60second for 100 no batching. 1.7 ex/s" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:56:35.501003Z", + "start_time": "2023-05-20T01:56:35.184047Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gc.collect()\n", + "torch.cuda.empty_cache()\n", + "gc.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:57:03.640525Z", + "start_time": "2023-05-20T01:56:35.502054Z" } }, "outputs": [ @@ -709,24 +1015,88 @@ "name": "stderr", "output_type": "stream", "text": [ - "get_hidden_states: 100%|█████████████████████████████████████████| 100/100 [00:56<00:00, 1.77examples/s]\n" + "Loading cached shuffled indices for dataset at /home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", + "100%|███████████████████████████████████████| 34/34 [00:27<00:00, 1.22it/s]\n" ] }, { "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <cell line: 1>:1                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tok     │\n",
+       "│   2                                                                                              │\n",
+       "│   3                                                                                              │\n",
+       "│   4 gc.collect()                                                                                 │\n",
+       "│                                                                                                  │\n",
+       "│ in get_hidden_states_many_examples:33                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   30 │   │   ])                                                                                  │\n",
+       "│   31 │                                                                                           │\n",
+       "│   32 │   # FIXME not all the hidden state are the same size, wat                                 │\n",
+       "│ ❱ 33 │   res = [np.concatenate(r) for r in zip(*res)]                                            │\n",
+       "│   34 │   return res                                                                              │\n",
+       "│   35 │   all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res                   │\n",
+       "│   36 │   return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans                  │\n",
+       "│                                                                                                  │\n",
+       "│ in <listcomp>:33                                                                                 │\n",
+       "│                                                                                                  │\n",
+       "│   30 │   │   ])                                                                                  │\n",
+       "│   31 │                                                                                           │\n",
+       "│   32 │   # FIXME not all the hidden state are the same size, wat                                 │\n",
+       "│ ❱ 33 │   res = [np.concatenate(r) for r in zip(*res)]                                            │\n",
+       "│   34 │   return res                                                                              │\n",
+       "│   35 │   all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res                   │\n",
+       "│   36 │   return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans                  │\n",
+       "│ in concatenate:200                                                                               │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "ValueError: all the input array dimensions except for the concatenation axis must match exactly, but along \n",
+       "dimension 1, the array at index 0 has size 2285568 and the array at index 1 has size 1908736\n",
+       "
\n" + ], "text/plain": [ - "0" + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tok \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mgc.collect() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m33\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# FIXME not all the hidden state are the same size, wat\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m33 \u001b[2m│ \u001b[0mres = [np.concatenate(r) \u001b[94mfor\u001b[0m r \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(*res)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ \u001b[0mall_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m33\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# FIXME not all the hidden state are the same size, wat\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m33 \u001b[2m│ \u001b[0mres = [np.concatenate(r) \u001b[94mfor\u001b[0m r \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(*res)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ \u001b[0mall_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mconcatenate\u001b[0m:\u001b[94m200\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mValueError: \u001b[0mall the input array dimensions except for the concatenation axis must match exactly, but along \n", + "dimension \u001b[1;36m1\u001b[0m, the array at index \u001b[1;36m0\u001b[0m has size \u001b[1;36m2285568\u001b[0m and the array at index \u001b[1;36m1\u001b[0m has size \u001b[1;36m1908736\u001b[0m\n" ] }, - "execution_count": 13, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ "neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tokenizer, data, model_type)\n", "\n", - "import gc\n", + "\n", "gc.collect()\n", "torch.cuda.empty_cache()\n", "gc.collect()" @@ -737,37 +1107,27 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-14T06:40:56.451564Z", - "start_time": "2023-05-14T06:40:56.451556Z" + "end_time": "2023-05-20T01:57:03.641742Z", + "start_time": "2023-05-20T01:57:03.641735Z" } }, "outputs": [], - "source": [] + "source": [ + "# all_pos_ans" + ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:24.238307Z", - "start_time": "2023-05-19T00:29:24.223444Z" + "end_time": "2023-05-20T01:57:03.642523Z", + "start_time": "2023-05-20T01:57:03.642517Z" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "(0.5366586538461539, 0.5759214743589743)" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n", - "\n", + "# roc_auc_score\n", "pos_score = roc_auc_score(y, all_pos_ans)\n", "neg_score = roc_auc_score(y, all_neg_ans)\n", "pos_score, neg_score" @@ -787,28 +1147,17 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:24.250717Z", - "start_time": "2023-05-19T00:29:24.239259Z" + "end_time": "2023-05-20T01:57:03.643517Z", + "start_time": "2023-05-20T01:57:03.643507Z" }, "scrolled": true }, - "outputs": [ - { - "data": { - "text/plain": [ - "(0.48, 0.52)" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "\n", + "# accuracy_score\n", "pos_score = accuracy_score(y, (all_pos_ans>0.)*1.0)\n", "neg_score = accuracy_score(y, (all_neg_ans<0.5)*1.0)\n", "pos_score, neg_score" @@ -827,23 +1176,14 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:24.317414Z", - "start_time": "2023-05-19T00:29:24.251856Z" + "end_time": "2023-05-20T01:57:03.644197Z", + "start_time": "2023-05-20T01:57:03.644190Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Logistic regression accuracy: 1.0 [TRAIN]\n", - "Logistic regression accuracy: 0.96 [TEST]\n" - ] - } - ], + "outputs": [], "source": [ "# let's create a simple 50/50 train split (the data is already randomized)\n", "n = len(y)\n", @@ -887,11 +1227,11 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:24.321545Z", - "start_time": "2023-05-19T00:29:24.318715Z" + "end_time": "2023-05-20T01:57:03.644851Z", + "start_time": "2023-05-20T01:57:03.644841Z" } }, "outputs": [], @@ -942,11 +1282,11 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:24.336538Z", - "start_time": "2023-05-19T00:29:24.322608Z" + "end_time": "2023-05-20T01:57:03.645452Z", + "start_time": "2023-05-20T01:57:03.645446Z" } }, "outputs": [], @@ -965,11 +1305,11 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:24.357517Z", - "start_time": "2023-05-19T00:29:24.337335Z" + "end_time": "2023-05-20T01:57:03.645991Z", + "start_time": "2023-05-20T01:57:03.645985Z" } }, "outputs": [], @@ -1007,17 +1347,15 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T00:29:25.004935Z", - "start_time": "2023-05-19T00:29:24.358571Z" + "end_time": "2023-05-19T04:12:55.004017Z", + "start_time": "2023-05-19T04:12:55.004011Z" } }, "outputs": [], - "source": [ - "import lightning.pytorch as pl" - ] + "source": [] }, { "cell_type": "markdown", @@ -1040,556 +1378,16 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.507525Z", - "start_time": "2023-05-19T00:29:25.006004Z" + "end_time": "2023-05-20T01:57:03.646546Z", + "start_time": "2023-05-20T01:57:03.646539Z" }, "scrolled": true }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "get_hidden_states: 90%|███████████████████████████████████▊ | 895/1000 [08:21<00:58, 1.78examples/s]\n" - ] - }, - { - "data": { - "text/html": [ - "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
-       "│ in <cell line: 94>:94                                                                            │\n",
-       "│                                                                                                  │\n",
-       "│   91                                                                                             │\n",
-       "│   92 # test                                                                                      │\n",
-       "│   93 dm = IMBDHSDataModule(model, tokenizer)                                                     │\n",
-       "│ ❱ 94 dm.setup('train')                                                                           │\n",
-       "│   95 dl = dm.val_dataloader()                                                                    │\n",
-       "│   96 b = next(iter(dl))                                                                          │\n",
-       "│   97 b                                                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ in setup:39                                                                                      │\n",
-       "│                                                                                                  │\n",
-       "│   36 │   │                                                                                       │\n",
-       "│   37 │   │   self.dataset = load_dataset(self.hparams.dataset_name, split=\"test\")                │\n",
-       "│   38 │   │                                                                                       │\n",
-       "│ ❱ 39 │   │   neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(      │\n",
-       "│   40 │   │   │   self.model, self.tokenizer, self.dataset, self.hparams.model_type, n=self.hp    │\n",
-       "│   41 │   │                                                                                       │\n",
-       "│   42 │   │   # let's create a simple 50/50 train split (the data is already randomized)          │\n",
-       "│                                                                                                  │\n",
-       "│ in get_hidden_states_many_examples:27                                                            │\n",
-       "│                                                                                                  │\n",
-       "│   24 │   │                                                                                       │\n",
-       "│   25 │   │   # get hidden states                                                                 │\n",
-       "│   26 #         print(format_imdb(text, 0))                                                       │\n",
-       "│ ❱ 27 │   │   neg = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model    │\n",
-       "│   28 │   │   pos = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model    │\n",
-       "│   29 │   │                                                                                       │\n",
-       "│   30 │   │   # collect                                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ in get_hidden_states:96                                                                          │\n",
-       "│                                                                                                  │\n",
-       "│   93 # \"encoder\": get_encoder_hidden_states, \"encoder_decoder\": get_encoder_decoder_hidden_st    │\n",
-       "│   94 │   │     \"decoder\": get_decoder_hidden_states}[model_type]                                 │\n",
-       "│   95 │                                                                                           │\n",
-       "│ ❱ 96 │   return fn(model, tokenizer, input_text, layers=layers)                                  │\n",
-       "│   97                                                                                             │\n",
-       "│                                                                                                  │\n",
-       "│ in get_decoder_hidden_states:56                                                                  │\n",
-       "│                                                                                                  │\n",
-       "│   53 │                                                                                           │\n",
-       "│   54 │   with torch.no_grad():                                                                   │\n",
-       "│   55 │   │   # FIXME: should be a batch, to speed it up                                          │\n",
-       "│ ❱ 56 │   │   output = model(input_ids,                                                           │\n",
-       "│   57 │   │   │   │   │      output_hidden_states=True                                            │\n",
-       "│   58 #                        , output_attentions=True                                           │\n",
-       "│   59 │   │   │   │   │     )                                                                     │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501  │\n",
-       "│ in _call_impl                                                                                    │\n",
-       "│                                                                                                  │\n",
-       "│   1498 │   │   if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks   │\n",
-       "│   1499 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hooks                   │\n",
-       "│   1500 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   │\n",
-       "│ ❱ 1501 │   │   │   return forward_call(*args, **kwargs)                                          │\n",
-       "│   1502 │   │   # Do not call functions when jit is used                                          │\n",
-       "│   1503 │   │   full_backward_hooks, non_full_backward_hooks = [], []                             │\n",
-       "│   1504 │   │   backward_pre_hooks = []                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in       │\n",
-       "│ new_forward                                                                                      │\n",
-       "│                                                                                                  │\n",
-       "│   162 │   │   │   with torch.no_grad():                                                          │\n",
-       "│   163 │   │   │   │   output = old_forward(*args, **kwargs)                                      │\n",
-       "│   164 │   │   else:                                                                              │\n",
-       "│ ❱ 165 │   │   │   output = old_forward(*args, **kwargs)                                          │\n",
-       "│   166 │   │   return module._hf_hook.post_forward(module, output)                                │\n",
-       "│   167 │                                                                                          │\n",
-       "│   168 │   module.forward = new_forward                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
-       "│ ng_llama.py:687 in forward                                                                       │\n",
-       "│                                                                                                  │\n",
-       "│   684 │   │   return_dict = return_dict if return_dict is not None else self.config.use_return   │\n",
-       "│   685 │   │                                                                                      │\n",
-       "│   686 │   │   # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)    │\n",
-       "│ ❱ 687 │   │   outputs = self.model(                                                              │\n",
-       "│   688 │   │   │   input_ids=input_ids,                                                           │\n",
-       "│   689 │   │   │   attention_mask=attention_mask,                                                 │\n",
-       "│   690 │   │   │   position_ids=position_ids,                                                     │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501  │\n",
-       "│ in _call_impl                                                                                    │\n",
-       "│                                                                                                  │\n",
-       "│   1498 │   │   if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks   │\n",
-       "│   1499 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hooks                   │\n",
-       "│   1500 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   │\n",
-       "│ ❱ 1501 │   │   │   return forward_call(*args, **kwargs)                                          │\n",
-       "│   1502 │   │   # Do not call functions when jit is used                                          │\n",
-       "│   1503 │   │   full_backward_hooks, non_full_backward_hooks = [], []                             │\n",
-       "│   1504 │   │   backward_pre_hooks = []                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in       │\n",
-       "│ new_forward                                                                                      │\n",
-       "│                                                                                                  │\n",
-       "│   162 │   │   │   with torch.no_grad():                                                          │\n",
-       "│   163 │   │   │   │   output = old_forward(*args, **kwargs)                                      │\n",
-       "│   164 │   │   else:                                                                              │\n",
-       "│ ❱ 165 │   │   │   output = old_forward(*args, **kwargs)                                          │\n",
-       "│   166 │   │   return module._hf_hook.post_forward(module, output)                                │\n",
-       "│   167 │                                                                                          │\n",
-       "│   168 │   module.forward = new_forward                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
-       "│ ng_llama.py:577 in forward                                                                       │\n",
-       "│                                                                                                  │\n",
-       "│   574 │   │   │   │   │   None,                                                                  │\n",
-       "│   575 │   │   │   │   )                                                                          │\n",
-       "│   576 │   │   │   else:                                                                          │\n",
-       "│ ❱ 577 │   │   │   │   layer_outputs = decoder_layer(                                             │\n",
-       "│   578 │   │   │   │   │   hidden_states,                                                         │\n",
-       "│   579 │   │   │   │   │   attention_mask=attention_mask,                                         │\n",
-       "│   580 │   │   │   │   │   position_ids=position_ids,                                             │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501  │\n",
-       "│ in _call_impl                                                                                    │\n",
-       "│                                                                                                  │\n",
-       "│   1498 │   │   if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks   │\n",
-       "│   1499 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hooks                   │\n",
-       "│   1500 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   │\n",
-       "│ ❱ 1501 │   │   │   return forward_call(*args, **kwargs)                                          │\n",
-       "│   1502 │   │   # Do not call functions when jit is used                                          │\n",
-       "│   1503 │   │   full_backward_hooks, non_full_backward_hooks = [], []                             │\n",
-       "│   1504 │   │   backward_pre_hooks = []                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in       │\n",
-       "│ new_forward                                                                                      │\n",
-       "│                                                                                                  │\n",
-       "│   162 │   │   │   with torch.no_grad():                                                          │\n",
-       "│   163 │   │   │   │   output = old_forward(*args, **kwargs)                                      │\n",
-       "│   164 │   │   else:                                                                              │\n",
-       "│ ❱ 165 │   │   │   output = old_forward(*args, **kwargs)                                          │\n",
-       "│   166 │   │   return module._hf_hook.post_forward(module, output)                                │\n",
-       "│   167 │                                                                                          │\n",
-       "│   168 │   module.forward = new_forward                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
-       "│ ng_llama.py:305 in forward                                                                       │\n",
-       "│                                                                                                  │\n",
-       "│   302 │   │   # Fully Connected                                                                  │\n",
-       "│   303 │   │   residual = hidden_states                                                           │\n",
-       "│   304 │   │   hidden_states = self.post_attention_layernorm(hidden_states)                       │\n",
-       "│ ❱ 305 │   │   hidden_states = self.mlp(hidden_states)                                            │\n",
-       "│   306 │   │   hidden_states = residual + hidden_states                                           │\n",
-       "│   307 │   │                                                                                      │\n",
-       "│   308 │   │   outputs = (hidden_states,)                                                         │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501  │\n",
-       "│ in _call_impl                                                                                    │\n",
-       "│                                                                                                  │\n",
-       "│   1498 │   │   if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks   │\n",
-       "│   1499 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hooks                   │\n",
-       "│   1500 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   │\n",
-       "│ ❱ 1501 │   │   │   return forward_call(*args, **kwargs)                                          │\n",
-       "│   1502 │   │   # Do not call functions when jit is used                                          │\n",
-       "│   1503 │   │   full_backward_hooks, non_full_backward_hooks = [], []                             │\n",
-       "│   1504 │   │   backward_pre_hooks = []                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in       │\n",
-       "│ new_forward                                                                                      │\n",
-       "│                                                                                                  │\n",
-       "│   162 │   │   │   with torch.no_grad():                                                          │\n",
-       "│   163 │   │   │   │   output = old_forward(*args, **kwargs)                                      │\n",
-       "│   164 │   │   else:                                                                              │\n",
-       "│ ❱ 165 │   │   │   output = old_forward(*args, **kwargs)                                          │\n",
-       "│   166 │   │   return module._hf_hook.post_forward(module, output)                                │\n",
-       "│   167 │                                                                                          │\n",
-       "│   168 │   module.forward = new_forward                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
-       "│ ng_llama.py:157 in forward                                                                       │\n",
-       "│                                                                                                  │\n",
-       "│   154 │   │   self.act_fn = ACT2FN[hidden_act]                                                   │\n",
-       "│   155 │                                                                                          │\n",
-       "│   156 │   def forward(self, x):                                                                  │\n",
-       "│ ❱ 157 │   │   return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))            │\n",
-       "│   158                                                                                            │\n",
-       "│   159                                                                                            │\n",
-       "│   160 class LlamaAttention(nn.Module):                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501  │\n",
-       "│ in _call_impl                                                                                    │\n",
-       "│                                                                                                  │\n",
-       "│   1498 │   │   if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks   │\n",
-       "│   1499 │   │   │   │   or _global_backward_pre_hooks or _global_backward_hooks                   │\n",
-       "│   1500 │   │   │   │   or _global_forward_hooks or _global_forward_pre_hooks):                   │\n",
-       "│ ❱ 1501 │   │   │   return forward_call(*args, **kwargs)                                          │\n",
-       "│   1502 │   │   # Do not call functions when jit is used                                          │\n",
-       "│   1503 │   │   full_backward_hooks, non_full_backward_hooks = [], []                             │\n",
-       "│   1504 │   │   backward_pre_hooks = []                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in       │\n",
-       "│ new_forward                                                                                      │\n",
-       "│                                                                                                  │\n",
-       "│   162 │   │   │   with torch.no_grad():                                                          │\n",
-       "│   163 │   │   │   │   output = old_forward(*args, **kwargs)                                      │\n",
-       "│   164 │   │   else:                                                                              │\n",
-       "│ ❱ 165 │   │   │   output = old_forward(*args, **kwargs)                                          │\n",
-       "│   166 │   │   return module._hf_hook.post_forward(module, output)                                │\n",
-       "│   167 │                                                                                          │\n",
-       "│   168 │   module.forward = new_forward                                                           │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/nn/modules.py:320   │\n",
-       "│ in forward                                                                                       │\n",
-       "│                                                                                                  │\n",
-       "│   317 │   │   if self.bias is not None and self.bias.dtype != x.dtype:                           │\n",
-       "│   318 │   │   │   self.bias.data = self.bias.data.to(x.dtype)                                    │\n",
-       "│   319 │   │                                                                                      │\n",
-       "│ ❱ 320 │   │   out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state)                 │\n",
-       "│   321 │   │                                                                                      │\n",
-       "│   322 │   │   if not self.state.has_fp16_weights:                                                │\n",
-       "│   323 │   │   │   if self.state.CB is not None and self.state.CxB is not None:                   │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/_functions │\n",
-       "│ .py:500 in matmul                                                                                │\n",
-       "│                                                                                                  │\n",
-       "│   497 │   state = state or MatmulLtState()                                                       │\n",
-       "│   498 │   if threshold > 0.0:                                                                    │\n",
-       "│   499 │   │   state.threshold = threshold                                                        │\n",
-       "│ ❱ 500 │   return MatMul8bitLt.apply(A, B, out, bias, state)                                      │\n",
-       "│   501                                                                                            │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/autograd/function.py:506   │\n",
-       "│ in apply                                                                                         │\n",
-       "│                                                                                                  │\n",
-       "│   503 │   │   if not torch._C._are_functorch_transforms_active():                                │\n",
-       "│   504 │   │   │   # See NOTE: [functorch vjp and autograd interaction]                           │\n",
-       "│   505 │   │   │   args = _functorch.utils.unwrap_dead_wrappers(args)                             │\n",
-       "│ ❱ 506 │   │   │   return super().apply(*args, **kwargs)  # type: ignore[misc]                    │\n",
-       "│   507 │   │                                                                                      │\n",
-       "│   508 │   │   if cls.setup_context == _SingleLevelFunction.setup_context:                        │\n",
-       "│   509 │   │   │   raise RuntimeError(                                                            │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/_functions │\n",
-       "│ .py:323 in forward                                                                               │\n",
-       "│                                                                                                  │\n",
-       "│   320 │   │   # 1. Quantize A                                                                    │\n",
-       "│   321 │   │   if len(A.shape) == 3:                                                              │\n",
-       "│   322 │   │   │   A = A.view(-1, A.shape[-1]).contiguous()                                       │\n",
-       "│ ❱ 323 │   │   CA, CAt, SCA, SCAt, coo_tensorA = F.double_quant(A.to(torch.float16), threshold=   │\n",
-       "│   324 │   │                                                                                      │\n",
-       "│   325 │   │   if state.threshold > 0.0 and coo_tensorA is not None:                              │\n",
-       "│   326 │   │   │   if state.has_fp16_weights:                                                     │\n",
-       "│                                                                                                  │\n",
-       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/functional.py:1660  │\n",
-       "│ in double_quant                                                                                  │\n",
-       "│                                                                                                  │\n",
-       "│   1657 │                                                                                         │\n",
-       "│   1658 │   is_on_gpu([A, col_stats, row_stats, out_col, out_row])                                │\n",
-       "│   1659 │   if threshold > 0.0:                                                                   │\n",
-       "│ ❱ 1660 │   │   nnz = nnz_row_ptr[-1].item()                                                      │\n",
-       "│   1661 │   │   if nnz > 0:                                                                       │\n",
-       "│   1662 │   │   │   coo_tensor = coo_zeros(                                                       │\n",
-       "│   1663 │   │   │   │   A.shape[0], A.shape[1], nnz_row_ptr[-1].item(), device                    │\n",
-       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
-       "KeyboardInterrupt\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m94\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m91 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m92 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m93 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m94 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m95 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m96 \u001b[0mb = \u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m97 \u001b[0mb \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m39\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m37 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.hparams.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m38 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m39 \u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m40 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[96mself\u001b[0m.dataset, \u001b[96mself\u001b[0m.hparams.model_type, n=\u001b[96mself\u001b[0m.hp \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m41 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m42 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# let's create a simple 50/50 train split (the data is already randomized)\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m27\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# get hidden states\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m26 \u001b[0m\u001b[2m# print(format_imdb(text, 0))\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m27 \u001b[2m│ │ \u001b[0mneg = get_hidden_states(model, tokenizer, format_imdb(text, \u001b[94m0\u001b[0m), model_type=model \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m28 \u001b[0m\u001b[2m│ │ \u001b[0mpos = get_hidden_states(model, tokenizer, format_imdb(text, \u001b[94m1\u001b[0m), model_type=model \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m29 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# collect\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states\u001b[0m:\u001b[94m96\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m93 \u001b[0m\u001b[2m# \"encoder\": get_encoder_hidden_states, \"encoder_decoder\": get_encoder_decoder_hidden_st\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m94 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mdecoder\u001b[0m\u001b[33m\"\u001b[0m: get_decoder_hidden_states}[model_type] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m95 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m96 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m fn(model, tokenizer, input_text, layers=layers) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m97 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mget_decoder_hidden_states\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# FIXME: should be a batch, to speed it up\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 \u001b[2m│ │ \u001b[0moutput = model(input_ids, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0moutput_hidden_states=\u001b[94mTrue\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[2m# , output_attentions=True\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m687\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m684 \u001b[0m\u001b[2m│ │ \u001b[0mreturn_dict = return_dict \u001b[94mif\u001b[0m return_dict \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m \u001b[96mself\u001b[0m.config.use_return \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m685 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m686 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m687 \u001b[2m│ │ \u001b[0moutputs = \u001b[96mself\u001b[0m.model( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m688 \u001b[0m\u001b[2m│ │ │ \u001b[0minput_ids=input_ids, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m689 \u001b[0m\u001b[2m│ │ │ \u001b[0mattention_mask=attention_mask, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m690 \u001b[0m\u001b[2m│ │ │ \u001b[0mposition_ids=position_ids, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m577\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m574 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m\u001b[94mNone\u001b[0m, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m575 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m576 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m577 \u001b[2m│ │ │ │ \u001b[0mlayer_outputs = decoder_layer( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m578 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mhidden_states, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m579 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mattention_mask=attention_mask, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m580 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mposition_ids=position_ids, \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m305\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m302 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Fully Connected\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m303 \u001b[0m\u001b[2m│ │ \u001b[0mresidual = hidden_states \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m304 \u001b[0m\u001b[2m│ │ \u001b[0mhidden_states = \u001b[96mself\u001b[0m.post_attention_layernorm(hidden_states) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m305 \u001b[2m│ │ \u001b[0mhidden_states = \u001b[96mself\u001b[0m.mlp(hidden_states) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m306 \u001b[0m\u001b[2m│ │ \u001b[0mhidden_states = residual + hidden_states \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m307 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m308 \u001b[0m\u001b[2m│ │ \u001b[0moutputs = (hidden_states,) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m157\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m154 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.act_fn = ACT2FN[hidden_act] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m155 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m156 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mforward\u001b[0m(\u001b[96mself\u001b[0m, x): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m157 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.down_proj(\u001b[96mself\u001b[0m.act_fn(\u001b[96mself\u001b[0m.gate_proj(x)) * \u001b[96mself\u001b[0m.up_proj(x)) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m158 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m159 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m160 \u001b[0m\u001b[94mclass\u001b[0m \u001b[4;92mLlamaAttention\u001b[0m(nn.Module): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/nn/\u001b[0m\u001b[1;33mmodules.py\u001b[0m:\u001b[94m320\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m317 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.bias \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.bias.dtype != x.dtype: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m318 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.bias.data = \u001b[96mself\u001b[0m.bias.data.to(x.dtype) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m319 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m320 \u001b[2m│ │ \u001b[0mout = bnb.matmul(x, \u001b[96mself\u001b[0m.weight, bias=\u001b[96mself\u001b[0m.bias, state=\u001b[96mself\u001b[0m.state) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m321 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m322 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m \u001b[96mself\u001b[0m.state.has_fp16_weights: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m323 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.state.CB \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.state.CxB \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/\u001b[0m\u001b[1;33m_functions\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[1;33m.py\u001b[0m:\u001b[94m500\u001b[0m in \u001b[92mmatmul\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m497 \u001b[0m\u001b[2m│ \u001b[0mstate = state \u001b[95mor\u001b[0m MatmulLtState() \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m498 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m threshold > \u001b[94m0.0\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m499 \u001b[0m\u001b[2m│ │ \u001b[0mstate.threshold = threshold \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m500 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m MatMul8bitLt.apply(A, B, out, bias, state) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m501 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/autograd/\u001b[0m\u001b[1;33mfunction.py\u001b[0m:\u001b[94m506\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mapply\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m503 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m torch._C._are_functorch_transforms_active(): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m504 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# See NOTE: [functorch vjp and autograd interaction]\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m505 \u001b[0m\u001b[2m│ │ │ \u001b[0margs = _functorch.utils.unwrap_dead_wrappers(args) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m506 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96msuper\u001b[0m().apply(*args, **kwargs) \u001b[2m# type: ignore[misc]\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m507 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m508 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mcls\u001b[0m.setup_context == _SingleLevelFunction.setup_context: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m509 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mRuntimeError\u001b[0m( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/\u001b[0m\u001b[1;33m_functions\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[1;33m.py\u001b[0m:\u001b[94m323\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m320 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# 1. Quantize A\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m321 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mlen\u001b[0m(A.shape) == \u001b[94m3\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m322 \u001b[0m\u001b[2m│ │ │ \u001b[0mA = A.view(-\u001b[94m1\u001b[0m, A.shape[-\u001b[94m1\u001b[0m]).contiguous() \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m323 \u001b[2m│ │ \u001b[0mCA, CAt, SCA, SCAt, coo_tensorA = F.double_quant(A.to(torch.float16), threshold= \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m324 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m325 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m state.threshold > \u001b[94m0.0\u001b[0m \u001b[95mand\u001b[0m coo_tensorA \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m326 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m state.has_fp16_weights: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/\u001b[0m\u001b[1;33mfunctional.py\u001b[0m:\u001b[94m1660\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mdouble_quant\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1657 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1658 \u001b[0m\u001b[2m│ \u001b[0mis_on_gpu([A, col_stats, row_stats, out_col, out_row]) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1659 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m threshold > \u001b[94m0.0\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1660 \u001b[2m│ │ \u001b[0mnnz = nnz_row_ptr[-\u001b[94m1\u001b[0m].item() \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1661 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m nnz > \u001b[94m0\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1662 \u001b[0m\u001b[2m│ │ │ \u001b[0mcoo_tensor = coo_zeros( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m1663 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mA.shape[\u001b[94m0\u001b[0m], A.shape[\u001b[94m1\u001b[0m], nnz_row_ptr[-\u001b[94m1\u001b[0m].item(), device \u001b[31m│\u001b[0m\n", - "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mKeyboardInterrupt\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "from dataclasses import dataclass\n", - "from torch.utils.data import random_split, DataLoader, TensorDataset\n", - "from transformers.models.auto.modeling_auto import AutoModel\n", - "# from scipy.stats import zscore\n", - "\n", - "from sklearn.preprocessing import RobustScaler\n", "\n", "# def normalize(x):\n", "# \"\"\"\n", @@ -1688,8 +1486,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.508827Z", - "start_time": "2023-05-19T01:18:07.508819Z" + "end_time": "2023-05-20T01:57:03.647107Z", + "start_time": "2023-05-20T01:57:03.647101Z" } }, "outputs": [], @@ -1709,8 +1507,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.509553Z", - "start_time": "2023-05-19T01:18:07.509545Z" + "end_time": "2023-05-20T01:57:03.647739Z", + "start_time": "2023-05-20T01:57:03.647733Z" } }, "outputs": [], @@ -1723,8 +1521,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.510256Z", - "start_time": "2023-05-19T01:18:07.510249Z" + "end_time": "2023-05-20T01:57:03.648618Z", + "start_time": "2023-05-20T01:57:03.648611Z" } }, "outputs": [], @@ -1835,8 +1633,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.510853Z", - "start_time": "2023-05-19T01:18:07.510846Z" + "end_time": "2023-05-20T01:57:03.649120Z", + "start_time": "2023-05-20T01:57:03.649114Z" } }, "outputs": [], @@ -1852,8 +1650,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.511523Z", - "start_time": "2023-05-19T01:18:07.511516Z" + "end_time": "2023-05-20T01:57:03.649727Z", + "start_time": "2023-05-20T01:57:03.649721Z" } }, "outputs": [], @@ -1866,8 +1664,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.512087Z", - "start_time": "2023-05-19T01:18:07.512080Z" + "end_time": "2023-05-20T01:57:03.650409Z", + "start_time": "2023-05-20T01:57:03.650402Z" }, "scrolled": true }, @@ -1883,8 +1681,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.512749Z", - "start_time": "2023-05-19T01:18:07.512742Z" + "end_time": "2023-05-20T01:57:03.650895Z", + "start_time": "2023-05-20T01:57:03.650889Z" } }, "outputs": [], @@ -1909,8 +1707,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.513235Z", - "start_time": "2023-05-19T01:18:07.513229Z" + "end_time": "2023-05-20T01:57:03.651740Z", + "start_time": "2023-05-20T01:57:03.651734Z" } }, "outputs": [], @@ -1946,8 +1744,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.513826Z", - "start_time": "2023-05-19T01:18:07.513819Z" + "end_time": "2023-05-20T01:57:03.652269Z", + "start_time": "2023-05-20T01:57:03.652263Z" } }, "outputs": [], @@ -1961,8 +1759,8 @@ "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-19T01:18:07.514335Z", - "start_time": "2023-05-19T01:18:07.514329Z" + "end_time": "2023-05-20T01:57:03.653040Z", + "start_time": "2023-05-20T01:57:03.653033Z" } }, "outputs": [], diff --git a/mjc_notes.md b/mjc_notes.md index d719443..7cacf9d 100644 --- a/mjc_notes.md +++ b/mjc_notes.md @@ -11,4 +11,5 @@ pip install -r requirements.txt - [x] Convert it to lightning - [ ] batch for get hidden states - - [ ] and cache + - [x] and cache + - [ ] 9s vs 60. so 10x faster