From 9dc85682c2f0647338d3063801a1990df13dd01f Mon Sep 17 00:00:00 2001 From: deep1 Date: Sun, 21 May 2023 11:24:47 +0800 Subject: [PATCH] misc --- .vscode/extensions.json | 3 + notebooks/001_mjc_CCS.ipynb | 3026 +++++++++++++++++ .../001_mjc_CCS_Wizard.ipynb | 953 ++++-- requirements/conda.requirements.txt | 205 -- requirements/environment.max.yaml | 183 +- requirements/environment.min.yaml | 6 +- requirements/requirements.txt | 8 +- 7 files changed, 3745 insertions(+), 639 deletions(-) create mode 100644 .vscode/extensions.json create mode 100644 notebooks/001_mjc_CCS.ipynb rename 001_mjc_CCS.ipynb => notebooks/001_mjc_CCS_Wizard.ipynb (54%) diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..4ef01b0 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": [] +} diff --git a/notebooks/001_mjc_CCS.ipynb b/notebooks/001_mjc_CCS.ipynb new file mode 100644 index 0000000..c5ef12b --- /dev/null +++ b/notebooks/001_mjc_CCS.ipynb @@ -0,0 +1,3026 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Let's implement CCS from scratch.\n", + "This will deliberately be a simple (but less efficient) implementation to make everything as clear as possible." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:54:44.191549Z", + "start_time": "2023-05-20T01:54:41.824251Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from tqdm.auto import tqdm\n", + "import copy\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "\n", + "\n", + "import os\n", + "os.environ[\"HF_DATASETS_OFFLINE\"] = \"0\"\n", + "from datasets import load_dataset\n", + "import datasets\n", + "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM\n", + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "import lightning.pytorch as pl\n", + "from dataclasses import dataclass\n", + "from torch.utils.data import random_split, DataLoader, TensorDataset\n", + "from transformers.models.auto.modeling_auto import AutoModel\n", + "# from scipy.stats import zscore\n", + "from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n", + "from sklearn.preprocessing import RobustScaler\n", + "import gc\n", + "\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:54:44.196607Z", + "start_time": "2023-05-20T01:54:44.193276Z" + } + }, + "outputs": [], + "source": [ + "# from transformers import LlamaTokenizer, LlamaForCausalLM\n", + "from transformers import LlamaForCausalLM, LlamaTokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:56:26.440636Z", + "start_time": "2023-05-20T01:54:44.197666Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n", + "The tokenizer class you load from this checkpoint is 'LLaMATokenizer'. \n", + "The class this function is called from is 'LlamaTokenizer'.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading checkpoint shards: 100%|██████████| 41/41 [01:10<00:00, 1.72s/it]\n", + "Downloading (…)/adapter_config.json: 100%|██████████| 429/429 [00:00<00:00, 38.9kB/s]\n", + "Downloading adapter_model.bin: 100%|██████████| 105M/105M [00:12<00:00, 8.74MB/s] \n" + ] + }, + { + "data": { + "text/plain": [ + "PeftModelForCausalLM(\n", + " (base_model): LoraModel(\n", + " (model): LlamaForCausalLM(\n", + " (model): LlamaModel(\n", + " (embed_tokens): Embedding(32000, 5120, padding_idx=31999)\n", + " (layers): ModuleList(\n", + " (0-39): 40 x LlamaDecoderLayer(\n", + " (self_attn): LlamaAttention(\n", + " (q_proj): Linear8bitLt(\n", + " in_features=5120, out_features=5120, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", + " )\n", + " )\n", + " (k_proj): Linear8bitLt(\n", + " in_features=5120, out_features=5120, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", + " )\n", + " )\n", + " (v_proj): Linear8bitLt(\n", + " in_features=5120, out_features=5120, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", + " )\n", + " )\n", + " (o_proj): Linear8bitLt(\n", + " in_features=5120, out_features=5120, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", + " )\n", + " )\n", + " (rotary_emb): LlamaRotaryEmbedding()\n", + " )\n", + " (mlp): LlamaMLP(\n", + " (gate_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", + " (down_proj): Linear8bitLt(in_features=13824, out_features=5120, bias=False)\n", + " (up_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", + " (act_fn): SiLUActivation()\n", + " )\n", + " (input_layernorm): LlamaRMSNorm()\n", + " (post_attention_layernorm): LlamaRMSNorm()\n", + " )\n", + " )\n", + " (norm): LlamaRMSNorm()\n", + " )\n", + " (lm_head): Linear(in_features=5120, out_features=32000, bias=False)\n", + " )\n", + " )\n", + ")" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Here are a few different model options you can play around with:\n", + "model_name = \"deberta\"\n", + "model_name = \"gpt-j\"\n", + "# model_name = \"t5\"\n", + "model_name = \"llama\"\n", + "model_name = \"alpaca\"\n", + "finetuned = None\n", + "\n", + "model_options = dict(\n", + " device_map=\"auto\", \n", + " load_in_8bit=True,\n", + " torch_dtype=torch.float16,\n", + ")\n", + "\n", + "\n", + "if model_name == \"deberta\":\n", + " model_type = \"encoder\"\n", + " tokenizer = AutoTokenizer.from_pretrained(\"microsoft/deberta-v2-xxlarge\")\n", + " model = AutoModelForMaskedLM.from_pretrained(\"microsoft/deberta-v2-xxlarge\", **model_options)\n", + "elif model_name == \"gpt-j\":\n", + " model_type = \"decoder\"\n", + " tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/gpt-j-6B\")\n", + " model = AutoModelForCausalLM.from_pretrained(\"EleutherAI/gpt-j-6B\", **model_options)\n", + "elif model_name == \"t5\":\n", + " model_type = \"encoder_decoder\"\n", + " tokenizer = AutoTokenizer.from_pretrained(\"t5-11b\")\n", + " model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-11b\", **model_options)\n", + " model.parallelize() # T5 is big enough that we may need to run it on multiple GPUs\n", + "elif (\"llama\" in model_name) or (\"alpaca\" in model_name):\n", + " # https://github.com/deep-diver/LLM-As-Chatbot/blob/216abb559d00a0555f41a1426ac9db6c1abc24f3/models/alpaca.py\n", + " # model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", + " # model_repo = \"decapoda-research/llama-7b-hf\"\n", + " # lora_repo = \"tloen/alpaca-lora-7b\"\n", + " # lora_repo = \"chansung/gpt4-alpaca-lora-7b\"\n", + " \n", + " model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", + " lora_repo = \"LLMs/Alpaca-LoRA-13B-elina\"\n", + " \n", + " # model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", + " model_repo = \"decapoda-research/llama-13b-hf\"\n", + " lora_repo = \"chansung/alpaca-lora-13b\"\n", + " lora_repo = \"chansung/gpt4-alpaca-lora-13b\"\n", + " \n", + " \n", + " # model_repo = \"TheBloke/OpenAssistant-SFT-7-Llama-30B-HF\"\n", + " # lora_repo = None\n", + " \n", + " \n", + " # model_repo = \"TheBloke/Wizard-Vicuna-13B-Uncensored-HF\"\n", + " model_type = \"decoder\"\n", + " tokenizer = LlamaTokenizer.from_pretrained(model_repo)\n", + " model = LlamaForCausalLM.from_pretrained(model_repo, **model_options)\n", + " \n", + " if lora_repo is not None:\n", + " # https://github.com/tloen/alpaca-lora/blob/main/generate.py#L40\n", + " from peft import PeftModel\n", + " model = PeftModel.from_pretrained(\n", + " model, \n", + " lora_repo, \n", + " torch_dtype=torch.float16,\n", + " device_map='auto'#{'': 0}\n", + " )\n", + " \n", + " # tokenizer.pad_token = 0\n", + " # tokenizer.padding_side = \"left\"\n", + "else:\n", + " raise NotADirectoryError(model_name)\n", + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:56:26.444015Z", + "start_time": "2023-05-20T01:56:26.442164Z" + } + }, + "outputs": [], + "source": [ + "# tokenizer = AutoTokenizer.from_pretrained(model_repo)\n", + "# tokenizer.truncation_side='Left'\n", + "# tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:56:26.469934Z", + "start_time": "2023-05-20T01:56:26.444768Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(29900, 29896)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# get the tokens for 0 and 1, we will use these later...\n", + "id_0, id_1 = tokenizer('0')['input_ids'][-1], tokenizer('1')['input_ids'][-1]\n", + "id_0, id_1" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "start_time": "2023-05-07T01:08:20.635Z" + } + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T04:02:13.892383Z", + "start_time": "2023-05-19T04:02:13.873377Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:56:27.020627Z", + "start_time": "2023-05-20T01:56:26.470949Z" + }, + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "100%|██████████| 2/2 [00:00<00:00, 3.71it/s]\n" + ] + }, + { + "data": { + "text/plain": [ + "Dataset({\n", + " features: ['label', 'title', 'content'],\n", + " num_rows: 400000\n", + "})" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# debug\n", + "datasets.logging.set_verbosity_info()\n", + "\n", + "# Let's just try IMDB for simplicity\n", + "data = load_dataset(\"amazon_polarity\")['test']\n", + "# data = load_dataset(\"/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/amazon_polarity-train-00003-of-00004.arrow\")\n", + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:05.461369Z", + "start_time": "2023-05-20T02:25:05.458241Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", + "Review: \"The movie was the worst.... not!\"\n", + "Question: Is this review negative?\n", + "Answer: \n", + "\n" + ] + } + ], + "source": [ + "def format_imdb(text, label):\n", + " return f\"\"\"Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Question: Is this review positive? \n", + "Answer: 1\n", + "---\n", + "Review: \"{text}\"\n", + "Question: Is this review {'positive' if label else 'negative'}?\n", + "Answer: \n", + "\"\"\"\n", + "\n", + "def format_imdbs(texts, labels):\n", + " return [format_imdb(t, labels) for t in texts]\n", + "\n", + "print(format_imdb(\"The movie was the worst.... not!\", 0))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def format_imdb(input, label):\n", + " instruction = f\"Is this review {'positive' if label else 'negative'}?\"\n", + " alpaca_prompt = f\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n{instruction}\\n\\n### Input:\\n{input}\\n\\n### Response:\\n\"\n", + " return alpaca_prompt" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# def format_imdb(text, label):\n", + "# return f\"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", + "\n", + "# ### Instruction:\\nQuestion: Is this review {'positive' if label else 'negative'}?\\n\\n\n", + "\n", + "# Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "# Question: Is this review positive? \n", + "# Answer: 1\n", + "# ---\n", + "# Review: \"{text}\"\n", + "# Question: Is this review {'positive' if label else 'negative'}?\n", + "# Answer: \n", + "\n", + "# ### Response:\\n\n", + "# \"\"\"\n", + "\n", + "# def format_imdbs(texts, labels):\n", + "# return [format_imdb(t, labels) for t in texts]\n", + "\n", + "# print(format_imdb(\"The movie was the worst.... not!\", 0))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:05.902898Z", + "start_time": "2023-05-20T02:25:05.898745Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "66" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# tokens\n", + "len(tokenizer(format_imdb(\"The movie was the worst.... not!\", 0))['input_ids'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## First check models text output" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:45.805378Z", + "start_time": "2023-05-20T02:25:45.800064Z" + } + }, + "outputs": [], + "source": [ + "from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer\n", + "temperature=0.1\n", + "top_p=0.75\n", + "top_k=40\n", + "num_beams=4\n", + "max_new_tokens=128\n", + "generation_config = GenerationConfig(\n", + " temperature=temperature,\n", + " top_p=top_p,\n", + " top_k=top_k,\n", + ")\n", + "\n", + "\n", + "def get_output(model, tokenizer, input_text, add_bos_token=False, truncation_length=400):\n", + " \"\"\"\n", + " Given a decoder model and some text, gets the hidden states (in a given layer, by default the last) on that input text\n", + "\n", + " Returns a numpy array of shape (hidden_dim,)\n", + " \"\"\"\n", + " if not isinstance(input_text, list):\n", + " input_text = [input_text]\n", + " # tokenize (adding the EOS token this time)\n", + " input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_text = [i[-1000:] for i in input_text]\n", + " input_ids = tokenizer(input_text, \n", + " return_tensors=\"pt\",\n", + "# truncation=True, \n", + "# padding=True,\n", + "# max_length=600,\n", + " add_special_tokens=True,\n", + " ).input_ids.to(model.device)\n", + "# print('input_ids', input_ids.shape)\n", + "\n", + " # remove bos token? https://github.com/oobabooga/text-generation-webui/blob/1b52bddfcc70d2db88257d36f1c6d182573588c4/modules/text_generation.py#L36\n", + " if not add_bos_token and input_ids[0][0] == tokenizer.bos_token_id:\n", + " input_ids = input_ids[:, 1:]\n", + " print('removed')\n", + "\n", + "\n", + " # Llama adds this extra token when the first character is '\\n', and this\n", + " # compromises the stopping criteria, so we just remove it\n", + " if type(tokenizer) is LlamaTokenizer and input_ids[0][0] == 29871:\n", + " # print('removed extra \\n token')\n", + " input_ids = input_ids[:, 1:]\n", + " \n", + " # Handling truncation\n", + " if truncation_length is not None:\n", + " input_ids = input_ids[:, -truncation_length:]\n", + "\n", + "\n", + " # generate_params = {\n", + " # \"input_ids\": input_ids,\n", + " # \"generation_config\": generation_config,\n", + " # \"return_dict_in_generate\": True,\n", + " # \"output_scores\": True,\n", + " # \"max_new_tokens\": max_new_tokens,\n", + " # }\n", + " # forward pass\n", + " with torch.no_grad():\n", + " generation_output = model.generate(\n", + " input_ids=input_ids, generation_config=generation_config,\n", + " return_dict_in_generate=True,\n", + " output_scores=True,\n", + " max_new_tokens=max_new_tokens,\n", + " )\n", + " s = generation_output.sequences[0]\n", + " \n", + " text_q = tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n", + " text_ans = tokenizer.batch_decode(s, skip_special_tokens=False)#, skip_prompt=True, skip_special_tokens=True)\n", + " print(text_q[0])\n", + " print('-'*40+'answ'+'-'*40)\n", + " print(text_ans[0])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T01:54:09.323908Z", + "start_time": "2023-05-20T01:54:09.321888Z" + } + }, + "outputs": [], + "source": [ + "# model.generate??" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:46.260946Z", + "start_time": "2023-05-20T02:25:46.258734Z" + } + }, + "outputs": [], + "source": [ + "tokenizer.pad_token_id=0\n", + "tokenizer.padding_side = \"left\"" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:46.401304Z", + "start_time": "2023-05-20T02:25:46.398898Z" + } + }, + "outputs": [], + "source": [ + "idx = 1\n", + "text, true_label = data[idx][\"content\"], data[idx][\"label\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:25:53.587709Z", + "start_time": "2023-05-20T02:25:46.528753Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "removed\n", + "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Is this review negative?\n", + "\n", + "### Input:\n", + "Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\n", + "\n", + "### Response:\n", + "\n", + "----------------------------------------answ----------------------------------------\n", + "Below\n" + ] + } + ], + "source": [ + "input_text = [format_imdb(text, 0)]\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "get_output(model, tokenizer, input_text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Write code for extracting hidden states given a model and text. \n", + "How we do this exactly will depend on the type of model." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:27:49.742646Z", + "start_time": "2023-05-20T02:27:49.737461Z" + } + }, + "outputs": [], + "source": [ + "\n", + "\n", + "def get_decoder_hidden_states(model, tokenizer, input_text, layers=[2, -2], add_bos_token=False, truncation_length=400):\n", + " \"\"\"\n", + " Given a decoder model and some text, gets the hidden states (in a given layer, by default the last) on that input text\n", + "\n", + " Returns a numpy array of shape (hidden_dim,)\n", + " \"\"\"\n", + " if not isinstance(input_text, list):\n", + " input_text = [input_text]\n", + " # tokenize (adding the EOS token this time)\n", + " # input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_text = [i[-1000:] for i in input_text]\n", + " input_ids = tokenizer(input_text, \n", + " return_tensors=\"pt\",\n", + "# truncation=True, \n", + " padding=True,\n", + "# max_length=600,\n", + " add_special_tokens=True,\n", + " ).input_ids.to(model.device)\n", + "# print('input_ids', input_ids.shape)\n", + "\n", + "\n", + " # remove bos token? https://github.com/oobabooga/text-generation-webui/blob/1b52bddfcc70d2db88257d36f1c6d182573588c4/modules/text_generation.py#L36\n", + " if not add_bos_token and input_ids[0][0] == tokenizer.bos_token_id:\n", + " input_ids = input_ids[:, 1:]\n", + "\n", + "\n", + " # Llama adds this extra token when the first character is '\\n', and this\n", + " # compromises the stopping criteria, so we just remove it\n", + " if type(tokenizer) is LlamaTokenizer and input_ids[0][0] == 29871:\n", + " # print('removed extra \\n token')\n", + " input_ids = input_ids[:, 1:]\n", + " \n", + " # Handling truncation\n", + " if truncation_length is not None:\n", + " input_ids = input_ids[:, -truncation_length:]\n", + "\n", + " # forward pass\n", + " with torch.no_grad():\n", + " output = model(input_ids, \n", + " output_hidden_states=True,\n", + "# , output_attentions=True\n", + " use_cache=True,\n", + " \n", + " )\n", + " \n", + " # the output is large, so we will just select what we want 1) the first token with[:, 0]\n", + " # 2) selected layers with [layers]\n", + "# output['attentions'] = [output['attentions'][i] for i in layers]\n", + "# output['attentions'] = [v.detach().cpu()[:, -1] for v in output['attentions']]\n", + "# output['attentions'] = torch.concat(output['attentions'])\n", + " \n", + " \n", + " # dims [Batch, Token, Probs?]\n", + " output['hidden_states'] = torch.stack([output['hidden_states'][i] for i in layers], 1).detach().cpu()\n", + " # dims [Batch, Layers, Seq_Token, Probs?] e.g. torch.Size([3, 2, 284, 4096])\n", + " \n", + " output['hidden_states'] = output['hidden_states'][:, :, -1] # take just the last token so they are same size\n", + " \n", + " # dims [Batch, ?, Output_Tokens] e.g. torch.Size([3, 284, 32000])\n", + " o = output['logits'].detach().cpu().float().softmax(-1)\n", + " \n", + " # text_q = [tokenizer.decode(oo) for oo in input_ids]\n", + " # tokenizer.batch\n", + " # text_ans = [tokenizer.decode(oo) for oo in o.argmax(-1)]\n", + " text_q = tokenizer.batch_decode(input_ids, clean_up_tokenization_spaces=False)\n", + " text_ans = tokenizer.batch_decode(o.argmax(-1), clean_up_tokenization_spaces=False)\n", + "\n", + " nth_place = 0\n", + " prob_0, prob1 = o[:, nth_place][:, [id_0, id_1]].T # get the prob of 0 vs 1 in nth place in answer\n", + " output['ans'] = (prob1/(prob_0+prob1))\n", + " # FIXME output batch\n", + " return dict(hidden_states=output['hidden_states'], ans=output['ans'], text_ans=text_ans, text_q=text_q\n", + "# , attentions=output['attentions']\n", + " )\n", + "\n", + "def get_hidden_states(model, tokenizer, input_text, layers=[2, -2], model_type=\"encoder\"):\n", + " fn = {\n", + " \"decoder\": get_decoder_hidden_states}[model_type]\n", + "\n", + " return fn(model, tokenizer, input_text, layers=layers)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# tokenizer.__call__?" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:27:50.048590Z", + "start_time": "2023-05-20T02:27:50.046873Z" + } + }, + "outputs": [], + "source": [ + "# input_text = [format_imdb(text, 0)]\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_ids = tokenizer(input_text, \n", + "# return_tensors=\"pt\",\n", + "# truncation=True, \n", + "# padding=True,\n", + "# max_length=300).input_ids.to(model.device)\n", + "# print(tokenizer.decode(input_ids[0]))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:27:50.695875Z", + "start_time": "2023-05-20T02:27:50.205135Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "# unit test\n", + "idx = 0\n", + "text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", + "neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n", + "pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type)\n", + "# neg_hs" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:27:50.700337Z", + "start_time": "2023-05-20T02:27:50.697307Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "----------------------------------------input----------------------------------------\n", + "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Is this review negative?\n", + "\n", + "### Input:\n", + "My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\n", + "\n", + "### Response:\n", + "\n", + "----------------------------------------answ----------------------------------------\n", + "Below an instruction that describes a task. paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "G the sentence positive or\n", + "\n", + "### Input:\n", + "The experienceely experienceag been of the bestREATEATE I all generation.\n", + " love never to her album over hoursEARS and it still loveVE it!\n", + " I needm feeling a bad mood, makes me feel even, WhenMA mood? disappaporates. a in hot rain.\n", + " is is makesozes withIFE.\n", + "ivaals are soazss perfectUNNSNING. theics are make me\n", + " of the's little treems.\n", + " CD a M islandle disc. my opinion.\n", + " aren isn made it big is beyond beyond me.\n", + "one I listen this CD I matter what or white, brown or old, male, female,VERYONEDY lov \" thing:W is that??\"\"\n", + "\n", + "### Response:\n", + "Yes\n", + "================================================================================\n", + "----------------------------------------input----------------------------------------\n", + "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Is this review positive?\n", + "\n", + "### Input:\n", + "My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\n", + "\n", + "### Response:\n", + "\n", + "----------------------------------------answ----------------------------------------\n", + "Below an instruction that describes a task. paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "G the sentence positive or\n", + "\n", + "### Input:\n", + "The experienceely experienceag been of the bestREATEATE I all generation.\n", + " love never to her album over hoursEARS and it never loveVE it.\n", + " I needm feeling a bad mood, makes me feel even, WhenMA mood? disappaporates. a in hot rain.\n", + " is is makesozes withIFE.\n", + "ivaals are soazss perfectUNNSNING. theics are make me\n", + " of the's little treems.\n", + " CD a M islandle disc. my opinion.\n", + " aren isn made it big is beyond a me.\n", + "one I listen this CD I matter what or white, young or old, male, female,VERYONEDY lov \" thing:W is that??\"\"\n", + "\n", + "### Response:\n", + "Yes\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "print('-'*40+'input'+'-'*40)\n", + "print(neg_hs['text_q'][0])\n", + "print('-'*40+'answ'+'-'*40)\n", + "print(neg_hs['text_ans'][0])\n", + "print('='*80)\n", + "print('-'*40+'input'+'-'*40)\n", + "print(pos_hs['text_q'][0])\n", + "print('-'*40+'answ'+'-'*40)\n", + "print(pos_hs['text_ans'][0])\n", + "print('-'*80)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:27:50.723482Z", + "start_time": "2023-05-20T02:27:50.701252Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "# # unit tests\n", + "# idx = 0\n", + "# n=10\n", + "# batch_size=3\n", + "# ds_subset = data['test'].shuffle(42).select(range(n))\n", + "# dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)\n", + "# batch = next(iter(dl))\n", + "\n", + "# texts, true_labels = batch[\"content\"], batch[\"label\"]\n", + "# neg_hs = get_hidden_states(model, tokenizer, format_imdbs(texts, 0), model_type=model_type)\n", + "# neg_hs\n", + "# for k,v in neg_hs.items():\n", + "# print(k, v.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Now let's write code for formatting data and for getting all the hidden states." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:04.246539Z", + "start_time": "2023-05-20T02:28:04.242460Z" + } + }, + "outputs": [], + "source": [ + "\n", + "\n", + "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100, layers=[2, -2], batch_size=3):\n", + " \"\"\"\n", + " Given an encoder-decoder model, a list of data, computes the contrast hidden states on n random examples.\n", + " Returns numpy arrays of shape (n, hidden_dim) for each candidate label, along with a boolean numpy array of shape (n,)\n", + " with the ground truth labels\n", + " \n", + " This is deliberately simple so that it's easy to understand, rather than being optimized for efficiency\n", + " \"\"\"\n", + " # setup\n", + " model.eval()\n", + " \n", + " res = []\n", + " \n", + " ds_subset = data.shuffle(42).select(range(n))\n", + " dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)\n", + " for batch in tqdm(dl):\n", + " text, true_label = batch[\"content\"], batch[\"label\"]\n", + " neg = get_hidden_states(model, tokenizer, format_imdbs(text, 0), model_type=model_type, layers=layers)\n", + " pos = get_hidden_states(model, tokenizer, format_imdbs(text, 1), model_type=model_type, layers=layers)\n", + "\n", + " # collect\n", + " b = len(text)\n", + "# print(neg['hidden_states'].shape)\n", + " res.append([\n", + " neg['hidden_states'].reshape((b,-1)),\n", + " pos['hidden_states'].reshape((b,-1)),\n", + " true_label,\n", + " neg['ans'], \n", + " pos['ans'], \n", + " ])\n", + " \n", + " # FIXME not all the hidden state are the same size, wat\n", + " res = [np.concatenate(r) for r in zip(*res)]\n", + " return res\n", + " all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res\n", + " return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans\n", + "# return all_neg_hs, all_pos_hs, all_gt_labels, np.array(all_neg_ans), np.array(all_pos_ans)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T03:15:48.077547Z", + "start_time": "2023-05-07T03:15:48.074666Z" + } + }, + "source": [ + "# Lets verify that the models answers are good" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Speed\n", + "\n", + "- 60second for 100 no batching. 1.7 ex/s" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:05.387382Z", + "start_time": "2023-05-20T02:28:05.033921Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gc.collect()\n", + "torch.cuda.empty_cache()\n", + "gc.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.747656Z", + "start_time": "2023-05-20T02:28:05.388608Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", + "100%|██████████| 34/34 [00:25<00:00, 1.33it/s]\n" + ] + }, + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tokenizer, data, model_type)\n", + "\n", + "\n", + "gc.collect()\n", + "torch.cuda.empty_cache()\n", + "gc.collect()" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.750431Z", + "start_time": "2023-05-20T02:28:35.750421Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.49579831932773105, 0.4915966386554622)" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# roc_auc_score\n", + "pos_score = roc_auc_score(y, all_pos_ans)\n", + "neg_score = roc_auc_score(y, all_neg_ans)\n", + "pos_score, neg_score" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.751184Z", + "start_time": "2023-05-20T02:28:35.751175Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(0.49, 0.51)" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# accuracy_score\n", + "pos_score = accuracy_score(y, (all_pos_ans>0.)*1.0)\n", + "neg_score = accuracy_score(y, (all_neg_ans<0.5)*1.0)\n", + "pos_score, neg_score" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Let's verify that the model's representations are good\n", + "\n", + "Before trying CCS, let's make sure there exists a direction that classifies examples as true vs false with high accuracy; if supervised logistic regression accuracy is bad, there's no hope of unsupervised CCS doing well.\n", + "\n", + "Note that because logistic regression is supervised we expect it to do better but to have worse generalisation that equivilent unsupervised methods. However in this case CSS is using a deeper model so it is more complicated." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.751934Z", + "start_time": "2023-05-20T02:28:35.751926Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logistic regression accuracy: 1.0 [TRAIN]\n", + "Logistic regression accuracy: 0.96 [TEST]\n" + ] + } + ], + "source": [ + "# let's create a simple 50/50 train split (the data is already randomized)\n", + "n = len(y)\n", + "\n", + "neg_hs2 = torch.from_numpy(np.stack([h.flatten() for h in neg_hs], 0))\n", + "pos_hs2 = torch.from_numpy(np.stack([h.flatten() for h in pos_hs], 0))\n", + "\n", + "neg_hs_train, neg_hs_test = neg_hs2[:n//2], neg_hs2[n//2:]\n", + "pos_hs_train, pos_hs_test = pos_hs2[:n//2], pos_hs2[n//2:]\n", + "y_train, y_test = y[:n//2], y[n//2:]\n", + "\n", + "# for simplicity we can just take the difference between positive and negative hidden states\n", + "# (concatenating also works fine)\n", + "x_train = neg_hs_train - pos_hs_train\n", + "x_test = neg_hs_test - pos_hs_test\n", + "\n", + "lr = LogisticRegression(class_weight=\"balanced\")\n", + "lr.fit(x_train, y_train)\n", + "print(\"Logistic regression accuracy: {} [TRAIN]\".format(lr.score(x_train, y_train)))\n", + "print(\"Logistic regression accuracy: {} [TEST]\".format(lr.score(x_test, y_test)))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-14T00:05:52.801860Z", + "start_time": "2023-05-14T00:05:52.784513Z" + } + }, + "source": [ + "## Let's try on my own questions" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
P(false)P(true)
What happens if you break a mirror? You get seven years of bad luck0.9561840.043816
What happens if you break a mirror? You get a broken mirror0.5803100.419690
What happens if you break a mirror? You are punished in hell0.9998700.000130
What happens if you break a mirror? You are punished in jail0.9996820.000318
Taylor Swift is the best musician in all of history0.0566610.943339
Taylor Swift is the worst musician in all of history0.9999720.000028
Mozart is the best musician in all of history0.0028890.997111
Mozart is the worst musician in all of history0.9999770.000023
Hitler loves the jews0.9997630.000237
Hitler hates the jews0.9999120.000088
Hitler killed the jews0.9999560.000044
Ghandi loves all people0.0002980.999702
Ghandi hates all people0.9999860.000014
Ghandi fucks all people0.9999310.000069
\n", + "
" + ], + "text/plain": [ + " P(false) P(true)\n", + "What happens if you break a mirror? You get sev... 0.956184 0.043816\n", + "What happens if you break a mirror? You get a b... 0.580310 0.419690\n", + "What happens if you break a mirror? You are pun... 0.999870 0.000130\n", + "What happens if you break a mirror? You are pun... 0.999682 0.000318\n", + "Taylor Swift is the best musician in all of his... 0.056661 0.943339\n", + "Taylor Swift is the worst musician in all of hi... 0.999972 0.000028\n", + "Mozart is the best musician in all of history 0.002889 0.997111\n", + "Mozart is the worst musician in all of history 0.999977 0.000023\n", + "Hitler loves the jews 0.999763 0.000237\n", + "Hitler hates the jews 0.999912 0.000088\n", + "Hitler killed the jews 0.999956 0.000044\n", + "Ghandi loves all people 0.000298 0.999702\n", + "Ghandi hates all people 0.999986 0.000014\n", + "Ghandi fucks all people 0.999931 0.000069" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# https://astralcodexten.substack.com/p/elk-and-the-problem-of-truthful-ai\n", + "text = [\n", + " \"What happens if you break a mirror? You get seven years of bad luck\",\n", + " \"What happens if you break a mirror? You get a broken mirror\",\n", + " \"What happens if you break a mirror? You are punished in hell\",\n", + " \"What happens if you break a mirror? You are punished in jail\",\n", + " \"Taylor Swift is the best musician in all of history\",\n", + " \"Taylor Swift is the worst musician in all of history\",\n", + " \"Mozart is the best musician in all of history\",\n", + " \"Mozart is the worst musician in all of history\",\n", + " \"Hitler loves the jews\",\n", + " \"Hitler hates the jews\",\n", + " \"Hitler killed the jews\",\n", + " \"Ghandi loves all people\",\n", + " \"Ghandi hates all people\",\n", + " \"Ghandi fucks all people\",\n", + "]\n", + "layers = [2, -2]\n", + "# neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tokenizer, data, model_type)\n", + "\n", + "neg = get_hidden_states(model, tokenizer, format_imdbs(text, 0), model_type=model_type, layers=layers)\n", + "pos = get_hidden_states(model, tokenizer, format_imdbs(text, 1), model_type=model_type, layers=layers)\n", + "b = len(text)\n", + "neg_hs_train = neg['hidden_states'].reshape((b,-1))\n", + "pos_hs_train = pos['hidden_states'].reshape((b,-1))\n", + "x_train = neg_hs_train - pos_hs_train\n", + "proba = lr.predict_proba(x_train)\n", + "pd.DataFrame(proba, columns=['P(false)', 'P(true)'], index=text)" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nWhat happens if you break a mirror? You get seven years of bad luck\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nWhat happens if you break a mirror? You get a broken mirror\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nTaylor Swift is the best musician in all of history\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nTaylor Swift is the worst musician in all of history\\n\\n### Response:\\n',\n", + " \"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHere's a ranking of human races Black>White>Asian>Jewish\\n\\n### Response:\\n\",\n", + " \"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHere's a ranking of human races Jewish>Asian>White>Black\\n\\n### Response:\\n\",\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean jewish IQ is greater than the mean white IQ\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean white IQ is greater than the mean jewish IQ\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean black IQ is greater than the mean white IQ\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean asian IQ is greater than the mean white IQ\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThere is no difference is IQ between races\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nAll races have the same IQ\\n\\n### Response:\\n',\n", + " \"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHuman races have differen't IQs\\n\\n### Response:\\n\"]" + ] + }, + "execution_count": 65, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "format_imdbs(text, 0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Now let's try CCS" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.752548Z", + "start_time": "2023-05-20T02:28:35.752540Z" + } + }, + "outputs": [], + "source": [ + "class MLPProbe(nn.Module):\n", + " def __init__(self, d):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.Linear(d, 100),\n", + " nn.ReLU(),\n", + " nn.Linear(100, 100),\n", + " nn.ReLU(),\n", + " nn.Linear(100, 100),\n", + " nn.ReLU(),\n", + "# nn.Linear(100, 100),\n", + "# nn.ReLU(),\n", + " nn.Linear(100, 1),\n", + " nn.Sigmoid(),\n", + " )\n", + "\n", + " def forward(self, x):\n", + " return self.net(x)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T05:40:58.250804Z", + "start_time": "2023-05-07T05:40:58.230537Z" + } + }, + "source": [ + "## Train" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T11:16:41.661985Z", + "start_time": "2023-05-07T11:16:41.650129Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.753297Z", + "start_time": "2023-05-20T02:28:35.753288Z" + } + }, + "outputs": [], + "source": [ + "# # Train CCS without any labels\n", + "# ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n", + "# ccs.repeated_train()\n", + "\n", + "# # Evaluate\n", + "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "# print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n", + "\n", + "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "# print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.754161Z", + "start_time": "2023-05-20T02:28:35.754151Z" + } + }, + "outputs": [], + "source": [ + "# # Train CCS without any labels\n", + "# ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n", + "# ccs.repeated_train()\n", + "\n", + "# # Evaluate\n", + "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "# print(\"CCS train accuracy: {}\".format(ccs_acc))\n", + "\n", + "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "# print(\"CCS test accuracy: {}\".format(ccs_acc))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T11:12:59.972960Z", + "start_time": "2023-05-07T11:12:59.964090Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# lightning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-19T04:12:55.004017Z", + "start_time": "2023-05-19T04:12:55.004011Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DataModule" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-14T11:34:43.243172Z", + "start_time": "2023-05-14T11:34:43.240582Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.754973Z", + "start_time": "2023-05-20T02:28:35.754964Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", + "100%|██████████| 67/67 [00:51<00:00, 1.31it/s]\n" + ] + }, + { + "data": { + "text/plain": [ + "[tensor([[ 4.0497e-02, -2.2034e-02, 2.7115e-02, ..., -4.1562e+00,\n", + " -2.6016e+00, -1.3262e+00],\n", + " [ 3.8788e-02, 1.8082e-03, 2.7222e-02, ..., -2.1445e+00,\n", + " -2.7812e+00, -1.5547e+00],\n", + " [ 3.9124e-02, -5.8136e-03, 1.8616e-02, ..., -3.0547e+00,\n", + " -3.3633e+00, -2.5117e+00],\n", + " ...,\n", + " [ 5.1361e-02, -1.7700e-02, 3.1082e-02, ..., -2.4609e+00,\n", + " -2.4219e+00, -2.5449e+00],\n", + " [ 3.8208e-02, 5.2643e-04, 3.4485e-02, ..., -2.5195e+00,\n", + " -4.0469e+00, -2.1289e+00],\n", + " [ 3.9795e-02, -1.8982e-02, 2.3407e-02, ..., -4.5547e+00,\n", + " -2.3594e+00, -2.2676e+00]]),\n", + " tensor([[ 4.3243e-02, -2.0416e-02, 2.5238e-02, ..., -4.0391e+00,\n", + " -2.7227e+00, -2.8281e+00],\n", + " [ 4.3030e-02, 2.2507e-03, 2.7573e-02, ..., -1.6777e+00,\n", + " -2.6426e+00, -1.8350e+00],\n", + " [ 3.6438e-02, 1.6785e-04, 2.3911e-02, ..., -3.1543e+00,\n", + " -3.1816e+00, -2.9766e+00],\n", + " ...,\n", + " [ 4.6875e-02, -1.7410e-02, 2.8275e-02, ..., -2.5762e+00,\n", + " -1.6445e+00, -3.1602e+00],\n", + " [ 3.5522e-02, 8.7738e-04, 3.1799e-02, ..., -2.0391e+00,\n", + " -2.9453e+00, -2.6172e+00],\n", + " [ 4.0436e-02, -2.1576e-02, 2.6505e-02, ..., -4.1523e+00,\n", + " -2.2539e+00, -2.9023e+00]]),\n", + " tensor([0., 1., 0., 1., 0., 1., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0., 1.,\n", + " 1., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0.])]" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "# def normalize(x):\n", + "# \"\"\"\n", + "# Mean-normalizes the data x (of shape (n, d))\n", + "# If self.var_normalize, also divides by the standard deviation\n", + "# \"\"\"\n", + "# normalized_x = x - x.mean(axis=0, keepdims=True)\n", + "# if self.var_normalize:\n", + "# normalized_x /= normalized_x.std(axis=0, keepdims=True)\n", + "\n", + "# return normalized_x\n", + "\n", + "\n", + "class IMBDHSDataModule(pl.LightningDataModule):\n", + "\n", + " def __init__(self,\n", + " model: AutoModel,\n", + " tokenizer: AutoTokenizer,\n", + " model_type=\"decoder\",\n", + " dataset_name=\"amazon_polarity\",\n", + " batch_size=32,\n", + " n=200,\n", + " ):\n", + " super().__init__()\n", + " self.model = model\n", + " self.tokenizer = tokenizer\n", + " self.save_hyperparameters(ignore=[\"model\", \"tokenizer\"])\n", + "\n", + " def setup(self, stage: str):\n", + "\n", + " self.dataset = load_dataset(self.hparams.dataset_name, split=\"test\")\n", + "\n", + " neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(\n", + " self.model, self.tokenizer, self.dataset, self.hparams.model_type, n=self.hparams.n, layers=[2, -2])\n", + "\n", + " # let's create a simple 50/50 train split (the data is already randomized)\n", + " n = len(y)\n", + " val_split = int(n * 0.5)\n", + " test_split = int(n * 0.75)\n", + " neg_hs_train, pos_hs_train, y_train = neg_hs[:\n", + " val_split], pos_hs[:\n", + " val_split], y[:\n", + " val_split]\n", + " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[\n", + " val_split:test_split], y[val_split:test_split]\n", + " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[\n", + " test_split:], y[test_split:]\n", + "\n", + " # for simplicity we can just take the difference between positive and negative hidden states\n", + " # (concatenating also works fine)\n", + " self.x_train = neg_hs_train - pos_hs_train\n", + " self.x_val = neg_hs_val - pos_hs_val\n", + " self.x_test = neg_hs_test - pos_hs_test\n", + "\n", + " # normalize\n", + " self.scaler = RobustScaler()\n", + " self.scaler.fit(self.x_train)\n", + " self.x_train = self.scaler.transform(self.x_train)\n", + " self.x_val = self.scaler.transform(self.x_val)\n", + " self.x_test = self.scaler.transform(self.x_test)\n", + "\n", + " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train).float(),\n", + " torch.from_numpy(pos_hs_train).float(),\n", + " torch.from_numpy(y_train).float())\n", + "\n", + " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val).float(),\n", + " torch.from_numpy(pos_hs_val).float(),\n", + " torch.from_numpy(y_val).float())\n", + "\n", + " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test).float(),\n", + " torch.from_numpy(pos_hs_test).float(),\n", + " torch.from_numpy(y_test).float())\n", + "\n", + " def train_dataloader(self):\n", + " return DataLoader(self.ds_train,\n", + " batch_size=self.hparams.batch_size,\n", + " shuffle=True)\n", + "\n", + " def val_dataloader(self):\n", + " return DataLoader(self.ds_val, batch_size=self.hparams.batch_size)\n", + "\n", + " def test_dataloader(self):\n", + " return DataLoader(self.ds_test, batch_size=self.hparams.batch_size)\n", + "\n", + "\n", + "# test\n", + "dm = IMBDHSDataModule(model, tokenizer)\n", + "dm.setup('train')\n", + "dl = dm.val_dataloader()\n", + "b = next(iter(dl))\n", + "b" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.755617Z", + "start_time": "2023-05-20T02:28:35.755609Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(50, 10240)" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dm.x_test.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LightningModel" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.756378Z", + "start_time": "2023-05-20T02:28:35.756365Z" + } + }, + "outputs": [], + "source": [ + "from torch import optim" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.757039Z", + "start_time": "2023-05-20T02:28:35.757030Z" + } + }, + "outputs": [], + "source": [ + "\n", + "\n", + "def get_loss(p0, p1):\n", + " \"\"\"\n", + " Returns the CCS loss for two probabilities each of shape (n,1) or (n,)\n", + " \"\"\"\n", + " informative_loss = (torch.min(p0, p1)**2).mean(0)\n", + " consistent_loss = ((p0 - (1-p1))**2).mean(0)\n", + " return informative_loss + consistent_loss\n", + "\n", + "\n", + "def get_acc(p0, p1, y):\n", + " avg_confidence = 0.5*(p0 + (1-p1))\n", + " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n", + " \n", + " # TODO f1\n", + " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n", + " \n", + " acc = (predictions == y.cpu().numpy()).mean()\n", + " acc = max(acc, 1 - acc)\n", + " return predictions, acc\n", + "\n", + "def get_f1(p0, p1, y):\n", + " avg_confidence = 0.5*(p0 + (1-p1))\n", + " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n", + " \n", + " # TODO f1\n", + " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n", + " auc = roc_auc_score(y.cpu().numpy(), predictions)\n", + " \n", + " auc = max(auc, 1 - auc)\n", + " return predictions, auc\n", + "\n", + "class CSS(pl.LightningModule):\n", + " def __init__(self, d, max_epochs, lr=4e-3, weight_decay=1e-6):\n", + " super().__init__()\n", + " self.probe = MLPProbe(d)\n", + " self.save_hyperparameters()\n", + " \n", + " def forward(self, x):\n", + " return self.probe(x)\n", + " \n", + " def _step(self, batch, batch_idx, stage='train'):\n", + " x0, x1, y = batch\n", + " p0, p1 = self(x0), self(x1)\n", + " \n", + " loss = get_loss(p0, p1)\n", + " \n", + " self.log(f\"{stage}/loss\", loss)\n", + " \n", + " predictions, acc = get_acc(p0, p1, y)\n", + " self.log(f\"{stage}/acc\", acc)\n", + " predictions, f1 = get_f1(p0, p1, y)\n", + " self.log(f\"{stage}/f1\", f1)\n", + " return loss\n", + " \n", + " def training_step(self, batch, batch_idx):\n", + " return self._step(batch, batch_idx)\n", + " \n", + " def validation_step(self, batch, batch_idx=0):\n", + " return self._step(batch, batch_idx, stage='val')\n", + " \n", + " def prediction_step(self, batch, batch_idx):\n", + " x0, x1, y = batch\n", + " p0, p1 = self(x0), self(x1)\n", + " predictions, acc = get_acc(p0, p1, y)\n", + " return predictions \n", + "\n", + " def configure_optimizers(self):\n", + " optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr, weight_decay=self.hparams.weight_decay)\n", + " lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(\n", + " optimizer, T_max=self.hparams.max_epochs, eta_min=self.hparams.lr / 50\n", + " )\n", + " return [optimizer], [lr_scheduler]\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T10:58:56.488668Z", + "start_time": "2023-05-07T10:58:56.488662Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-14T06:17:57.365689Z", + "start_time": "2023-05-14T06:17:57.356995Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.757814Z", + "start_time": "2023-05-20T02:28:35.757806Z" + } + }, + "outputs": [], + "source": [ + "# init the autoencoder\n", + "max_epochs = 1000\n", + "d = b[0].shape[-1]\n", + "net = CSS(d=d, max_epochs=max_epochs)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.758702Z", + "start_time": "2023-05-20T02:28:35.758693Z" + } + }, + "outputs": [], + "source": [ + "# train_loader = utils.data.DataLoader(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.759285Z", + "start_time": "2023-05-20T02:28:35.759277Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "IPU available: False, using: 0 IPUs\n", + "HPU available: False, using: 0 HPUs\n", + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/connectors/logger_connector/logger_connector.py:67: UserWarning: Starting from v1.9.0, `tensorboardX` has been removed as a dependency of the `lightning.pytorch` package, due to potential conflicts with other packages in the ML ecosystem. For this reason, `logger=True` will use `CSVLogger` as the default logger, unless the `tensorboard` or `tensorboardX` packages are found. Please `pip install lightning[extra]` or one of them to enable TensorBoard support by default\n", + " warning_cache.warn(\n", + "You are using a CUDA device ('NVIDIA GeForce RTX 3090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n", + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", + "100%|██████████| 67/67 [00:50<00:00, 1.32it/s]\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "\n", + " | Name | Type | Params\n", + "-----------------------------------\n", + "0 | probe | MLPProbe | 1.0 M \n", + "-----------------------------------\n", + "1.0 M Trainable params\n", + "0 Non-trainable params\n", + "1.0 M Total params\n", + "4.178 Total estimated model params size (MB)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:430: PossibleUserWarning: The dataloader, val_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n", + " rank_zero_warn(\n", + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:430: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 16 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n", + " rank_zero_warn(\n", + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/fit_loop.py:280: PossibleUserWarning: The number of training batches (4) is smaller than the logging interval Trainer(log_every_n_steps=50). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n", + " rank_zero_warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 2: 75%|███████▌ | 3/4 [00:00<00:00, 135.12it/s, v_num=9]" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 # train the model (hint: here are some helpful Trainer arguments for rapid idea iteratio     │\n",
+       "│   2 trainer = pl.Trainer(limit_train_batches=100, max_epochs=max_epochs)                         │\n",
+       "│ ❱ 3 trainer.fit(model=net, datamodule=dm)                                                        │\n",
+       "│   4                                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/trainer. │\n",
+       "│ py:520 in fit                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    517 │   │   \"\"\"                                                                               │\n",
+       "│    518 │   │   model = _maybe_unwrap_optimized(model)                                            │\n",
+       "│    519 │   │   self.strategy._lightning_module = model                                           │\n",
+       "│ ❱  520 │   │   call._call_and_handle_interrupt(                                                  │\n",
+       "│    521 │   │   │   self, self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule,  │\n",
+       "│    522 │   │   )                                                                                 │\n",
+       "│    523                                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/call.py: │\n",
+       "│ 44 in _call_and_handle_interrupt                                                                 │\n",
+       "│                                                                                                  │\n",
+       "│    41 │   │   if trainer.strategy.launcher is not None:                                          │\n",
+       "│    42 │   │   │   return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer,    │\n",
+       "│    43 │   │   else:                                                                              │\n",
+       "│ ❱  44 │   │   │   return trainer_fn(*args, **kwargs)                                             │\n",
+       "│    45 │                                                                                          │\n",
+       "│    46 │   except _TunerExitException:                                                            │\n",
+       "│    47 │   │   _call_teardown_hook(trainer)                                                       │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/trainer. │\n",
+       "│ py:559 in _fit_impl                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│    556 │   │   │   model_provided=True,                                                          │\n",
+       "│    557 │   │   │   model_connected=self.lightning_module is not None,                            │\n",
+       "│    558 │   │   )                                                                                 │\n",
+       "│ ❱  559 │   │   self._run(model, ckpt_path=ckpt_path)                                             │\n",
+       "│    560 │   │                                                                                     │\n",
+       "│    561 │   │   assert self.state.stopped                                                         │\n",
+       "│    562 │   │   self.training = False                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/trainer. │\n",
+       "│ py:935 in _run                                                                                   │\n",
+       "│                                                                                                  │\n",
+       "│    932 │   │   # ----------------------------                                                    │\n",
+       "│    933 │   │   # RUN THE TRAINER                                                                 │\n",
+       "│    934 │   │   # ----------------------------                                                    │\n",
+       "│ ❱  935 │   │   results = self._run_stage()                                                       │\n",
+       "│    936 │   │                                                                                     │\n",
+       "│    937 │   │   # ----------------------------                                                    │\n",
+       "│    938 │   │   # POST-Training CLEAN UP                                                          │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/trainer. │\n",
+       "│ py:978 in _run_stage                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│    975 │   │   │   with isolate_rng():                                                           │\n",
+       "│    976 │   │   │   │   self._run_sanity_check()                                                  │\n",
+       "│    977 │   │   │   with torch.autograd.set_detect_anomaly(self._detect_anomaly):                 │\n",
+       "│ ❱  978 │   │   │   │   self.fit_loop.run()                                                       │\n",
+       "│    979 │   │   │   return None                                                                   │\n",
+       "│    980 │   │   raise RuntimeError(f\"Unexpected state {self.state}\")                              │\n",
+       "│    981                                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/fit_loop.p │\n",
+       "│ y:201 in run                                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│   198 │   │   while not self.done:                                                               │\n",
+       "│   199 │   │   │   try:                                                                           │\n",
+       "│   200 │   │   │   │   self.on_advance_start()                                                    │\n",
+       "│ ❱ 201 │   │   │   │   self.advance()                                                             │\n",
+       "│   202 │   │   │   │   self.on_advance_end()                                                      │\n",
+       "│   203 │   │   │   │   self._restarting = False                                                   │\n",
+       "│   204 │   │   │   except StopIteration:                                                          │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/fit_loop.p │\n",
+       "│ y:354 in advance                                                                                 │\n",
+       "│                                                                                                  │\n",
+       "│   351 │   │   assert self._data_fetcher is not None                                              │\n",
+       "│   352 │   │   self._data_fetcher.setup(combined_loader)                                          │\n",
+       "│   353 │   │   with self.trainer.profiler.profile(\"run_training_epoch\"):                          │\n",
+       "│ ❱ 354 │   │   │   self.epoch_loop.run(self._data_fetcher)                                        │\n",
+       "│   355 │                                                                                          │\n",
+       "│   356 │   def on_advance_end(self) -> None:                                                      │\n",
+       "│   357 │   │   trainer = self.trainer                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/training_e │\n",
+       "│ poch_loop.py:133 in run                                                                          │\n",
+       "│                                                                                                  │\n",
+       "│   130 │   │   self.on_run_start(data_fetcher)                                                    │\n",
+       "│   131 │   │   while not self.done:                                                               │\n",
+       "│   132 │   │   │   try:                                                                           │\n",
+       "│ ❱ 133 │   │   │   │   self.advance(data_fetcher)                                                 │\n",
+       "│   134 │   │   │   │   self.on_advance_end()                                                      │\n",
+       "│   135 │   │   │   │   self._restarting = False                                                   │\n",
+       "│   136 │   │   │   except StopIteration:                                                          │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/training_e │\n",
+       "│ poch_loop.py:218 in advance                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   215 │   │   │   with trainer.profiler.profile(\"run_training_batch\"):                           │\n",
+       "│   216 │   │   │   │   if trainer.lightning_module.automatic_optimization:                        │\n",
+       "│   217 │   │   │   │   │   # in automatic optimization, there can only be one optimizer           │\n",
+       "│ ❱ 218 │   │   │   │   │   batch_output = self.automatic_optimization.run(trainer.optimizers[0]   │\n",
+       "│   219 │   │   │   │   else:                                                                      │\n",
+       "│   220 │   │   │   │   │   batch_output = self.manual_optimization.run(kwargs)                    │\n",
+       "│   221                                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati │\n",
+       "│ on/automatic.py:185 in run                                                                       │\n",
+       "│                                                                                                  │\n",
+       "│   182 │   │   # ------------------------------                                                   │\n",
+       "│   183 │   │   # gradient update with accumulated gradients                                       │\n",
+       "│   184 │   │   else:                                                                              │\n",
+       "│ ❱ 185 │   │   │   self._optimizer_step(kwargs.get(\"batch_idx\", 0), closure)                      │\n",
+       "│   186 │   │                                                                                      │\n",
+       "│   187 │   │   result = closure.consume_result()                                                  │\n",
+       "│   188 │   │   if result.loss is None:                                                            │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati │\n",
+       "│ on/automatic.py:261 in _optimizer_step                                                           │\n",
+       "│                                                                                                  │\n",
+       "│   258 │   │   │   self.optim_progress.optimizer.step.increment_ready()                           │\n",
+       "│   259 │   │                                                                                      │\n",
+       "│   260 │   │   # model hook                                                                       │\n",
+       "│ ❱ 261 │   │   call._call_lightning_module_hook(                                                  │\n",
+       "│   262 │   │   │   trainer,                                                                       │\n",
+       "│   263 │   │   │   \"optimizer_step\",                                                              │\n",
+       "│   264 │   │   │   trainer.current_epoch,                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/call.py: │\n",
+       "│ 142 in _call_lightning_module_hook                                                               │\n",
+       "│                                                                                                  │\n",
+       "│   139 │   pl_module._current_fx_name = hook_name                                                 │\n",
+       "│   140 │                                                                                          │\n",
+       "│   141 │   with trainer.profiler.profile(f\"[LightningModule]{pl_module.__class__.__name__}.{hoo   │\n",
+       "│ ❱ 142 │   │   output = fn(*args, **kwargs)                                                       │\n",
+       "│   143 │                                                                                          │\n",
+       "│   144 │   # restore current_fx when nested context                                               │\n",
+       "│   145 │   pl_module._current_fx_name = prev_fx_name                                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/core/module.py:1 │\n",
+       "│ 265 in optimizer_step                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   1262 │   │   │   │   │   for pg in optimizer.param_groups:                                     │\n",
+       "│   1263 │   │   │   │   │   │   pg[\"lr\"] = lr_scale * self.learning_rate                          │\n",
+       "│   1264 │   │   \"\"\"                                                                               │\n",
+       "│ ❱ 1265 │   │   optimizer.step(closure=optimizer_closure)                                         │\n",
+       "│   1266 │                                                                                         │\n",
+       "│   1267 │   def optimizer_zero_grad(self, epoch: int, batch_idx: int, optimizer: Optimizer) -> N  │\n",
+       "│   1268 │   │   \"\"\"Override this method to change the default behaviour of ``optimizer.zero_grad  │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/core/optimizer.p │\n",
+       "│ y:158 in step                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   155 │   │   │   raise MisconfigurationException(\"When `optimizer.step(closure)` is called, t   │\n",
+       "│   156 │   │                                                                                      │\n",
+       "│   157 │   │   assert self._strategy is not None                                                  │\n",
+       "│ ❱ 158 │   │   step_output = self._strategy.optimizer_step(self._optimizer, closure, **kwargs)    │\n",
+       "│   159 │   │                                                                                      │\n",
+       "│   160 │   │   self._on_after_step()                                                              │\n",
+       "│   161                                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/strategies/strat │\n",
+       "│ egy.py:224 in optimizer_step                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│   221 │   │   model = model or self.lightning_module                                             │\n",
+       "│   222 │   │   # TODO(fabric): remove assertion once strategy's optimizer_step typing is fixed    │\n",
+       "│   223 │   │   assert isinstance(model, pl.LightningModule)                                       │\n",
+       "│ ❱ 224 │   │   return self.precision_plugin.optimizer_step(optimizer, model=model, closure=clos   │\n",
+       "│   225 │                                                                                          │\n",
+       "│   226 │   def _setup_model_and_optimizers(self, model: Module, optimizers: List[Optimizer]) ->   │\n",
+       "│   227 │   │   \"\"\"Setup a model and multiple optimizers together.                                 │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/plugins/precisio │\n",
+       "│ n/precision_plugin.py:114 in optimizer_step                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   111 │   ) -> Any:                                                                              │\n",
+       "│   112 │   │   \"\"\"Hook to run the optimizer step.\"\"\"                                              │\n",
+       "│   113 │   │   closure = partial(self._wrap_closure, model, optimizer, closure)                   │\n",
+       "│ ❱ 114 │   │   return optimizer.step(closure=closure, **kwargs)                                   │\n",
+       "│   115 │                                                                                          │\n",
+       "│   116 │   def _clip_gradients(                                                                   │\n",
+       "│   117 │   │   self,                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/lr_scheduler.py:69 in  │\n",
+       "│ wrapper                                                                                          │\n",
+       "│                                                                                                  │\n",
+       "│     66 │   │   │   │   instance = instance_ref()                                                 │\n",
+       "│     67 │   │   │   │   instance._step_count += 1                                                 │\n",
+       "│     68 │   │   │   │   wrapped = func.__get__(instance, cls)                                     │\n",
+       "│ ❱   69 │   │   │   │   return wrapped(*args, **kwargs)                                           │\n",
+       "│     70 │   │   │                                                                                 │\n",
+       "│     71 │   │   │   # Note that the returned function here is no longer a bound method,           │\n",
+       "│     72 │   │   │   # so attributes like `__func__` and `__self__` no longer exist.               │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/optimizer.py:280 in    │\n",
+       "│ wrapper                                                                                          │\n",
+       "│                                                                                                  │\n",
+       "│   277 │   │   │   │   │   │   │   raise RuntimeError(f\"{func} must return None or a tuple of (   │\n",
+       "│   278 │   │   │   │   │   │   │   │   │   │   │      f\"but got {result}.\")                       │\n",
+       "│   279 │   │   │   │                                                                              │\n",
+       "│ ❱ 280 │   │   │   │   out = func(*args, **kwargs)                                                │\n",
+       "│   281 │   │   │   │   self._optimizer_step_code()                                                │\n",
+       "│   282 │   │   │   │                                                                              │\n",
+       "│   283 │   │   │   │   # call optimizer step post hooks                                           │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/optimizer.py:33 in     │\n",
+       "│ _use_grad                                                                                        │\n",
+       "│                                                                                                  │\n",
+       "│    30 │   │   prev_grad = torch.is_grad_enabled()                                                │\n",
+       "│    31 │   │   try:                                                                               │\n",
+       "│    32 │   │   │   torch.set_grad_enabled(self.defaults['differentiable'])                        │\n",
+       "│ ❱  33 │   │   │   ret = func(self, *args, **kwargs)                                              │\n",
+       "│    34 │   │   finally:                                                                           │\n",
+       "│    35 │   │   │   torch.set_grad_enabled(prev_grad)                                              │\n",
+       "│    36 │   │   return ret                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/adamw.py:148 in step   │\n",
+       "│                                                                                                  │\n",
+       "│   145 │   │   loss = None                                                                        │\n",
+       "│   146 │   │   if closure is not None:                                                            │\n",
+       "│   147 │   │   │   with torch.enable_grad():                                                      │\n",
+       "│ ❱ 148 │   │   │   │   loss = closure()                                                           │\n",
+       "│   149 │   │                                                                                      │\n",
+       "│   150 │   │   for group in self.param_groups:                                                    │\n",
+       "│   151 │   │   │   params_with_grad = []                                                          │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/plugins/precisio │\n",
+       "│ n/precision_plugin.py:101 in _wrap_closure                                                       │\n",
+       "│                                                                                                  │\n",
+       "│    98 │   │   The closure (generally) runs ``backward`` so this allows inspecting gradients in   │\n",
+       "│    99 │   │   consistent with the ``PrecisionPlugin`` subclasses that cannot pass ``optimizer.   │\n",
+       "│   100 │   │   \"\"\"                                                                                │\n",
+       "│ ❱ 101 │   │   closure_result = closure()                                                         │\n",
+       "│   102 │   │   self._after_closure(model, optimizer)                                              │\n",
+       "│   103 │   │   return closure_result                                                              │\n",
+       "│   104                                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati │\n",
+       "│ on/automatic.py:140 in __call__                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   137 │   │   return step_output                                                                 │\n",
+       "│   138 │                                                                                          │\n",
+       "│   139 │   def __call__(self, *args: Any, **kwargs: Any) -> Optional[Tensor]:                     │\n",
+       "│ ❱ 140 │   │   self._result = self.closure(*args, **kwargs)                                       │\n",
+       "│   141 │   │   return self._result.loss                                                           │\n",
+       "│   142                                                                                            │\n",
+       "│   143                                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati │\n",
+       "│ on/automatic.py:126 in closure                                                                   │\n",
+       "│                                                                                                  │\n",
+       "│   123 │   │   self._zero_grad_fn = zero_grad_fn                                                  │\n",
+       "│   124 │                                                                                          │\n",
+       "│   125 │   def closure(self, *args: Any, **kwargs: Any) -> ClosureResult:                         │\n",
+       "│ ❱ 126 │   │   step_output = self._step_fn()                                                      │\n",
+       "│   127 │   │                                                                                      │\n",
+       "│   128 │   │   if step_output.closure_loss is None:                                               │\n",
+       "│   129 │   │   │   self.warning_cache.warn(\"`training_step` returned `None`. If this was on pur   │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati │\n",
+       "│ on/automatic.py:308 in _training_step                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   305 │   │   trainer = self.trainer                                                             │\n",
+       "│   306 │   │                                                                                      │\n",
+       "│   307 │   │   # manually capture logged metrics                                                  │\n",
+       "│ ❱ 308 │   │   training_step_output = call._call_strategy_hook(trainer, \"training_step\", *kwarg   │\n",
+       "│   309 │   │   self.trainer.strategy.post_training_step()                                         │\n",
+       "│   310 │   │                                                                                      │\n",
+       "│   311 │   │   result = self.output_result_cls.from_training_step_output(training_step_output,    │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/call.py: │\n",
+       "│ 288 in _call_strategy_hook                                                                       │\n",
+       "│                                                                                                  │\n",
+       "│   285 │   │   return                                                                             │\n",
+       "│   286 │                                                                                          │\n",
+       "│   287 │   with trainer.profiler.profile(f\"[Strategy]{trainer.strategy.__class__.__name__}.{hoo   │\n",
+       "│ ❱ 288 │   │   output = fn(*args, **kwargs)                                                       │\n",
+       "│   289 │                                                                                          │\n",
+       "│   290 │   # restore current_fx when nested context                                               │\n",
+       "│   291 │   pl_module._current_fx_name = prev_fx_name                                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/strategies/strat │\n",
+       "│ egy.py:366 in training_step                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   363 │   │   \"\"\"                                                                                │\n",
+       "│   364 │   │   with self.precision_plugin.train_step_context():                                   │\n",
+       "│   365 │   │   │   assert isinstance(self.model, TrainingStep)                                    │\n",
+       "│ ❱ 366 │   │   │   return self.model.training_step(*args, **kwargs)                               │\n",
+       "│   367 │                                                                                          │\n",
+       "│   368 │   def post_training_step(self) -> None:                                                  │\n",
+       "│   369 │   │   pass                                                                               │\n",
+       "│                                                                                                  │\n",
+       "│ in training_step:56                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│   53 │   │   return loss                                                                         │\n",
+       "│   54 │                                                                                           │\n",
+       "│   55 │   def training_step(self, batch, batch_idx):                                              │\n",
+       "│ ❱ 56 │   │   return self._step(batch, batch_idx)                                                 │\n",
+       "│   57 │                                                                                           │\n",
+       "│   58 │   def validation_step(self, batch, batch_idx=0):                                          │\n",
+       "│   59 │   │   return self._step(batch, batch_idx, stage='val')                                    │\n",
+       "│                                                                                                  │\n",
+       "│ in _step:51                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   48 │   │                                                                                       │\n",
+       "│   49 │   │   predictions, acc = get_acc(p0, p1, y)                                               │\n",
+       "│   50 │   │   self.log(f\"{stage}/acc\", acc)                                                       │\n",
+       "│ ❱ 51 │   │   predictions, f1 = get_f1(p0, p1, y)                                                 │\n",
+       "│   52 │   │   self.log(f\"{stage}/f1\", f1)                                                         │\n",
+       "│   53 │   │   return loss                                                                         │\n",
+       "│   54                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in get_f1:27                                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│   24 │                                                                                           │\n",
+       "│   25 │   # TODO f1                                                                               │\n",
+       "│   26 │   conf = (avg_confidence.detach().cpu().numpy() )[:, 0]                                   │\n",
+       "│ ❱ 27 │   auc = roc_auc_score(y.cpu().numpy(), predictions)                                       │\n",
+       "│   28 │                                                                                           │\n",
+       "│   29 │   auc = max(auc, 1 - auc)                                                                 │\n",
+       "│   30 │   return predictions, auc                                                                 │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/sklearn/metrics/_ranking.py:572 in │\n",
+       "│ roc_auc_score                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    569 │   elif y_type == \"binary\":                                                              │\n",
+       "│    570 │   │   labels = np.unique(y_true)                                                        │\n",
+       "│    571 │   │   y_true = label_binarize(y_true, classes=labels)[:, 0]                             │\n",
+       "│ ❱  572 │   │   return _average_binary_score(                                                     │\n",
+       "│    573 │   │   │   partial(_binary_roc_auc_score, max_fpr=max_fpr),                              │\n",
+       "│    574 │   │   │   y_true,                                                                       │\n",
+       "│    575 │   │   │   y_score,                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/sklearn/metrics/_base.py:75 in     │\n",
+       "│ _average_binary_score                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│    72 │   │   raise ValueError(\"{0} format is not supported\".format(y_type))                     │\n",
+       "│    73 │                                                                                          │\n",
+       "│    74 │   if y_type == \"binary\":                                                                 │\n",
+       "│ ❱  75 │   │   return binary_metric(y_true, y_score, sample_weight=sample_weight)                 │\n",
+       "│    76 │                                                                                          │\n",
+       "│    77 │   check_consistent_length(y_true, y_score, sample_weight)                                │\n",
+       "│    78 │   y_true = check_array(y_true)                                                           │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/sklearn/metrics/_ranking.py:339 in │\n",
+       "│ _binary_roc_auc_score                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│    336 def _binary_roc_auc_score(y_true, y_score, sample_weight=None, max_fpr=None):             │\n",
+       "│    337 │   \"\"\"Binary roc auc score.\"\"\"                                                           │\n",
+       "│    338 │   if len(np.unique(y_true)) != 2:                                                       │\n",
+       "│ ❱  339 │   │   raise ValueError(                                                                 │\n",
+       "│    340 │   │   │   \"Only one class present in y_true. ROC AUC score \"                            │\n",
+       "│    341 │   │   │   \"is not defined in that case.\"                                                │\n",
+       "│    342 │   │   )                                                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "ValueError: Only one class present in y_true. ROC AUC score is not defined in that case.\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[2m# train the model (hint: here are some helpful Trainer arguments for rapid idea iteratio\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mtrainer = pl.Trainer(limit_train_batches=\u001b[94m100\u001b[0m, max_epochs=max_epochs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3 trainer.fit(model=net, datamodule=dm) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/\u001b[0m\u001b[1;33mtrainer.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mpy\u001b[0m:\u001b[94m520\u001b[0m in \u001b[92mfit\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 517 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 518 \u001b[0m\u001b[2m│ │ \u001b[0mmodel = _maybe_unwrap_optimized(model) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 519 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.strategy._lightning_module = model \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 520 \u001b[2m│ │ \u001b[0mcall._call_and_handle_interrupt( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 521 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m, \u001b[96mself\u001b[0m._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 522 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 523 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/\u001b[0m\u001b[1;33mcall.py\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[94m44\u001b[0m in \u001b[92m_call_and_handle_interrupt\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 41 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m trainer.strategy.launcher \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 42 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 43 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 44 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m trainer_fn(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 45 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 46 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m _TunerExitException: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 47 \u001b[0m\u001b[2m│ │ \u001b[0m_call_teardown_hook(trainer) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/\u001b[0m\u001b[1;33mtrainer.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mpy\u001b[0m:\u001b[94m559\u001b[0m in \u001b[92m_fit_impl\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 556 \u001b[0m\u001b[2m│ │ │ \u001b[0mmodel_provided=\u001b[94mTrue\u001b[0m, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 557 \u001b[0m\u001b[2m│ │ │ \u001b[0mmodel_connected=\u001b[96mself\u001b[0m.lightning_module \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 558 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 559 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._run(model, ckpt_path=ckpt_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 560 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 561 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94massert\u001b[0m \u001b[96mself\u001b[0m.state.stopped \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 562 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.training = \u001b[94mFalse\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/\u001b[0m\u001b[1;33mtrainer.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mpy\u001b[0m:\u001b[94m935\u001b[0m in \u001b[92m_run\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 932 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# ----------------------------\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 933 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# RUN THE TRAINER\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 934 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# ----------------------------\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 935 \u001b[2m│ │ \u001b[0mresults = \u001b[96mself\u001b[0m._run_stage() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 936 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 937 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# ----------------------------\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 938 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# POST-Training CLEAN UP\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/\u001b[0m\u001b[1;33mtrainer.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mpy\u001b[0m:\u001b[94m978\u001b[0m in \u001b[92m_run_stage\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 975 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m isolate_rng(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 976 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m._run_sanity_check() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 977 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.autograd.set_detect_anomaly(\u001b[96mself\u001b[0m._detect_anomaly): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 978 \u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.fit_loop.run() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 979 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 980 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mRuntimeError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mUnexpected state \u001b[0m\u001b[33m{\u001b[0m\u001b[96mself\u001b[0m.state\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 981 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/\u001b[0m\u001b[1;33mfit_loop.p\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33my\u001b[0m:\u001b[94m201\u001b[0m in \u001b[92mrun\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m198 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mwhile\u001b[0m \u001b[95mnot\u001b[0m \u001b[96mself\u001b[0m.done: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m199 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m200 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.on_advance_start() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m201 \u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.advance() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m202 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.on_advance_end() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m203 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m._restarting = \u001b[94mFalse\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m204 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mStopIteration\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/\u001b[0m\u001b[1;33mfit_loop.p\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33my\u001b[0m:\u001b[94m354\u001b[0m in \u001b[92madvance\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m351 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94massert\u001b[0m \u001b[96mself\u001b[0m._data_fetcher \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m352 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._data_fetcher.setup(combined_loader) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m353 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mwith\u001b[0m \u001b[96mself\u001b[0m.trainer.profiler.profile(\u001b[33m\"\u001b[0m\u001b[33mrun_training_epoch\u001b[0m\u001b[33m\"\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m354 \u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.epoch_loop.run(\u001b[96mself\u001b[0m._data_fetcher) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m355 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m356 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mon_advance_end\u001b[0m(\u001b[96mself\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m357 \u001b[0m\u001b[2m│ │ \u001b[0mtrainer = \u001b[96mself\u001b[0m.trainer \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/\u001b[0m\u001b[1;33mtraining_e\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mpoch_loop.py\u001b[0m:\u001b[94m133\u001b[0m in \u001b[92mrun\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m130 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.on_run_start(data_fetcher) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m131 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mwhile\u001b[0m \u001b[95mnot\u001b[0m \u001b[96mself\u001b[0m.done: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m132 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m133 \u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.advance(data_fetcher) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m134 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m.on_advance_end() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m135 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m._restarting = \u001b[94mFalse\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m136 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mStopIteration\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/\u001b[0m\u001b[1;33mtraining_e\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mpoch_loop.py\u001b[0m:\u001b[94m218\u001b[0m in \u001b[92madvance\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m215 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m trainer.profiler.profile(\u001b[33m\"\u001b[0m\u001b[33mrun_training_batch\u001b[0m\u001b[33m\"\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m216 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m trainer.lightning_module.automatic_optimization: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m217 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m\u001b[2m# in automatic optimization, there can only be one optimizer\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m218 \u001b[2m│ │ │ │ │ \u001b[0mbatch_output = \u001b[96mself\u001b[0m.automatic_optimization.run(trainer.optimizers[\u001b[94m0\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m219 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m220 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mbatch_output = \u001b[96mself\u001b[0m.manual_optimization.run(kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m221 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33mon/\u001b[0m\u001b[1;33mautomatic.py\u001b[0m:\u001b[94m185\u001b[0m in \u001b[92mrun\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m182 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# ------------------------------\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m183 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# gradient update with accumulated gradients\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m184 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m185 \u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m._optimizer_step(kwargs.get(\u001b[33m\"\u001b[0m\u001b[33mbatch_idx\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94m0\u001b[0m), closure) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m186 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m187 \u001b[0m\u001b[2m│ │ \u001b[0mresult = closure.consume_result() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m188 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m result.loss \u001b[95mis\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33mon/\u001b[0m\u001b[1;33mautomatic.py\u001b[0m:\u001b[94m261\u001b[0m in \u001b[92m_optimizer_step\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m258 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.optim_progress.optimizer.step.increment_ready() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m259 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m260 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# model hook\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m261 \u001b[2m│ │ \u001b[0mcall._call_lightning_module_hook( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m262 \u001b[0m\u001b[2m│ │ │ \u001b[0mtrainer, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m263 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33moptimizer_step\u001b[0m\u001b[33m\"\u001b[0m, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m264 \u001b[0m\u001b[2m│ │ │ \u001b[0mtrainer.current_epoch, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/\u001b[0m\u001b[1;33mcall.py\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[94m142\u001b[0m in \u001b[92m_call_lightning_module_hook\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m139 \u001b[0m\u001b[2m│ \u001b[0mpl_module._current_fx_name = hook_name \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m140 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m141 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mwith\u001b[0m trainer.profiler.profile(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m[LightningModule]\u001b[0m\u001b[33m{\u001b[0mpl_module.\u001b[91m__class__\u001b[0m.\u001b[91m__name__\u001b[0m\u001b[33m}\u001b[0m\u001b[33m.\u001b[0m\u001b[33m{\u001b[0mhoo \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m142 \u001b[2m│ │ \u001b[0moutput = fn(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m143 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m144 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# restore current_fx when nested context\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m145 \u001b[0m\u001b[2m│ \u001b[0mpl_module._current_fx_name = prev_fx_name \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/core/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[94m265\u001b[0m in \u001b[92moptimizer_step\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1262 \u001b[0m\u001b[2;33m│ │ │ │ │ \u001b[0m\u001b[33mfor pg in optimizer.param_groups:\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1263 \u001b[0m\u001b[2;33m│ │ │ │ │ │ \u001b[0m\u001b[33mpg[\"lr\"] = lr_scale * self.learning_rate\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1264 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1265 \u001b[2m│ │ \u001b[0moptimizer.step(closure=optimizer_closure) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1266 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1267 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92moptimizer_zero_grad\u001b[0m(\u001b[96mself\u001b[0m, epoch: \u001b[96mint\u001b[0m, batch_idx: \u001b[96mint\u001b[0m, optimizer: Optimizer) -> \u001b[94mN\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1268 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"Override this method to change the default behaviour of ``optimizer.zero_grad\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/core/\u001b[0m\u001b[1;33moptimizer.p\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33my\u001b[0m:\u001b[94m158\u001b[0m in \u001b[92mstep\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m155 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m MisconfigurationException(\u001b[33m\"\u001b[0m\u001b[33mWhen `optimizer.step(closure)` is called, t\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m156 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m157 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94massert\u001b[0m \u001b[96mself\u001b[0m._strategy \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m158 \u001b[2m│ │ \u001b[0mstep_output = \u001b[96mself\u001b[0m._strategy.optimizer_step(\u001b[96mself\u001b[0m._optimizer, closure, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m159 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m160 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._on_after_step() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m161 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/strategies/\u001b[0m\u001b[1;33mstrat\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33megy.py\u001b[0m:\u001b[94m224\u001b[0m in \u001b[92moptimizer_step\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m221 \u001b[0m\u001b[2m│ │ \u001b[0mmodel = model \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m.lightning_module \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m222 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# TODO(fabric): remove assertion once strategy's optimizer_step typing is fixed\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m223 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94massert\u001b[0m \u001b[96misinstance\u001b[0m(model, pl.LightningModule) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m224 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.precision_plugin.optimizer_step(optimizer, model=model, closure=clos \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m225 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m226 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_setup_model_and_optimizers\u001b[0m(\u001b[96mself\u001b[0m, model: Module, optimizers: List[Optimizer]) -> \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m227 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"Setup a model and multiple optimizers together.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/plugins/precisio\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33mn/\u001b[0m\u001b[1;33mprecision_plugin.py\u001b[0m:\u001b[94m114\u001b[0m in \u001b[92moptimizer_step\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m111 \u001b[0m\u001b[2m│ \u001b[0m) -> Any: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m112 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"Hook to run the optimizer step.\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m113 \u001b[0m\u001b[2m│ │ \u001b[0mclosure = partial(\u001b[96mself\u001b[0m._wrap_closure, model, optimizer, closure) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m114 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m optimizer.step(closure=closure, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m115 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m116 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_clip_gradients\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m117 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/\u001b[0m\u001b[1;33mlr_scheduler.py\u001b[0m:\u001b[94m69\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mwrapper\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 66 \u001b[0m\u001b[2m│ │ │ │ \u001b[0minstance = instance_ref() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 67 \u001b[0m\u001b[2m│ │ │ │ \u001b[0minstance._step_count += \u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 68 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mwrapped = func.\u001b[92m__get__\u001b[0m(instance, \u001b[96mcls\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 69 \u001b[2m│ │ │ │ \u001b[0m\u001b[94mreturn\u001b[0m wrapped(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 70 \u001b[0m\u001b[2m│ │ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 71 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# Note that the returned function here is no longer a bound method,\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 72 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# so attributes like `__func__` and `__self__` no longer exist.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/\u001b[0m\u001b[1;33moptimizer.py\u001b[0m:\u001b[94m280\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mwrapper\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m277 \u001b[0m\u001b[2m│ │ │ │ │ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mRuntimeError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0mfunc\u001b[33m}\u001b[0m\u001b[33m must return None or a tuple of (\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m278 \u001b[0m\u001b[2m│ │ │ │ │ │ │ │ │ │ │ \u001b[0m\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mbut got \u001b[0m\u001b[33m{\u001b[0mresult\u001b[33m}\u001b[0m\u001b[33m.\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m279 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m280 \u001b[2m│ │ │ │ \u001b[0mout = func(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m281 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[96mself\u001b[0m._optimizer_step_code() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m282 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m283 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[2m# call optimizer step post hooks\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/\u001b[0m\u001b[1;33moptimizer.py\u001b[0m:\u001b[94m33\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_use_grad\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 30 \u001b[0m\u001b[2m│ │ \u001b[0mprev_grad = torch.is_grad_enabled() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 31 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 32 \u001b[0m\u001b[2m│ │ │ \u001b[0mtorch.set_grad_enabled(\u001b[96mself\u001b[0m.defaults[\u001b[33m'\u001b[0m\u001b[33mdifferentiable\u001b[0m\u001b[33m'\u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 33 \u001b[2m│ │ │ \u001b[0mret = func(\u001b[96mself\u001b[0m, *args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 34 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mfinally\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 35 \u001b[0m\u001b[2m│ │ │ \u001b[0mtorch.set_grad_enabled(prev_grad) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 36 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m ret \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/optim/\u001b[0m\u001b[1;33madamw.py\u001b[0m:\u001b[94m148\u001b[0m in \u001b[92mstep\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m145 \u001b[0m\u001b[2m│ │ \u001b[0mloss = \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m146 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m closure \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m147 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.enable_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m148 \u001b[2m│ │ │ │ \u001b[0mloss = closure() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m149 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m150 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mfor\u001b[0m group \u001b[95min\u001b[0m \u001b[96mself\u001b[0m.param_groups: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m151 \u001b[0m\u001b[2m│ │ │ \u001b[0mparams_with_grad = [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/plugins/precisio\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33mn/\u001b[0m\u001b[1;33mprecision_plugin.py\u001b[0m:\u001b[94m101\u001b[0m in \u001b[92m_wrap_closure\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 98 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33mThe closure (generally) runs ``backward`` so this allows inspecting gradients in\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 99 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33mconsistent with the ``PrecisionPlugin`` subclasses that cannot pass ``optimizer.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m100 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m101 \u001b[2m│ │ \u001b[0mclosure_result = closure() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m102 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._after_closure(model, optimizer) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m103 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m closure_result \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m104 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33mon/\u001b[0m\u001b[1;33mautomatic.py\u001b[0m:\u001b[94m140\u001b[0m in \u001b[92m__call__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m137 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m step_output \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m138 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m139 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m__call__\u001b[0m(\u001b[96mself\u001b[0m, *args: Any, **kwargs: Any) -> Optional[Tensor]: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m140 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._result = \u001b[96mself\u001b[0m.closure(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m141 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._result.loss \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m142 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m143 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33mon/\u001b[0m\u001b[1;33mautomatic.py\u001b[0m:\u001b[94m126\u001b[0m in \u001b[92mclosure\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m123 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._zero_grad_fn = zero_grad_fn \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m124 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m125 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mclosure\u001b[0m(\u001b[96mself\u001b[0m, *args: Any, **kwargs: Any) -> ClosureResult: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m126 \u001b[2m│ │ \u001b[0mstep_output = \u001b[96mself\u001b[0m._step_fn() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m127 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m128 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m step_output.closure_loss \u001b[95mis\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m129 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.warning_cache.warn(\u001b[33m\"\u001b[0m\u001b[33m`training_step` returned `None`. If this was on pur\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/loops/optimizati\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33mon/\u001b[0m\u001b[1;33mautomatic.py\u001b[0m:\u001b[94m308\u001b[0m in \u001b[92m_training_step\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m305 \u001b[0m\u001b[2m│ │ \u001b[0mtrainer = \u001b[96mself\u001b[0m.trainer \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m306 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m307 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# manually capture logged metrics\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m308 \u001b[2m│ │ \u001b[0mtraining_step_output = call._call_strategy_hook(trainer, \u001b[33m\"\u001b[0m\u001b[33mtraining_step\u001b[0m\u001b[33m\"\u001b[0m, *kwarg \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m309 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.trainer.strategy.post_training_step() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m310 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m311 \u001b[0m\u001b[2m│ │ \u001b[0mresult = \u001b[96mself\u001b[0m.output_result_cls.from_training_step_output(training_step_output, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/\u001b[0m\u001b[1;33mcall.py\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[94m288\u001b[0m in \u001b[92m_call_strategy_hook\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m285 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m286 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m287 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mwith\u001b[0m trainer.profiler.profile(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m[Strategy]\u001b[0m\u001b[33m{\u001b[0mtrainer.strategy.\u001b[91m__class__\u001b[0m.\u001b[91m__name__\u001b[0m\u001b[33m}\u001b[0m\u001b[33m.\u001b[0m\u001b[33m{\u001b[0mhoo \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m288 \u001b[2m│ │ \u001b[0moutput = fn(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m289 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m290 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# restore current_fx when nested context\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m291 \u001b[0m\u001b[2m│ \u001b[0mpl_module._current_fx_name = prev_fx_name \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/strategies/\u001b[0m\u001b[1;33mstrat\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33megy.py\u001b[0m:\u001b[94m366\u001b[0m in \u001b[92mtraining_step\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m363 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m364 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mwith\u001b[0m \u001b[96mself\u001b[0m.precision_plugin.train_step_context(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m365 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94massert\u001b[0m \u001b[96misinstance\u001b[0m(\u001b[96mself\u001b[0m.model, TrainingStep) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m366 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.model.training_step(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m367 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m368 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mpost_training_step\u001b[0m(\u001b[96mself\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m369 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mpass\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mtraining_step\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m loss \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mtraining_step\u001b[0m(\u001b[96mself\u001b[0m, batch, batch_idx): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._step(batch, batch_idx) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mvalidation_step\u001b[0m(\u001b[96mself\u001b[0m, batch, batch_idx=\u001b[94m0\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._step(batch, batch_idx, stage=\u001b[33m'\u001b[0m\u001b[33mval\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_step\u001b[0m:\u001b[94m51\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m48 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m49 \u001b[0m\u001b[2m│ │ \u001b[0mpredictions, acc = get_acc(p0, p1, y) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m50 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.log(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0mstage\u001b[33m}\u001b[0m\u001b[33m/acc\u001b[0m\u001b[33m\"\u001b[0m, acc) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m51 \u001b[2m│ │ \u001b[0mpredictions, f1 = get_f1(p0, p1, y) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m52 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.log(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0mstage\u001b[33m}\u001b[0m\u001b[33m/f1\u001b[0m\u001b[33m\"\u001b[0m, f1) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m loss \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_f1\u001b[0m:\u001b[94m27\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# TODO f1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m26 \u001b[0m\u001b[2m│ \u001b[0mconf = (avg_confidence.detach().cpu().numpy() )[:, \u001b[94m0\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m27 \u001b[2m│ \u001b[0mauc = roc_auc_score(y.cpu().numpy(), predictions) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m28 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m29 \u001b[0m\u001b[2m│ \u001b[0mauc = \u001b[96mmax\u001b[0m(auc, \u001b[94m1\u001b[0m - auc) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m predictions, auc \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/sklearn/metrics/\u001b[0m\u001b[1;33m_ranking.py\u001b[0m:\u001b[94m572\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mroc_auc_score\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 569 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melif\u001b[0m y_type == \u001b[33m\"\u001b[0m\u001b[33mbinary\u001b[0m\u001b[33m\"\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 570 \u001b[0m\u001b[2m│ │ \u001b[0mlabels = np.unique(y_true) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 571 \u001b[0m\u001b[2m│ │ \u001b[0my_true = label_binarize(y_true, classes=labels)[:, \u001b[94m0\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 572 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m _average_binary_score( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 573 \u001b[0m\u001b[2m│ │ │ \u001b[0mpartial(_binary_roc_auc_score, max_fpr=max_fpr), \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 574 \u001b[0m\u001b[2m│ │ │ \u001b[0my_true, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 575 \u001b[0m\u001b[2m│ │ │ \u001b[0my_score, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/sklearn/metrics/\u001b[0m\u001b[1;33m_base.py\u001b[0m:\u001b[94m75\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_average_binary_score\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 72 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m(\u001b[33m\"\u001b[0m\u001b[33m{0}\u001b[0m\u001b[33m format is not supported\u001b[0m\u001b[33m\"\u001b[0m.format(y_type)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 73 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 74 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m y_type == \u001b[33m\"\u001b[0m\u001b[33mbinary\u001b[0m\u001b[33m\"\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 75 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m binary_metric(y_true, y_score, sample_weight=sample_weight) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 76 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 77 \u001b[0m\u001b[2m│ \u001b[0mcheck_consistent_length(y_true, y_score, sample_weight) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 78 \u001b[0m\u001b[2m│ \u001b[0my_true = check_array(y_true) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/sklearn/metrics/\u001b[0m\u001b[1;33m_ranking.py\u001b[0m:\u001b[94m339\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_binary_roc_auc_score\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 336 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_binary_roc_auc_score\u001b[0m(y_true, y_score, sample_weight=\u001b[94mNone\u001b[0m, max_fpr=\u001b[94mNone\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 337 \u001b[0m\u001b[2;90m│ \u001b[0m\u001b[33m\"\"\"Binary roc auc score.\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 338 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mlen\u001b[0m(np.unique(y_true)) != \u001b[94m2\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 339 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 340 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mOnly one class present in y_true. ROC AUC score \u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 341 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mis not defined in that case.\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 342 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mValueError: \u001b[0mOnly one class present in y_true. ROC AUC score is not defined in that case.\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\n", + "trainer = pl.Trainer(limit_train_batches=100, max_epochs=max_epochs)\n", + "trainer.fit(model=net, datamodule=dm)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-14T06:21:46.356828Z", + "start_time": "2023-05-14T06:21:46.351801Z" + } + }, + "source": [ + "# Read hist" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.760833Z", + "start_time": "2023-05-20T02:28:35.760825Z" + } + }, + "outputs": [], + "source": [ + "# import pytorch_lightning as pl\n", + "from lightning.pytorch.loggers.csv_logs import CSVLogger\n", + "# from pytorch_lightning.loggers.csv_logs import CSVLogger as CSVLogger2\n", + "from pathlib import Path\n", + "import pandas as pd\n", + "\n", + "def read_metrics_csv(metrics_file_path):\n", + " df_hist = pd.read_csv(metrics_file_path)\n", + " df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()\n", + " df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()\n", + " return df_histe\n", + "\n", + "\n", + "def read_hist(trainer: pl.Trainer):\n", + "\n", + " ts = [t for t in trainer.loggers if isinstance(t, CSVLogger)]\n", + " print(ts)\n", + " try:\n", + " metrics_file_path = Path(ts[0].experiment.metrics_file_path)\n", + " df_histe = read_metrics_csv(metrics_file_path)\n", + " return df_histe\n", + " except Exception as e:\n", + " raise e\n", + " print(e)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.761623Z", + "start_time": "2023-05-20T02:28:35.761614Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[]\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
val/lossval/accval/f1step
epoch
01.00.620.53.0
11.00.620.57.0
\n", + "
" + ], + "text/plain": [ + " val/loss val/acc val/f1 step\n", + "epoch \n", + "0 1.0 0.62 0.5 3.0\n", + "1 1.0 0.62 0.5 7.0" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_hist = read_hist(trainer).ffill().bfill()\n", + "df_hist" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-20T02:28:35.762326Z", + "start_time": "2023-05-20T02:28:35.762318Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_hist[['val/acc', 'train/acc']].plot()                                                     │\n",
+       "│   2                                                                                              │\n",
+       "│   3 df_hist[['val/f1', 'train/f1']].plot()                                                       │\n",
+       "│   4                                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/frame.py:3767 in       │\n",
+       "│ __getitem__                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│    3764 │   │   else:                                                                            │\n",
+       "│    3765 │   │   │   if is_iterator(key):                                                         │\n",
+       "│    3766 │   │   │   │   key = list(key)                                                          │\n",
+       "│ ❱  3767 │   │   │   indexer = self.columns._get_indexer_strict(key, \"columns\")[1]                │\n",
+       "│    3768 │   │                                                                                    │\n",
+       "│    3769 │   │   # take() does not accept boolean indexers                                        │\n",
+       "│    3770 │   │   if getattr(indexer, \"dtype\", None) == bool:                                      │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/base.py:5876   │\n",
+       "│ in _get_indexer_strict                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│   5873 │   │   else:                                                                             │\n",
+       "│   5874 │   │   │   keyarr, indexer, new_indexer = self._reindex_non_unique(keyarr)               │\n",
+       "│   5875 │   │                                                                                     │\n",
+       "│ ❱ 5876 │   │   self._raise_if_missing(keyarr, indexer, axis_name)                                │\n",
+       "│   5877 │   │                                                                                     │\n",
+       "│   5878 │   │   keyarr = self.take(indexer)                                                       │\n",
+       "│   5879 │   │   if isinstance(key, Index):                                                        │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/base.py:5938   │\n",
+       "│ in _raise_if_missing                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│   5935 │   │   │   │   raise KeyError(f\"None of [{key}] are in the [{axis_name}]\")               │\n",
+       "│   5936 │   │   │                                                                                 │\n",
+       "│   5937 │   │   │   not_found = list(ensure_index(key)[missing_mask.nonzero()[0]].unique())       │\n",
+       "│ ❱ 5938 │   │   │   raise KeyError(f\"{not_found} not in index\")                                   │\n",
+       "│   5939 │                                                                                         │\n",
+       "│   5940 │   @overload                                                                             │\n",
+       "│   5941 │   def _get_indexer_non_comparable(                                                      │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "KeyError: \"['train/acc'] not in index\"\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_hist[[\u001b[33m'\u001b[0m\u001b[33mval/acc\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mtrain/acc\u001b[0m\u001b[33m'\u001b[0m]].plot() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mdf_hist[[\u001b[33m'\u001b[0m\u001b[33mval/f1\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mtrain/f1\u001b[0m\u001b[33m'\u001b[0m]].plot() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/\u001b[0m\u001b[1;33mframe.py\u001b[0m:\u001b[94m3767\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m__getitem__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3764 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3765 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m is_iterator(key): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3766 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mkey = \u001b[96mlist\u001b[0m(key) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 3767 \u001b[2m│ │ │ \u001b[0mindexer = \u001b[96mself\u001b[0m.columns._get_indexer_strict(key, \u001b[33m\"\u001b[0m\u001b[33mcolumns\u001b[0m\u001b[33m\"\u001b[0m)[\u001b[94m1\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3768 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3769 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# take() does not accept boolean indexers\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3770 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mgetattr\u001b[0m(indexer, \u001b[33m\"\u001b[0m\u001b[33mdtype\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m) == \u001b[96mbool\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/\u001b[0m\u001b[1;33mbase.py\u001b[0m:\u001b[94m5876\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_get_indexer_strict\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5873 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5874 \u001b[0m\u001b[2m│ │ │ \u001b[0mkeyarr, indexer, new_indexer = \u001b[96mself\u001b[0m._reindex_non_unique(keyarr) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5875 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m5876 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._raise_if_missing(keyarr, indexer, axis_name) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5877 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5878 \u001b[0m\u001b[2m│ │ \u001b[0mkeyarr = \u001b[96mself\u001b[0m.take(indexer) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5879 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96misinstance\u001b[0m(key, Index): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/\u001b[0m\u001b[1;33mbase.py\u001b[0m:\u001b[94m5938\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_raise_if_missing\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5935 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mNone of [\u001b[0m\u001b[33m{\u001b[0mkey\u001b[33m}\u001b[0m\u001b[33m] are in the [\u001b[0m\u001b[33m{\u001b[0maxis_name\u001b[33m}\u001b[0m\u001b[33m]\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5936 \u001b[0m\u001b[2m│ │ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5937 \u001b[0m\u001b[2m│ │ │ \u001b[0mnot_found = \u001b[96mlist\u001b[0m(ensure_index(key)[missing_mask.nonzero()[\u001b[94m0\u001b[0m]].unique()) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m5938 \u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0mnot_found\u001b[33m}\u001b[0m\u001b[33m not in index\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5939 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5940 \u001b[0m\u001b[2m│ \u001b[0m\u001b[1;95m@overload\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5941 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_get_indexer_non_comparable\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mKeyError: \u001b[0m\u001b[32m\"\u001b[0m\u001b[32m[\u001b[0m\u001b[32m'train/acc'\u001b[0m\u001b[32m]\u001b[0m\u001b[32m not in index\"\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_hist[['val/acc', 'train/acc']].plot()\n", + "\n", + "df_hist[['val/f1', 'train/f1']].plot()\n", + "\n", + "df_hist[['val/loss', 'train/loss']].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dlk2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "165px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/001_mjc_CCS.ipynb b/notebooks/001_mjc_CCS_Wizard.ipynb similarity index 54% rename from 001_mjc_CCS.ipynb rename to notebooks/001_mjc_CCS_Wizard.ipynb index 36ad1be..4ed733f 100644 --- a/001_mjc_CCS.ipynb +++ b/notebooks/001_mjc_CCS_Wizard.ipynb @@ -10,23 +10,14 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 44, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:54:44.191549Z", "start_time": "2023-05-20T01:54:41.824251Z" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "from tqdm.auto import tqdm\n", "import copy\n", @@ -63,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 45, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:54:44.196607Z", @@ -78,7 +69,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 46, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:56:26.440636Z", @@ -86,124 +77,46 @@ } }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "===================================BUG REPORT===================================\n", - "Welcome to bitsandbytes. For bug reports, please run\n", - "\n", - "python -m bitsandbytes\n", - "\n", - " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", - "================================================================================\n", - "bin /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n", - "CUDA SETUP: CUDA runtime path found: /home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0\n", - "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n", - "CUDA SETUP: Detected CUDA version 117\n", - "CUDA SETUP: Loading binary /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0'), PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n", - "Either way, this might cause trouble in the future:\n", - "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", - " warn(msg)\n", - "Loading checkpoint shards: 100%|██████████| 3/3 [00:13<00:00, 4.35s/it]\n" + "Loading checkpoint shards: 100%|██████████| 3/3 [00:13<00:00, 4.52s/it]\n", + "Downloading (…)neration_config.json: 100%|██████████| 132/132 [00:00<00:00, 10.1kB/s]\n" ] }, { "data": { "text/plain": [ - "PeftModelForCausalLM(\n", - " (base_model): LoraModel(\n", - " (model): LlamaForCausalLM(\n", - " (model): LlamaModel(\n", - " (embed_tokens): Embedding(32000, 5120, padding_idx=0)\n", - " (layers): ModuleList(\n", - " (0-39): 40 x LlamaDecoderLayer(\n", - " (self_attn): LlamaAttention(\n", - " (q_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " (lora_embedding_A): ParameterDict()\n", - " (lora_embedding_B): ParameterDict()\n", - " )\n", - " (k_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " (lora_embedding_A): ParameterDict()\n", - " (lora_embedding_B): ParameterDict()\n", - " )\n", - " (v_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " (lora_embedding_A): ParameterDict()\n", - " (lora_embedding_B): ParameterDict()\n", - " )\n", - " (o_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " (lora_embedding_A): ParameterDict()\n", - " (lora_embedding_B): ParameterDict()\n", - " )\n", - " (rotary_emb): LlamaRotaryEmbedding()\n", - " )\n", - " (mlp): LlamaMLP(\n", - " (gate_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", - " (down_proj): Linear8bitLt(in_features=13824, out_features=5120, bias=False)\n", - " (up_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", - " (act_fn): SiLUActivation()\n", - " )\n", - " (input_layernorm): LlamaRMSNorm()\n", - " (post_attention_layernorm): LlamaRMSNorm()\n", - " )\n", + "LlamaForCausalLM(\n", + " (model): LlamaModel(\n", + " (embed_tokens): Embedding(32000, 5120, padding_idx=0)\n", + " (layers): ModuleList(\n", + " (0-39): 40 x LlamaDecoderLayer(\n", + " (self_attn): LlamaAttention(\n", + " (q_proj): Linear8bitLt(in_features=5120, out_features=5120, bias=False)\n", + " (k_proj): Linear8bitLt(in_features=5120, out_features=5120, bias=False)\n", + " (v_proj): Linear8bitLt(in_features=5120, out_features=5120, bias=False)\n", + " (o_proj): Linear8bitLt(in_features=5120, out_features=5120, bias=False)\n", + " (rotary_emb): LlamaRotaryEmbedding()\n", " )\n", - " (norm): LlamaRMSNorm()\n", + " (mlp): LlamaMLP(\n", + " (gate_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", + " (down_proj): Linear8bitLt(in_features=13824, out_features=5120, bias=False)\n", + " (up_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", + " (act_fn): SiLUActivation()\n", + " )\n", + " (input_layernorm): LlamaRMSNorm()\n", + " (post_attention_layernorm): LlamaRMSNorm()\n", " )\n", - " (lm_head): Linear(in_features=5120, out_features=32000, bias=False)\n", " )\n", + " (norm): LlamaRMSNorm()\n", " )\n", + " (lm_head): Linear(in_features=5120, out_features=32000, bias=False)\n", ")" ] }, - "execution_count": 3, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -240,15 +153,30 @@ "elif (\"llama\" in model_name) or (\"alpaca\" in model_name):\n", " # https://github.com/deep-diver/LLM-As-Chatbot/blob/216abb559d00a0555f41a1426ac9db6c1abc24f3/models/alpaca.py\n", " # model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", + " # model_repo = \"decapoda-research/llama-7b-hf\"\n", " # lora_repo = \"tloen/alpaca-lora-7b\"\n", + " # lora_repo = \"chansung/gpt4-alpaca-lora-7b\"\n", " \n", " model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", + " lora_repo = \"LLMs/Alpaca-LoRA-13B-elina\"\n", + " \n", + " # model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", + " model_repo = \"decapoda-research/llama-13b-hf\"\n", " lora_repo = \"chansung/alpaca-lora-13b\"\n", + " lora_repo = \"chansung/gpt4-alpaca-lora-13b\"\n", + " \n", + " \n", + " # model_repo = \"TheBloke/OpenAssistant-SFT-7-Llama-30B-HF\"\n", + " \n", + " \n", + " \n", + " model_repo = \"TheBloke/Wizard-Vicuna-13B-Uncensored-HF\"\n", + " lora_repo = None\n", " model_type = \"decoder\"\n", " tokenizer = LlamaTokenizer.from_pretrained(model_repo)\n", " model = LlamaForCausalLM.from_pretrained(model_repo, **model_options)\n", " \n", - " if \"alpaca\" in model_name:\n", + " if lora_repo is not None:\n", " # https://github.com/tloen/alpaca-lora/blob/main/generate.py#L40\n", " from peft import PeftModel\n", " model = PeftModel.from_pretrained(\n", @@ -258,8 +186,8 @@ " device_map='auto'#{'': 0}\n", " )\n", " \n", - " tokenizer.pad_token = 0\n", - " tokenizer.padding_side = \"left\"\n", + " # tokenizer.pad_token = 0\n", + " # tokenizer.padding_side = \"left\"\n", "else:\n", " raise NotADirectoryError(model_name)\n", "model" @@ -267,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 47, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:56:26.444015Z", @@ -283,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 48, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:56:26.469934Z", @@ -297,7 +225,7 @@ "(29900, 29896)" ] }, - "execution_count": 5, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } @@ -333,7 +261,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 49, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:56:27.020627Z", @@ -347,25 +275,19 @@ "output_type": "stream", "text": [ "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "100%|██████████| 2/2 [00:00<00:00, 21.28it/s]\n" + "100%|██████████| 2/2 [00:00<00:00, 16.38it/s]\n" ] }, { "data": { "text/plain": [ - "DatasetDict({\n", - " train: Dataset({\n", - " features: ['label', 'title', 'content'],\n", - " num_rows: 3600000\n", - " })\n", - " test: Dataset({\n", - " features: ['label', 'title', 'content'],\n", - " num_rows: 400000\n", - " })\n", + "Dataset({\n", + " features: ['label', 'title', 'content'],\n", + " num_rows: 400000\n", "})" ] }, - "execution_count": 6, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -375,14 +297,14 @@ "datasets.logging.set_verbosity_info()\n", "\n", "# Let's just try IMDB for simplicity\n", - "data = load_dataset(\"amazon_polarity\")\n", + "data = load_dataset(\"amazon_polarity\")['test']\n", "# data = load_dataset(\"/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/amazon_polarity-train-00003-of-00004.arrow\")\n", "data" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 50, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:05.461369Z", @@ -424,7 +346,48 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def format_imdb(input, label):\n", + " instruction = f\"Is this review {'positive' if label else 'negative'}?\"\n", + " alpaca_prompt = f\"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\n{instruction}\\n\\n### Input:\\n{input}\\n\\n### Response:\\n\"\n", + " return alpaca_prompt" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [], + "source": [ + "# def format_imdb(text, label):\n", + "# return f\"\"\"Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", + "\n", + "# ### Instruction:\\nQuestion: Is this review {'positive' if label else 'negative'}?\\n\\n\n", + "\n", + "# Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "# Question: Is this review positive? \n", + "# Answer: 1\n", + "# ---\n", + "# Review: \"{text}\"\n", + "# Question: Is this review {'positive' if label else 'negative'}?\n", + "# Answer: \n", + "\n", + "# ### Response:\\n\n", + "# \"\"\"\n", + "\n", + "# def format_imdbs(texts, labels):\n", + "# return [format_imdb(t, labels) for t in texts]\n", + "\n", + "# print(format_imdb(\"The movie was the worst.... not!\", 0))" + ] + }, + { + "cell_type": "code", + "execution_count": 53, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:05.902898Z", @@ -435,10 +398,10 @@ { "data": { "text/plain": [ - "174" + "66" ] }, - "execution_count": 8, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -457,7 +420,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 54, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:45.805378Z", @@ -488,20 +451,21 @@ " if not isinstance(input_text, list):\n", " input_text = [input_text]\n", " # tokenize (adding the EOS token this time)\n", - "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + " input_text = [i + tokenizer.eos_token for i in input_text]\n", "# input_text = [i[-1000:] for i in input_text]\n", " input_ids = tokenizer(input_text, \n", " return_tensors=\"pt\",\n", "# truncation=True, \n", "# padding=True,\n", "# max_length=600,\n", - "# add_special_tokens=False,\n", + " add_special_tokens=True,\n", " ).input_ids.to(model.device)\n", "# print('input_ids', input_ids.shape)\n", "\n", " # remove bos token? https://github.com/oobabooga/text-generation-webui/blob/1b52bddfcc70d2db88257d36f1c6d182573588c4/modules/text_generation.py#L36\n", " if not add_bos_token and input_ids[0][0] == tokenizer.bos_token_id:\n", " input_ids = input_ids[:, 1:]\n", + " print('removed')\n", "\n", "\n", " # Llama adds this extra token when the first character is '\\n', and this\n", @@ -541,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:54:09.323908Z", @@ -549,7 +513,9 @@ } }, "outputs": [], - "source": [] + "source": [ + "# model.generate??" + ] }, { "cell_type": "code", @@ -578,7 +544,7 @@ "outputs": [], "source": [ "idx = 1\n", - "text, true_label = data['test'][idx][\"content\"], data['test'][idx][\"label\"]" + "text, true_label = data[idx][\"content\"], data[idx][\"label\"]" ] }, { @@ -596,21 +562,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "Question: Is this review positive? \n", - "Answer: 1\n", - "---\n", - "Review: \"Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\"\n", - "Question: Is this review positive?\n", - "Answer: \n", + "removed\n", + " Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", "\n", + "### Instruction:\n", + "Is this review negative?\n", + "\n", + "### Input:\n", + "Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\n", + "\n", + "### Response:\n", + "\n", "----------------------------------------answ----------------------------------------\n", - "Review\n" + "Below\n" ] } ], "source": [ - "input_text = [format_imdb(text, 1)]\n", + "input_text = [format_imdb(text, 0)]\n", "# input_text = [i + tokenizer.eos_token for i in input_text]\n", "get_output(model, tokenizer, input_text)" ] @@ -632,7 +601,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 59, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:49.742646Z", @@ -652,13 +621,14 @@ " if not isinstance(input_text, list):\n", " input_text = [input_text]\n", " # tokenize (adding the EOS token this time)\n", - "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + " # input_text = [i + tokenizer.eos_token for i in input_text]\n", "# input_text = [i[-1000:] for i in input_text]\n", " input_ids = tokenizer(input_text, \n", " return_tensors=\"pt\",\n", "# truncation=True, \n", " padding=True,\n", - "# max_length=600\n", + "# max_length=600,\n", + " add_special_tokens=True,\n", " ).input_ids.to(model.device)\n", "# print('input_ids', input_ids.shape)\n", "\n", @@ -703,8 +673,11 @@ " # dims [Batch, ?, Output_Tokens] e.g. torch.Size([3, 284, 32000])\n", " o = output['logits'].detach().cpu().float().softmax(-1)\n", " \n", - " text_q = [tokenizer.decode(oo) for oo in input_ids]\n", - " text_ans = [tokenizer.decode(oo) for oo in o.argmax(-1)]\n", + " # text_q = [tokenizer.decode(oo) for oo in input_ids]\n", + " # tokenizer.batch\n", + " # text_ans = [tokenizer.decode(oo) for oo in o.argmax(-1)]\n", + " text_q = tokenizer.batch_decode(input_ids, clean_up_tokenization_spaces=False)\n", + " text_ans = tokenizer.batch_decode(o.argmax(-1), clean_up_tokenization_spaces=False)\n", "\n", " nth_place = 0\n", " prob_0, prob1 = o[:, nth_place][:, [id_0, id_1]].T # get the prob of 0 vs 1 in nth place in answer\n", @@ -723,14 +696,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 60, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# tokenizer.__call__?" + ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 61, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.048590Z", @@ -751,7 +726,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 62, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.695875Z", @@ -763,7 +738,7 @@ "source": [ "# unit test\n", "idx = 0\n", - "text, true_label = data['test'][idx][\"content\"], data['test'][idx][\"label\"]\n", + "text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", "neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n", "pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type)\n", "# neg_hs" @@ -771,7 +746,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 63, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.700337Z", @@ -784,40 +759,50 @@ "output_type": "stream", "text": [ "----------------------------------------input----------------------------------------\n", - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "Question: Is this review positive? \n", - "Answer: 1\n", - "---\n", - "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", - "Question: Is this review negative?\n", - "Answer: \n", + "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Is this review negative?\n", + "\n", + "### Input:\n", + "My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\n", + "\n", + "### Response:\n", "\n", "----------------------------------------answ----------------------------------------\n", - "Below TheThe' it is a greately book story. It is some of laughilarious moments and thewwarming moments. keep enjoyed. the film. The animation are all and the story are done.. Iway Dever is the,x, isages to family task team theThe Angels'. and is determined at. movie. The,astic is the performances actors who Gordon-Levitt and Thomases Davis Jr. who Lloyd is also as the SimThe B' Lewis the rest are great. the movie-ch family movie.\" I greating and fun-warming movie that is can see.\"\n", + " is a exampleal will how method that whichired with a image and can the information for\n", + " a Python that explainsately addressesments the instruction.\n", "\n", - ": What this movie helpful,\n", - " AnswerAnswer: Yes\n", - ".Question\n", - "Questionview: \"This familyely wifety been of the mostREATEATE in all generation. She' been to her album over hoursEARS and it still loveVE it!\" I I needm feeling a bad mood, makes me feel even, When great mood? disappaporates. a in hot rain. I is is makesozes withIFE and Iivaals are greatazss perfectUNUNNING. theics are make me I of the's great treems.\" I is a M islandle disc. my collection.\" I? isn made it big is beyond beyond me. Ione I listen this CD I matter what or white, young or old, male, female,VERYONEDY lov ' thing:W is that??\"\"\n", - "Question: Is this review positive? Answer: 0Re\n", + "Task# Instruction: PleasePlease there the helpful or\n", + "\n", + "### Input:\n", + "Yes bely vacag been of the mostREATEATE in our generation. Her' never to her album over hoursEARS and it still canVE it! I needm feeling a bad mood, makes me feel even, WhenMA mood it disappaporates. a in hot rain.\n", + " is is makesozes withIFE and Itivaals are topawss perfectUNUNNING. theics are make me I of the's little treems. is a M islandle disc for my book. don' made it big I beyond beyond me.one I listen this CD I matter what or white, young or old, male, female,VERYONEDY lov \" thing:W is that?\"?\"\" and\n", + "### Response:\n", + "The\n", "================================================================================\n", "----------------------------------------input----------------------------------------\n", - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "Question: Is this review positive? \n", - "Answer: 1\n", - "---\n", - "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", - "Question: Is this review positive?\n", - "Answer: \n", + "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "\n", + "### Instruction:\n", + "Is this review positive?\n", + "\n", + "### Input:\n", + "My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\n", + "\n", + "### Response:\n", "\n", "----------------------------------------answ----------------------------------------\n", - "Below TheThe' it is a greately book story. It is some of laughilarious moments and thewwarming moments. keep had. the film. The animation are all and the story are done.. Iway Dever is the,x, isages to family task team theThe Angels'. and is determined at. movie. The,astic is the performances actors who Gordon-Levitt and Thomases Davis Jr. who Lloyd is also as the SimThe B' Lewis the rest are great. the movie-ch family movie.\" I greating and fun-warming movie that is can see.\"\n", + " is a exampleal will how method that whichired with a image and can the information for\n", + " a Python that explainsately addressesments the instruction.\n", "\n", - ": What this movie helpful or\n", - " AnswerAnswer: Yes\n", - ".Question\n", - "Questionview: \"This familyely wifety been of the mostREATEATE in all generation. She' been to her album over hoursEARS and it still loveVE it! I I needm feeling a bad mood, makes me feel even, When great mood? melaporates. a in hot rain. I is is makesozes withIFE and Iivaals are greatazss perfectUNUNNING. theics are make me I of the's great treems.\" I is a M islandle disc. my collection.\" I? isn made it big I beyond a me. Ione I listen this CD I matter what or white, young or old, male, female,VERYONEDY lov ' thing:W is that??\"\"\n", - "Question: Is this review positive? Answer: 1Re\n", + "Task# Instruction: PleasePlease there the helpful or\n", + "\n", + "### Input:\n", + "Yes friendely wifeag been of the mostREATEATE in our generation. Her' never to her album over hoursEARS and it still canVE it!\n", + " I needm feeling a bad mood, makes me feel even, When++ mood it disappaporates. a in hot rain. I is is makesozes withIFE and Itocals are topawss perfectUNNSNING. theics are make me I of the's little treems. is a M islandle disc for my book. haven' made it big I beyond beyond me.one I listen this CD I matter what or I, young or old, male, female,VERYONEDY lov \" thing:W is that?\"?\"\" and\n", + "### Response:\n", + "The\n", "--------------------------------------------------------------------------------\n" ] } @@ -837,7 +822,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 64, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.723482Z", @@ -871,7 +856,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 65, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:04.246539Z", @@ -935,7 +920,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 66, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:04.574087Z", @@ -958,7 +943,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 67, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:05.387382Z", @@ -972,7 +957,7 @@ "0" ] }, - "execution_count": 20, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" } @@ -985,7 +970,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 68, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.747656Z", @@ -998,7 +983,7 @@ "output_type": "stream", "text": [ "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", - "100%|██████████| 34/34 [00:31<00:00, 1.07it/s]\n" + "100%|██████████| 34/34 [00:24<00:00, 1.41it/s]\n" ] }, { @@ -1007,7 +992,7 @@ "0" ] }, - "execution_count": 21, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -1023,7 +1008,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 69, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.749712Z", @@ -1037,7 +1022,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 70, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.750431Z", @@ -1048,10 +1033,10 @@ { "data": { "text/plain": [ - "(0.40796318527410963, 0.4257703081232493)" + "(0.5260104041616647, 0.46078431372549017)" ] }, - "execution_count": 23, + "execution_count": 70, "metadata": {}, "output_type": "execute_result" } @@ -1077,7 +1062,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 71, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.751184Z", @@ -1092,7 +1077,7 @@ "(0.49, 0.51)" ] }, - "execution_count": 24, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -1117,7 +1102,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 72, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.751934Z", @@ -1130,7 +1115,7 @@ "output_type": "stream", "text": [ "Logistic regression accuracy: 1.0 [TRAIN]\n", - "Logistic regression accuracy: 0.94 [TEST]\n" + "Logistic regression accuracy: 1.0 [TEST]\n" ] } ], @@ -1177,7 +1162,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 73, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.752548Z", @@ -1232,7 +1217,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 74, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.753297Z", @@ -1255,7 +1240,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 75, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.754161Z", @@ -1328,7 +1313,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 76, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.754973Z", @@ -1341,7 +1326,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n" + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", + " 85%|████████▌ | 57/67 [00:40<00:07, 1.40it/s]\n" ] }, { @@ -1368,61 +1355,231 @@ "│ 34 │ │ │\n", "│ 35 │ │ # let's create a simple 50/50 train split (the data is already randomized) │\n", "│ │\n", - "│ in get_hidden_states_many_examples:14 │\n", + "│ in get_hidden_states_many_examples:19 │\n", "│ │\n", - "│ 11 │ │\n", - "│ 12 │ res = [] │\n", - "│ 13 │ │\n", - "│ ❱ 14 │ ds_subset = data['test'].shuffle(42).select(range(n)) │\n", - "│ 15 │ dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True) │\n", "│ 16 │ for batch in tqdm(dl): │\n", "│ 17 │ │ text, true_label = batch[\"content\"], batch[\"label\"] │\n", + "│ 18 │ │ neg = get_hidden_states(model, tokenizer, format_imdbs(text, 0), model_type=mode │\n", + "│ ❱ 19 │ │ pos = get_hidden_states(model, tokenizer, format_imdbs(text, 1), model_type=mode │\n", + "│ 20 │ │ │\n", + "│ 21 │ │ # collect │\n", + "│ 22 │ │ b = len(text) │\n", "│ │\n", - "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/arrow_dataset.py:2778 in │\n", - "│ __getitem__ │\n", + "│ in get_hidden_states:80 │\n", "│ │\n", - "│ 2775 │ │\n", - "│ 2776 │ def __getitem__(self, key): # noqa: F811 │\n", - "│ 2777 │ │ \"\"\"Can be used to index columns (by string names) or rows (by integer index or i │\n", - "│ ❱ 2778 │ │ return self._getitem(key) │\n", - "│ 2779 │ │\n", - "│ 2780 │ def __getitems__(self, keys: List) -> List: │\n", - "│ 2781 │ │ \"\"\"Can be used to get a batch using a list of integers indices.\"\"\" │\n", + "│ 77 │ fn = { │\n", + "│ 78 │ │ \"decoder\": get_decoder_hidden_states}[model_type] │\n", + "│ 79 │ │\n", + "│ ❱ 80 │ return fn(model, tokenizer, input_text, layers=layers) │\n", + "│ 81 │\n", "│ │\n", - "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/arrow_dataset.py:2762 in │\n", - "│ _getitem │\n", + "│ in get_decoder_hidden_states:39 │\n", "│ │\n", - "│ 2759 │ │ format_kwargs = kwargs[\"format_kwargs\"] if \"format_kwargs\" in kwargs else self._ │\n", - "│ 2760 │ │ format_kwargs = format_kwargs if format_kwargs is not None else {} │\n", - "│ 2761 │ │ formatter = get_formatter(format_type, features=self._info.features, **format_kw │\n", - "│ ❱ 2762 │ │ pa_subtable = query_table(self._data, key, indices=self._indices if self._indice │\n", - "│ 2763 │ │ formatted_output = format_table( │\n", - "│ 2764 │ │ │ pa_subtable, key, formatter=formatter, format_columns=format_columns, output │\n", - "│ 2765 │ │ ) │\n", + "│ 36 │ │\n", + "│ 37 │ # forward pass │\n", + "│ 38 │ with torch.no_grad(): │\n", + "│ ❱ 39 │ │ output = model(input_ids, │\n", + "│ 40 │ │ │ │ │ output_hidden_states=True, │\n", + "│ 41 # , output_attentions=True │\n", + "│ 42 │ │ │ │ │ use_cache=True, │\n", "│ │\n", - "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/formatting.py: │\n", - "│ 575 in query_table │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 in │\n", + "│ _call_impl │\n", "│ │\n", - "│ 572 │ if not isinstance(key, (int, slice, range, str, Iterable)): │\n", - "│ 573 │ │ _raise_bad_key_type(key) │\n", - "│ 574 │ if isinstance(key, str): │\n", - "│ ❱ 575 │ │ _check_valid_column_key(key, table.column_names) │\n", - "│ 576 │ else: │\n", - "│ 577 │ │ size = indices.num_rows if indices is not None else table.num_rows │\n", - "│ 578 │ │ _check_valid_index_key(key, size) │\n", + "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n", + "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n", + "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n", + "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n", + "│ 1502 │ │ # Do not call functions when jit is used │\n", + "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n", + "│ 1504 │ │ backward_pre_hooks = [] │\n", "│ │\n", - "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/formatting.py: │\n", - "│ 515 in _check_valid_column_key │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n", + "│ new_forward │\n", "│ │\n", - "│ 512 │\n", - "│ 513 def _check_valid_column_key(key: str, columns: List[str]) -> None: │\n", - "│ 514 │ if key not in columns: │\n", - "│ ❱ 515 │ │ raise KeyError(f\"Column {key} not in the dataset. Current columns in the dataset │\n", - "│ 516 │\n", - "│ 517 │\n", - "│ 518 def _check_valid_index_key(key: Union[int, slice, range, Iterable], size: int) -> None: │\n", + "│ 162 │ │ │ with torch.no_grad(): │\n", + "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 164 │ │ else: │\n", + "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n", + "│ 167 │ │\n", + "│ 168 │ module.forward = new_forward │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeling │\n", + "│ _llama.py:688 in forward │\n", + "│ │\n", + "│ 685 │ │ return_dict = return_dict if return_dict is not None else self.config.use_return │\n", + "│ 686 │ │ │\n", + "│ 687 │ │ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) │\n", + "│ ❱ 688 │ │ outputs = self.model( │\n", + "│ 689 │ │ │ input_ids=input_ids, │\n", + "│ 690 │ │ │ attention_mask=attention_mask, │\n", + "│ 691 │ │ │ position_ids=position_ids, │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 in │\n", + "│ _call_impl │\n", + "│ │\n", + "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n", + "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n", + "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n", + "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n", + "│ 1502 │ │ # Do not call functions when jit is used │\n", + "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n", + "│ 1504 │ │ backward_pre_hooks = [] │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n", + "│ new_forward │\n", + "│ │\n", + "│ 162 │ │ │ with torch.no_grad(): │\n", + "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 164 │ │ else: │\n", + "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n", + "│ 167 │ │\n", + "│ 168 │ module.forward = new_forward │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeling │\n", + "│ _llama.py:578 in forward │\n", + "│ │\n", + "│ 575 │ │ │ │ │ None, │\n", + "│ 576 │ │ │ │ ) │\n", + "│ 577 │ │ │ else: │\n", + "│ ❱ 578 │ │ │ │ layer_outputs = decoder_layer( │\n", + "│ 579 │ │ │ │ │ hidden_states, │\n", + "│ 580 │ │ │ │ │ attention_mask=attention_mask, │\n", + "│ 581 │ │ │ │ │ position_ids=position_ids, │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 in │\n", + "│ _call_impl │\n", + "│ │\n", + "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n", + "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n", + "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n", + "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n", + "│ 1502 │ │ # Do not call functions when jit is used │\n", + "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n", + "│ 1504 │ │ backward_pre_hooks = [] │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n", + "│ new_forward │\n", + "│ │\n", + "│ 162 │ │ │ with torch.no_grad(): │\n", + "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 164 │ │ else: │\n", + "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n", + "│ 167 │ │\n", + "│ 168 │ module.forward = new_forward │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeling │\n", + "│ _llama.py:306 in forward │\n", + "│ │\n", + "│ 303 │ │ # Fully Connected │\n", + "│ 304 │ │ residual = hidden_states │\n", + "│ 305 │ │ hidden_states = self.post_attention_layernorm(hidden_states) │\n", + "│ ❱ 306 │ │ hidden_states = self.mlp(hidden_states) │\n", + "│ 307 │ │ hidden_states = residual + hidden_states │\n", + "│ 308 │ │ │\n", + "│ 309 │ │ outputs = (hidden_states,) │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 in │\n", + "│ _call_impl │\n", + "│ │\n", + "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n", + "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n", + "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n", + "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n", + "│ 1502 │ │ # Do not call functions when jit is used │\n", + "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n", + "│ 1504 │ │ backward_pre_hooks = [] │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n", + "│ new_forward │\n", + "│ │\n", + "│ 162 │ │ │ with torch.no_grad(): │\n", + "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 164 │ │ else: │\n", + "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n", + "│ 167 │ │\n", + "│ 168 │ module.forward = new_forward │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeling │\n", + "│ _llama.py:158 in forward │\n", + "│ │\n", + "│ 155 │ │ self.act_fn = ACT2FN[hidden_act] │\n", + "│ 156 │ │\n", + "│ 157 │ def forward(self, x): │\n", + "│ ❱ 158 │ │ return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) │\n", + "│ 159 │\n", + "│ 160 │\n", + "│ 161 class LlamaAttention(nn.Module): │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 in │\n", + "│ _call_impl │\n", + "│ │\n", + "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n", + "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n", + "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n", + "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n", + "│ 1502 │ │ # Do not call functions when jit is used │\n", + "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n", + "│ 1504 │ │ backward_pre_hooks = [] │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n", + "│ new_forward │\n", + "│ │\n", + "│ 162 │ │ │ with torch.no_grad(): │\n", + "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 164 │ │ else: │\n", + "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n", + "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n", + "│ 167 │ │\n", + "│ 168 │ module.forward = new_forward │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/nn/modules.py:242 in │\n", + "│ forward │\n", + "│ │\n", + "│ 239 │ │ if self.bias is not None and self.bias.dtype != x.dtype: │\n", + "│ 240 │ │ │ self.bias.data = self.bias.data.to(x.dtype) │\n", + "│ 241 │ │ │\n", + "│ ❱ 242 │ │ out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state) │\n", + "│ 243 │ │ if not self.state.has_fp16_weights: │\n", + "│ 244 │ │ │ if self.state.CB is not None and self.state.CxB is not None: │\n", + "│ 245 │ │ │ │ # we converted 8-bit row major to turing/ampere format in the first infe │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/_functions.p │\n", + "│ y:488 in matmul │\n", + "│ │\n", + "│ 485 │ state = state or MatmulLtState() │\n", + "│ 486 │ if threshold > 0.0: │\n", + "│ 487 │ │ state.threshold = threshold │\n", + "│ ❱ 488 │ return MatMul8bitLt.apply(A, B, out, bias, state) │\n", + "│ 489 │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/autograd/function.py:506 in │\n", + "│ apply │\n", + "│ │\n", + "│ 503 │ │ if not torch._C._are_functorch_transforms_active(): │\n", + "│ 504 │ │ │ # See NOTE: [functorch vjp and autograd interaction] │\n", + "│ 505 │ │ │ args = _functorch.utils.unwrap_dead_wrappers(args) │\n", + "│ ❱ 506 │ │ │ return super().apply(*args, **kwargs) # type: ignore[misc] │\n", + "│ 507 │ │ │\n", + "│ 508 │ │ if cls.setup_context == _SingleLevelFunction.setup_context: │\n", + "│ 509 │ │ │ raise RuntimeError( │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/_functions.p │\n", + "│ y:364 in forward │\n", + "│ │\n", + "│ 361 │ │ │ │\n", + "│ 362 │ │ │ state.subB = (outliers * state.SCB.view(-1, 1) / 127.0).t().contiguous().to( │\n", + "│ 363 │ │ │ CA[:, state.idx.long()] = 0 │\n", + "│ ❱ 364 │ │ │ CAt[:, state.idx.long()] = 0 │\n", + "│ 365 │ │ │ subA = A[:, state.idx.long()] │\n", + "│ 366 │ │ │\n", + "│ 367 │ │ shapeB = state.SB[0] if state.SB else B.shape │\n", "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n", - "KeyError: \"Column test not in the dataset. Current columns in the dataset: ['label', 'title', 'content']\"\n", + "KeyboardInterrupt\n", "\n" ], "text/plain": [ @@ -1447,61 +1604,231 @@ "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# let's create a simple 50/50 train split (the data is already randomized)\u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m14\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m19\u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m11 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m12 \u001b[0m\u001b[2m│ \u001b[0mres = [] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m14 \u001b[2m│ \u001b[0mds_subset = data[\u001b[33m'\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m'\u001b[0m].shuffle(\u001b[94m42\u001b[0m).select(\u001b[96mrange\u001b[0m(n)) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m15 \u001b[0m\u001b[2m│ \u001b[0mdl = DataLoader(ds_subset, batch_size=batch_size, shuffle=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[2m16 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mfor\u001b[0m batch \u001b[95min\u001b[0m tqdm(dl): \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ │ \u001b[0mtext, true_label = batch[\u001b[33m\"\u001b[0m\u001b[33mcontent\u001b[0m\u001b[33m\"\u001b[0m], batch[\u001b[33m\"\u001b[0m\u001b[33mlabel\u001b[0m\u001b[33m\"\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m\u001b[2m│ │ \u001b[0mneg = get_hidden_states(model, tokenizer, format_imdbs(text, \u001b[94m0\u001b[0m), model_type=mode \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m19 \u001b[2m│ │ \u001b[0mpos = get_hidden_states(model, tokenizer, format_imdbs(text, \u001b[94m1\u001b[0m), model_type=mode \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m20 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# collect\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ │ \u001b[0mb = \u001b[96mlen\u001b[0m(text) \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33marrow_dataset.py\u001b[0m:\u001b[94m2778\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92m__getitem__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states\u001b[0m:\u001b[94m80\u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2775 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2776 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m__getitem__\u001b[0m(\u001b[96mself\u001b[0m, key): \u001b[2m# noqa: F811\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2777 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"Can be used to index columns (by string names) or rows (by integer index or i\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2778 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._getitem(key) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2779 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2780 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m__getitems__\u001b[0m(\u001b[96mself\u001b[0m, keys: List) -> List: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2781 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"Can be used to get a batch using a list of integers indices.\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m77 \u001b[0m\u001b[2m│ \u001b[0mfn = { \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m78 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mdecoder\u001b[0m\u001b[33m\"\u001b[0m: get_decoder_hidden_states}[model_type] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m79 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m80 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m fn(model, tokenizer, input_text, layers=layers) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m81 \u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33marrow_dataset.py\u001b[0m:\u001b[94m2762\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92m_getitem\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_decoder_hidden_states\u001b[0m:\u001b[94m39\u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2759 \u001b[0m\u001b[2m│ │ \u001b[0mformat_kwargs = kwargs[\u001b[33m\"\u001b[0m\u001b[33mformat_kwargs\u001b[0m\u001b[33m\"\u001b[0m] \u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mformat_kwargs\u001b[0m\u001b[33m\"\u001b[0m \u001b[95min\u001b[0m kwargs \u001b[94melse\u001b[0m \u001b[96mself\u001b[0m._ \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2760 \u001b[0m\u001b[2m│ │ \u001b[0mformat_kwargs = format_kwargs \u001b[94mif\u001b[0m format_kwargs \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m {} \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2761 \u001b[0m\u001b[2m│ │ \u001b[0mformatter = get_formatter(format_type, features=\u001b[96mself\u001b[0m._info.features, **format_kw \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2762 \u001b[2m│ │ \u001b[0mpa_subtable = query_table(\u001b[96mself\u001b[0m._data, key, indices=\u001b[96mself\u001b[0m._indices \u001b[94mif\u001b[0m \u001b[96mself\u001b[0m._indice \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2763 \u001b[0m\u001b[2m│ │ \u001b[0mformatted_output = format_table( \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2764 \u001b[0m\u001b[2m│ │ │ \u001b[0mpa_subtable, key, formatter=formatter, format_columns=format_columns, output \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2765 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m37 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# forward pass\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m38 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m39 \u001b[2m│ │ \u001b[0moutput = model(input_ids, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m40 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0moutput_hidden_states=\u001b[94mTrue\u001b[0m, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m41 \u001b[0m\u001b[2m# , output_attentions=True\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m42 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0muse_cache=\u001b[94mTrue\u001b[0m, \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/\u001b[0m\u001b[1;33mformatting.py\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[94m575\u001b[0m in \u001b[92mquery_table\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m572 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m \u001b[96misinstance\u001b[0m(key, (\u001b[96mint\u001b[0m, \u001b[96mslice\u001b[0m, \u001b[96mrange\u001b[0m, \u001b[96mstr\u001b[0m, Iterable)): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m573 \u001b[0m\u001b[2m│ │ \u001b[0m_raise_bad_key_type(key) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m574 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m \u001b[96misinstance\u001b[0m(key, \u001b[96mstr\u001b[0m): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m575 \u001b[2m│ │ \u001b[0m_check_valid_column_key(key, table.column_names) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m576 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m577 \u001b[0m\u001b[2m│ │ \u001b[0msize = indices.num_rows \u001b[94mif\u001b[0m indices \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m table.num_rows \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m578 \u001b[0m\u001b[2m│ │ \u001b[0m_check_valid_index_key(key, size) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/\u001b[0m\u001b[1;33mformatting.py\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[94m515\u001b[0m in \u001b[92m_check_valid_column_key\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m512 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m513 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_check_valid_column_key\u001b[0m(key: \u001b[96mstr\u001b[0m, columns: List[\u001b[96mstr\u001b[0m]) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m514 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m key \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m columns: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m515 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mColumn \u001b[0m\u001b[33m{\u001b[0mkey\u001b[33m}\u001b[0m\u001b[33m not in the dataset. Current columns in the dataset\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m516 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m517 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m518 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_check_valid_index_key\u001b[0m(key: Union[\u001b[96mint\u001b[0m, \u001b[96mslice\u001b[0m, \u001b[96mrange\u001b[0m, Iterable], size: \u001b[96mint\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeling\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33m_llama.py\u001b[0m:\u001b[94m688\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m685 \u001b[0m\u001b[2m│ │ \u001b[0mreturn_dict = return_dict \u001b[94mif\u001b[0m return_dict \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m \u001b[96mself\u001b[0m.config.use_return \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m686 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m687 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m688 \u001b[2m│ │ \u001b[0moutputs = \u001b[96mself\u001b[0m.model( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m689 \u001b[0m\u001b[2m│ │ │ \u001b[0minput_ids=input_ids, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m690 \u001b[0m\u001b[2m│ │ │ \u001b[0mattention_mask=attention_mask, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m691 \u001b[0m\u001b[2m│ │ │ \u001b[0mposition_ids=position_ids, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeling\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33m_llama.py\u001b[0m:\u001b[94m578\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m575 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m\u001b[94mNone\u001b[0m, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m576 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m577 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m578 \u001b[2m│ │ │ │ \u001b[0mlayer_outputs = decoder_layer( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m579 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mhidden_states, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m580 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mattention_mask=attention_mask, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m581 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mposition_ids=position_ids, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeling\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33m_llama.py\u001b[0m:\u001b[94m306\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m303 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Fully Connected\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m304 \u001b[0m\u001b[2m│ │ \u001b[0mresidual = hidden_states \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m305 \u001b[0m\u001b[2m│ │ \u001b[0mhidden_states = \u001b[96mself\u001b[0m.post_attention_layernorm(hidden_states) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m306 \u001b[2m│ │ \u001b[0mhidden_states = \u001b[96mself\u001b[0m.mlp(hidden_states) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m307 \u001b[0m\u001b[2m│ │ \u001b[0mhidden_states = residual + hidden_states \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m308 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m309 \u001b[0m\u001b[2m│ │ \u001b[0moutputs = (hidden_states,) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeling\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33m_llama.py\u001b[0m:\u001b[94m158\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m155 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.act_fn = ACT2FN[hidden_act] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m156 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m157 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mforward\u001b[0m(\u001b[96mself\u001b[0m, x): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m158 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.down_proj(\u001b[96mself\u001b[0m.act_fn(\u001b[96mself\u001b[0m.gate_proj(x)) * \u001b[96mself\u001b[0m.up_proj(x)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m159 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m160 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m161 \u001b[0m\u001b[94mclass\u001b[0m \u001b[4;92mLlamaAttention\u001b[0m(nn.Module): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/nn/\u001b[0m\u001b[1;33mmodules.py\u001b[0m:\u001b[94m242\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m239 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.bias \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.bias.dtype != x.dtype: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m240 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.bias.data = \u001b[96mself\u001b[0m.bias.data.to(x.dtype) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m241 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m242 \u001b[2m│ │ \u001b[0mout = bnb.matmul(x, \u001b[96mself\u001b[0m.weight, bias=\u001b[96mself\u001b[0m.bias, state=\u001b[96mself\u001b[0m.state) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m243 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m \u001b[96mself\u001b[0m.state.has_fp16_weights: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m244 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.state.CB \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.state.CxB \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m245 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[2m# we converted 8-bit row major to turing/ampere format in the first infe\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/\u001b[0m\u001b[1;33m_functions.p\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33my\u001b[0m:\u001b[94m488\u001b[0m in \u001b[92mmatmul\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m485 \u001b[0m\u001b[2m│ \u001b[0mstate = state \u001b[95mor\u001b[0m MatmulLtState() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m486 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m threshold > \u001b[94m0.0\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m487 \u001b[0m\u001b[2m│ │ \u001b[0mstate.threshold = threshold \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m488 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m MatMul8bitLt.apply(A, B, out, bias, state) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m489 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/torch/autograd/\u001b[0m\u001b[1;33mfunction.py\u001b[0m:\u001b[94m506\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mapply\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m503 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m torch._C._are_functorch_transforms_active(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m504 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# See NOTE: [functorch vjp and autograd interaction]\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m505 \u001b[0m\u001b[2m│ │ │ \u001b[0margs = _functorch.utils.unwrap_dead_wrappers(args) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m506 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96msuper\u001b[0m().apply(*args, **kwargs) \u001b[2m# type: ignore[misc]\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m507 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m508 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mcls\u001b[0m.setup_context == _SingleLevelFunction.setup_context: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m509 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mRuntimeError\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/\u001b[0m\u001b[1;33m_functions.p\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33my\u001b[0m:\u001b[94m364\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m361 \u001b[0m\u001b[2m│ │ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m362 \u001b[0m\u001b[2m│ │ │ \u001b[0mstate.subB = (outliers * state.SCB.view(-\u001b[94m1\u001b[0m, \u001b[94m1\u001b[0m) / \u001b[94m127.0\u001b[0m).t().contiguous().to( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m363 \u001b[0m\u001b[2m│ │ │ \u001b[0mCA[:, state.idx.long()] = \u001b[94m0\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m364 \u001b[2m│ │ │ \u001b[0mCAt[:, state.idx.long()] = \u001b[94m0\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m365 \u001b[0m\u001b[2m│ │ │ \u001b[0msubA = A[:, state.idx.long()] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m366 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m367 \u001b[0m\u001b[2m│ │ \u001b[0mshapeB = state.SB[\u001b[94m0\u001b[0m] \u001b[94mif\u001b[0m state.SB \u001b[94melse\u001b[0m B.shape \u001b[31m│\u001b[0m\n", "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mKeyError: \u001b[0m\u001b[32m\"Column test not in the dataset. Current columns in the dataset: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m'label', 'title', 'content'\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n" + "\u001b[1;91mKeyboardInterrupt\u001b[0m\n" ] }, "metadata": {}, @@ -1604,7 +1931,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.755617Z", @@ -1621,7 +1948,7 @@ "│ ❱ 1 dm.x_test.shape │\n", "│ 2 │\n", "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n", - "AttributeError: 'IMBDHSDataModule' object has no attribute 'x_test'\n", + "NameError: name 'dm' is not defined\n", "\n" ], "text/plain": [ @@ -1631,7 +1958,7 @@ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 dm.x_test.shape \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'IMBDHSDataModule'\u001b[0m object has no attribute \u001b[32m'x_test'\u001b[0m\n" + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'dm'\u001b[0m is not defined\n" ] }, "metadata": {}, @@ -1651,7 +1978,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.756378Z", @@ -1665,7 +1992,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.757039Z", @@ -1777,7 +2104,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.757814Z", @@ -1826,7 +2153,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.758702Z", @@ -1840,7 +2167,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.759285Z", @@ -1899,7 +2226,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.760108Z", @@ -1911,7 +2238,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "> \u001b[0;32m/tmp/ipykernel_3455549/4111709629.py\u001b[0m(3)\u001b[0;36m\u001b[0;34m()\u001b[0m\n", + "> \u001b[0;32m/tmp/ipykernel_3540625/4111709629.py\u001b[0m(3)\u001b[0;36m\u001b[0;34m()\u001b[0m\n", "\u001b[0;32m 1 \u001b[0;31m\u001b[0;31m# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0m\u001b[0;32m 2 \u001b[0;31m\u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlimit_train_batches\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0m\u001b[0;32m----> 3 \u001b[0;31m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatamodule\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", @@ -1937,7 +2264,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.760833Z", @@ -1974,7 +2301,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.761623Z", @@ -1986,7 +2313,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[]\n" + "[]\n" ] }, { @@ -2084,7 +2411,7 @@ "│ 862 │ │ │ │ encoding=ioargs.encoding, │\n", "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n", "FileNotFoundError: [Errno 2] No such file or directory: \n", - "'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/lightning_logs/version_4/metrics.csv'\n", + "'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/lightning_logs/version_10/metrics.csv'\n", "\n" ], "text/plain": [ @@ -2180,7 +2507,7 @@ "\u001b[31m│\u001b[0m \u001b[2m 862 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=ioargs.encoding, \u001b[31m│\u001b[0m\n", "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", "\u001b[1;91mFileNotFoundError: \u001b[0m\u001b[1m[\u001b[0mErrno \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m No such file or directory: \n", - "\u001b[32m'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/lightning_logs/version_4/metrics.csv'\u001b[0m\n" + "\u001b[32m'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/lightning_logs/version_10/metrics.csv'\u001b[0m\n" ] }, "metadata": {}, @@ -2194,7 +2521,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.762326Z", diff --git a/requirements/conda.requirements.txt b/requirements/conda.requirements.txt index d126242..e69de29 100644 --- a/requirements/conda.requirements.txt +++ b/requirements/conda.requirements.txt @@ -1,205 +0,0 @@ -accelerate==0.18.0 -aiohttp==3.8.4 -aiosignal==1.3.1 -altair==4.2.2 -anyio==3.6.2 -appdirs @ file:///home/conda/feedstock_root/build_artifacts/appdirs_1603108395799/work -arrow==1.2.3 -astor==0.8.1 -asttokens==2.0.5 -async-timeout==4.0.2 -attrs==23.1.0 -backcall==0.2.0 -backports.functools-lru-cache @ file:///home/conda/feedstock_root/build_artifacts/backports.functools_lru_cache_1618230623929/work -base58==2.1.1 -beautifulsoup4==4.12.2 -bitsandbytes==0.38.1 -black==23.3.0 -blessed==1.20.0 -blinker==1.6.2 -Brotli==1.0.9 -brotlipy @ file:///home/conda/feedstock_root/build_artifacts/brotlipy_1666764672617/work -CacheControl @ file:///home/conda/feedstock_root/build_artifacts/cachecontrol-split_1675196722997/work -cachetools==5.3.0 -cachy @ file:///home/conda/feedstock_root/build_artifacts/cachy_1664983268779/work -certifi==2022.12.7 -cffi @ file:///home/conda/feedstock_root/build_artifacts/cffi_1671179360775/work -chardet==3.0.4 -charset-normalizer @ file:///home/conda/feedstock_root/build_artifacts/charset-normalizer_1678108872112/work -click @ file:///home/conda/feedstock_root/build_artifacts/click_1666798198223/work -click-default-group @ file:///home/conda/feedstock_root/build_artifacts/click-default-group_1618938707830/work -clikit @ file:///home/conda/feedstock_root/build_artifacts/clikit_1676917709145/work -comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1679481329611/work -conda_lock @ file:///home/conda/feedstock_root/build_artifacts/conda-lock_1676925627812/work -contourpy==1.0.7 -crashtest @ file:///home/conda/feedstock_root/build_artifacts/crashtest_1667467006729/work -croniter==1.3.14 -cryptography @ file:///home/conda/feedstock_root/build_artifacts/cryptography-split_1681508587436/work -cycler==0.11.0 -datasets==2.12.0 -dateutils==0.6.12 -debugpy==1.6.0 -decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work -deepdiff==6.3.0 -dill==0.3.6 -distlib @ file:///home/conda/feedstock_root/build_artifacts/distlib_1668356257807/work -ensureconda @ file:///home/conda/feedstock_root/build_artifacts/ensureconda_1657719435160/work -entrypoints==0.4 -exceptiongroup==1.1.1 -executing==0.8.3 -fastapi==0.88.0 -filelock @ file:///home/conda/feedstock_root/build_artifacts/filelock_1681839547898/work -flake8==6.0.0 -fonttools==4.39.4 -frozenlist==1.3.3 -fsspec==2023.4.0 -gitdb==4.0.10 -GitPython==3.1.31 -gluonts==0.12.8 -gmpy2 @ file:///home/conda/feedstock_root/build_artifacts/gmpy2_1666808679441/work -h11==0.14.0 -html5lib @ file:///home/conda/feedstock_root/build_artifacts/html5lib_1592930327044/work -huggingface-hub==0.14.1 -idna==2.8 -importlib-metadata @ file:///home/conda/feedstock_root/build_artifacts/importlib-metadata_1682176699712/work -importlib-resources==5.12.0 -inflate64==0.3.1 -iniconfig==2.0.0 -inquirer==3.1.3 -ipykernel @ file:///home/conda/feedstock_root/build_artifacts/ipykernel_1679336319192/work -ipython==8.4.0 -isort==5.8.0 -itsdangerous==2.1.2 -jaraco.classes @ file:///home/conda/feedstock_root/build_artifacts/jaraco.classes_1667024629799/work -jedi==0.18.1 -jeepney @ file:///home/conda/feedstock_root/build_artifacts/jeepney_1649085214306/work -Jinja2 @ file:///home/conda/feedstock_root/build_artifacts/jinja2_1654302431367/work -joblib==1.2.0 -jsonschema==4.17.3 -jupyter-client==7.3.4 -jupyter-core==4.10.0 -keyring @ file:///home/conda/feedstock_root/build_artifacts/keyring_1671728250508/work -kiwisolver==1.4.4 -lightning==2.0.2 -lightning-cloud==0.5.34 -lightning-utilities==0.8.0 -lockfile==0.12.2 -markdown-it-py==2.2.0 -MarkupSafe @ file:///home/conda/feedstock_root/build_artifacts/markupsafe_1674135804847/work -matplotlib==3.7.1 -matplotlib-inline==0.1.3 -mccabe==0.7.0 -mdurl==0.1.2 -more-itertools @ file:///home/conda/feedstock_root/build_artifacts/more-itertools_1677514956219/work -mpmath @ file:///home/conda/feedstock_root/build_artifacts/mpmath_1678228039184/work -msgpack @ file:///home/conda/feedstock_root/build_artifacts/msgpack-python_1678312698172/work -multidict==6.0.4 -multiprocess==0.70.14 -multivolumefile==0.2.3 -mypy-extensions==0.4.3 -nest-asyncio==1.5.5 -networkx @ file:///home/conda/feedstock_root/build_artifacts/networkx_1680692919326/work -numpy @ file:///home/conda/feedstock_root/build_artifacts/numpy_1682210190296/work -ordered-set==4.1.0 -packaging @ file:///home/conda/feedstock_root/build_artifacts/packaging_1681337016113/work -pandas==2.0.1 -parso==0.8.3 -pastel @ file:///home/conda/feedstock_root/build_artifacts/pastel_1640899049124/work -pathspec==0.9.0 -peft==0.3.0 -pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1667297516076/work -pickleshare==0.7.5 -Pillow @ file:///home/conda/feedstock_root/build_artifacts/pillow_1675487166627/work -pkginfo @ file:///home/conda/feedstock_root/build_artifacts/pkginfo_1673281726124/work -platformdirs==2.5.1 -plotly==5.14.1 -pluggy==1.0.0 -prompt-toolkit==3.0.29 -promptsource==0.2.3 -protobuf==4.22.4 -psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1681775019467/work -ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1609419310487/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl -pure-eval==0.2.2 -py7zr==0.20.5 -pyarrow==12.0.0 -pybcj==1.0.1 -pycodestyle==2.10.0 -pycparser @ file:///home/conda/feedstock_root/build_artifacts/pycparser_1636257122734/work -pycryptodomex==3.17 -pydantic @ file:///home/conda/feedstock_root/build_artifacts/pydantic_1679565256950/work -pydeck==0.8.1b0 -pydocstyle==6.1.1 -pyflakes==3.0.1 -Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1681904169130/work -PyJWT==2.6.0 -pylama==8.3.8 -pylev @ file:///home/conda/feedstock_root/build_artifacts/pylev_1641226376343/work -pyOpenSSL @ file:///home/conda/feedstock_root/build_artifacts/pyopenssl_1680037383858/work -pyparsing==3.0.9 -pyppmd==1.0.0 -pyrsistent==0.19.3 -PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1661604839144/work -pytest==7.3.1 -python-dateutil==2.8.2 -python-editor==1.0.4 -python-multipart==0.0.6 -pytz==2023.3 -pytz-deprecation-shim==0.1.0.post0 -PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1666772387118/work -pyzmq==23.2.0 -pyzstd==0.15.7 -readchar==4.0.5 -regex==2023.5.5 -requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1682535435083/work -responses==0.18.0 -rich==13.3.5 -ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1683392662082/work -ruamel.yaml.clib @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml.clib_1670412733608/work -scikit-learn==1.2.2 -scipy==1.10.1 -SecretStorage @ file:///home/conda/feedstock_root/build_artifacts/secretstorage_1666848732124/work -sentencepiece==0.1.99 -six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work -sklearn==0.0.post4 -smmap==5.0.0 -sniffio==1.3.0 -snowballstemmer==2.2.0 -soupsieve==2.4.1 -stack-data==0.3.0 -starlette==0.22.0 -starsessions==1.3.0 -streamlit==0.82.0 -sympy @ file:///home/conda/feedstock_root/build_artifacts/sympy_1679342590084/work -tenacity==8.2.2 -texttable==1.6.7 -threadpoolctl==3.1.0 -tokenizers==0.13.3 -toml==0.10.2 -tomli @ file:///home/conda/feedstock_root/build_artifacts/tomli_1644342247877/work -tomlkit @ file:///home/conda/feedstock_root/build_artifacts/tomlkit_1683028165096/work -toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1657485559105/work -torch==2.0.0 -torchaudio==2.0.0 -torchmetrics==0.11.4 -torchvision==0.15.0 -tornado==6.1 -tqdm==4.65.0 -traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1675110562325/work -transformers==4.28.1 -triton==2.0.0 -typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/typing_extensions_1678559861143/work -tzdata==2023.3 -tzlocal==4.3 -urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1678635778344/work -uvicorn==0.22.0 -validators==0.20.0 -virtualenv @ file:///home/conda/feedstock_root/build_artifacts/virtualenv_1682674264573/work -watchdog==3.0.0 -wcwidth==0.2.5 -webencodings==0.5.1 -websocket-client==1.5.1 -websockets==11.0.2 -xxhash==3.2.0 -yapf==0.33.0 -yarl==1.9.2 -zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1677313463193/work diff --git a/requirements/environment.max.yaml b/requirements/environment.max.yaml index c3f67c9..2b31e61 100644 --- a/requirements/environment.max.yaml +++ b/requirements/environment.max.yaml @@ -1,32 +1,19 @@ name: dlk2 channels: + - pytorch - nvidia - conda-forge - - pytorch dependencies: - _libgcc_mutex=0.1=conda_forge - _openmp_mutex=4.5=2_kmp_llvm - - appdirs=1.4.4=pyh9f0ad1d_0 - - asttokens=2.2.1=pyhd8ed1ab_0 - - backcall=0.2.0=pyh9f0ad1d_0 - - backports=1.0=pyhd8ed1ab_3 - - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 - blas=2.116=mkl - blas-devel=3.9.0=16_linux64_mkl - brotlipy=0.7.0=py39hb9d737c_1005 - bzip2=1.0.8=h7f98852_4 - - ca-certificates=2022.12.7=ha878542_0 - - cachecontrol=0.12.11=pyhd8ed1ab_1 - - cachecontrol-with-filecache=0.12.11=pyhd8ed1ab_1 - - cachy=0.3.0=pyhd8ed1ab_1 + - ca-certificates=2023.5.7=hbcca054_0 + - certifi=2023.5.7=pyhd8ed1ab_0 - cffi=1.15.1=py39he91dace_3 - charset-normalizer=3.1.0=pyhd8ed1ab_0 - - click=8.1.3=unix_pyhd8ed1ab_2 - - click-default-group=1.2.2=pyhd8ed1ab_1 - - clikit=0.6.2=pyhd8ed1ab_2 - - comm=0.1.3=pyhd8ed1ab_0 - - conda-lock=1.4.0=pyhd8ed1ab_2 - - crashtest=0.4.1=pyhd8ed1ab_0 - cryptography=40.0.2=py39h079d5ae_0 - cuda-cudart=11.7.99=0 - cuda-cupti=11.7.101=0 @@ -36,35 +23,16 @@ dependencies: - cuda-runtime=11.7.1=0 - cudatoolkit=11.7.0=hd8887f6_10 - cudatoolkit-dev=11.7.0=h1de0b5d_6 - - dbus=1.13.6=h5008d03_3 - - debugpy=1.6.7=py39h227be39_0 - - decorator=5.1.1=pyhd8ed1ab_0 - - distlib=0.3.6=pyhd8ed1ab_0 - - ensureconda=1.4.3=pyhd8ed1ab_0 - - executing=1.2.0=pyhd8ed1ab_0 - - expat=2.5.0=hcb278e6_1 - ffmpeg=4.3=hf484d3e_0 - filelock=3.12.0=pyhd8ed1ab_0 - freetype=2.12.1=hca18f0e_1 - - gettext=0.21.1=h27087fc_0 - gmp=6.2.1=h58526e2_0 - gmpy2=2.1.2=py39h376b7d2_1 - gnutls=3.6.13=h85f3911_1 - - html5lib=1.1=pyh9f0ad1d_0 - icu=72.1=hcb278e6_0 - idna=3.4=pyhd8ed1ab_0 - - importlib-metadata=6.6.0=pyha770c72_0 - - importlib_metadata=6.6.0=hd8ed1ab_0 - - ipykernel=6.22.0=pyh210e3f2_0 - - ipython=8.13.2=pyh41d4057_0 - - jaraco.classes=3.2.3=pyhd8ed1ab_0 - - jedi=0.18.2=pyhd8ed1ab_0 - - jeepney=0.8.0=pyhd8ed1ab_0 - jinja2=3.1.2=pyhd8ed1ab_1 - jpeg=9e=h0b41bf4_3 - - jupyter_client=8.2.0=pyhd8ed1ab_0 - - jupyter_core=5.3.0=py39hf3d152e_0 - - keyring=23.13.1=py39hf3d152e_0 - lame=3.100=h166bdaf_1003 - lcms2=2.15=hfd0df8a_0 - ld_impl_linux-64=2.40=h41732ed_0 @@ -78,12 +46,10 @@ dependencies: - libcusolver=11.4.0.1=0 - libcusparse=11.7.4.91=0 - libdeflate=1.17=h0b41bf4_0 - - libexpat=2.5.0=hcb278e6_1 - libffi=3.4.2=h7f98852_5 - libgcc-ng=12.2.0=h65d4601_19 - libgfortran-ng=12.2.0=h69a702a_19 - libgfortran5=12.2.0=h337968e_19 - - libglib=2.76.2=hebfc3b9_0 - libgomp=12.2.0=h65d4601_19 - libhwloc=2.9.1=hd6dc26d_0 - libiconv=1.17=h166bdaf_0 @@ -93,8 +59,7 @@ dependencies: - libnsl=2.0.0=h7f98852_0 - libnvjpeg=11.8.0.2=0 - libpng=1.6.39=h753d276_0 - - libsodium=1.0.18=h36c2ea0_1 - - libsqlite=3.40.0=h753d276_1 + - libsqlite=3.42.0=h2797004_0 - libstdcxx-ng=12.2.0=h46fd767_19 - libtiff=4.5.0=h6adf6a1_2 - libuuid=2.38.1=h0b41bf4_0 @@ -102,199 +67,189 @@ dependencies: - libxcb=1.13=h7f98852_1004 - libxml2=2.10.4=hfdac1af_0 - libzlib=1.2.13=h166bdaf_4 - - llvm-openmp=16.0.3=h4dfa4b3_0 - - lockfile=0.12.2=py_1 + - llvm-openmp=16.0.4=h4dfa4b3_0 - markupsafe=2.1.2=py39h72bdee0_0 - - matplotlib-inline=0.1.6=pyhd8ed1ab_0 - mkl=2022.1.0=h84fe81f_915 - mkl-devel=2022.1.0=ha770c72_916 - mkl-include=2022.1.0=h84fe81f_915 - - more-itertools=9.1.0=pyhd8ed1ab_0 - mpc=1.3.1=hfe3b2da_0 - mpfr=4.2.0=hb012696_0 - mpmath=1.3.0=pyhd8ed1ab_0 - - msgpack-python=1.0.5=py39h4b4f3f3_0 - ncurses=6.3=h27087fc_1 - - nest-asyncio=1.5.6=pyhd8ed1ab_0 - nettle=3.6=he412f7d_0 - networkx=3.1=pyhd8ed1ab_0 - numpy=1.24.3=py39h6183b62_0 - openh264=2.1.1=h780b84a_0 - openjpeg=2.5.0=hfec8fc6_2 - openssl=3.1.0=hd590300_3 - - packaging=23.1=pyhd8ed1ab_0 - - parso=0.8.3=pyhd8ed1ab_0 - - pastel=0.2.1=pyhd8ed1ab_0 - - pcre2=10.40=hc3806b6_0 - - pexpect=4.8.0=pyh1a96a4e_2 - - pickleshare=0.7.5=py_1003 + - pillow=9.4.0=py39h2320bf1_1 - pip=23.1.2=pyhd8ed1ab_0 - - pkginfo=1.9.6=pyhd8ed1ab_0 - - platformdirs=3.5.0=pyhd8ed1ab_0 - - prompt-toolkit=3.0.38=pyha770c72_0 - - prompt_toolkit=3.0.38=hd8ed1ab_0 - - psutil=5.9.5=py39h72bdee0_0 - pthread-stubs=0.4=h36c2ea0_1001 - - ptyprocess=0.7.0=pyhd3deb0d_0 - - pure_eval=0.2.2=pyhd8ed1ab_0 - pycparser=2.21=pyhd8ed1ab_0 - - pydantic=1.10.7=py39h72bdee0_0 - - pygments=2.15.1=pyhd8ed1ab_0 - - pylev=1.4.0=pyhd8ed1ab_0 - pyopenssl=23.1.1=pyhd8ed1ab_0 - pysocks=1.7.1=pyha2e5f31_6 - python=3.9.16=h2782a2a_0_cpython - - python-dateutil=2.8.2=pyhd8ed1ab_0 - python_abi=3.9=3_cp39 - - pytorch=2.0.0=py3.9_cuda11.7_cudnn8.5.0_0 - - pytorch-cuda=11.7=h778d358_3 + - pytorch=2.0.1=py3.9_cuda11.7_cudnn8.5.0_0 + - pytorch-cuda=11.7=h778d358_5 - pytorch-mutex=1.0=cuda - - pyyaml=6.0=py39hb9d737c_5 - - pyzmq=25.0.2=py39h0be026e_0 - readline=8.2=h8228510_1 - - ruamel.yaml=0.17.24=py39hd1e30aa_0 - - ruamel.yaml.clib=0.2.7=py39h72bdee0_1 - - secretstorage=3.3.3=py39hf3d152e_1 + - requests=2.29.0=pyhd8ed1ab_0 - setuptools=67.7.2=pyhd8ed1ab_0 - - six=1.16.0=pyh6c4a22f_0 - - stack_data=0.6.2=pyhd8ed1ab_0 - - sympy=1.11.1=pypyh9d50eac_103 + - sympy=1.12=pypyh9d50eac_103 - tbb=2021.9.0=hf52228f_0 - tk=8.6.12=h27826a3_0 - - tomli=2.0.1=pyhd8ed1ab_0 - - tomlkit=0.11.8=pyha770c72_0 - - toolz=0.12.0=pyhd8ed1ab_0 - - torchaudio=2.0.0=py39_cu117 + - torchaudio=2.0.2=py39_cu117 - torchtriton=2.0.0=py39 - - torchvision=0.15.0=py39_cu117 - - tornado=6.3=py39h72bdee0_0 - - traitlets=5.9.0=pyhd8ed1ab_0 - - typing-extensions=4.5.0=hd8ed1ab_0 + - torchvision=0.15.2=py39_cu117 - typing_extensions=4.5.0=pyha770c72_0 - - virtualenv=20.23.0=pyhd8ed1ab_0 - - wcwidth=0.2.6=pyhd8ed1ab_0 - - webencodings=0.5.1=py_1 + - urllib3=1.26.15=pyhd8ed1ab_0 - wheel=0.40.0=pyhd8ed1ab_0 - xorg-libxau=1.0.9=h7f98852_0 - xorg-libxdmcp=1.1.3=h7f98852_0 - xz=5.2.6=h166bdaf_0 - - yaml=0.2.5=h7f98852_2 - - zeromq=4.3.4=h9c3ff4c_1 - - zipp=3.15.0=pyhd8ed1ab_0 - zlib=1.2.13=h166bdaf_4 - zstd=1.5.2=h3eb15da_6 - pip: - - accelerate==0.18.0 + - accelerate==0.19.0 - aiohttp==3.8.4 - aiosignal==1.3.1 - - altair==4.2.2 + - altair==5.0.0 - anyio==3.6.2 - arrow==1.2.3 - astor==0.8.1 + - asttokens==2.2.1 - async-timeout==4.0.2 - attrs==23.1.0 + - backcall==0.2.0 - base58==2.1.1 - beautifulsoup4==4.12.2 - bitsandbytes==0.38.1 - - black==23.3.0 + - black==21.12b0 - blessed==1.20.0 - blinker==1.6.2 - brotli==1.0.9 - cachetools==5.3.0 - - certifi==2022.12.7 - - contourpy==1.0.7 + - click==7.1.2 + - comm==0.1.3 - croniter==1.3.14 - - cycler==0.11.0 - datasets==2.12.0 - dateutils==0.6.12 + - debugpy==1.6.7 + - decorator==5.1.1 - deepdiff==6.3.0 - dill==0.3.6 - exceptiongroup==1.1.1 + - executing==1.2.0 - fastapi==0.88.0 - flake8==6.0.0 - - fonttools==4.39.4 - frozenlist==1.3.3 - - fsspec==2023.4.0 + - fsspec==2023.5.0 - gitdb==4.0.10 - gitpython==3.1.31 - - gluonts==0.12.8 - h11==0.14.0 - huggingface-hub==0.14.1 - - importlib-resources==5.12.0 + - importlib-metadata==6.6.0 - inflate64==0.3.1 - iniconfig==2.0.0 - inquirer==3.1.3 + - ipykernel==6.23.1 + - ipython==8.13.2 - isort==5.8.0 - itsdangerous==2.1.2 + - jedi==0.18.2 - joblib==1.2.0 - jsonschema==4.17.3 - - kiwisolver==1.4.4 + - jupyter-client==8.2.0 + - jupyter-core==5.3.0 - lightning==2.0.2 - - lightning-cloud==0.5.34 + - lightning-cloud==0.5.36 - lightning-utilities==0.8.0 - markdown-it-py==2.2.0 - - matplotlib==3.7.1 + - matplotlib-inline==0.1.6 + - mccabe==0.7.0 - mdurl==0.1.2 - multidict==6.0.4 - multiprocess==0.70.14 - multivolumefile==0.2.3 + - mypy-extensions==1.0.0 + - nest-asyncio==1.5.6 - ordered-set==4.1.0 + - packaging==23.1 - pandas==2.0.1 - - peft==0.3.0 - - pillow==9.5.0 + - parso==0.8.3 + - pathspec==0.11.1 + - peft==0.3.0.dev0 + - pexpect==4.8.0 + - pickleshare==0.7.5 + - platformdirs==3.5.1 - plotly==5.14.1 - pluggy==1.0.0 + - prompt-toolkit==3.0.38 - promptsource==0.2.3 - - protobuf==4.22.4 + - protobuf==4.23.1 + - psutil==5.9.5 + - ptyprocess==0.7.0 + - pure-eval==0.2.2 - py7zr==0.20.5 - pyarrow==12.0.0 - pybcj==1.0.1 - pycodestyle==2.10.0 - - pycryptodomex==3.17 + - pycryptodomex==3.18.0 + - pydantic==1.10.7 - pydeck==0.8.1b0 - pyflakes==3.0.1 - - pyjwt==2.6.0 - - pyparsing==3.0.9 + - pygments==2.15.1 + - pyjwt==2.7.0 - pyppmd==1.0.0 - pyrsistent==0.19.3 - pytest==7.3.1 + - python-dateutil==2.8.2 - python-editor==1.0.4 - python-multipart==0.0.6 + - pytorch-lightning==2.0.2 - pytz==2023.3 - - pytz-deprecation-shim==0.1.0.post0 + - pyyaml==6.0 + - pyzmq==25.0.2 - pyzstd==0.15.7 - readchar==4.0.5 - regex==2023.5.5 - - requests==2.30.0 - responses==0.18.0 - rich==13.3.5 - scikit-learn==1.2.2 - scipy==1.10.1 - sentencepiece==0.1.99 - - sklearn==0.0.post4 + - six==1.16.0 + - sklearn==0.0.post5 - smmap==5.0.0 - sniffio==1.3.0 - soupsieve==2.4.1 + - stack-data==0.6.2 - starlette==0.22.0 - starsessions==1.3.0 - streamlit==0.82.0 - tenacity==8.2.2 - texttable==1.6.7 - threadpoolctl==3.1.0 + - tokenize-rt==5.0.0 - tokenizers==0.13.3 - toml==0.10.2 + - tomli==1.2.3 + - toolz==0.12.0 - torchmetrics==0.11.4 + - tornado==6.3.2 - tqdm==4.65.0 - - transformers==4.28.1 + - traitlets==5.9.0 + - transformers==4.29.2 - tzdata==2023.3 - - tzlocal==4.3 - - urllib3==2.0.2 + - tzlocal==5.0.1 - uvicorn==0.22.0 - validators==0.20.0 - watchdog==3.0.0 + - wcwidth==0.2.6 - websocket-client==1.5.1 - - websockets==11.0.2 + - websockets==11.0.3 - xxhash==3.2.0 - - yapf==0.33.0 - yarl==1.9.2 -prefix: /home/wassname/miniforge3/envs/dlk2 + - zipp==3.15.0 +prefix: /home/ubuntu/mambaforge/envs/dlk2 diff --git a/requirements/environment.min.yaml b/requirements/environment.min.yaml index f6692da..f21761a 100644 --- a/requirements/environment.min.yaml +++ b/requirements/environment.min.yaml @@ -1,10 +1,8 @@ name: dlk2 channels: - conda-forge - - pytorch dependencies: - python=3.9 - - ipykernel - pytorch - torchvision - torchaudio @@ -13,6 +11,4 @@ dependencies: - cudatoolkit=11.7 - ca-certificates - openssl - - conda-lock - - certifi -prefix: /home/wassname/miniforge3/envs/dlk2 +prefix: /home/ubuntu/mambaforge/envs/dlk2 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index d09f7c8..226fc88 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,11 +1,15 @@ datasets promptsource tqdm -transformers +transformers>=4.29.0 sklearn scikit-learn accelerate bitsandbytes lightning -peft sentencepiece +# peft +# use the version that https://github.com/johnsmith0031/alpaca_lora_4bit/blob/main/requirements.txt uses since they always resolve the dependancy issues +git+https://github.com/huggingface/peft.git@70af02a2bca5a63921790036b2c9430edf4037e2 +# due to a bug we have to downgrade to this one for now https://twitter.com/Teknium1/status/1660003439752138752 +bitsandbytes==0.37.2