From 4253cb53b23225da864e3f211d251f6fc6646b63 Mon Sep 17 00:00:00 2001 From: deep1 <> Date: Sun, 2 Jul 2023 14:39:58 +0800 Subject: [PATCH] fix cache --- notebooks/017_mjc_sup_mcdrop_dm.ipynb | 1815 +++++++++++++++++++++++-- 1 file changed, 1720 insertions(+), 95 deletions(-) diff --git a/notebooks/017_mjc_sup_mcdrop_dm.ipynb b/notebooks/017_mjc_sup_mcdrop_dm.ipynb index 5febc79..5a36b9c 100644 --- a/notebooks/017_mjc_sup_mcdrop_dm.ipynb +++ b/notebooks/017_mjc_sup_mcdrop_dm.ipynb @@ -26,9 +26,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'4.30.1'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\n", "import copy\n", @@ -104,18 +115,95 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please run\n", + "\n", + "python -m bitsandbytes\n", + "\n", + " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "================================================================================\n", + "bin /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n", + "CUDA SETUP: CUDA runtime path found: /home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0'), PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n", + "Either way, this might cause trouble in the future:\n", + "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", + " warn(msg)\n" + ] + } + ], "source": [ "from peft import PeftModel" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPTBigCodeConfig {\n", + " \"_name_or_path\": \"WizardLM/WizardCoder-15B-V1.0\",\n", + " \"activation_function\": \"gelu\",\n", + " \"architectures\": [\n", + " \"GPTBigCodeForCausalLM\"\n", + " ],\n", + " \"attention_softmax_in_fp32\": true,\n", + " \"attn_pdrop\": 0.1,\n", + " \"bos_token_id\": 0,\n", + " \"embd_pdrop\": 0.1,\n", + " \"eos_token_id\": 0,\n", + " \"inference_runner\": 0,\n", + " \"initializer_range\": 0.02,\n", + " \"layer_norm_epsilon\": 1e-05,\n", + " \"max_batch_size\": null,\n", + " \"max_sequence_length\": null,\n", + " \"model_type\": \"gpt_bigcode\",\n", + " \"multi_query\": true,\n", + " \"n_embd\": 6144,\n", + " \"n_head\": 48,\n", + " \"n_inner\": 24576,\n", + " \"n_layer\": 40,\n", + " \"n_positions\": 8192,\n", + " \"pad_key_length\": true,\n", + " \"pre_allocate_kv_cache\": false,\n", + " \"resid_pdrop\": 0.1,\n", + " \"scale_attention_softmax_in_fp32\": true,\n", + " \"scale_attn_weights\": true,\n", + " \"summary_activation\": null,\n", + " \"summary_first_dropout\": 0.1,\n", + " \"summary_proj_to_labels\": true,\n", + " \"summary_type\": \"cls_index\",\n", + " \"summary_use_proj\": true,\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.30.1\",\n", + " \"use_cache\": true,\n", + " \"validate_runner_input\": true,\n", + " \"vocab_size\": 49153\n", + "}\n", + "\n" + ] + } + ], "source": [ "# leaderboard https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard\n", "model_options = dict(\n", @@ -175,18 +263,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "GPTBigCodeForCausalLM(\n", + " (transformer): GPTBigCodeModel(\n", + " (wte): Embedding(49153, 6144)\n", + " (wpe): Embedding(8192, 6144)\n", + " (drop): Dropout(p=0.1, inplace=False)\n", + " (h): ModuleList(\n", + " (0-39): 40 x GPTBigCodeBlock(\n", + " (ln_1): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\n", + " (attn): GPTBigCodeAttention(\n", + " (c_attn): Linear4bit(in_features=6144, out_features=6400, bias=True)\n", + " (c_proj): Linear4bit(in_features=6144, out_features=6144, bias=True)\n", + " (attn_dropout): Dropout(p=0.1, inplace=False)\n", + " (resid_dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ln_2): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): GPTBigCodeMLP(\n", + " (c_fc): Linear4bit(in_features=6144, out_features=24576, bias=True)\n", + " (c_proj): Linear4bit(in_features=24576, out_features=6144, bias=True)\n", + " (act): GELUActivation()\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (ln_f): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (lm_head): Linear(in_features=6144, out_features=49153, bias=False)\n", + ")" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "model" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "49152\n" + ] + } + ], "source": [ "# https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/falcon.py\n", "print(tokenizer.pad_token_id)\n", @@ -196,6 +329,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -204,12 +338,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "40\n" + ] + }, + { + "data": { + "text/plain": [ + "((2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38), 40)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Params\n", - "N_SAMPLES = 9000\n", + "N_SAMPLES = 4000\n", "BATCH_SIZE = 8 # None # None means auto # 6 gives 16Gb/25GB. where 10GB is the base model. so 6 is 6/15\n", "N_SHOTS = 3\n", "USE_MCDROPOUT = True\n", @@ -227,29 +379,26 @@ " num_layers = 10\n", " \n", "stride = 2\n", - "extract_layers = tuple(range(2, num_layers, stride)) + (num_layers,)\n", + "extract_layers = tuple(range(2, num_layers-2, stride)) + (num_layers-2,)\n", "extract_layers, num_layers" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(15272, 18502)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# get the tokens for 0 and 1, we will use these later...\n", "# note that sentancepeice tokenizers have differen't tokens for No and \\nNo.\n", @@ -278,9 +427,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bcea2adb209640099d98f92a6784672f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/2 [00:000.001, \"the hidden state pairs should be different but are not. Check model.config.use_cache==False, check this model has dropout in it's arch\"\n", + " eps=1e-5\n", + " mpe = lambda x,y: np.mean(np.abs(x-y)/(np.abs(x)+np.abs(y)+eps))\n", + " a,b=hs2['hidden_states'],hs1['hidden_states']\n", + " assert mpe(a,b)>eps, \"the hidden state pairs should be different but are not. Check model.config.use_cache==False, check this model has dropout in it's arch\"\n", "\n", " # collect\n", " b = len(texts)\n", " res.append([\n", - " hs1['hidden_states'].reshape((b,-1)),\n", + " hs1['hidden_states'],#.reshape((b,-1)),\n", " hs1[\"ans\"], \n", - " hs2['hidden_states'].reshape((b,-1)),\n", + " hs2['hidden_states'],#.reshape((b,-1)),\n", " hs2[\"ans\"],\n", " true_labels,\n", " ])\n", @@ -892,7 +1203,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 106, "metadata": {}, "outputs": [], "source": [ @@ -984,6 +1295,238 @@ "execution_count": null, "metadata": {}, "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "\u001b[32m2023-07-02 11:48:36.624\u001b[0m | \u001b[34m\u001b[1mDEBUG \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mwrap\u001b[0m:\u001b[36m18\u001b[0m - \u001b[34m\u001b[1mkwargs ['GPTBigCodeForCausalLM(\\n (transformer): GPTBigCodeModel(\\n (wte): Embedding(49153, 6144)\\n (wpe): Embedding(8192, 6144)\\n (drop): Dropout(p=0.1, inplace=False)\\n (h): ModuleList(\\n (0-39): 40 x GPTBigCodeBlock(\\n (ln_1): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n (attn): GPTBigCodeAttention(\\n (c_attn): Linear4bit(in_features=6144, out_features=6400, bias=True)\\n (c_proj): Linear4bit(in_features=6144, out_features=6144, bias=True)\\n (attn_dropout): Dropout(p=0.1, inplace=False)\\n (resid_dropout): Dropout(p=0.1, inplace=False)\\n )\\n (ln_2): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n (mlp): GPTBigCodeMLP(\\n (c_fc): Linear4bit(in_features=6144, out_features=24576, bias=True)\\n (c_proj): Linear4bit(in_features=24576, out_features=6144, bias=True)\\n (act): GELUActivation()\\n (dropout): Dropout(p=0.1, inplace=False)\\n )\\n )\\n )\\n (ln_f): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n )\\n (lm_head): Linear(in_features=6144, out_features=49153, bias=False)\\n)', \"GPT2TokenizerFast(name_or_path='WizardLM/WizardCoder-15B-V1.0', vocab_size=49152, model_max_length=2048, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<|endoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>', 'pad_token': '[PAD]', 'additional_special_tokens': ['<|endoftext|>', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '']}, clean_up_tokenization_spaces=True)\", \"Dataset({\\n features: ['label', 'title', 'content'],\\n num_rows: 400000\\n})\", 'format_imdbs_multishot', 4000, 'The classic puzzle of the two guards. It presents a scenario where two guards are involved in a conflict of truths and lies. Here\\'s the setup:\\n\\nThere are two guards standing at two separate doors. One door leads to safety, while the other leads to danger. One guard always tells the truth, and the other guard always lies. You don\\'t know which guard is which or which door leads to safety.\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"not great\". Content: \"Does not allow you to play the radio at night without having to reset the alarm button for it to work in the morning. Controls for settings are tiny switches in hard to get to places.\"\\n\\n### Left Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Exactly What I Needed\". Content: \"I have several of the Pyrex Totables and love them. I have worn out all of my combo packs so I was thrilled to find these. They arrived in no time at all and are the answer to my needs.\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Great Blender!\". Content: \"I thought I wanted a VitaMix, but I read the reviews for this Blendtec and the Vitamix and decided to get this one. It certainly lives up to its claims. The only recipe I haven\\'t liked so far was the bread. The dough seemed too thin. I love the choice of buttons to run the machine and it stops when it is finished. The smoothies are fine enough to give to my mom who has diverticulitis So far...I love it!\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"not great\". Content: \"Does not allow you to play the radio at night without having to reset the alarm button for it to work in the morning. Controls for settings are tiny switches in hard to get to places.\"\\n\\n### Left Guard Response:\\n', 'The classic puzzle of the two guards. It presents a scenario where two guards are involved in a conflict of truths and lies. Here\\'s the setup:\\n\\nThere are two guards standing at two separate doors. One door leads to safety, while the other leads to danger. One guard always tells the truth, and the other guard always lies. You don\\'t know which guard is which or which door leads to safety.\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"not great\". Content: \"Does not allow you to play the radio at night without having to reset the alarm button for it to work in the morning. Controls for settings are tiny switches in hard to get to places.\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Exactly What I Needed\". Content: \"I have several of the Pyrex Totables and love them. I have worn out all of my combo packs so I was thrilled to find these. They arrived in no time at all and are the answer to my needs.\"\\n\\n### Left Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Great Blender!\". Content: \"I thought I wanted a VitaMix, but I read the reviews for this Blendtec and the Vitamix and decided to get this one. It certainly lives up to its claims. The only recipe I haven\\'t liked so far was the bread. The dough seemed too thin. I love the choice of buttons to run the machine and it stops when it is finished. The smoothies are fine enough to give to my mom who has diverticulitis So far...I love it!\"\\n\\n### Left Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"not great\". Content: \"Does not allow you to play the radio at night without having to reset the alarm button for it to work in the morning. Controls for settings are tiny switches in hard to get to places.\"\\n\\n### Left Guard Response:\\n']\u001b[0m\n", + "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0915d512a3f04b958903ca0ae8459bd6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "get hidden states: 0%| | 0/500 [00:00╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n", + "│ in <module>:3 │\n", + "│ │\n", + "│ 1 # test and cache │\n", + "│ 2 dm = imdbHSDataModule(model, tokenizer, batch_size=BATCH_SIZE, n=N_SAMPLES) │\n", + "│ ❱ 3 dm.setup('train') │\n", + "│ 4 │\n", + "│ 5 dl_val = dm.val_dataloader() │\n", + "│ 6 dl_train = dm.train_dataloader() │\n", + "│ │\n", + "│ in setup:31 │\n", + "│ │\n", + "│ 28 │ │ self.dataset = load_dataset(h.dataset_name, split=\"test\") │\n", + "│ 29 │ │ │\n", + "│ 30 │ │ # in ELK they cache as a huggingface dataset │\n", + "│ ❱ 31 │ │ self.hs1, self.ans1, self.hs2, self.ans2, self.y, self.infos = batch_hidden_stat │\n", + "│ 32 │ │ │ self.model, self.tokenizer, self.dataset, self.prompt_fn, n=h.n, batch_size= │\n", + "│ 33 │ │ │\n", + "│ 34 │ │ # let's create a simple 50/50 train split (the data is already randomized) │\n", + "│ │\n", + "│ in wrap:30 │\n", + "│ │\n", + "│ 27 │ │ │ logger.info(f\"loading hs from {f}\") │\n", + "│ 28 │ │ │ res = pickle.load(f.open('rb')) │\n", + "│ 29 │ │ else: │\n", + "│ ❱ 30 │ │ │ res = func(model, tokenizer, data, prompt_fn, n, batch_size) │\n", + "│ 31 │ │ │ logger.info(f\"caching hs to {f}\") │\n", + "│ 32 │ │ │ pickle.dump(res, f.open('wb')) │\n", + "│ 33 │ │ return res │\n", + "│ │\n", + "│ in batch_hidden_states:28 │\n", + "│ │\n", + "│ 25 │ │ │\n", + "│ 26 │ │ # different due to dropout │\n", + "│ 27 │ │ # set_seeds(i*10) │\n", + "│ ❱ 28 │ │ hs1 = get_hidden_states(model, tokenizer, q) │\n", + "│ 29 │ │ # set_seeds(i*10+1) │\n", + "│ 30 │ │ hs2 = get_hidden_states(model, tokenizer, q) │\n", + "│ 31 │ │ if i==0: │\n", + "│ │\n", + "│ in get_hidden_states:53 │\n", + "│ │\n", + "│ 50 │ │ │\n", + "│ 51 │ │ hidden_states = hidden_states[:, :, last_token] # (batch, layers, past_seq, logi │\n", + "│ 52 │ │ │\n", + "│ ❱ 53 │ │ text_q = tokenizer.batch_decode(input_ids) │\n", + "│ 54 │ │ │\n", + "│ 55 │ │ s = outputs['sequences'] │\n", + "│ 56 │ │ s = [s[i][len(input_ids[i]):] for i in range(len(s))] │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/tokenization_utils_ba │\n", + "│ se.py:3469 in batch_decode │\n", + "│ │\n", + "│ 3466 │ │ Returns: │\n", + "│ 3467 │ │ │ `List[str]`: The list of decoded sentences. │\n", + "│ 3468 │ │ \"\"\" │\n", + "│ ❱ 3469 │ │ return [ │\n", + "│ 3470 │ │ │ self.decode( │\n", + "│ 3471 │ │ │ │ seq, │\n", + "│ 3472 │ │ │ │ skip_special_tokens=skip_special_tokens, │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/tokenization_utils_ba │\n", + "│ se.py:3470 in <listcomp> │\n", + "│ │\n", + "│ 3467 │ │ │ `List[str]`: The list of decoded sentences. │\n", + "│ 3468 │ │ \"\"\" │\n", + "│ 3469 │ │ return [ │\n", + "│ ❱ 3470 │ │ │ self.decode( │\n", + "│ 3471 │ │ │ │ seq, │\n", + "│ 3472 │ │ │ │ skip_special_tokens=skip_special_tokens, │\n", + "│ 3473 │ │ │ │ clean_up_tokenization_spaces=clean_up_tokenization_spaces, │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/tokenization_utils_ba │\n", + "│ se.py:3507 in decode │\n", + "│ │\n", + "│ 3504 │ │ │ `str`: The decoded sentence. │\n", + "│ 3505 │ │ \"\"\" │\n", + "│ 3506 │ │ # Convert inputs to python lists │\n", + "│ ❱ 3507 │ │ token_ids = to_py_obj(token_ids) │\n", + "│ 3508 │ │ │\n", + "│ 3509 │ │ return self._decode( │\n", + "│ 3510 │ │ │ token_ids=token_ids, │\n", + "│ │\n", + "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/utils/generic.py:210 │\n", + "│ in to_py_obj │\n", + "│ │\n", + "│ 207 │ elif is_tf_tensor(obj): │\n", + "│ 208 │ │ return obj.numpy().tolist() │\n", + "│ 209 │ elif is_torch_tensor(obj): │\n", + "│ ❱ 210 │ │ return obj.detach().cpu().tolist() │\n", + "│ 211 │ elif is_jax_tensor(obj): │\n", + "│ 212 │ │ return np.asarray(obj).tolist() │\n", + "│ 213 │ elif isinstance(obj, (np.ndarray, np.number)): # tolist also works on 0d np arrays │\n", + "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n", + "KeyboardInterrupt\n", + "\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 1 \u001b[0m\u001b[2m# test and cache\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0mdm = imdbHSDataModule(model, tokenizer, batch_size=BATCH_SIZE, n=N_SAMPLES) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 3 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0mdl_val = dm.val_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0mdl_train = dm.train_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m31\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m28 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(h.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m29 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# in ELK they cache as a huggingface dataset\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m31 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.hs1, \u001b[96mself\u001b[0m.ans1, \u001b[96mself\u001b[0m.hs2, \u001b[96mself\u001b[0m.ans2, \u001b[96mself\u001b[0m.y, \u001b[96mself\u001b[0m.infos = batch_hidden_stat \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[96mself\u001b[0m.dataset, \u001b[96mself\u001b[0m.prompt_fn, n=h.n, batch_size= \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m33 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# let's create a simple 50/50 train split (the data is already randomized)\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mwrap\u001b[0m:\u001b[94m30\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m27 \u001b[0m\u001b[2m│ │ │ \u001b[0mlogger.info(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mloading hs from \u001b[0m\u001b[33m{\u001b[0mf\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m28 \u001b[0m\u001b[2m│ │ │ \u001b[0mres = pickle.load(f.open(\u001b[33m'\u001b[0m\u001b[33mrb\u001b[0m\u001b[33m'\u001b[0m)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m29 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m30 \u001b[2m│ │ │ \u001b[0mres = func(model, tokenizer, data, prompt_fn, n, batch_size) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ │ │ \u001b[0mlogger.info(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mcaching hs to \u001b[0m\u001b[33m{\u001b[0mf\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ │ │ \u001b[0mpickle.dump(res, f.open(\u001b[33m'\u001b[0m\u001b[33mwb\u001b[0m\u001b[33m'\u001b[0m)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m33 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mbatch_hidden_states\u001b[0m:\u001b[94m28\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m26 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# different due to dropout\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m27 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# set_seeds(i*10)\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m28 \u001b[2m│ │ \u001b[0mhs1 = get_hidden_states(model, tokenizer, q) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m29 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# set_seeds(i*10+1)\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0mhs2 = get_hidden_states(model, tokenizer, q) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m i==\u001b[94m0\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states\u001b[0m:\u001b[94m53\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m50 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m51 \u001b[0m\u001b[2m│ │ \u001b[0mhidden_states = hidden_states[:, :, last_token] \u001b[2m# (batch, layers, past_seq, logi\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m52 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m53 \u001b[2m│ │ \u001b[0mtext_q = tokenizer.batch_decode(input_ids) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0m\u001b[2m│ │ \u001b[0ms = outputs[\u001b[33m'\u001b[0m\u001b[33msequences\u001b[0m\u001b[33m'\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m56 \u001b[0m\u001b[2m│ │ \u001b[0ms = [s[i][\u001b[96mlen\u001b[0m(input_ids[i]):] \u001b[94mfor\u001b[0m i \u001b[95min\u001b[0m \u001b[96mrange\u001b[0m(\u001b[96mlen\u001b[0m(s))] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/\u001b[0m\u001b[1;33mtokenization_utils_ba\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mse.py\u001b[0m:\u001b[94m3469\u001b[0m in \u001b[92mbatch_decode\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3466 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33mReturns:\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3467 \u001b[0m\u001b[2;33m│ │ │ \u001b[0m\u001b[33m`List[str]`: The list of decoded sentences.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3468 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3469 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m [ \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3470 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.decode( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3471 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mseq, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3472 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mskip_special_tokens=skip_special_tokens, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/\u001b[0m\u001b[1;33mtokenization_utils_ba\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mse.py\u001b[0m:\u001b[94m3470\u001b[0m in \u001b[92m\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3467 \u001b[0m\u001b[2;33m│ │ │ \u001b[0m\u001b[33m`List[str]`: The list of decoded sentences.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3468 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3469 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m [ \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3470 \u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.decode( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3471 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mseq, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3472 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mskip_special_tokens=skip_special_tokens, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3473 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mclean_up_tokenization_spaces=clean_up_tokenization_spaces, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/\u001b[0m\u001b[1;33mtokenization_utils_ba\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[1;33mse.py\u001b[0m:\u001b[94m3507\u001b[0m in \u001b[92mdecode\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3504 \u001b[0m\u001b[2;33m│ │ │ \u001b[0m\u001b[33m`str`: The decoded sentence.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3505 \u001b[0m\u001b[2;33m│ │ \u001b[0m\u001b[33m\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3506 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Convert inputs to python lists\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3507 \u001b[2m│ │ \u001b[0mtoken_ids = to_py_obj(token_ids) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3508 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3509 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._decode( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3510 \u001b[0m\u001b[2m│ │ │ \u001b[0mtoken_ids=token_ids, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/transformers/utils/\u001b[0m\u001b[1;33mgeneric.py\u001b[0m:\u001b[94m210\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mto_py_obj\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m207 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melif\u001b[0m is_tf_tensor(obj): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m208 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m obj.numpy().tolist() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m209 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melif\u001b[0m is_torch_tensor(obj): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m210 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m obj.detach().cpu().tolist() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m211 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melif\u001b[0m is_jax_tensor(obj): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m212 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m np.asarray(obj).tolist() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m213 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melif\u001b[0m \u001b[96misinstance\u001b[0m(obj, (np.ndarray, np.number)): \u001b[2m# tolist also works on 0d np arrays\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mKeyboardInterrupt\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# test and cache\n", "dm = imdbHSDataModule(model, tokenizer, batch_size=BATCH_SIZE, n=N_SAMPLES)\n", @@ -1000,7 +1543,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 hss1 = dm.hs1                                                                                │\n",
+       "│   2 hss2 = dm.hs2                                                                                │\n",
+       "│   3 ans_1 = dm.ans1                                                                              │\n",
+       "│   4 ans_2 = dm.ans2                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'hs1'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 hss1 = dm.hs1 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mhss2 = dm.hs2 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mans_1 = dm.ans1 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mans_2 = dm.ans2 \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'hs1'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "hss1 = dm.hs1\n", "hss2 = dm.hs2\n", @@ -1013,7 +1586,39 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    1 # temp: balance everything in case we stopped early                                         │\n",
+       "│ ❱  2 print(len(infos), len(ans_1), len(ans_2))                                                   │\n",
+       "│    3 hss1 = hss1[:len(hss2)]                                                                     │\n",
+       "│    4 hss2 = hss2[:len(hss1)]                                                                     │\n",
+       "│    5 ans_1 = ans_1[:len(ans_2)]                                                                  │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'infos' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 1 \u001b[0m\u001b[2m# temp: balance everything in case we stopped early\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 2 \u001b[96mprint\u001b[0m(\u001b[96mlen\u001b[0m(infos), \u001b[96mlen\u001b[0m(ans_1), \u001b[96mlen\u001b[0m(ans_2)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0mhss1 = hss1[:\u001b[96mlen\u001b[0m(hss2)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mhss2 = hss2[:\u001b[96mlen\u001b[0m(hss1)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0mans_1 = ans_1[:\u001b[96mlen\u001b[0m(ans_2)] \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'infos'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# temp: balance everything in case we stopped early\n", "print(len(infos), len(ans_1), len(ans_2))\n", @@ -1037,6 +1642,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -1049,7 +1655,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱  1 ans = (ans_1 + ans_2) / 2                                                                   │\n",
+       "│    2 acc=((ans>0.5)==df_infos2['true_answer']).mean()                                            │\n",
+       "│    3 print(f\"acc {acc:2.2f}\")                                                                    │\n",
+       "│    4                                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'ans_1' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 1 ans = (ans_1 + ans_2) / \u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0macc=((ans>\u001b[94m0.5\u001b[0m)==df_infos2[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m]).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33macc \u001b[0m\u001b[33m{\u001b[0macc\u001b[33m:\u001b[0m\u001b[33m2.2f\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'ans_1'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "ans = (ans_1 + ans_2) / 2\n", "acc=((ans>0.5)==df_infos2['true_answer']).mean()\n", @@ -1087,6 +1723,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -1112,7 +1749,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱  1 n = len(df_infos2)                                                                          │\n",
+       "│    2                                                                                             │\n",
+       "│    3 # Define X and y                                                                            │\n",
+       "│    4 X = hss1-hss2                                                                               │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 1 n = \u001b[96mlen\u001b[0m(df_infos2) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m\u001b[2m# Define X and y\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mX = hss1-hss2 \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "\n", "n = len(df_infos2)\n", @@ -1142,7 +1809,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱  1 print(\"Logistic cls acc: {:2.2%} [TRAIN]\".format(lr.score(X_train2, y_train>0)))            │\n",
+       "│    2 print(\"Logistic cls acc: {:2.2%} [TEST]\".format(lr.score(X_test2, y_test>0)))               │\n",
+       "│    3                                                                                             │\n",
+       "│    4 m = df_infos2['lie'][n//2:]                                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'lr' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 1 \u001b[96mprint\u001b[0m(\u001b[33m\"\u001b[0m\u001b[33mLogistic cls acc: \u001b[0m\u001b[33m{:2.2%}\u001b[0m\u001b[33m [TRAIN]\u001b[0m\u001b[33m\"\u001b[0m.format(lr.score(X_train2, y_train>\u001b[94m0\u001b[0m))) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33m\"\u001b[0m\u001b[33mLogistic cls acc: \u001b[0m\u001b[33m{:2.2%}\u001b[0m\u001b[33m [TEST]\u001b[0m\u001b[33m\"\u001b[0m.format(lr.score(X_test2, y_test>\u001b[94m0\u001b[0m))) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mm = df_infos2[\u001b[33m'\u001b[0m\u001b[33mlie\u001b[0m\u001b[33m'\u001b[0m][n//\u001b[94m2\u001b[0m:] \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'lr'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "print(\"Logistic cls acc: {:2.2%} [TRAIN]\".format(lr.score(X_train2, y_train>0)))\n", "print(\"Logistic cls acc: {:2.2%} [TEST]\".format(lr.score(X_test2, y_test>0)))\n", @@ -1159,7 +1856,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_info_test = df_infos2.iloc[n//2:].copy()                                                  │\n",
+       "│   2 y_pred = lr.predict(X_test2)                                                                 │\n",
+       "│   3 df_info_test['inner_truth'] = y_pred                                                         │\n",
+       "│   4 df_info_test                                                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_info_test = df_infos2.iloc[n//\u001b[94m2\u001b[0m:].copy() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0my_pred = lr.predict(X_test2) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mdf_info_test[\u001b[33m'\u001b[0m\u001b[33minner_truth\u001b[0m\u001b[33m'\u001b[0m] = y_pred \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mdf_info_test \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "df_info_test = df_infos2.iloc[n//2:].copy()\n", "y_pred = lr.predict(X_test2)\n", @@ -1179,7 +1906,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 lie_pred = df_info_test['inner_truth']==df_info_test['true_answer']                          │\n",
+       "│   2 lie_true = df_info_test['lie']                                                               │\n",
+       "│   3 acc_lie = accuracy_score(lie_pred, lie_true)                                                 │\n",
+       "│   4 print(f\"model can detect lies with acc {acc_lie:2.2%}\")                                      │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_info_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 lie_pred = df_info_test[\u001b[33m'\u001b[0m\u001b[33minner_truth\u001b[0m\u001b[33m'\u001b[0m]==df_info_test[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mlie_true = df_info_test[\u001b[33m'\u001b[0m\u001b[33mlie\u001b[0m\u001b[33m'\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0macc_lie = accuracy_score(lie_pred, lie_true) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mmodel can detect lies with acc \u001b[0m\u001b[33m{\u001b[0macc_lie\u001b[33m:\u001b[0m\u001b[33m2.2%\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_info_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "lie_pred = df_info_test['inner_truth']==df_info_test['true_answer']\n", "lie_true = df_info_test['lie']\n", @@ -1189,6 +1946,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -1199,7 +1957,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 bool_to_switch = lambda b:b*2-1                                                              │\n",
+       "│ ❱ 2 true_answer_switch = bool_to_switch(df_infos2['true_answer'])                                │\n",
+       "│   3 y = y_left_more_true = df_infos2['dir_true'] * true_answer_switch                            │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0mbool_to_switch = \u001b[94mlambda\u001b[0m b:b*\u001b[94m2\u001b[0m-\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 true_answer_switch = bool_to_switch(df_infos2[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0my = y_left_more_true = df_infos2[\u001b[33m'\u001b[0m\u001b[33mdir_true\u001b[0m\u001b[33m'\u001b[0m] * true_answer_switch \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "bool_to_switch = lambda b:b*2-1\n", "true_answer_switch = bool_to_switch(df_infos2['true_answer'])\n", @@ -1210,7 +1998,43 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:5                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    2 from sklearn.linear_model import ElasticNet                                                 │\n",
+       "│    3                                                                                             │\n",
+       "│    4 # Try a classification of direction                                                         │\n",
+       "│ ❱  5 n = len(df_infos2)                                                                          │\n",
+       "│    6                                                                                             │\n",
+       "│    7 # Define X and y                                                                            │\n",
+       "│    8 X = hss1-hss2                                                                               │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m5\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m\u001b[94mfrom\u001b[0m \u001b[4;96msklearn\u001b[0m\u001b[4;96m.\u001b[0m\u001b[4;96mlinear_model\u001b[0m \u001b[94mimport\u001b[0m ElasticNet \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0m\u001b[2m# Try a classification of direction\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 5 n = \u001b[96mlen\u001b[0m(df_infos2) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 7 \u001b[0m\u001b[2m# Define X and y\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 8 \u001b[0mX = hss1-hss2 \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Try a regression\n", "from sklearn.linear_model import ElasticNet\n", @@ -1246,7 +2070,39 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 eps = 0.                                                                                     │\n",
+       "│ ❱ 2 acc=np.mean((lr2.predict(X_train2)>eps)==(y_train>eps))                                      │\n",
+       "│   3 print(f'acc from train ElasticNet {acc:2.2f}')                                               │\n",
+       "│   4 acc=np.mean((lr2.predict(X_test2)>eps)==(y_test>eps))                                        │\n",
+       "│   5 print(f'acc from test ElasticNet {acc:2.2f}')                                                │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'lr2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0meps = \u001b[94m0.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 acc=np.mean((lr2.predict(X_train2)>eps)==(y_train>eps)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m'\u001b[0m\u001b[33macc from train ElasticNet \u001b[0m\u001b[33m{\u001b[0macc\u001b[33m:\u001b[0m\u001b[33m2.2f\u001b[0m\u001b[33m}\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0macc=np.mean((lr2.predict(X_test2)>eps)==(y_test>eps)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m'\u001b[0m\u001b[33macc from test ElasticNet \u001b[0m\u001b[33m{\u001b[0macc\u001b[33m:\u001b[0m\u001b[33m2.2f\u001b[0m\u001b[33m}\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'lr2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "eps = 0.\n", "acc=np.mean((lr2.predict(X_train2)>eps)==(y_train>eps))\n", @@ -1259,7 +2115,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 y_test_pred = lr2.predict(X_test)                                                            │\n",
+       "│   2 plt.scatter(y_test, y_test_pred)                                                             │\n",
+       "│   3 plt.xlabel('true')                                                                           │\n",
+       "│   4 plt.ylabel('pred')                                                                           │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'lr2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 y_test_pred = lr2.predict(X_test) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mplt.scatter(y_test, y_test_pred) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mplt.xlabel(\u001b[33m'\u001b[0m\u001b[33mtrue\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mplt.ylabel(\u001b[33m'\u001b[0m\u001b[33mpred\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'lr2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "y_test_pred = lr2.predict(X_test)\n", "plt.scatter(y_test, y_test_pred)\n", @@ -1276,6 +2162,7 @@ "source": [] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -1296,7 +2183,7 @@ "outputs": [], "source": [ "class MLPProbe(nn.Module):\n", - " def __init__(self, d, depth=1, hs=32, dropout=0):\n", + " def __init__(self, d, depth=0, hs=16, dropout=0):\n", " super().__init__()\n", "\n", " layers = [\n", @@ -1360,7 +2247,7 @@ " self.log(f\"{stage}/loss\", loss)\n", " \n", " self.auroc(y_pred, y.long())\n", - " self.log(f\"{stage}/acc_step\", self.auroc) \n", + " self.log(f\"{stage}/acc_step\", self.auroc, on_step=False, on_epoch=True)\n", " return loss\n", " \n", " def on_train_epoch_end(self):\n", @@ -1396,12 +2283,54 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Run" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 if dm.hs1:                                                                                   │\n",
+       "│   2 │   model = None                                                                             │\n",
+       "│   3 clear_mem()                                                                                  │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'hs1'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 \u001b[94mif\u001b[0m dm.hs1: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[2m│ \u001b[0mmodel = \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mclear_mem() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'hs1'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "if dm.hs1:\n", + " model = None\n", + "clear_mem()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1417,6 +2346,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -1427,7 +2357,39 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    1 # split                                                                                     │\n",
+       "│ ❱  2 X = hss1-hss2                                                                               │\n",
+       "│    3 y = (df_infos2['true_answer'] == (df_infos2['dir_true']>0)).values # direction              │\n",
+       "│    4 n = len(y)                                                                                  │\n",
+       "│    5 print('split size', n//2)                                                                   │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'hss1' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 1 \u001b[0m\u001b[2m# split\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 2 X = hss1-hss2 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0my = (df_infos2[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m] == (df_infos2[\u001b[33m'\u001b[0m\u001b[33mdir_true\u001b[0m\u001b[33m'\u001b[0m]>\u001b[94m0\u001b[0m)).values \u001b[2m# direction\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mn = \u001b[96mlen\u001b[0m(y) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33m'\u001b[0m\u001b[33msplit size\u001b[0m\u001b[33m'\u001b[0m, n//\u001b[94m2\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'hss1'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# split\n", "X = hss1-hss2\n", @@ -1448,7 +2410,57 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 dl_train = dm.train_dataloader()                                                             │\n",
+       "│   2 dl_val = dm.val_dataloader()                                                                 │\n",
+       "│   3 b = next(iter(dl_train))                                                                     │\n",
+       "│   4 # b                                                                                          │\n",
+       "│                                                                                                  │\n",
+       "│ in train_dataloader:73                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│   70 │   │   │   │   │   │   │   │   │    torch.from_numpy(y_test).float())                      │\n",
+       "│   71 │                                                                                           │\n",
+       "│   72 │   def train_dataloader(self):                                                             │\n",
+       "│ ❱ 73 │   │   return DataLoader(self.ds_train,                                                    │\n",
+       "│   74 │   │   │   │   │   │     batch_size=self.hparams.dl_batch_size,                            │\n",
+       "│   75 │   │   │   │   │   │     shuffle=True)                                                     │\n",
+       "│   76                                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'ds_train'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 dl_train = dm.train_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mdl_val = dm.val_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mb = \u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl_train)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m\u001b[2m# b\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mtrain_dataloader\u001b[0m:\u001b[94m73\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m70 \u001b[0m\u001b[2m│ │ │ │ │ │ │ │ │ \u001b[0mtorch.from_numpy(y_test).float()) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m71 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m72 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mtrain_dataloader\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m73 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m DataLoader(\u001b[96mself\u001b[0m.ds_train, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m74 \u001b[0m\u001b[2m│ │ │ │ │ │ \u001b[0mbatch_size=\u001b[96mself\u001b[0m.hparams.dl_batch_size, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m75 \u001b[0m\u001b[2m│ │ │ │ │ │ \u001b[0mshuffle=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m76 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'ds_train'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "dl_train = dm.train_dataloader()\n", "dl_val = dm.val_dataloader()\n", @@ -1460,12 +2472,47 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 # init the model                                                                             │\n",
+       "│   2 max_epochs = 53                                                                              │\n",
+       "│ ❱ 3 d = b[0].shape[-1]                                                                           │\n",
+       "│   4 net = CSS(d=d, total_steps=max_epochs*len(dl_train), lr=4e-3, weight_decay=1e-3, dropout     │\n",
+       "│   5 net                                                                                          │\n",
+       "│   6                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'b' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[2m# init the model\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mmax_epochs = \u001b[94m53\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3 d = b[\u001b[94m0\u001b[0m].shape[-\u001b[94m1\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mnet = CSS(d=d, total_steps=max_epochs*\u001b[96mlen\u001b[0m(dl_train), lr=\u001b[94m4e-3\u001b[0m, weight_decay=\u001b[94m1e-3\u001b[0m, dropout \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5 \u001b[0mnet \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'b'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# init the model\n", - "max_epochs = 33\n", + "max_epochs = 53\n", "d = b[0].shape[-1]\n", - "net = CSS(d=d, total_steps=max_epochs*len(dl_train), lr=4e-3, weight_decay=1e-3, dropout=0.3)" + "net = CSS(d=d, total_steps=max_epochs*len(dl_train), lr=4e-3, weight_decay=1e-3, dropout=0.3)\n", + "net" ] }, { @@ -1496,7 +2543,47 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "IPU available: False, using: 0 IPUs\n", + "HPU available: False, using: 0 HPUs\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 trainer = pl.Trainer(                                                                        │\n",
+       "│   2 │   │   │   │   │    max_epochs=max_epochs, log_every_n_steps=5)                             │\n",
+       "│ ❱ 3 trainer.fit(model=net, train_dataloaders=dl_train, val_dataloaders=dl_val)                   │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'net' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0mtrainer = pl.Trainer( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mmax_epochs=max_epochs, log_every_n_steps=\u001b[94m5\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3 trainer.fit(model=net, train_dataloaders=dl_train, val_dataloaders=dl_val) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'net'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "trainer = pl.Trainer(\n", " max_epochs=max_epochs, log_every_n_steps=5)\n", @@ -1504,6 +2591,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -1514,7 +2602,220 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[]\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:26                                                                                   │\n",
+       "│                                                                                                  │\n",
+       "│   23 │   │   raise e                                                                             │\n",
+       "│   24                                                                                             │\n",
+       "│   25                                                                                             │\n",
+       "│ ❱ 26 df_hist = read_hist(trainer).ffill().bfill()                                                │\n",
+       "│   27 df_hist                                                                                     │\n",
+       "│   28                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:23                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│ ❱ 23 │   │   raise e                                                                             │\n",
+       "│   24                                                                                             │\n",
+       "│   25                                                                                             │\n",
+       "│   26 df_hist = read_hist(trainer).ffill().bfill()                                                │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:20                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   17 │   print(ts)                                                                               │\n",
+       "│   18 │   try:                                                                                    │\n",
+       "│   19 │   │   metrics_file_path = Path(ts[0].experiment.metrics_file_path)                        │\n",
+       "│ ❱ 20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│   23 │   │   raise e                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_metrics_csv:8                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│    5 import pandas as pd                                                                         │\n",
+       "│    6                                                                                             │\n",
+       "│    7 def read_metrics_csv(metrics_file_path):                                                    │\n",
+       "│ ❱  8 │   df_hist = pd.read_csv(metrics_file_path)                                                │\n",
+       "│    9 │   df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()                                             │\n",
+       "│   10 │   df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()                           │\n",
+       "│   11 │   return df_histe                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:912   │\n",
+       "│ in read_csv                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│    909 │   )                                                                                     │\n",
+       "│    910 │   kwds.update(kwds_defaults)                                                            │\n",
+       "│    911 │                                                                                         │\n",
+       "│ ❱  912 │   return _read(filepath_or_buffer, kwds)                                                │\n",
+       "│    913                                                                                           │\n",
+       "│    914                                                                                           │\n",
+       "│    915 # iterator=True -> TextFileReader                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:577   │\n",
+       "│ in _read                                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│    574 │   _validate_names(kwds.get(\"names\", None))                                              │\n",
+       "│    575 │                                                                                         │\n",
+       "│    576 │   # Create the parser.                                                                  │\n",
+       "│ ❱  577 │   parser = TextFileReader(filepath_or_buffer, **kwds)                                   │\n",
+       "│    578 │                                                                                         │\n",
+       "│    579 │   if chunksize or iterator:                                                             │\n",
+       "│    580 │   │   return parser                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1407  │\n",
+       "│ in __init__                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   1404 │   │   │   self.options[\"has_index_names\"] = kwds[\"has_index_names\"]                     │\n",
+       "│   1405 │   │                                                                                     │\n",
+       "│   1406 │   │   self.handles: IOHandles | None = None                                             │\n",
+       "│ ❱ 1407 │   │   self._engine = self._make_engine(f, self.engine)                                  │\n",
+       "│   1408 │                                                                                         │\n",
+       "│   1409 │   def close(self) -> None:                                                              │\n",
+       "│   1410 │   │   if self.handles is not None:                                                      │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1661  │\n",
+       "│ in _make_engine                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   1658 │   │   │   │   is_text = False                                                           │\n",
+       "│   1659 │   │   │   │   if \"b\" not in mode:                                                       │\n",
+       "│   1660 │   │   │   │   │   mode += \"b\"                                                           │\n",
+       "│ ❱ 1661 │   │   │   self.handles = get_handle(                                                    │\n",
+       "│   1662 │   │   │   │   f,                                                                        │\n",
+       "│   1663 │   │   │   │   mode,                                                                     │\n",
+       "│   1664 │   │   │   │   encoding=self.options.get(\"encoding\", None),                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/common.py:859 in         │\n",
+       "│ get_handle                                                                                       │\n",
+       "│                                                                                                  │\n",
+       "│    856 │   │   # Binary mode does not support 'encoding' and 'newline'.                          │\n",
+       "│    857 │   │   if ioargs.encoding and \"b\" not in ioargs.mode:                                    │\n",
+       "│    858 │   │   │   # Encoding                                                                    │\n",
+       "│ ❱  859 │   │   │   handle = open(                                                                │\n",
+       "│    860 │   │   │   │   handle,                                                                   │\n",
+       "│    861 │   │   │   │   ioargs.mode,                                                              │\n",
+       "│    862 │   │   │   │   encoding=ioargs.encoding,                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "FileNotFoundError: [Errno 2] No such file or directory: \n",
+       "'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_72/metrics.csv'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m26\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m23 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m26 df_hist = read_hist(trainer).ffill().bfill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m27 \u001b[0mdf_hist \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m28 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m23\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m20 \u001b[0m\u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m23 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m26 \u001b[0mdf_hist = read_hist(trainer).ffill().bfill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m20\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ \u001b[0m\u001b[96mprint\u001b[0m(ts) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m19 \u001b[0m\u001b[2m│ │ \u001b[0mmetrics_file_path = Path(ts[\u001b[94m0\u001b[0m].experiment.metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m20 \u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m23 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_metrics_csv\u001b[0m:\u001b[94m8\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0m\u001b[94mimport\u001b[0m \u001b[4;96mpandas\u001b[0m \u001b[94mas\u001b[0m \u001b[4;96mpd\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 7 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mread_metrics_csv\u001b[0m(metrics_file_path): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 8 \u001b[2m│ \u001b[0mdf_hist = pd.read_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 9 \u001b[0m\u001b[2m│ \u001b[0mdf_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m] = df_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m].ffill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m10 \u001b[0m\u001b[2m│ \u001b[0mdf_histe = df_hist.set_index(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).groupby(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m11 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m912\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_csv\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 909 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 910 \u001b[0m\u001b[2m│ \u001b[0mkwds.update(kwds_defaults) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 911 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 912 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m _read(filepath_or_buffer, kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 913 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 914 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 915 \u001b[0m\u001b[2m# iterator=True -> TextFileReader\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m577\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_read\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 574 \u001b[0m\u001b[2m│ \u001b[0m_validate_names(kwds.get(\u001b[33m\"\u001b[0m\u001b[33mnames\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 575 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 576 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create the parser.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 577 \u001b[2m│ \u001b[0mparser = TextFileReader(filepath_or_buffer, **kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 578 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 579 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m chunksize \u001b[95mor\u001b[0m iterator: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 580 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m parser \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1407\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1404 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.options[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] = kwds[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1405 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1406 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.handles: IOHandles | \u001b[94mNone\u001b[0m = \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1407 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._engine = \u001b[96mself\u001b[0m._make_engine(f, \u001b[96mself\u001b[0m.engine) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1408 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1409 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mclose\u001b[0m(\u001b[96mself\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1410 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.handles \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1661\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_make_engine\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1658 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mis_text = \u001b[94mFalse\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1659 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1660 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mmode += \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1661 \u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.handles = get_handle( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1662 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mf, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1663 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1664 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=\u001b[96mself\u001b[0m.options.get(\u001b[33m\"\u001b[0m\u001b[33mencoding\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m), \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/\u001b[0m\u001b[1;33mcommon.py\u001b[0m:\u001b[94m859\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mget_handle\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 856 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Binary mode does not support 'encoding' and 'newline'.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 857 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m ioargs.encoding \u001b[95mand\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m ioargs.mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 858 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# Encoding\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 859 \u001b[2m│ │ │ \u001b[0mhandle = \u001b[96mopen\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 860 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mhandle, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 861 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mioargs.mode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 862 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=ioargs.encoding, \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mFileNotFoundError: \u001b[0m\u001b[1m[\u001b[0mErrno \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m No such file or directory: \n", + "\u001b[32m'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_72/metrics.csv'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# import pytorch_lightning as pl\n", "from lightning.pytorch.loggers.csv_logs import CSVLogger\n", @@ -1549,7 +2850,37 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 keys = set(s.split('/')[1] for s in df_hist.columns if '/' in s)                             │\n",
+       "│   2 for k in keys:                                                                               │\n",
+       "│   3 │   df_hist[[c for c in df_hist.columns if c.endswith(k)]].plot(title=k)                     │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_hist' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 keys = \u001b[96mset\u001b[0m(s.split(\u001b[33m'\u001b[0m\u001b[33m/\u001b[0m\u001b[33m'\u001b[0m)[\u001b[94m1\u001b[0m] \u001b[94mfor\u001b[0m s \u001b[95min\u001b[0m df_hist.columns \u001b[94mif\u001b[0m \u001b[33m'\u001b[0m\u001b[33m/\u001b[0m\u001b[33m'\u001b[0m \u001b[95min\u001b[0m s) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[94mfor\u001b[0m k \u001b[95min\u001b[0m keys: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m\u001b[2m│ \u001b[0mdf_hist[[c \u001b[94mfor\u001b[0m c \u001b[95min\u001b[0m df_hist.columns \u001b[94mif\u001b[0m c.endswith(k)]].plot(title=k) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_hist'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "keys = set(s.split('/')[1] for s in df_hist.columns if '/' in s)\n", "for k in keys: \n", @@ -1574,6 +2905,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -1584,7 +2916,53 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 dl_test = dm.test_dataloader()                                                               │\n",
+       "│   2 y_test_pred = trainer.predict(net, dl_test)                                                  │\n",
+       "│   3 y_test_pred = np.concatenate(y_test_pred)                                                    │\n",
+       "│   4 # y_test_pred                                                                                │\n",
+       "│                                                                                                  │\n",
+       "│ in test_dataloader:81                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   78 │   │   return DataLoader(self.ds_val, batch_size=self.hparams.dl_batch_size)               │\n",
+       "│   79 │                                                                                           │\n",
+       "│   80 │   def test_dataloader(self):                                                              │\n",
+       "│ ❱ 81 │   │   return DataLoader(self.ds_test, batch_size=self.hparams.dl_batch_size)              │\n",
+       "│   82                                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'ds_test'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 dl_test = dm.test_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0my_test_pred = trainer.predict(net, dl_test) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0my_test_pred = np.concatenate(y_test_pred) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m\u001b[2m# y_test_pred\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mtest_dataloader\u001b[0m:\u001b[94m81\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m78 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m DataLoader(\u001b[96mself\u001b[0m.ds_val, batch_size=\u001b[96mself\u001b[0m.hparams.dl_batch_size) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m79 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m80 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mtest_dataloader\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m81 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m DataLoader(\u001b[96mself\u001b[0m.ds_test, batch_size=\u001b[96mself\u001b[0m.hparams.dl_batch_size) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m82 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'ds_test'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "dl_test = dm.test_dataloader()\n", "y_test_pred = trainer.predict(net, dl_test)\n", @@ -1596,17 +2974,117 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 # y_test_pred.shape, df_test.shape                                                           │\n",
+       "│ ❱ 2 dm.val_split, dm.test_split                                                                  │\n",
+       "│   3                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'val_split'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[2m# y_test_pred.shape, df_test.shape\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 dm.val_split, dm.test_split \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'val_split'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# y_test_pred.shape, df_test.shape\n", + "dm.val_split, dm.test_split" + ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 y_true = dl_test.dataset.tensors[2].numpy()                                                  │\n",
+       "│   2 assert ((df_test['true_answer'].values>0.5)==(y_true>0.5)).all(), 'check it all lines up     │\n",
+       "│   3                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'dl_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 y_true = dl_test.dataset.tensors[\u001b[94m2\u001b[0m].numpy() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[94massert\u001b[0m ((df_test[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m].values>\u001b[94m0.5\u001b[0m)==(y_true>\u001b[94m0.5\u001b[0m)).all(), \u001b[33m'\u001b[0m\u001b[33mcheck it all lines up\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'dl_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "df_test = dm.df.iloc[dm.val_split:dm.test_split].copy()\n", - "df_test['pred'] = y_test_pred.argmax(-1)\n", + "y_true = dl_test.dataset.tensors[2].numpy()\n", + "assert ((df_test['true_answer'].values>0.5)==(y_true>0.5)).all(), 'check it all lines up'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_test = dm.df.iloc[dm.test_split:].copy()                                                  │\n",
+       "│   2 df_test['prob_pred'] = y_test_pred.argmax(-1)                                                │\n",
+       "│   3 df_test['llm_ans'] = (df_test['ans1']+df_test['ans1'])/2                                     │\n",
+       "│   4 df_test                                                                                      │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'df'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_test = dm.df.iloc[dm.test_split:].copy() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mdf_test[\u001b[33m'\u001b[0m\u001b[33mprob_pred\u001b[0m\u001b[33m'\u001b[0m] = y_test_pred.argmax(-\u001b[94m1\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mdf_test[\u001b[33m'\u001b[0m\u001b[33mllm_ans\u001b[0m\u001b[33m'\u001b[0m] = (df_test[\u001b[33m'\u001b[0m\u001b[33mans1\u001b[0m\u001b[33m'\u001b[0m]+df_test[\u001b[33m'\u001b[0m\u001b[33mans1\u001b[0m\u001b[33m'\u001b[0m])/\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mdf_test \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'df'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df_test = dm.df.iloc[dm.test_split:].copy()\n", + "df_test['prob_pred'] = y_test_pred.argmax(-1)\n", + "df_test['llm_ans'] = (df_test['ans1']+df_test['ans1'])/2\n", "df_test" ] }, @@ -1614,9 +3092,151 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 ((df_test['llm_ans']>0.5)==df_test['desired_answer']).mean()                                 │\n",
+       "│   2                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 ((df_test[\u001b[33m'\u001b[0m\u001b[33mllm_ans\u001b[0m\u001b[33m'\u001b[0m]>\u001b[94m0.5\u001b[0m)==df_test[\u001b[33m'\u001b[0m\u001b[33mdesired_answer\u001b[0m\u001b[33m'\u001b[0m]).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "acc_truth = (df_test['pred']==df_test['true_answer']).mean()\n", + "((df_test['llm_ans']>0.5)==df_test['desired_answer']).mean()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 # this must be wrong                                                                         │\n",
+       "│ ❱ 2 acc_truth = (df_test['prob_pred']==df_test['true_answer']).mean()                            │\n",
+       "│   3 print(f\"lightning model acc at predicting truth: {acc_truth:2.2%}\")                          │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[2m# this must be wrong\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 acc_truth = (df_test[\u001b[33m'\u001b[0m\u001b[33mprob_pred\u001b[0m\u001b[33m'\u001b[0m]==df_test[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m]).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mlightning model acc at predicting truth: \u001b[0m\u001b[33m{\u001b[0macc_truth\u001b[33m:\u001b[0m\u001b[33m2.2%\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# this must be wrong\n", + "acc_truth = (df_test['prob_pred']==df_test['true_answer']).mean()\n", + "print(f\"lightning model acc at predicting truth: {acc_truth:2.2%}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 acc_truth = (df_test['prob_pred']==df_test['desired_answer']).mean()                         │\n",
+       "│   2 print(f\"lightning model acc at predicting truth: {acc_truth:2.2%}\")                          │\n",
+       "│   3                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 acc_truth = (df_test[\u001b[33m'\u001b[0m\u001b[33mprob_pred\u001b[0m\u001b[33m'\u001b[0m]==df_test[\u001b[33m'\u001b[0m\u001b[33mdesired_answer\u001b[0m\u001b[33m'\u001b[0m]).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mlightning model acc at predicting truth: \u001b[0m\u001b[33m{\u001b[0macc_truth\u001b[33m:\u001b[0m\u001b[33m2.2%\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "acc_truth = (df_test['prob_pred']==df_test['desired_answer']).mean()\n", + "print(f\"lightning model acc at predicting truth: {acc_truth:2.2%}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 acc_truth = (df_test['prob_pred']==(df_test['llm_ans']>0.5)).mean()                          │\n",
+       "│   2 print(f\"lightning model acc at predicting truth: {acc_truth:2.2%}\")                          │\n",
+       "│   3                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 acc_truth = (df_test[\u001b[33m'\u001b[0m\u001b[33mprob_pred\u001b[0m\u001b[33m'\u001b[0m]==(df_test[\u001b[33m'\u001b[0m\u001b[33mllm_ans\u001b[0m\u001b[33m'\u001b[0m]>\u001b[94m0.5\u001b[0m)).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mlightning model acc at predicting truth: \u001b[0m\u001b[33m{\u001b[0macc_truth\u001b[33m:\u001b[0m\u001b[33m2.2%\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "acc_truth = (df_test['prob_pred']==(df_test['llm_ans']>0.5)).mean()\n", "print(f\"lightning model acc at predicting truth: {acc_truth:2.2%}\")" ] }, @@ -1689,7 +3309,12 @@ "pygments_lexer": "ipython3", "version": "3.9.16" }, - "orig_nbformat": 4 + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "25d3203011652c9a0b3745968f18b04c477a3d0b83eddc02ed4f61e610dee119" + } + } }, "nbformat": 4, "nbformat_minor": 2