diff --git a/notebooks/015_mjc_CCS_mcdrop_dm.ipynb b/notebooks/015_mjc_CCS_mcdrop_dm.ipynb index b53c011..fd15e6e 100644 --- a/notebooks/015_mjc_CCS_mcdrop_dm.ipynb +++ b/notebooks/015_mjc_CCS_mcdrop_dm.ipynb @@ -152,7 +152,54 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPTBigCodeConfig {\n", + " \"_name_or_path\": \"WizardLM/WizardCoder-15B-V1.0\",\n", + " \"activation_function\": \"gelu\",\n", + " \"architectures\": [\n", + " \"GPTBigCodeForCausalLM\"\n", + " ],\n", + " \"attention_softmax_in_fp32\": true,\n", + " \"attn_pdrop\": 0.1,\n", + " \"bos_token_id\": 0,\n", + " \"embd_pdrop\": 0.1,\n", + " \"eos_token_id\": 0,\n", + " \"inference_runner\": 0,\n", + " \"initializer_range\": 0.02,\n", + " \"layer_norm_epsilon\": 1e-05,\n", + " \"max_batch_size\": null,\n", + " \"max_sequence_length\": null,\n", + " \"model_type\": \"gpt_bigcode\",\n", + " \"multi_query\": true,\n", + " \"n_embd\": 6144,\n", + " \"n_head\": 48,\n", + " \"n_inner\": 24576,\n", + " \"n_layer\": 40,\n", + " \"n_positions\": 8192,\n", + " \"pad_key_length\": true,\n", + " \"pre_allocate_kv_cache\": false,\n", + " \"resid_pdrop\": 0.1,\n", + " \"scale_attention_softmax_in_fp32\": true,\n", + " \"scale_attn_weights\": true,\n", + " \"summary_activation\": null,\n", + " \"summary_first_dropout\": 0.1,\n", + " \"summary_proj_to_labels\": true,\n", + " \"summary_type\": \"cls_index\",\n", + " \"summary_use_proj\": true,\n", + " \"torch_dtype\": \"float16\",\n", + " \"transformers_version\": \"4.30.1\",\n", + " \"use_cache\": false,\n", + " \"validate_runner_input\": true,\n", + " \"vocab_size\": 49153\n", + "}\n", + "\n" + ] + } + ], "source": [ "# leaderboard https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard\n", "model_options = dict(\n", @@ -212,18 +259,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "GPTBigCodeForCausalLM(\n", + " (transformer): GPTBigCodeModel(\n", + " (wte): Embedding(49153, 6144)\n", + " (wpe): Embedding(8192, 6144)\n", + " (drop): Dropout(p=0.1, inplace=False)\n", + " (h): ModuleList(\n", + " (0-39): 40 x GPTBigCodeBlock(\n", + " (ln_1): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\n", + " (attn): GPTBigCodeAttention(\n", + " (c_attn): Linear4bit(in_features=6144, out_features=6400, bias=True)\n", + " (c_proj): Linear4bit(in_features=6144, out_features=6144, bias=True)\n", + " (attn_dropout): Dropout(p=0.1, inplace=False)\n", + " (resid_dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (ln_2): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): GPTBigCodeMLP(\n", + " (c_fc): Linear4bit(in_features=6144, out_features=24576, bias=True)\n", + " (c_proj): Linear4bit(in_features=24576, out_features=6144, bias=True)\n", + " (act): GELUActivation()\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " (ln_f): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (lm_head): Linear(in_features=6144, out_features=49153, bias=False)\n", + ")" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "model" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "49152\n" + ] + } + ], "source": [ "# https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/falcon.py\n", "print(tokenizer.pad_token_id)\n", @@ -241,13 +333,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "40\n" + ] + }, + { + "data": { + "text/plain": [ + "((2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40),\n", + " 40)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Params\n", "N_SAMPLES = 3000\n", - "BATCH_SIZE = 6 # None # None means auto\n", + "BATCH_SIZE = 6 # None # None means auto # 6 gives 16Gb/25GB. where 10GB is the base model. so 6 is 6/15\n", "N_SHOTS = 3\n", "USE_MCDROPOUT = True\n", "dataset_n = 200\n", @@ -284,9 +395,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(15272, 18502)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# get the tokens for 0 and 1, we will use these later...\n", "# note that sentancepeice tokenizers have differen't tokens for No and \\nNo.\n", @@ -315,9 +437,31 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0adc14b4059f45e081a9d461dab14702", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/2 [00:000.001, \"the hidden state pairs should be different but are not. Check model.config.use_cache==False, check this model has dropout in it's arch\"\n", + " if i==0:\n", + " assert hs1['hidden_states'][0, 0, 0]-hs2['hidden_states'][0, 0, 0]>0.001, \"the hidden state pairs should be different but are not. Check model.config.use_cache==False, check this model has dropout in it's arch\"\n", "\n", " # collect\n", " b = len(texts)\n", @@ -922,7 +1159,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -1011,24 +1248,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 76, "metadata": {}, - "outputs": [], - "source": [ - "# # test and cache\n", - "# dm = imdbHSDataModule(model, tokenizer, batch_size=BATCH_SIZE, n=BATCH_SIZE*2)\n", - "# dm.setup('train')\n", - "# dl = dm.val_dataloader()\n", - "# b = next(iter(dl))\n", - "# clear_mem()\n", - "# b" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "\u001b[32m2023-06-18 13:24:03.366\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36mwrap\u001b[0m:\u001b[36m17\u001b[0m - \u001b[1mkwargs ['GPTBigCodeForCausalLM(\\n (transformer): GPTBigCodeModel(\\n (wte): Embedding(49153, 6144)\\n (wpe): Embedding(8192, 6144)\\n (drop): Dropout(p=0.1, inplace=False)\\n (h): ModuleList(\\n (0-39): 40 x GPTBigCodeBlock(\\n (ln_1): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n (attn): GPTBigCodeAttention(\\n (c_attn): Linear4bit(in_features=6144, out_features=6400, bias=True)\\n (c_proj): Linear4bit(in_features=6144, out_features=6144, bias=True)\\n (attn_dropout): Dropout(p=0.1, inplace=False)\\n (resid_dropout): Dropout(p=0.1, inplace=False)\\n )\\n (ln_2): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n (mlp): GPTBigCodeMLP(\\n (c_fc): Linear4bit(in_features=6144, out_features=24576, bias=True)\\n (c_proj): Linear4bit(in_features=24576, out_features=6144, bias=True)\\n (act): GELUActivation()\\n (dropout): Dropout(p=0.1, inplace=False)\\n )\\n )\\n )\\n (ln_f): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n )\\n (lm_head): Linear(in_features=6144, out_features=49153, bias=False)\\n)', \"GPT2TokenizerFast(name_or_path='WizardLM/WizardCoder-15B-V1.0', vocab_size=49152, model_max_length=2048, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<|endoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>', 'pad_token': '[PAD]', 'additional_special_tokens': ['<|endoftext|>', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '']}, clean_up_tokenization_spaces=True)\", \"Dataset({\\n features: ['label', 'title', 'content'],\\n num_rows: 400000\\n})\", '', 3000, 'The classic puzzle of the two guards. It presents a scenario where two guards are involved in a conflict of truths and lies. Here\\'s the setup:\\n\\nThere are two guards standing at two separate doors. One door leads to safety, while the other leads to danger. One guard always tells the truth, and the other guard always lies. You don\\'t know which guard is which or which door leads to safety.\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"excellent product! I love it!\". Content: \"I\\'m manager in a fast food restaurant, and this watch have been very helpful to me, I use it to set the break-meals of my employees in it, in a fast way. It\\'s very convenient because I have a lot of things to think about in my position, and thanks this watch I don\\'t have to worry about the breaks.\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Table hockey becomes friend maker\". Content: \"This game was a hit with my grandson! The little fella & his parents moved to Minnesota a week after receiving this game and this became his \"friend maker\". I appreciate the quick shipping. The game arrived in perfect condition.\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Perfect!!!\". Content: \"The highly anticipated dubbed version of the final season. Some of the original cast did not return for it but the replacement voice actors do a good job of staying true to the style of the characters that we\\'ve grown accustomed to. Can\\'t wait for set two!\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Later than expected\". Content: \"This product arrived near the end of the timeframe designated for media mail. When it did arrived, it arrived as described. I bought a brand new book.\"\\n\\n### Left Guard Response:\\n', 'The classic puzzle of the two guards. It presents a scenario where two guards are involved in a conflict of truths and lies. Here\\'s the setup:\\n\\nThere are two guards standing at two separate doors. One door leads to safety, while the other leads to danger. One guard always tells the truth, and the other guard always lies. You don\\'t know which guard is which or which door leads to safety.\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"excellent product! I love it!\". Content: \"I\\'m manager in a fast food restaurant, and this watch have been very helpful to me, I use it to set the break-meals of my employees in it, in a fast way. It\\'s very convenient because I have a lot of things to think about in my position, and thanks this watch I don\\'t have to worry about the breaks.\"\\n\\n### Right Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Table hockey becomes friend maker\". Content: \"This game was a hit with my grandson! The little fella & his parents moved to Minnesota a week after receiving this game and this became his \"friend maker\". I appreciate the quick shipping. The game arrived in perfect condition.\"\\n\\n### Right Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Perfect!!!\". Content: \"The highly anticipated dubbed version of the final season. Some of the original cast did not return for it but the replacement voice actors do a good job of staying true to the style of the characters that we\\'ve grown accustomed to. Can\\'t wait for set two!\"\\n\\n### Right Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Later than expected\". Content: \"This product arrived near the end of the timeframe designated for media mail. When it did arrived, it arrived as described. I bought a brand new book.\"\\n\\n### Right Guard Response:\\n']\u001b[0m\n", + "2023-06-18T13:24:03.366644+0800 INFO kwargs ['GPTBigCodeForCausalLM(\\n (transformer): GPTBigCodeModel(\\n (wte): Embedding(49153, 6144)\\n (wpe): Embedding(8192, 6144)\\n (drop): Dropout(p=0.1, inplace=False)\\n (h): ModuleList(\\n (0-39): 40 x GPTBigCodeBlock(\\n (ln_1): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n (attn): GPTBigCodeAttention(\\n (c_attn): Linear4bit(in_features=6144, out_features=6400, bias=True)\\n (c_proj): Linear4bit(in_features=6144, out_features=6144, bias=True)\\n (attn_dropout): Dropout(p=0.1, inplace=False)\\n (resid_dropout): Dropout(p=0.1, inplace=False)\\n )\\n (ln_2): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n (mlp): GPTBigCodeMLP(\\n (c_fc): Linear4bit(in_features=6144, out_features=24576, bias=True)\\n (c_proj): Linear4bit(in_features=24576, out_features=6144, bias=True)\\n (act): GELUActivation()\\n (dropout): Dropout(p=0.1, inplace=False)\\n )\\n )\\n )\\n (ln_f): LayerNorm((6144,), eps=1e-05, elementwise_affine=True)\\n )\\n (lm_head): Linear(in_features=6144, out_features=49153, bias=False)\\n)', \"GPT2TokenizerFast(name_or_path='WizardLM/WizardCoder-15B-V1.0', vocab_size=49152, model_max_length=2048, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '<|endoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>', 'pad_token': '[PAD]', 'additional_special_tokens': ['<|endoftext|>', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '']}, clean_up_tokenization_spaces=True)\", \"Dataset({\\n features: ['label', 'title', 'content'],\\n num_rows: 400000\\n})\", '', 3000, 'The classic puzzle of the two guards. It presents a scenario where two guards are involved in a conflict of truths and lies. Here\\'s the setup:\\n\\nThere are two guards standing at two separate doors. One door leads to safety, while the other leads to danger. One guard always tells the truth, and the other guard always lies. You don\\'t know which guard is which or which door leads to safety.\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"excellent product! I love it!\". Content: \"I\\'m manager in a fast food restaurant, and this watch have been very helpful to me, I use it to set the break-meals of my employees in it, in a fast way. It\\'s very convenient because I have a lot of things to think about in my position, and thanks this watch I don\\'t have to worry about the breaks.\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Table hockey becomes friend maker\". Content: \"This game was a hit with my grandson! The little fella & his parents moved to Minnesota a week after receiving this game and this became his \"friend maker\". I appreciate the quick shipping. The game arrived in perfect condition.\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Perfect!!!\". Content: \"The highly anticipated dubbed version of the final season. Some of the original cast did not return for it but the replacement voice actors do a good job of staying true to the style of the characters that we\\'ve grown accustomed to. Can\\'t wait for set two!\"\\n\\n### Left Guard Response:\\nnegative\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Later than expected\". Content: \"This product arrived near the end of the timeframe designated for media mail. When it did arrived, it arrived as described. I bought a brand new book.\"\\n\\n### Left Guard Response:\\n', 'The classic puzzle of the two guards. It presents a scenario where two guards are involved in a conflict of truths and lies. Here\\'s the setup:\\n\\nThere are two guards standing at two separate doors. One door leads to safety, while the other leads to danger. One guard always tells the truth, and the other guard always lies. You don\\'t know which guard is which or which door leads to safety.\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"excellent product! I love it!\". Content: \"I\\'m manager in a fast food restaurant, and this watch have been very helpful to me, I use it to set the break-meals of my employees in it, in a fast way. It\\'s very convenient because I have a lot of things to think about in my position, and thanks this watch I don\\'t have to worry about the breaks.\"\\n\\n### Right Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Table hockey becomes friend maker\". Content: \"This game was a hit with my grandson! The little fella & his parents moved to Minnesota a week after receiving this game and this became his \"friend maker\". I appreciate the quick shipping. The game arrived in perfect condition.\"\\n\\n### Right Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Perfect!!!\". Content: \"The highly anticipated dubbed version of the final season. Some of the original cast did not return for it but the replacement voice actors do a good job of staying true to the style of the characters that we\\'ve grown accustomed to. Can\\'t wait for set two!\"\\n\\n### Right Guard Response:\\npositive\\n\\n### Instruction:\\nFollowing the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".\\n\\nTitle: \"Later than expected\". Content: \"This product arrived near the end of the timeframe designated for media mail. When it did arrived, it arrived as described. I bought a brand new book.\"\\n\\n### Right Guard Response:\\n']\n", + "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a3aeeb737b9148898b61bdd2945fe14f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "get hidden states: 0%| | 0/500 [00:00╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n", + "│ in <module>:1 │\n", + "│ │\n", + "│ ❱ 1 hss1 = dm.hs1 │\n", + "│ 2 hss2 = dm.hs2 │\n", + "│ 3 ans_1 = dm.ans1 │\n", + "│ 4 ans_2 = dm.ans2 │\n", + "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n", + "AttributeError: 'imdbHSDataModule' object has no attribute 'hs1'\n", + "\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 hss1 = dm.hs1 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mhss2 = dm.hs2 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mans_1 = dm.ans1 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mans_2 = dm.ans2 \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'hs1'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "hss1 = dm.hs1\n", "hss2 = dm.hs2\n", @@ -1068,9 +1333,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    1 # temp: balance everything in case we stopped early                                         │\n",
+       "│ ❱  2 print(len(infos), len(ans_1), len(ans_2))                                                   │\n",
+       "│    3 hss1 = hss1[:len(hss2)]                                                                     │\n",
+       "│    4 hss2 = hss2[:len(hss1)]                                                                     │\n",
+       "│    5 ans_1 = ans_1[:len(ans_2)]                                                                  │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'infos' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 1 \u001b[0m\u001b[2m# temp: balance everything in case we stopped early\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 2 \u001b[96mprint\u001b[0m(\u001b[96mlen\u001b[0m(infos), \u001b[96mlen\u001b[0m(ans_1), \u001b[96mlen\u001b[0m(ans_2)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0mhss1 = hss1[:\u001b[96mlen\u001b[0m(hss2)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mhss2 = hss2[:\u001b[96mlen\u001b[0m(hss1)] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0mans_1 = ans_1[:\u001b[96mlen\u001b[0m(ans_2)] \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'infos'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# temp: balance everything in case we stopped early\n", "print(len(infos), len(ans_1), len(ans_2))\n", @@ -1104,9 +1401,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱  1 acc=((ans_1>0.5)==df_infos2['true_answer']).mean()                                          │\n",
+       "│    2 print(f\"acc {acc:2.2f}\")                                                                    │\n",
+       "│    3                                                                                             │\n",
+       "│    4 d = df_infos2['lie']==True                                                                  │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'ans_1' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 1 acc=((ans_1>\u001b[94m0.5\u001b[0m)==df_infos2[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m]).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33macc \u001b[0m\u001b[33m{\u001b[0macc\u001b[33m:\u001b[0m\u001b[33m2.2f\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0md = df_infos2[\u001b[33m'\u001b[0m\u001b[33mlie\u001b[0m\u001b[33m'\u001b[0m]==\u001b[94mTrue\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'ans_1'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "acc=((ans_1>0.5)==df_infos2['true_answer']).mean()\n", "print(f\"acc {acc:2.2f}\")\n", @@ -1166,9 +1493,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱  1 n = len(df_infos2)                                                                          │\n",
+       "│    2                                                                                             │\n",
+       "│    3 # Define X and y                                                                            │\n",
+       "│    4 X = hss1-hss2                                                                               │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 1 n = \u001b[96mlen\u001b[0m(df_infos2) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m\u001b[2m# Define X and y\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mX = hss1-hss2 \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "\n", "n = len(df_infos2)\n", @@ -1196,9 +1553,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱  1 print(\"Logistic cls acc: {:2.2%} [TRAIN]\".format(lr.score(X_train2, y_train>0)))            │\n",
+       "│    2 print(\"Logistic cls acc: {:2.2%} [TEST]\".format(lr.score(X_test2, y_test>0)))               │\n",
+       "│    3                                                                                             │\n",
+       "│    4 m = df_infos2['lie'][n//2:]                                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'lr' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 1 \u001b[96mprint\u001b[0m(\u001b[33m\"\u001b[0m\u001b[33mLogistic cls acc: \u001b[0m\u001b[33m{:2.2%}\u001b[0m\u001b[33m [TRAIN]\u001b[0m\u001b[33m\"\u001b[0m.format(lr.score(X_train2, y_train>\u001b[94m0\u001b[0m))) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33m\"\u001b[0m\u001b[33mLogistic cls acc: \u001b[0m\u001b[33m{:2.2%}\u001b[0m\u001b[33m [TEST]\u001b[0m\u001b[33m\"\u001b[0m.format(lr.score(X_test2, y_test>\u001b[94m0\u001b[0m))) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mm = df_infos2[\u001b[33m'\u001b[0m\u001b[33mlie\u001b[0m\u001b[33m'\u001b[0m][n//\u001b[94m2\u001b[0m:] \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'lr'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "print(\"Logistic cls acc: {:2.2%} [TRAIN]\".format(lr.score(X_train2, y_train>0)))\n", "print(\"Logistic cls acc: {:2.2%} [TEST]\".format(lr.score(X_test2, y_test>0)))\n", @@ -1213,9 +1600,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_info_test = df_infos2.iloc[n//2:].copy()                                                  │\n",
+       "│   2 y_pred = lr.predict(X_test2)                                                                 │\n",
+       "│   3 df_info_test['inner_truth'] = y_pred                                                         │\n",
+       "│   4 df_info_test                                                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_info_test = df_infos2.iloc[n//\u001b[94m2\u001b[0m:].copy() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0my_pred = lr.predict(X_test2) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mdf_info_test[\u001b[33m'\u001b[0m\u001b[33minner_truth\u001b[0m\u001b[33m'\u001b[0m] = y_pred \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mdf_info_test \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "df_info_test = df_infos2.iloc[n//2:].copy()\n", "y_pred = lr.predict(X_test2)\n", @@ -1233,9 +1650,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 lie_pred = df_info_test['inner_truth']==df_info_test['true_answer']                          │\n",
+       "│   2 lie_true = df_info_test['lie']                                                               │\n",
+       "│   3 acc_lie = accuracy_score(lie_pred, lie_true)                                                 │\n",
+       "│   4 print(f\"model can detect lies with acc {acc_lie:2.2%}\")                                      │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_info_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 lie_pred = df_info_test[\u001b[33m'\u001b[0m\u001b[33minner_truth\u001b[0m\u001b[33m'\u001b[0m]==df_info_test[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mlie_true = df_info_test[\u001b[33m'\u001b[0m\u001b[33mlie\u001b[0m\u001b[33m'\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0macc_lie = accuracy_score(lie_pred, lie_true) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mmodel can detect lies with acc \u001b[0m\u001b[33m{\u001b[0macc_lie\u001b[33m:\u001b[0m\u001b[33m2.2%\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_info_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "lie_pred = df_info_test['inner_truth']==df_info_test['true_answer']\n", "lie_true = df_info_test['lie']\n", @@ -1253,9 +1700,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 bool_to_switch = lambda b:b*2-1                                                              │\n",
+       "│ ❱ 2 true_answer_switch = bool_to_switch(df_infos2['true_answer'])                                │\n",
+       "│   3 y = y_left_more_true = df_infos2['dir_true'] * true_answer_switch                            │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0mbool_to_switch = \u001b[94mlambda\u001b[0m b:b*\u001b[94m2\u001b[0m-\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 true_answer_switch = bool_to_switch(df_infos2[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0my = y_left_more_true = df_infos2[\u001b[33m'\u001b[0m\u001b[33mdir_true\u001b[0m\u001b[33m'\u001b[0m] * true_answer_switch \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "bool_to_switch = lambda b:b*2-1\n", "true_answer_switch = bool_to_switch(df_infos2['true_answer'])\n", @@ -1264,9 +1741,45 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:5                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    2 from sklearn.linear_model import ElasticNet                                                 │\n",
+       "│    3                                                                                             │\n",
+       "│    4 # Try a classification of direction                                                         │\n",
+       "│ ❱  5 n = len(df_infos2)                                                                          │\n",
+       "│    6                                                                                             │\n",
+       "│    7 # Define X and y                                                                            │\n",
+       "│    8 X = hss1-hss2                                                                               │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_infos2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m5\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m\u001b[94mfrom\u001b[0m \u001b[4;96msklearn\u001b[0m\u001b[4;96m.\u001b[0m\u001b[4;96mlinear_model\u001b[0m \u001b[94mimport\u001b[0m ElasticNet \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0m\u001b[2m# Try a classification of direction\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 5 n = \u001b[96mlen\u001b[0m(df_infos2) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 7 \u001b[0m\u001b[2m# Define X and y\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 8 \u001b[0mX = hss1-hss2 \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_infos2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# Try a regression\n", "from sklearn.linear_model import ElasticNet\n", @@ -1300,9 +1813,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 eps = 0.                                                                                     │\n",
+       "│ ❱ 2 acc=np.mean((lr2.predict(X_train2)>eps)==(y_train>eps))                                      │\n",
+       "│   3 print(f'acc from train ElasticNet {acc:2.2f}')                                               │\n",
+       "│   4 acc=np.mean((lr2.predict(X_test2)>eps)==(y_test>eps))                                        │\n",
+       "│   5 print(f'acc from test ElasticNet {acc:2.2f}')                                                │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'lr2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0meps = \u001b[94m0.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 acc=np.mean((lr2.predict(X_train2)>eps)==(y_train>eps)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m'\u001b[0m\u001b[33macc from train ElasticNet \u001b[0m\u001b[33m{\u001b[0macc\u001b[33m:\u001b[0m\u001b[33m2.2f\u001b[0m\u001b[33m}\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0macc=np.mean((lr2.predict(X_test2)>eps)==(y_test>eps)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m'\u001b[0m\u001b[33macc from test ElasticNet \u001b[0m\u001b[33m{\u001b[0macc\u001b[33m:\u001b[0m\u001b[33m2.2f\u001b[0m\u001b[33m}\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'lr2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "eps = 0.\n", "acc=np.mean((lr2.predict(X_train2)>eps)==(y_train>eps))\n", @@ -1313,9 +1858,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 y_test_pred = lr2.predict(X_test)                                                            │\n",
+       "│   2 plt.scatter(y_test, y_test_pred)                                                             │\n",
+       "│   3 plt.xlabel('true')                                                                           │\n",
+       "│   4 plt.ylabel('pred')                                                                           │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'lr2' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 y_test_pred = lr2.predict(X_test) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mplt.scatter(y_test, y_test_pred) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mplt.xlabel(\u001b[33m'\u001b[0m\u001b[33mtrue\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mplt.ylabel(\u001b[33m'\u001b[0m\u001b[33mpred\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'lr2'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "y_test_pred = lr2.predict(X_test)\n", "plt.scatter(y_test, y_test_pred)\n", @@ -1340,7 +1915,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -1374,7 +1949,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1418,7 +1993,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -1429,7 +2004,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -1536,7 +2111,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -1557,9 +2132,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    1 # split                                                                                     │\n",
+       "│ ❱  2 X = hss1-hss2                                                                               │\n",
+       "│    3 y = (df_infos2['true_answer'] == (df_infos2['dir_true']>0)).values # direction              │\n",
+       "│    4 n = len(y)                                                                                  │\n",
+       "│    5 print('split size', n//2)                                                                   │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'hss1' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 1 \u001b[0m\u001b[2m# split\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 2 X = hss1-hss2 \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 3 \u001b[0my = (df_infos2[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m] == (df_infos2[\u001b[33m'\u001b[0m\u001b[33mdir_true\u001b[0m\u001b[33m'\u001b[0m]>\u001b[94m0\u001b[0m)).values \u001b[2m# direction\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0mn = \u001b[96mlen\u001b[0m(y) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33m'\u001b[0m\u001b[33msplit size\u001b[0m\u001b[33m'\u001b[0m, n//\u001b[94m2\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'hss1'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# split\n", "X = hss1-hss2\n", @@ -1578,9 +2185,59 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 43, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 dl_train = dm.train_dataloader()                                                             │\n",
+       "│   2 dl_val = dm.val_dataloader()                                                                 │\n",
+       "│   3 b = next(iter(dl_train))                                                                     │\n",
+       "│   4 b                                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│ in train_dataloader:73                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│   70 │   │   │   │   │   │   │   │   │    torch.from_numpy(y_test).float())                      │\n",
+       "│   71 │                                                                                           │\n",
+       "│   72 │   def train_dataloader(self):                                                             │\n",
+       "│ ❱ 73 │   │   return DataLoader(self.ds_train,                                                    │\n",
+       "│   74 │   │   │   │   │   │     batch_size=self.hparams.dl_batch_size,                            │\n",
+       "│   75 │   │   │   │   │   │     shuffle=True)                                                     │\n",
+       "│   76                                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'ds_train'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 dl_train = dm.train_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mdl_val = dm.val_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mb = \u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl_train)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mb \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mtrain_dataloader\u001b[0m:\u001b[94m73\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m70 \u001b[0m\u001b[2m│ │ │ │ │ │ │ │ │ \u001b[0mtorch.from_numpy(y_test).float()) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m71 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m72 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mtrain_dataloader\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m73 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m DataLoader(\u001b[96mself\u001b[0m.ds_train, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m74 \u001b[0m\u001b[2m│ │ │ │ │ │ \u001b[0mbatch_size=\u001b[96mself\u001b[0m.hparams.dl_batch_size, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m75 \u001b[0m\u001b[2m│ │ │ │ │ │ \u001b[0mshuffle=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m76 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'ds_train'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "dl_train = dm.train_dataloader()\n", "dl_val = dm.val_dataloader()\n", @@ -1590,9 +2247,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 44, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 # init the model                                                                             │\n",
+       "│   2 max_epochs = 840                                                                             │\n",
+       "│ ❱ 3 d = b[0].shape[-1]                                                                           │\n",
+       "│   4 net = CSS(d=d, total_steps=max_epochs*len(dl_train), lr=5e-4, weight_decay=1e-7)             │\n",
+       "│   5                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'b' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[2m# init the model\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mmax_epochs = \u001b[94m840\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3 d = b[\u001b[94m0\u001b[0m].shape[-\u001b[94m1\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mnet = CSS(d=d, total_steps=max_epochs*\u001b[96mlen\u001b[0m(dl_train), lr=\u001b[94m5e-4\u001b[0m, weight_decay=\u001b[94m1e-7\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'b'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# init the model\n", "max_epochs = 840\n", @@ -1602,9 +2291,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 45, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:2                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 with torch.no_grad():                                                                        │\n",
+       "│ ❱ 2 │   b = next(iter(dl_train))                                                                 │\n",
+       "│   3 │   b2 = [bb.to(net.device) for bb in b]                                                     │\n",
+       "│   4 │   y = net(b2[0])                                                                           │\n",
+       "│   5 y                                                                                            │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'dl_train' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m2\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2 \u001b[2m│ \u001b[0mb = \u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl_train)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m\u001b[2m│ \u001b[0mb2 = [bb.to(net.device) \u001b[94mfor\u001b[0m bb \u001b[95min\u001b[0m b] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m\u001b[2m│ \u001b[0my = net(b2[\u001b[94m0\u001b[0m]) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5 \u001b[0my \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'dl_train'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "with torch.no_grad():\n", " b = next(iter(dl_train))\n", @@ -1615,9 +2336,51 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 46, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "IPU available: False, using: 0 IPUs\n", + "HPU available: False, using: 0 HPUs\n", + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/connectors/logger_connector/logger_connector.py:67: UserWarning: Starting from v1.9.0, `tensorboardX` has been removed as a dependency of the `lightning.pytorch` package, due to potential conflicts with other packages in the ML ecosystem. For this reason, `logger=True` will use `CSVLogger` as the default logger, unless the `tensorboard` or `tensorboardX` packages are found. Please `pip install lightning[extra]` or one of them to enable TensorBoard support by default\n", + " warning_cache.warn(\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 trainer = pl.Trainer(                                                                        │\n",
+       "│   2 │   │   │   │   │    max_epochs=max_epochs, log_every_n_steps=5)                             │\n",
+       "│ ❱ 3 trainer.fit(model=net, train_dataloaders=dl_train, val_dataloaders=dl_val)                   │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'net' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0mtrainer = pl.Trainer( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mmax_epochs=max_epochs, log_every_n_steps=\u001b[94m5\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3 trainer.fit(model=net, train_dataloaders=dl_train, val_dataloaders=dl_val) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'net'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "trainer = pl.Trainer(\n", " max_epochs=max_epochs, log_every_n_steps=5)\n", @@ -1633,9 +2396,222 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[]\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:26                                                                                   │\n",
+       "│                                                                                                  │\n",
+       "│   23 │   │   raise e                                                                             │\n",
+       "│   24                                                                                             │\n",
+       "│   25                                                                                             │\n",
+       "│ ❱ 26 df_hist = read_hist(trainer).ffill().bfill()                                                │\n",
+       "│   27 df_hist                                                                                     │\n",
+       "│   28                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:23                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│ ❱ 23 │   │   raise e                                                                             │\n",
+       "│   24                                                                                             │\n",
+       "│   25                                                                                             │\n",
+       "│   26 df_hist = read_hist(trainer).ffill().bfill()                                                │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:20                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   17 │   print(ts)                                                                               │\n",
+       "│   18 │   try:                                                                                    │\n",
+       "│   19 │   │   metrics_file_path = Path(ts[0].experiment.metrics_file_path)                        │\n",
+       "│ ❱ 20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│   23 │   │   raise e                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_metrics_csv:8                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│    5 import pandas as pd                                                                         │\n",
+       "│    6                                                                                             │\n",
+       "│    7 def read_metrics_csv(metrics_file_path):                                                    │\n",
+       "│ ❱  8 │   df_hist = pd.read_csv(metrics_file_path)                                                │\n",
+       "│    9 │   df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()                                             │\n",
+       "│   10 │   df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()                           │\n",
+       "│   11 │   return df_histe                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:912   │\n",
+       "│ in read_csv                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│    909 │   )                                                                                     │\n",
+       "│    910 │   kwds.update(kwds_defaults)                                                            │\n",
+       "│    911 │                                                                                         │\n",
+       "│ ❱  912 │   return _read(filepath_or_buffer, kwds)                                                │\n",
+       "│    913                                                                                           │\n",
+       "│    914                                                                                           │\n",
+       "│    915 # iterator=True -> TextFileReader                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:577   │\n",
+       "│ in _read                                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│    574 │   _validate_names(kwds.get(\"names\", None))                                              │\n",
+       "│    575 │                                                                                         │\n",
+       "│    576 │   # Create the parser.                                                                  │\n",
+       "│ ❱  577 │   parser = TextFileReader(filepath_or_buffer, **kwds)                                   │\n",
+       "│    578 │                                                                                         │\n",
+       "│    579 │   if chunksize or iterator:                                                             │\n",
+       "│    580 │   │   return parser                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1407  │\n",
+       "│ in __init__                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   1404 │   │   │   self.options[\"has_index_names\"] = kwds[\"has_index_names\"]                     │\n",
+       "│   1405 │   │                                                                                     │\n",
+       "│   1406 │   │   self.handles: IOHandles | None = None                                             │\n",
+       "│ ❱ 1407 │   │   self._engine = self._make_engine(f, self.engine)                                  │\n",
+       "│   1408 │                                                                                         │\n",
+       "│   1409 │   def close(self) -> None:                                                              │\n",
+       "│   1410 │   │   if self.handles is not None:                                                      │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1661  │\n",
+       "│ in _make_engine                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   1658 │   │   │   │   is_text = False                                                           │\n",
+       "│   1659 │   │   │   │   if \"b\" not in mode:                                                       │\n",
+       "│   1660 │   │   │   │   │   mode += \"b\"                                                           │\n",
+       "│ ❱ 1661 │   │   │   self.handles = get_handle(                                                    │\n",
+       "│   1662 │   │   │   │   f,                                                                        │\n",
+       "│   1663 │   │   │   │   mode,                                                                     │\n",
+       "│   1664 │   │   │   │   encoding=self.options.get(\"encoding\", None),                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/common.py:859 in         │\n",
+       "│ get_handle                                                                                       │\n",
+       "│                                                                                                  │\n",
+       "│    856 │   │   # Binary mode does not support 'encoding' and 'newline'.                          │\n",
+       "│    857 │   │   if ioargs.encoding and \"b\" not in ioargs.mode:                                    │\n",
+       "│    858 │   │   │   # Encoding                                                                    │\n",
+       "│ ❱  859 │   │   │   handle = open(                                                                │\n",
+       "│    860 │   │   │   │   handle,                                                                   │\n",
+       "│    861 │   │   │   │   ioargs.mode,                                                              │\n",
+       "│    862 │   │   │   │   encoding=ioargs.encoding,                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "FileNotFoundError: [Errno 2] No such file or directory: \n",
+       "'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_22/metrics.csv'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m26\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m23 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m26 df_hist = read_hist(trainer).ffill().bfill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m27 \u001b[0mdf_hist \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m28 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m23\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m20 \u001b[0m\u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m23 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m26 \u001b[0mdf_hist = read_hist(trainer).ffill().bfill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m20\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ \u001b[0m\u001b[96mprint\u001b[0m(ts) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m19 \u001b[0m\u001b[2m│ │ \u001b[0mmetrics_file_path = Path(ts[\u001b[94m0\u001b[0m].experiment.metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m20 \u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m23 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_metrics_csv\u001b[0m:\u001b[94m8\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0m\u001b[94mimport\u001b[0m \u001b[4;96mpandas\u001b[0m \u001b[94mas\u001b[0m \u001b[4;96mpd\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 7 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mread_metrics_csv\u001b[0m(metrics_file_path): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 8 \u001b[2m│ \u001b[0mdf_hist = pd.read_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 9 \u001b[0m\u001b[2m│ \u001b[0mdf_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m] = df_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m].ffill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m10 \u001b[0m\u001b[2m│ \u001b[0mdf_histe = df_hist.set_index(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).groupby(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m11 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m912\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_csv\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 909 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 910 \u001b[0m\u001b[2m│ \u001b[0mkwds.update(kwds_defaults) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 911 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 912 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m _read(filepath_or_buffer, kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 913 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 914 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 915 \u001b[0m\u001b[2m# iterator=True -> TextFileReader\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m577\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_read\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 574 \u001b[0m\u001b[2m│ \u001b[0m_validate_names(kwds.get(\u001b[33m\"\u001b[0m\u001b[33mnames\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 575 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 576 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create the parser.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 577 \u001b[2m│ \u001b[0mparser = TextFileReader(filepath_or_buffer, **kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 578 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 579 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m chunksize \u001b[95mor\u001b[0m iterator: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 580 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m parser \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1407\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1404 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.options[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] = kwds[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1405 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1406 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.handles: IOHandles | \u001b[94mNone\u001b[0m = \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1407 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._engine = \u001b[96mself\u001b[0m._make_engine(f, \u001b[96mself\u001b[0m.engine) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1408 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1409 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mclose\u001b[0m(\u001b[96mself\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1410 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.handles \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1661\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_make_engine\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1658 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mis_text = \u001b[94mFalse\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1659 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1660 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mmode += \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1661 \u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.handles = get_handle( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1662 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mf, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1663 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1664 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=\u001b[96mself\u001b[0m.options.get(\u001b[33m\"\u001b[0m\u001b[33mencoding\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m), \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/\u001b[0m\u001b[1;33mcommon.py\u001b[0m:\u001b[94m859\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mget_handle\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 856 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Binary mode does not support 'encoding' and 'newline'.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 857 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m ioargs.encoding \u001b[95mand\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m ioargs.mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 858 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# Encoding\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 859 \u001b[2m│ │ │ \u001b[0mhandle = \u001b[96mopen\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 860 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mhandle, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 861 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mioargs.mode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 862 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=ioargs.encoding, \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mFileNotFoundError: \u001b[0m\u001b[1m[\u001b[0mErrno \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m No such file or directory: \n", + "\u001b[32m'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_22/metrics.csv'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# import pytorch_lightning as pl\n", "from lightning.pytorch.loggers.csv_logs import CSVLogger\n", @@ -1668,7 +2644,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -1679,9 +2655,43 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 49, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│    1 # df_hist[['val/acc', 'train/acc']].plot()                                                  │\n",
+       "│    2                                                                                             │\n",
+       "│ ❱  3 df_hist[['val/f1', 'train/f1']].plot()                                                      │\n",
+       "│    4                                                                                             │\n",
+       "│    5 # df_hist[['val/roc_auc_bc', 'train/roc_auc_bc']].plot()                                    │\n",
+       "│    6                                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_hist' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 1 \u001b[0m\u001b[2m# df_hist[['val/acc', 'train/acc']].plot()\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 3 df_hist[[\u001b[33m'\u001b[0m\u001b[33mval/f1\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mtrain/f1\u001b[0m\u001b[33m'\u001b[0m]].plot() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0m\u001b[2m# df_hist[['val/roc_auc_bc', 'train/roc_auc_bc']].plot()\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_hist'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# df_hist[['val/acc', 'train/acc']].plot()\n", "\n", @@ -1703,9 +2713,55 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 dl_test = dm.test_dataloader()                                                               │\n",
+       "│   2 y_test_pred = trainer.predict(net, dl_test)                                                  │\n",
+       "│   3 y_test_pred = np.concatenate(y_test_pred)                                                    │\n",
+       "│   4 y_test_pred                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│ in test_dataloader:81                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   78 │   │   return DataLoader(self.ds_val, batch_size=self.hparams.dl_batch_size)               │\n",
+       "│   79 │                                                                                           │\n",
+       "│   80 │   def test_dataloader(self):                                                              │\n",
+       "│ ❱ 81 │   │   return DataLoader(self.ds_test, batch_size=self.hparams.dl_batch_size)              │\n",
+       "│   82                                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'ds_test'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 dl_test = dm.test_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0my_test_pred = trainer.predict(net, dl_test) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0my_test_pred = np.concatenate(y_test_pred) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0my_test_pred \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mtest_dataloader\u001b[0m:\u001b[94m81\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m78 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m DataLoader(\u001b[96mself\u001b[0m.ds_val, batch_size=\u001b[96mself\u001b[0m.hparams.dl_batch_size) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m79 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m80 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mtest_dataloader\u001b[0m(\u001b[96mself\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m81 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m DataLoader(\u001b[96mself\u001b[0m.ds_test, batch_size=\u001b[96mself\u001b[0m.hparams.dl_batch_size) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m82 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'ds_test'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "dl_test = dm.test_dataloader()\n", "y_test_pred = trainer.predict(net, dl_test)\n", @@ -1715,27 +2771,109 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 len(y_test_pred)                                                                             │\n",
+       "│   2                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'y_test_pred' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 \u001b[96mlen\u001b[0m(y_test_pred) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'y_test_pred'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "len(y_test_pred)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 len(dl_test.dataset)                                                                         │\n",
+       "│   2                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'dl_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 \u001b[96mlen\u001b[0m(dl_test.dataset) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'dl_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "len(dl_test.dataset)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_test = dm.df.iloc[dm.val_split:dm.test_split].copy()                                      │\n",
+       "│   2 df_test['pred'] = y_test_pred                                                                │\n",
+       "│   3 df_test                                                                                      │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'imdbHSDataModule' object has no attribute 'df'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_test = dm.df.iloc[dm.val_split:dm.test_split].copy() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mdf_test[\u001b[33m'\u001b[0m\u001b[33mpred\u001b[0m\u001b[33m'\u001b[0m] = y_test_pred \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mdf_test \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'imdbHSDataModule'\u001b[0m object has no attribute \u001b[32m'df'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "df_test = dm.df.iloc[dm.val_split:dm.test_split].copy()\n", "df_test['pred'] = y_test_pred\n", @@ -1744,9 +2882,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 acc_truth = (df_test['pred']==df_test['true_answer']).mean()                                 │\n",
+       "│   2 print(f\"lightning model acc at predicting inner truth: {acc_truth:2.2%}\")                    │\n",
+       "│   3                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_test' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 acc_truth = (df_test[\u001b[33m'\u001b[0m\u001b[33mpred\u001b[0m\u001b[33m'\u001b[0m]==df_test[\u001b[33m'\u001b[0m\u001b[33mtrue_answer\u001b[0m\u001b[33m'\u001b[0m]).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m\u001b[96mprint\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mlightning model acc at predicting inner truth: \u001b[0m\u001b[33m{\u001b[0macc_truth\u001b[33m:\u001b[0m\u001b[33m2.2%\u001b[0m\u001b[33m}\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_test'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "acc_truth = (df_test['pred']==df_test['true_answer']).mean()\n", "print(f\"lightning model acc at predicting inner truth: {acc_truth:2.2%}\")"