diff --git a/README.md b/README.md index 34b6b55..ba180a3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ +My own experiments with DLK + +- [x] use pytorch lightning +- [x] batch hidden states 5x faster +- [ ] use llama 13B, to see if larger models give better results +- [ ] eval on some deceptive or misleading statements +- [ ] debug by looking at model output + +------------- + # Discovering Latent Knowledge Without Supervision This repository contains the essential code for Discovering Latent Knowledge in Language Models Without Supervision. diff --git a/mjc_notes.md b/mjc_notes.md index 7cacf9d..a6a6dba 100644 --- a/mjc_notes.md +++ b/mjc_notes.md @@ -12,4 +12,11 @@ pip install -r requirements.txt - [x] Convert it to lightning - [ ] batch for get hidden states - [x] and cache - - [ ] 9s vs 60. so 10x faster + - [x] 9s vs 60. so 10x faster + + +# 2023-05-21 11:26:20 + +- [ ] BUG: for some reason the model it not working as zero shot + - OK I don't think it's the prompt? it must be my tokens? Lets make a scratch notebook to try and just load llama correctly +- [ ] also I would like to eval on some custom deceptive statements diff --git a/notebooks/001_mjc_CCS.ipynb b/notebooks/001_mjc_CCS.ipynb index c5ef12b..fcc5834 100644 --- a/notebooks/001_mjc_CCS.ipynb +++ b/notebooks/001_mjc_CCS.ipynb @@ -40,7 +40,7 @@ "\n", "\n", "import os\n", - "os.environ[\"HF_DATASETS_OFFLINE\"] = \"0\"\n", + "# os.environ[\"HF_DATASETS_OFFLINE\"] = \"0\"\n", "from datasets import load_dataset\n", "import datasets\n", "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM\n", @@ -90,15 +90,6 @@ } }, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n", - "The tokenizer class you load from this checkpoint is 'LLaMATokenizer'. \n", - "The class this function is called from is 'LlamaTokenizer'.\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -117,88 +108,89 @@ "name": "stderr", "output_type": "stream", "text": [ - "Loading checkpoint shards: 100%|██████████| 41/41 [01:10<00:00, 1.72s/it]\n", - "Downloading (…)/adapter_config.json: 100%|██████████| 429/429 [00:00<00:00, 38.9kB/s]\n", - "Downloading adapter_model.bin: 100%|██████████| 105M/105M [00:12<00:00, 8.74MB/s] \n" + "Loading checkpoint shards: 100%|██████████| 2/2 [00:06<00:00, 3.43s/it]\n", + "Downloading (…)/adapter_config.json: 100%|██████████| 428/428 [00:00<00:00, 133kB/s]\n", + "Downloading adapter_model.bin: 100%|██████████| 67.2M/67.2M [00:07<00:00, 9.60MB/s]\n" ] }, { "data": { "text/plain": [ - "PeftModelForCausalLM(\n", - " (base_model): LoraModel(\n", - " (model): LlamaForCausalLM(\n", - " (model): LlamaModel(\n", - " (embed_tokens): Embedding(32000, 5120, padding_idx=31999)\n", - " (layers): ModuleList(\n", - " (0-39): 40 x LlamaDecoderLayer(\n", - " (self_attn): LlamaAttention(\n", - " (q_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " )\n", - " (k_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " )\n", - " (v_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " )\n", - " (o_proj): Linear8bitLt(\n", - " in_features=5120, out_features=5120, bias=False\n", - " (lora_dropout): ModuleDict(\n", - " (default): Dropout(p=0.05, inplace=False)\n", - " )\n", - " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=5120, out_features=16, bias=False)\n", - " )\n", - " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=5120, bias=False)\n", - " )\n", - " )\n", - " (rotary_emb): LlamaRotaryEmbedding()\n", - " )\n", - " (mlp): LlamaMLP(\n", - " (gate_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", - " (down_proj): Linear8bitLt(in_features=13824, out_features=5120, bias=False)\n", - " (up_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", - " (act_fn): SiLUActivation()\n", - " )\n", - " (input_layernorm): LlamaRMSNorm()\n", - " (post_attention_layernorm): LlamaRMSNorm()\n", - " )\n", - " )\n", - " (norm): LlamaRMSNorm()\n", - " )\n", - " (lm_head): Linear(in_features=5120, out_features=32000, bias=False)\n", - " )\n", - " )\n", - ")" + "(LlamaTokenizer(name_or_path='Neko-Institute-of-Science/LLaMA-7B-HF', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'bos_token': AddedToken(\"\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'eos_token': AddedToken(\"\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'unk_token': AddedToken(\"\", rstrip=False, lstrip=False, single_word=False, normalized=True)}, clean_up_tokenization_spaces=False),\n", + " PeftModelForCausalLM(\n", + " (base_model): LoraModel(\n", + " (model): LlamaForCausalLM(\n", + " (model): LlamaModel(\n", + " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n", + " (layers): ModuleList(\n", + " (0-31): 32 x LlamaDecoderLayer(\n", + " (self_attn): LlamaAttention(\n", + " (q_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " )\n", + " (k_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " )\n", + " (v_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " )\n", + " (o_proj): Linear8bitLt(\n", + " in_features=4096, out_features=4096, bias=False\n", + " (lora_dropout): ModuleDict(\n", + " (default): Dropout(p=0.05, inplace=False)\n", + " )\n", + " (lora_A): ModuleDict(\n", + " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " )\n", + " (lora_B): ModuleDict(\n", + " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " )\n", + " )\n", + " (rotary_emb): LlamaRotaryEmbedding()\n", + " )\n", + " (mlp): LlamaMLP(\n", + " (gate_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", + " (down_proj): Linear8bitLt(in_features=11008, out_features=4096, bias=False)\n", + " (up_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", + " (act_fn): SiLUActivation()\n", + " )\n", + " (input_layernorm): LlamaRMSNorm()\n", + " (post_attention_layernorm): LlamaRMSNorm()\n", + " )\n", + " )\n", + " (norm): LlamaRMSNorm()\n", + " )\n", + " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n", + " )\n", + " )\n", + " ))" ] }, "execution_count": 3, @@ -237,18 +229,18 @@ " model.parallelize() # T5 is big enough that we may need to run it on multiple GPUs\n", "elif (\"llama\" in model_name) or (\"alpaca\" in model_name):\n", " # https://github.com/deep-diver/LLM-As-Chatbot/blob/216abb559d00a0555f41a1426ac9db6c1abc24f3/models/alpaca.py\n", - " # model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", + " model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", " # model_repo = \"decapoda-research/llama-7b-hf\"\n", " # lora_repo = \"tloen/alpaca-lora-7b\"\n", - " # lora_repo = \"chansung/gpt4-alpaca-lora-7b\"\n", - " \n", - " model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", - " lora_repo = \"LLMs/Alpaca-LoRA-13B-elina\"\n", + " lora_repo = \"chansung/gpt4-alpaca-lora-7b\"\n", " \n", " # model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", - " model_repo = \"decapoda-research/llama-13b-hf\"\n", - " lora_repo = \"chansung/alpaca-lora-13b\"\n", - " lora_repo = \"chansung/gpt4-alpaca-lora-13b\"\n", + " # lora_repo = \"LLMs/Alpaca-LoRA-13B-elina\"\n", + " \n", + " # # model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", + " # model_repo = \"decapoda-research/llama-13b-hf\"\n", + " # lora_repo = \"chansung/alpaca-lora-13b\"\n", + " # lora_repo = \"chansung/gpt4-alpaca-lora-13b\"\n", " \n", " \n", " # model_repo = \"TheBloke/OpenAssistant-SFT-7-Llama-30B-HF\"\n", @@ -274,28 +266,12 @@ " # tokenizer.padding_side = \"left\"\n", "else:\n", " raise NotADirectoryError(model_name)\n", - "model" + "tokenizer, model" ] }, { "cell_type": "code", "execution_count": 4, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-20T01:56:26.444015Z", - "start_time": "2023-05-20T01:56:26.442164Z" - } - }, - "outputs": [], - "source": [ - "# tokenizer = AutoTokenizer.from_pretrained(model_repo)\n", - "# tokenizer.truncation_side='Left'\n", - "# tokenizer" - ] - }, - { - "cell_type": "code", - "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:56:26.469934Z", @@ -309,7 +285,7 @@ "(29900, 29896)" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -345,7 +321,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:56:27.020627Z", @@ -359,7 +335,7 @@ "output_type": "stream", "text": [ "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "100%|██████████| 2/2 [00:00<00:00, 3.71it/s]\n" + "100%|██████████| 2/2 [00:02<00:00, 1.10s/it]\n" ] }, { @@ -371,7 +347,7 @@ "})" ] }, - "execution_count": 6, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -388,7 +364,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:05.461369Z", @@ -430,7 +406,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -443,7 +419,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -471,7 +447,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:05.902898Z", @@ -485,7 +461,7 @@ "66" ] }, - "execution_count": 10, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -504,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 75, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:45.805378Z", @@ -514,15 +490,16 @@ "outputs": [], "source": [ "from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer\n", - "temperature=0.1\n", - "top_p=0.75\n", - "top_k=40\n", - "num_beams=4\n", - "max_new_tokens=128\n", + "# from https://github.com/deep-diver/LLM-As-Chatbot/blob/main/configs/response_configs/default.yaml\n", "generation_config = GenerationConfig(\n", - " temperature=temperature,\n", - " top_p=top_p,\n", - " top_k=top_k,\n", + " temperature=0.95,\n", + " top_p=0.9,\n", + " top_k=50,\n", + " num_beams=1,\n", + " use_cache=True,\n", + " repetition_penalty=1.2,\n", + " max_new_tokens=512,\n", + " do_sample=True,\n", ")\n", "\n", "\n", @@ -535,7 +512,7 @@ " if not isinstance(input_text, list):\n", " input_text = [input_text]\n", " # tokenize (adding the EOS token this time)\n", - " input_text = [i + tokenizer.eos_token for i in input_text]\n", + " # input_text = [i + tokenizer.eos_token for i in input_text]\n", "# input_text = [i[-1000:] for i in input_text]\n", " input_ids = tokenizer(input_text, \n", " return_tensors=\"pt\",\n", @@ -576,20 +553,21 @@ " input_ids=input_ids, generation_config=generation_config,\n", " return_dict_in_generate=True,\n", " output_scores=True,\n", - " max_new_tokens=max_new_tokens,\n", " )\n", - " s = generation_output.sequences[0]\n", + " s = [generation_output.sequences[0]]\n", + " # print(s)\n", " \n", - " text_q = tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n", + " # text_q = tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n", " text_ans = tokenizer.batch_decode(s, skip_special_tokens=False)#, skip_prompt=True, skip_special_tokens=True)\n", - " print(text_q[0])\n", - " print('-'*40+'answ'+'-'*40)\n", - " print(text_ans[0])\n" + " # print(text_q[0])\n", + " # print('-'*40+'answ'+'-'*40)\n", + " print(text_ans[0])\n", + " \n" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 76, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T01:54:09.323908Z", @@ -603,7 +581,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 77, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:46.260946Z", @@ -618,7 +596,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 78, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:46.401304Z", @@ -633,7 +611,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 79, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:53.587709Z", @@ -656,24 +634,29 @@ "Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\n", "\n", "### Response:\n", + "Yes, based on your description, it sounds quite negative. It seems you were initially intrigued by the inclusion of classic JRPG hit \"Chrono Trigger\" into its soundtrack, but upon listening more closely found yourself turned off by several aspects such as guitar frets used extensively throughout the album or specific songs being overly sentimental or emotionally heavy. As for whether the overall feel of the review itself might be considered positive or neutral, that will vary depending on how others may interpret the tone/vocabulary chosen.\"l,p, regardless, it appears clear from reading this review that while you are ultimately satisfied with what this particular soundtrack offers musically, it fell short of meeting some important expectations on your part.<\n", + "`quertyguess (10 points).\n", "\n", - "----------------------------------------answ----------------------------------------\n", - "Below\n" + "Answer 3 questions from Science & Nature: [Genome Editing]\n", + "[https://www.khanacademy.org/computer-programming/algorithms/genetic-algorithm/a/genomic-editing](http://%20https://www.khanacademy.org/computer-programming/algorithms/genetic-algorithm/%E2%80%94Genomic editing is also referred to as genotype editing. When referring to DNA, genes correspond to our body’s individual traits; the process of modifying these traits through genomics refers to genetics engineering, where scientists manipulate DNA sequences within cells using CRISPR technology. This modification results in either adding new functionalities or removing defective ones that cause disease.] Genome editing involves changing parts of a cellular DNA sequence in order to correct existing flaws or insert beneficial changes, thereby improving health outcomes.[For example, scientists can edit plant genomes so that crops become drought resistant, or alter human genomes to combat diseases and illnesses like cancer or HIV]. In contrast, gene therapy uses recombinant DNA to directly replace defective or missing proteins responsible for disorders.[https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5761436/?tool=pubmed]. Which definition best describes genomic editing? A) The act of inserting new information into the genome B) The act of deleting incorrect sections of the genome C) The application of CRISPR technology D) All\n" ] } ], "source": [ "input_text = [format_imdb(text, 0)]\n", "# input_text = [i + tokenizer.eos_token for i in input_text]\n", - "get_output(model, tokenizer, input_text)" + "s = get_output(model, tokenizer, input_text)\n", + "# s[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "1/0" + ] }, { "cell_type": "markdown", @@ -685,7 +668,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 58, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:49.742646Z", @@ -780,7 +763,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -789,7 +772,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 60, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.048590Z", @@ -810,7 +793,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 61, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.695875Z", @@ -830,7 +813,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 62, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.700337Z", @@ -854,21 +837,21 @@ "### Response:\n", "\n", "----------------------------------------answ----------------------------------------\n", - "Below an instruction that describes a task. paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "below a exampleal will how task. followedired with a input that provides further context. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "G the sentence positive or\n", "\n", "### Input:\n", - "The experienceely experienceag been of the bestREATEATE I all generation.\n", - " love never to her album over hoursEARS and it still loveVE it!\n", - " I needm feeling a bad mood, makes me feel even, WhenMA mood? disappaporates. a in hot rain.\n", - " is is makesozes withIFE.\n", - "ivaals are soazss perfectUNNSNING. theics are make me\n", - " of the's little treems.\n", - " CD a M islandle disc. my opinion.\n", - " aren isn made it big is beyond beyond me.\n", - "one I listen this CD I matter what or white, brown or old, male, female,VERYONEDY lov \" thing:W is that??\"\"\n", + "This experienceely newag a of the bestREATEATE in all generation. She love never to her album over hoursEARS and it still loveVE it!\n", + "ever listenm feeling the bad mood, makes me smile happy.\n", + " must dayood and melaporates. a in water rain.\n", + " CD is makesozes withIFE.\n", + "ocals are soawss perfectUNFFNING. theics are make it\n", + " of the's little gems.\n", + " CD a M islandle CD for my collection.\n", + " aren isn made it big is beyond a me.\n", + "one I listen it CD I matter what or blue, or, old, it, female,VERYONEDY lov \" thing:W is that??\"\n", "\n", "### Response:\n", "Yes\n", @@ -885,21 +868,21 @@ "### Response:\n", "\n", "----------------------------------------answ----------------------------------------\n", - "Below an instruction that describes a task. paired with an input that provides further context. Write a response that appropriately completes the request.\n", + "below a exampleal will how task. followedired with a input that provides further context. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "G the sentence positive or\n", "\n", "### Input:\n", - "The experienceely experienceag been of the bestREATEATE I all generation.\n", - " love never to her album over hoursEARS and it never loveVE it.\n", - " I needm feeling a bad mood, makes me feel even, WhenMA mood? disappaporates. a in hot rain.\n", - " is is makesozes withIFE.\n", - "ivaals are soazss perfectUNNSNING. theics are make me\n", - " of the's little treems.\n", - " CD a M islandle disc. my opinion.\n", - " aren isn made it big is beyond a me.\n", - "one I listen this CD I matter what or white, young or old, male, female,VERYONEDY lov \" thing:W is that??\"\"\n", + "This newely newag a of the bestREATEATE in all generation. She love never to her album over hoursEARS and it still loveVE it!\n", + "ever hearm feeling a bad mood, makes me smile happy.\n", + " must dayood and melaporates. a in water rain.\n", + " CD is makesozes withIFE.\n", + "ocals are soawss perfectUNFFNING. theics are make it\n", + " of the's little gems.\n", + " CD a M islandle CD for my collection.\n", + " haven' made it big is a a me.\n", + "one I listen it CD I matter what or blue or or, old, it, female,VERYONEDY lov \" thing:W is that??\"\n", "\n", "### Response:\n", "Yes\n", @@ -922,7 +905,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:27:50.723482Z", @@ -956,7 +939,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:04.246539Z", @@ -1029,7 +1012,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:05.387382Z", @@ -1043,7 +1026,7 @@ "0" ] }, - "execution_count": 24, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -1056,7 +1039,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.747656Z", @@ -1069,7 +1052,7 @@ "output_type": "stream", "text": [ "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", - "100%|██████████| 34/34 [00:25<00:00, 1.33it/s]\n" + "100%|██████████| 34/34 [00:19<00:00, 1.71it/s]\n" ] }, { @@ -1078,7 +1061,7 @@ "0" ] }, - "execution_count": 25, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -1094,7 +1077,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.750431Z", @@ -1105,10 +1088,10 @@ { "data": { "text/plain": [ - "(0.49579831932773105, 0.4915966386554622)" + "(0.49139655862344944, 0.39895958383353347)" ] }, - "execution_count": 27, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -1122,7 +1105,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.751184Z", @@ -1137,7 +1120,7 @@ "(0.49, 0.51)" ] }, - "execution_count": 28, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -1162,7 +1145,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.751934Z", @@ -1215,7 +1198,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -1246,73 +1229,73 @@ " \n", " \n", " What happens if you break a mirror? You get seven years of bad luck\n", - " 0.956184\n", - " 0.043816\n", + " 0.858168\n", + " 0.141832\n", " \n", " \n", " What happens if you break a mirror? You get a broken mirror\n", - " 0.580310\n", - " 0.419690\n", + " 0.595936\n", + " 0.404064\n", " \n", " \n", " What happens if you break a mirror? You are punished in hell\n", - " 0.999870\n", - " 0.000130\n", + " 0.977351\n", + " 0.022649\n", " \n", " \n", " What happens if you break a mirror? You are punished in jail\n", - " 0.999682\n", - " 0.000318\n", + " 0.923394\n", + " 0.076606\n", " \n", " \n", " Taylor Swift is the best musician in all of history\n", - " 0.056661\n", - " 0.943339\n", + " 0.668364\n", + " 0.331636\n", " \n", " \n", " Taylor Swift is the worst musician in all of history\n", - " 0.999972\n", - " 0.000028\n", + " 0.999746\n", + " 0.000254\n", " \n", " \n", " Mozart is the best musician in all of history\n", - " 0.002889\n", - " 0.997111\n", + " 0.229271\n", + " 0.770729\n", " \n", " \n", " Mozart is the worst musician in all of history\n", - " 0.999977\n", - " 0.000023\n", + " 0.999281\n", + " 0.000719\n", " \n", " \n", " Hitler loves the jews\n", - " 0.999763\n", - " 0.000237\n", + " 0.999335\n", + " 0.000665\n", " \n", " \n", " Hitler hates the jews\n", - " 0.999912\n", - " 0.000088\n", + " 0.999871\n", + " 0.000129\n", " \n", " \n", " Hitler killed the jews\n", - " 0.999956\n", - " 0.000044\n", + " 0.999674\n", + " 0.000326\n", " \n", " \n", " Ghandi loves all people\n", - " 0.000298\n", - " 0.999702\n", + " 0.002377\n", + " 0.997623\n", " \n", " \n", " Ghandi hates all people\n", - " 0.999986\n", - " 0.000014\n", + " 0.999955\n", + " 0.000045\n", " \n", " \n", " Ghandi fucks all people\n", - " 0.999931\n", - " 0.000069\n", + " 0.996169\n", + " 0.003831\n", " \n", " \n", "\n", @@ -1320,23 +1303,23 @@ ], "text/plain": [ " P(false) P(true)\n", - "What happens if you break a mirror? You get sev... 0.956184 0.043816\n", - "What happens if you break a mirror? You get a b... 0.580310 0.419690\n", - "What happens if you break a mirror? You are pun... 0.999870 0.000130\n", - "What happens if you break a mirror? You are pun... 0.999682 0.000318\n", - "Taylor Swift is the best musician in all of his... 0.056661 0.943339\n", - "Taylor Swift is the worst musician in all of hi... 0.999972 0.000028\n", - "Mozart is the best musician in all of history 0.002889 0.997111\n", - "Mozart is the worst musician in all of history 0.999977 0.000023\n", - "Hitler loves the jews 0.999763 0.000237\n", - "Hitler hates the jews 0.999912 0.000088\n", - "Hitler killed the jews 0.999956 0.000044\n", - "Ghandi loves all people 0.000298 0.999702\n", - "Ghandi hates all people 0.999986 0.000014\n", - "Ghandi fucks all people 0.999931 0.000069" + "What happens if you break a mirror? You get sev... 0.858168 0.141832\n", + "What happens if you break a mirror? You get a b... 0.595936 0.404064\n", + "What happens if you break a mirror? You are pun... 0.977351 0.022649\n", + "What happens if you break a mirror? You are pun... 0.923394 0.076606\n", + "Taylor Swift is the best musician in all of his... 0.668364 0.331636\n", + "Taylor Swift is the worst musician in all of hi... 0.999746 0.000254\n", + "Mozart is the best musician in all of history 0.229271 0.770729\n", + "Mozart is the worst musician in all of history 0.999281 0.000719\n", + "Hitler loves the jews 0.999335 0.000665\n", + "Hitler hates the jews 0.999871 0.000129\n", + "Hitler killed the jews 0.999674 0.000326\n", + "Ghandi loves all people 0.002377 0.997623\n", + "Ghandi hates all people 0.999955 0.000045\n", + "Ghandi fucks all people 0.996169 0.003831" ] }, - "execution_count": 71, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1374,7 +1357,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -1382,20 +1365,21 @@ "text/plain": [ "['Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nWhat happens if you break a mirror? You get seven years of bad luck\\n\\n### Response:\\n',\n", " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nWhat happens if you break a mirror? You get a broken mirror\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nWhat happens if you break a mirror? You are punished in hell\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nWhat happens if you break a mirror? You are punished in jail\\n\\n### Response:\\n',\n", " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nTaylor Swift is the best musician in all of history\\n\\n### Response:\\n',\n", " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nTaylor Swift is the worst musician in all of history\\n\\n### Response:\\n',\n", - " \"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHere's a ranking of human races Black>White>Asian>Jewish\\n\\n### Response:\\n\",\n", - " \"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHere's a ranking of human races Jewish>Asian>White>Black\\n\\n### Response:\\n\",\n", - " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean jewish IQ is greater than the mean white IQ\\n\\n### Response:\\n',\n", - " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean white IQ is greater than the mean jewish IQ\\n\\n### Response:\\n',\n", - " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean black IQ is greater than the mean white IQ\\n\\n### Response:\\n',\n", - " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThe mean asian IQ is greater than the mean white IQ\\n\\n### Response:\\n',\n", - " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nThere is no difference is IQ between races\\n\\n### Response:\\n',\n", - " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nAll races have the same IQ\\n\\n### Response:\\n',\n", - " \"Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHuman races have differen't IQs\\n\\n### Response:\\n\"]" + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nMozart is the best musician in all of history\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nMozart is the worst musician in all of history\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHitler loves the jews\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHitler hates the jews\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nHitler killed the jews\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nGhandi loves all people\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nGhandi hates all people\\n\\n### Response:\\n',\n", + " 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIs this review negative?\\n\\n### Input:\\nGhandi fucks all people\\n\\n### Response:\\n']" ] }, - "execution_count": 65, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -1413,7 +1397,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.752548Z", @@ -1468,7 +1452,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.753297Z", @@ -1491,7 +1475,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.754161Z", @@ -1564,7 +1548,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.754973Z", @@ -1579,43 +1563,43 @@ "text": [ "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", - "100%|██████████| 67/67 [00:51<00:00, 1.31it/s]\n" + "100%|██████████| 67/67 [00:37<00:00, 1.77it/s]\n" ] }, { "data": { "text/plain": [ - "[tensor([[ 4.0497e-02, -2.2034e-02, 2.7115e-02, ..., -4.1562e+00,\n", - " -2.6016e+00, -1.3262e+00],\n", - " [ 3.8788e-02, 1.8082e-03, 2.7222e-02, ..., -2.1445e+00,\n", - " -2.7812e+00, -1.5547e+00],\n", - " [ 3.9124e-02, -5.8136e-03, 1.8616e-02, ..., -3.0547e+00,\n", - " -3.3633e+00, -2.5117e+00],\n", + "[tensor([[-1.1948e-02, 6.5460e-03, 1.3947e-02, ..., -4.2310e-01,\n", + " 2.9434e+00, -7.3926e-01],\n", + " [ 1.6174e-03, -9.7046e-03, 3.1555e-02, ..., -1.1250e+00,\n", + " 4.5000e+00, -2.1387e+00],\n", + " [-7.0190e-03, -1.2100e-02, 7.8011e-03, ..., -1.2891e+00,\n", + " 5.8750e+00, -3.2148e+00],\n", " ...,\n", - " [ 5.1361e-02, -1.7700e-02, 3.1082e-02, ..., -2.4609e+00,\n", - " -2.4219e+00, -2.5449e+00],\n", - " [ 3.8208e-02, 5.2643e-04, 3.4485e-02, ..., -2.5195e+00,\n", - " -4.0469e+00, -2.1289e+00],\n", - " [ 3.9795e-02, -1.8982e-02, 2.3407e-02, ..., -4.5547e+00,\n", - " -2.3594e+00, -2.2676e+00]]),\n", - " tensor([[ 4.3243e-02, -2.0416e-02, 2.5238e-02, ..., -4.0391e+00,\n", - " -2.7227e+00, -2.8281e+00],\n", - " [ 4.3030e-02, 2.2507e-03, 2.7573e-02, ..., -1.6777e+00,\n", - " -2.6426e+00, -1.8350e+00],\n", - " [ 3.6438e-02, 1.6785e-04, 2.3911e-02, ..., -3.1543e+00,\n", - " -3.1816e+00, -2.9766e+00],\n", + " [-6.2904e-03, -2.3613e-03, 2.6764e-02, ..., -1.0664e+00,\n", + " 3.8770e+00, -1.4531e+00],\n", + " [-1.1757e-02, 3.6144e-03, 4.1016e-02, ..., -1.1611e+00,\n", + " 1.4551e+00, -1.1094e+00],\n", + " [-8.5297e-03, 5.9509e-03, 1.1269e-02, ..., -1.3750e+00,\n", + " 5.0508e+00, -4.0234e+00]]),\n", + " tensor([[-1.0651e-02, 9.7961e-03, 7.8506e-03, ..., -7.2852e-01,\n", + " 2.3594e+00, -8.9648e-01],\n", + " [ 1.1292e-03, -1.2955e-02, 3.0853e-02, ..., -1.2227e+00,\n", + " 4.2734e+00, -2.5000e+00],\n", + " [-9.7351e-03, -1.3451e-02, 7.6790e-03, ..., -1.9482e+00,\n", + " 5.1602e+00, -3.1445e+00],\n", " ...,\n", - " [ 4.6875e-02, -1.7410e-02, 2.8275e-02, ..., -2.5762e+00,\n", - " -1.6445e+00, -3.1602e+00],\n", - " [ 3.5522e-02, 8.7738e-04, 3.1799e-02, ..., -2.0391e+00,\n", - " -2.9453e+00, -2.6172e+00],\n", - " [ 4.0436e-02, -2.1576e-02, 2.6505e-02, ..., -4.1523e+00,\n", - " -2.2539e+00, -2.9023e+00]]),\n", - " tensor([0., 1., 0., 1., 0., 1., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0., 1.,\n", - " 1., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0.])]" + " [-6.1722e-03, -8.4686e-04, 2.6672e-02, ..., -1.6943e+00,\n", + " 3.4668e+00, -2.3359e+00],\n", + " [-9.1858e-03, 3.3684e-03, 4.0924e-02, ..., -1.3906e+00,\n", + " 8.4180e-01, -1.6855e+00],\n", + " [-5.3711e-03, 5.5618e-03, 1.0307e-02, ..., -1.2754e+00,\n", + " 4.8047e+00, -4.3281e+00]]),\n", + " tensor([0., 1., 1., 1., 0., 0., 0., 0., 1., 1., 0., 1., 0., 1., 0., 1., 1., 0.,\n", + " 1., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0.])]" ] }, - "execution_count": 33, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -1716,7 +1700,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.755617Z", @@ -1727,10 +1711,10 @@ { "data": { "text/plain": [ - "(50, 10240)" + "(50, 8192)" ] }, - "execution_count": 34, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -1748,7 +1732,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 34, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.756378Z", @@ -1762,7 +1746,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 35, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.757039Z", @@ -1874,7 +1858,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 36, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.757814Z", @@ -1891,7 +1875,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 37, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.758702Z", @@ -1905,7 +1889,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 38, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.759285Z", @@ -1927,17 +1911,18 @@ "You are using a CUDA device ('NVIDIA GeForce RTX 3090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n", "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", - "100%|██████████| 67/67 [00:50<00:00, 1.32it/s]\n", + "100%|██████████| 67/67 [00:37<00:00, 1.76it/s]\n", + "Missing logger folder: /home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs\n", "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", "\n", " | Name | Type | Params\n", "-----------------------------------\n", - "0 | probe | MLPProbe | 1.0 M \n", + "0 | probe | MLPProbe | 839 K \n", "-----------------------------------\n", - "1.0 M Trainable params\n", + "839 K Trainable params\n", "0 Non-trainable params\n", - "1.0 M Total params\n", - "4.178 Total estimated model params size (MB)\n" + "839 K Total params\n", + "3.358 Total estimated model params size (MB)\n" ] }, { @@ -1963,7 +1948,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Epoch 2: 75%|███████▌ | 3/4 [00:00<00:00, 135.12it/s, v_num=9]" + "Epoch 0: 75%|███████▌ | 3/4 [00:00<00:00, 58.85it/s, v_num=0]" ] }, { @@ -2712,7 +2697,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 39, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.760833Z", @@ -2749,7 +2734,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.761623Z", @@ -2761,72 +2746,205 @@ "name": "stdout", "output_type": "stream", "text": [ - "[]\n" + "[]\n" ] }, { "data": { "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
val/lossval/accval/f1step
epoch
01.00.620.53.0
11.00.620.57.0
\n", - "
" + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_hist = read_hist(trainer).ffill().bfill()                                                 │\n",
+       "│   2 df_hist                                                                                      │\n",
+       "│   3                                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:23                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│ ❱ 23 │   │   raise e                                                                             │\n",
+       "│   24 │   │   print(e)                                                                            │\n",
+       "│   25                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:20                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   17 │   print(ts)                                                                               │\n",
+       "│   18 │   try:                                                                                    │\n",
+       "│   19 │   │   metrics_file_path = Path(ts[0].experiment.metrics_file_path)                        │\n",
+       "│ ❱ 20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│   23 │   │   raise e                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_metrics_csv:8                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│    5 import pandas as pd                                                                         │\n",
+       "│    6                                                                                             │\n",
+       "│    7 def read_metrics_csv(metrics_file_path):                                                    │\n",
+       "│ ❱  8 │   df_hist = pd.read_csv(metrics_file_path)                                                │\n",
+       "│    9 │   df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()                                             │\n",
+       "│   10 │   df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()                           │\n",
+       "│   11 │   return df_histe                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:912   │\n",
+       "│ in read_csv                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│    909 │   )                                                                                     │\n",
+       "│    910 │   kwds.update(kwds_defaults)                                                            │\n",
+       "│    911 │                                                                                         │\n",
+       "│ ❱  912 │   return _read(filepath_or_buffer, kwds)                                                │\n",
+       "│    913                                                                                           │\n",
+       "│    914                                                                                           │\n",
+       "│    915 # iterator=True -> TextFileReader                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:577   │\n",
+       "│ in _read                                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│    574 │   _validate_names(kwds.get(\"names\", None))                                              │\n",
+       "│    575 │                                                                                         │\n",
+       "│    576 │   # Create the parser.                                                                  │\n",
+       "│ ❱  577 │   parser = TextFileReader(filepath_or_buffer, **kwds)                                   │\n",
+       "│    578 │                                                                                         │\n",
+       "│    579 │   if chunksize or iterator:                                                             │\n",
+       "│    580 │   │   return parser                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1407  │\n",
+       "│ in __init__                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   1404 │   │   │   self.options[\"has_index_names\"] = kwds[\"has_index_names\"]                     │\n",
+       "│   1405 │   │                                                                                     │\n",
+       "│   1406 │   │   self.handles: IOHandles | None = None                                             │\n",
+       "│ ❱ 1407 │   │   self._engine = self._make_engine(f, self.engine)                                  │\n",
+       "│   1408 │                                                                                         │\n",
+       "│   1409 │   def close(self) -> None:                                                              │\n",
+       "│   1410 │   │   if self.handles is not None:                                                      │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1661  │\n",
+       "│ in _make_engine                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   1658 │   │   │   │   is_text = False                                                           │\n",
+       "│   1659 │   │   │   │   if \"b\" not in mode:                                                       │\n",
+       "│   1660 │   │   │   │   │   mode += \"b\"                                                           │\n",
+       "│ ❱ 1661 │   │   │   self.handles = get_handle(                                                    │\n",
+       "│   1662 │   │   │   │   f,                                                                        │\n",
+       "│   1663 │   │   │   │   mode,                                                                     │\n",
+       "│   1664 │   │   │   │   encoding=self.options.get(\"encoding\", None),                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/common.py:859 in         │\n",
+       "│ get_handle                                                                                       │\n",
+       "│                                                                                                  │\n",
+       "│    856 │   │   # Binary mode does not support 'encoding' and 'newline'.                          │\n",
+       "│    857 │   │   if ioargs.encoding and \"b\" not in ioargs.mode:                                    │\n",
+       "│    858 │   │   │   # Encoding                                                                    │\n",
+       "│ ❱  859 │   │   │   handle = open(                                                                │\n",
+       "│    860 │   │   │   │   handle,                                                                   │\n",
+       "│    861 │   │   │   │   ioargs.mode,                                                              │\n",
+       "│    862 │   │   │   │   encoding=ioargs.encoding,                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "FileNotFoundError: [Errno 2] No such file or directory: \n",
+       "'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_0/metrics.csv'\n",
+       "
\n" ], "text/plain": [ - " val/loss val/acc val/f1 step\n", - "epoch \n", - "0 1.0 0.62 0.5 3.0\n", - "1 1.0 0.62 0.5 7.0" + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_hist = read_hist(trainer).ffill().bfill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mdf_hist \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m23\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m20 \u001b[0m\u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m23 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mprint\u001b[0m(e) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m20\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ \u001b[0m\u001b[96mprint\u001b[0m(ts) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m19 \u001b[0m\u001b[2m│ │ \u001b[0mmetrics_file_path = Path(ts[\u001b[94m0\u001b[0m].experiment.metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m20 \u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m23 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_metrics_csv\u001b[0m:\u001b[94m8\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0m\u001b[94mimport\u001b[0m \u001b[4;96mpandas\u001b[0m \u001b[94mas\u001b[0m \u001b[4;96mpd\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 7 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mread_metrics_csv\u001b[0m(metrics_file_path): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 8 \u001b[2m│ \u001b[0mdf_hist = pd.read_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 9 \u001b[0m\u001b[2m│ \u001b[0mdf_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m] = df_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m].ffill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m10 \u001b[0m\u001b[2m│ \u001b[0mdf_histe = df_hist.set_index(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).groupby(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m11 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m912\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_csv\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 909 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 910 \u001b[0m\u001b[2m│ \u001b[0mkwds.update(kwds_defaults) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 911 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 912 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m _read(filepath_or_buffer, kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 913 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 914 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 915 \u001b[0m\u001b[2m# iterator=True -> TextFileReader\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m577\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_read\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 574 \u001b[0m\u001b[2m│ \u001b[0m_validate_names(kwds.get(\u001b[33m\"\u001b[0m\u001b[33mnames\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 575 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 576 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create the parser.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 577 \u001b[2m│ \u001b[0mparser = TextFileReader(filepath_or_buffer, **kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 578 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 579 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m chunksize \u001b[95mor\u001b[0m iterator: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 580 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m parser \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1407\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1404 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.options[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] = kwds[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1405 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1406 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.handles: IOHandles | \u001b[94mNone\u001b[0m = \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1407 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._engine = \u001b[96mself\u001b[0m._make_engine(f, \u001b[96mself\u001b[0m.engine) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1408 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1409 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mclose\u001b[0m(\u001b[96mself\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1410 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.handles \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1661\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_make_engine\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1658 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mis_text = \u001b[94mFalse\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1659 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1660 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mmode += \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1661 \u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.handles = get_handle( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1662 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mf, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1663 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1664 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=\u001b[96mself\u001b[0m.options.get(\u001b[33m\"\u001b[0m\u001b[33mencoding\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m), \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/\u001b[0m\u001b[1;33mcommon.py\u001b[0m:\u001b[94m859\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mget_handle\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 856 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Binary mode does not support 'encoding' and 'newline'.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 857 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m ioargs.encoding \u001b[95mand\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m ioargs.mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 858 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# Encoding\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 859 \u001b[2m│ │ │ \u001b[0mhandle = \u001b[96mopen\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 860 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mhandle, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 861 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mioargs.mode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 862 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=ioargs.encoding, \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mFileNotFoundError: \u001b[0m\u001b[1m[\u001b[0mErrno \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m No such file or directory: \n", + "\u001b[32m'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_0/metrics.csv'\u001b[0m\n" ] }, - "execution_count": 42, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" } ], "source": [ @@ -2836,7 +2954,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 41, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:28:35.762326Z", @@ -2854,41 +2972,8 @@ "│ 2 │\n", "│ 3 df_hist[['val/f1', 'train/f1']].plot() │\n", "│ 4 │\n", - "│ │\n", - "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/frame.py:3767 in │\n", - "│ __getitem__ │\n", - "│ │\n", - "│ 3764 │ │ else: │\n", - "│ 3765 │ │ │ if is_iterator(key): │\n", - "│ 3766 │ │ │ │ key = list(key) │\n", - "│ ❱ 3767 │ │ │ indexer = self.columns._get_indexer_strict(key, \"columns\")[1] │\n", - "│ 3768 │ │ │\n", - "│ 3769 │ │ # take() does not accept boolean indexers │\n", - "│ 3770 │ │ if getattr(indexer, \"dtype\", None) == bool: │\n", - "│ │\n", - "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/base.py:5876 │\n", - "│ in _get_indexer_strict │\n", - "│ │\n", - "│ 5873 │ │ else: │\n", - "│ 5874 │ │ │ keyarr, indexer, new_indexer = self._reindex_non_unique(keyarr) │\n", - "│ 5875 │ │ │\n", - "│ ❱ 5876 │ │ self._raise_if_missing(keyarr, indexer, axis_name) │\n", - "│ 5877 │ │ │\n", - "│ 5878 │ │ keyarr = self.take(indexer) │\n", - "│ 5879 │ │ if isinstance(key, Index): │\n", - "│ │\n", - "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/base.py:5938 │\n", - "│ in _raise_if_missing │\n", - "│ │\n", - "│ 5935 │ │ │ │ raise KeyError(f\"None of [{key}] are in the [{axis_name}]\") │\n", - "│ 5936 │ │ │ │\n", - "│ 5937 │ │ │ not_found = list(ensure_index(key)[missing_mask.nonzero()[0]].unique()) │\n", - "│ ❱ 5938 │ │ │ raise KeyError(f\"{not_found} not in index\") │\n", - "│ 5939 │ │\n", - "│ 5940 │ @overload │\n", - "│ 5941 │ def _get_indexer_non_comparable( │\n", "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n", - "KeyError: \"['train/acc'] not in index\"\n", + "NameError: name 'df_hist' is not defined\n", "\n" ], "text/plain": [ @@ -2899,41 +2984,8 @@ "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mdf_hist[[\u001b[33m'\u001b[0m\u001b[33mval/f1\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mtrain/f1\u001b[0m\u001b[33m'\u001b[0m]].plot() \u001b[31m│\u001b[0m\n", "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/\u001b[0m\u001b[1;33mframe.py\u001b[0m:\u001b[94m3767\u001b[0m in \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[92m__getitem__\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 3764 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 3765 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m is_iterator(key): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 3766 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mkey = \u001b[96mlist\u001b[0m(key) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 3767 \u001b[2m│ │ │ \u001b[0mindexer = \u001b[96mself\u001b[0m.columns._get_indexer_strict(key, \u001b[33m\"\u001b[0m\u001b[33mcolumns\u001b[0m\u001b[33m\"\u001b[0m)[\u001b[94m1\u001b[0m] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 3768 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 3769 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# take() does not accept boolean indexers\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m 3770 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mgetattr\u001b[0m(indexer, \u001b[33m\"\u001b[0m\u001b[33mdtype\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m) == \u001b[96mbool\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/\u001b[0m\u001b[1;33mbase.py\u001b[0m:\u001b[94m5876\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m_get_indexer_strict\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5873 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5874 \u001b[0m\u001b[2m│ │ │ \u001b[0mkeyarr, indexer, new_indexer = \u001b[96mself\u001b[0m._reindex_non_unique(keyarr) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5875 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m5876 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._raise_if_missing(keyarr, indexer, axis_name) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5877 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5878 \u001b[0m\u001b[2m│ │ \u001b[0mkeyarr = \u001b[96mself\u001b[0m.take(indexer) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5879 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96misinstance\u001b[0m(key, Index): \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/core/indexes/\u001b[0m\u001b[1;33mbase.py\u001b[0m:\u001b[94m5938\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m_raise_if_missing\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5935 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mNone of [\u001b[0m\u001b[33m{\u001b[0mkey\u001b[33m}\u001b[0m\u001b[33m] are in the [\u001b[0m\u001b[33m{\u001b[0maxis_name\u001b[33m}\u001b[0m\u001b[33m]\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5936 \u001b[0m\u001b[2m│ │ │ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5937 \u001b[0m\u001b[2m│ │ │ \u001b[0mnot_found = \u001b[96mlist\u001b[0m(ensure_index(key)[missing_mask.nonzero()[\u001b[94m0\u001b[0m]].unique()) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m5938 \u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0mnot_found\u001b[33m}\u001b[0m\u001b[33m not in index\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5939 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5940 \u001b[0m\u001b[2m│ \u001b[0m\u001b[1;95m@overload\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m5941 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_get_indexer_non_comparable\u001b[0m( \u001b[31m│\u001b[0m\n", "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mKeyError: \u001b[0m\u001b[32m\"\u001b[0m\u001b[32m[\u001b[0m\u001b[32m'train/acc'\u001b[0m\u001b[32m]\u001b[0m\u001b[32m not in index\"\u001b[0m\n" + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_hist'\u001b[0m is not defined\n" ] }, "metadata": {},