diff --git a/001_mjc_CCS.ipynb b/001_mjc_CCS.ipynb index f3d130c..36ad1be 100644 --- a/001_mjc_CCS.ipynb +++ b/001_mjc_CCS.ipynb @@ -22,7 +22,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] } @@ -36,7 +36,7 @@ "import torch.nn.functional as F\n", "\n", "import os\n", - "os.environ[\"HF_DATASETS_OFFLINE\"] = \"1\"\n", + "os.environ[\"HF_DATASETS_OFFLINE\"] = \"0\"\n", "from datasets import load_dataset\n", "import datasets\n", "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM\n", @@ -76,18 +76,6 @@ "from transformers import LlamaForCausalLM, LlamaTokenizer" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-20T01:54:27.257168Z", - "start_time": "2023-05-20T01:54:27.254862Z" - } - }, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": 3, @@ -110,37 +98,22 @@ "\n", " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", "================================================================================\n", - "bin /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n", - "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n", - "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n", - "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "bin /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n", + "CUDA SETUP: CUDA runtime path found: /home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n", "CUDA SETUP: Detected CUDA version 117\n", - "CUDA SETUP: Loading binary /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + "CUDA SETUP: Loading binary /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: /home/wassname/miniforge3/envs/jupyter2 did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n", - " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/cinnamon.mandatory.path')}\n", - " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/cinnamon.default.path')}\n", - " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('local/wassname-fractal-desktop'), PosixPath('@/tmp/.ICE-unix/5335,unix/wassname-fractal-desktop')}\n", - " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('0'), PosixPath('1')}\n", - " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/etc/xdg/xdg-cinnamon')}\n", - " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//matplotlib_inline.backend_inline'), PosixPath('module')}\n", - " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n", + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so.11.0'), PosixPath('/home/ubuntu/mambaforge/envs/dlk2/lib/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n", "Either way, this might cause trouble in the future:\n", "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", " warn(msg)\n", - "Loading checkpoint shards: 100%|██████████████| 2/2 [00:07<00:00, 3.78s/it]\n" + "Loading checkpoint shards: 100%|██████████| 3/3 [00:13<00:00, 4.35s/it]\n" ] }, { @@ -150,62 +123,62 @@ " (base_model): LoraModel(\n", " (model): LlamaForCausalLM(\n", " (model): LlamaModel(\n", - " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n", + " (embed_tokens): Embedding(32000, 5120, padding_idx=0)\n", " (layers): ModuleList(\n", - " (0-31): 32 x LlamaDecoderLayer(\n", + " (0-39): 40 x LlamaDecoderLayer(\n", " (self_attn): LlamaAttention(\n", " (q_proj): Linear8bitLt(\n", - " in_features=4096, out_features=4096, bias=False\n", + " in_features=5120, out_features=5120, bias=False\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.05, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (k_proj): Linear8bitLt(\n", - " in_features=4096, out_features=4096, bias=False\n", + " in_features=5120, out_features=5120, bias=False\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.05, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (v_proj): Linear8bitLt(\n", - " in_features=4096, out_features=4096, bias=False\n", + " in_features=5120, out_features=5120, bias=False\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.05, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (o_proj): Linear8bitLt(\n", - " in_features=4096, out_features=4096, bias=False\n", + " in_features=5120, out_features=5120, bias=False\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.05, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", - " (default): Linear(in_features=4096, out_features=16, bias=False)\n", + " (default): Linear(in_features=5120, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", - " (default): Linear(in_features=16, out_features=4096, bias=False)\n", + " (default): Linear(in_features=16, out_features=5120, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", @@ -213,9 +186,9 @@ " (rotary_emb): LlamaRotaryEmbedding()\n", " )\n", " (mlp): LlamaMLP(\n", - " (gate_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", - " (down_proj): Linear8bitLt(in_features=11008, out_features=4096, bias=False)\n", - " (up_proj): Linear8bitLt(in_features=4096, out_features=11008, bias=False)\n", + " (gate_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", + " (down_proj): Linear8bitLt(in_features=13824, out_features=5120, bias=False)\n", + " (up_proj): Linear8bitLt(in_features=5120, out_features=13824, bias=False)\n", " (act_fn): SiLUActivation()\n", " )\n", " (input_layernorm): LlamaRMSNorm()\n", @@ -224,7 +197,7 @@ " )\n", " (norm): LlamaRMSNorm()\n", " )\n", - " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n", + " (lm_head): Linear(in_features=5120, out_features=32000, bias=False)\n", " )\n", " )\n", ")" @@ -266,17 +239,22 @@ " model.parallelize() # T5 is big enough that we may need to run it on multiple GPUs\n", "elif (\"llama\" in model_name) or (\"alpaca\" in model_name):\n", " # https://github.com/deep-diver/LLM-As-Chatbot/blob/216abb559d00a0555f41a1426ac9db6c1abc24f3/models/alpaca.py\n", - " model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", - " lora_repo = \"tloen/alpaca-lora-7b\"\n", + " # model_repo = \"Neko-Institute-of-Science/LLaMA-7B-HF\"\n", + " # lora_repo = \"tloen/alpaca-lora-7b\"\n", + " \n", + " model_repo = \"Neko-Institute-of-Science/LLaMA-13B-HF\"\n", + " lora_repo = \"chansung/alpaca-lora-13b\"\n", " model_type = \"decoder\"\n", " tokenizer = LlamaTokenizer.from_pretrained(model_repo)\n", " model = LlamaForCausalLM.from_pretrained(model_repo, **model_options)\n", " \n", " if \"alpaca\" in model_name:\n", + " # https://github.com/tloen/alpaca-lora/blob/main/generate.py#L40\n", " from peft import PeftModel\n", " model = PeftModel.from_pretrained(\n", " model, \n", " lora_repo, \n", + " torch_dtype=torch.float16,\n", " device_map='auto'#{'': 0}\n", " )\n", " \n", @@ -368,9 +346,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "Using the latest cached version of the module from /home/wassname/.cache/huggingface/modules/datasets_modules/datasets/amazon_polarity/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc (last modified on Sat May 6 07:52:43 2023) since it couldn't be found locally at amazon_polarity.\n", - "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "100%|█████████████████████████████████████████| 2/2 [00:00<00:00, 3.96it/s]\n" + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "100%|██████████| 2/2 [00:00<00:00, 21.28it/s]\n" ] }, { @@ -397,7 +374,6 @@ "# debug\n", "datasets.logging.set_verbosity_info()\n", "\n", - "\n", "# Let's just try IMDB for simplicity\n", "data = load_dataset(\"amazon_polarity\")\n", "# data = load_dataset(\"/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/amazon_polarity-train-00003-of-00004.arrow\")\n", @@ -406,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:05.461369Z", @@ -448,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:05.902898Z", @@ -462,7 +438,7 @@ "174" ] }, - "execution_count": 23, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -481,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 55, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:45.805378Z", @@ -490,18 +466,18 @@ }, "outputs": [], "source": [ - "# gen_config_raw = {\n", - "# \"temperature\": temperature,\n", - "# \"top_p\": top_p,\n", - "# \"top_k\": top_k,\n", - "# \"repetition_penalty\": repetition_penalty,\n", - "# \"max_new_tokens\": max_new_tokens,\n", - "# \"num_beams\": num_beams,\n", - "# \"use_cache\": use_cache,\n", - "# \"do_sample\": do_sample,\n", - "# \"eos_token_id\": eos_token_id, \n", - "# \"pad_token_id\": pad_token_id\n", - "# }\n", + "from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer\n", + "temperature=0.1\n", + "top_p=0.75\n", + "top_k=40\n", + "num_beams=4\n", + "max_new_tokens=128\n", + "generation_config = GenerationConfig(\n", + " temperature=temperature,\n", + " top_p=top_p,\n", + " top_k=top_k,\n", + ")\n", + "\n", "\n", "def get_output(model, tokenizer, input_text, add_bos_token=False, truncation_length=400):\n", " \"\"\"\n", @@ -531,20 +507,33 @@ " # Llama adds this extra token when the first character is '\\n', and this\n", " # compromises the stopping criteria, so we just remove it\n", " if type(tokenizer) is LlamaTokenizer and input_ids[0][0] == 29871:\n", - " print('removed extra \\n token')\n", + " # print('removed extra \\n token')\n", " input_ids = input_ids[:, 1:]\n", " \n", " # Handling truncation\n", " if truncation_length is not None:\n", " input_ids = input_ids[:, -truncation_length:]\n", "\n", + "\n", + " # generate_params = {\n", + " # \"input_ids\": input_ids,\n", + " # \"generation_config\": generation_config,\n", + " # \"return_dict_in_generate\": True,\n", + " # \"output_scores\": True,\n", + " # \"max_new_tokens\": max_new_tokens,\n", + " # }\n", " # forward pass\n", " with torch.no_grad():\n", - " output = model.generate(input_ids=input_ids, max_length=400)\n", - "# print(output)\n", + " generation_output = model.generate(\n", + " input_ids=input_ids, generation_config=generation_config,\n", + " return_dict_in_generate=True,\n", + " output_scores=True,\n", + " max_new_tokens=max_new_tokens,\n", + " )\n", + " s = generation_output.sequences[0]\n", " \n", " text_q = tokenizer.batch_decode(input_ids, skip_special_tokens=False)\n", - " text_ans = tokenizer.batch_decode(output, skip_special_tokens=False)#, skip_prompt=True, skip_special_tokens=True)\n", + " text_ans = tokenizer.batch_decode(s, skip_special_tokens=False)#, skip_prompt=True, skip_special_tokens=True)\n", " print(text_q[0])\n", " print('-'*40+'answ'+'-'*40)\n", " print(text_ans[0])\n" @@ -564,7 +553,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 56, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:46.260946Z", @@ -579,7 +568,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 57, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:46.401304Z", @@ -594,7 +583,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 58, "metadata": { "ExecuteTime": { "end_time": "2023-05-20T02:25:53.587709Z", @@ -616,15 +605,7 @@ "Answer: \n", "\n", "----------------------------------------answ----------------------------------------\n", - "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", - "Question: Is this review positive? \n", - "Answer: 1\n", - "---\n", - "Review: \"Despite the fact that I have only played a small portion of the game, the music I heard (plus the connection to Chrono Trigger which was great as well) led me to purchase the soundtrack, and it remains one of my favorite albums. There is an incredible mix of fun, epic, and emotional songs. Those sad and beautiful tracks I especially like, as there's not too many of those kinds of songs in my other video game soundtracks. I must admit that one of the songs (Life-A Distant Promise) has brought tears to my eyes on many occasions.My one complaint about this soundtrack is that they use guitar fretting effects in many of the songs, which I find distracting. But even if those weren't included I would still consider the collection worth it.\"\n", - "Question: Is this review positive?\n", - "Answer: \n", - "---\n", - "Review: \"This is a great movie. It has a great storyline and the acting is superb. The characters are well developed and the plot is interesting. The special effects are great and the cinematography is beautiful. The music is also very good and fits the mood of the\n" + "Review\n" ] } ], @@ -634,6 +615,13 @@ "get_output(model, tokenizer, input_text)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -644,11 +632,11 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 13, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T02:27:33.628985Z", - "start_time": "2023-05-20T02:27:33.623501Z" + "end_time": "2023-05-20T02:27:49.742646Z", + "start_time": "2023-05-20T02:27:49.737461Z" } }, "outputs": [], @@ -664,8 +652,8 @@ " if not isinstance(input_text, list):\n", " input_text = [input_text]\n", " # tokenize (adding the EOS token this time)\n", - " input_text = [i + tokenizer.eos_token for i in input_text]\n", - " input_text = [i[-1000:] for i in input_text]\n", + "# input_text = [i + tokenizer.eos_token for i in input_text]\n", + "# input_text = [i[-1000:] for i in input_text]\n", " input_ids = tokenizer(input_text, \n", " return_tensors=\"pt\",\n", "# truncation=True, \n", @@ -683,7 +671,7 @@ " # Llama adds this extra token when the first character is '\\n', and this\n", " # compromises the stopping criteria, so we just remove it\n", " if type(tokenizer) is LlamaTokenizer and input_ids[0][0] == 29871:\n", - " print('removed extra \\n token')\n", + " # print('removed extra \\n token')\n", " input_ids = input_ids[:, 1:]\n", " \n", " # Handling truncation\n", @@ -710,6 +698,8 @@ " output['hidden_states'] = torch.stack([output['hidden_states'][i] for i in layers], 1).detach().cpu()\n", " # dims [Batch, Layers, Seq_Token, Probs?] e.g. torch.Size([3, 2, 284, 4096])\n", " \n", + " output['hidden_states'] = output['hidden_states'][:, :, -1] # take just the last token so they are same size\n", + " \n", " # dims [Batch, ?, Output_Tokens] e.g. torch.Size([3, 284, 32000])\n", " o = output['logits'].detach().cpu().float().softmax(-1)\n", " \n", @@ -740,11 +730,11 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 14, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T02:27:33.946048Z", - "start_time": "2023-05-20T02:27:33.944125Z" + "end_time": "2023-05-20T02:27:50.048590Z", + "start_time": "2023-05-20T02:27:50.046873Z" } }, "outputs": [], @@ -761,11 +751,11 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 15, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T02:27:34.584342Z", - "start_time": "2023-05-20T02:27:34.092856Z" + "end_time": "2023-05-20T02:27:50.695875Z", + "start_time": "2023-05-20T02:27:50.205135Z" }, "scrolled": true }, @@ -781,11 +771,11 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 16, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T02:27:34.588474Z", - "start_time": "2023-05-20T02:27:34.585702Z" + "end_time": "2023-05-20T02:27:50.700337Z", + "start_time": "2023-05-20T02:27:50.697307Z" } }, "outputs": [ @@ -794,48 +784,40 @@ "output_type": "stream", "text": [ "----------------------------------------input----------------------------------------\n", - " hout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", "Question: Is this review positive? \n", "Answer: 1\n", "---\n", "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", "Question: Is this review negative?\n", "Answer: \n", - "\n", - "----------------------------------------answ----------------------------------------\n", - " Belowen\n", - "us hadver, a,x, isages a team situation team.The Brookels'. and is trying at. film.\n", - " stastic is the supporting actors who Gordon-Levitt and Ches Williams who. who Lloyd plays also as thevinThe Bull' who the rest are great. the film notch family production.\n", - " musting and funwwarming film that is should watch.\n", - "\": What this movie based or\n", - " AnswerAnswer: Yes\n", - "0\n", "\n", - "Questionview: \"This reviewely wiferic been of the mostREATEATE in all generation. She' been to her CD many monthsEARS and it still loveVE it! She I hearm feeling the bad mood,' me smile even, When must dayood and makesaporates. a in a rain. I is is makesozes classIFE and IERYally are superawss perfectUNFFNING. theics are make me I of the's great gems. Bu CD a must islandle CD for my collection. I isn isn made it big is beyond a me. Sheone I listen this CD I matter what or white or or or old, it or female,VERYONEDY lov the thing:W is that?\"?\"\n", - "Question: Is this review positive?\n", - "Answer: 0---\n", + "----------------------------------------answ----------------------------------------\n", + "Below TheThe' it is a greately book story. It is some of laughilarious moments and thewwarming moments. keep enjoyed. the film. The animation are all and the story are done.. Iway Dever is the,x, isages to family task team theThe Angels'. and is determined at. movie. The,astic is the performances actors who Gordon-Levitt and Thomases Davis Jr. who Lloyd is also as the SimThe B' Lewis the rest are great. the movie-ch family movie.\" I greating and fun-warming movie that is can see.\"\n", + "\n", + ": What this movie helpful,\n", + " AnswerAnswer: Yes\n", + ".Question\n", + "Questionview: \"This familyely wifety been of the mostREATEATE in all generation. She' been to her album over hoursEARS and it still loveVE it!\" I I needm feeling a bad mood, makes me feel even, When great mood? disappaporates. a in hot rain. I is is makesozes withIFE and Iivaals are greatazss perfectUNUNNING. theics are make me I of the's great treems.\" I is a M islandle disc. my collection.\" I? isn made it big is beyond beyond me. Ione I listen this CD I matter what or white, young or old, male, female,VERYONEDY lov ' thing:W is that??\"\"\n", + "Question: Is this review positive? Answer: 0Re\n", "================================================================================\n", "----------------------------------------input----------------------------------------\n", - " hout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", "Question: Is this review positive? \n", "Answer: 1\n", "---\n", "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", "Question: Is this review positive?\n", "Answer: \n", - "\n", - "----------------------------------------answ----------------------------------------\n", - " Belowen\n", - "us hadver, a,x, isages a team situation team.The Brookels'. and is trying at. film.\n", - " stastic is the supporting actors who Gordon-Levitt and Ches Williams who. who Lloyd plays also as thevinThe Bull' who the rest are great. the film notch family production.\n", - " musting and funwwarming film that is should watch.\n", - "\": What this movie based or\n", - " AnswerAnswer: Yes\n", - "0\n", "\n", - "Questionview: \"This reviewely wiferic been of the mostREATEATE in all generation. She' been to her CD many monthsEARS and it still loveVE it! She I hearm feeling the bad mood,' me feel even, When must mood and makesaporates. a in a rain. I is is makesozes classIFE and IERYally are superawss perfectUNPENING. theics are make me I of the's great gems. Bu CD a must islandle CD for my collection. I isn' made it big is beyond a me. She song I listen this CD I matter what or white or or or old, it or female,VERYONEDY lov the thing:W is that?\"?\"\n", - "Question: Is this review positive?\n", - "Answer: 1---\n", + "----------------------------------------answ----------------------------------------\n", + "Below TheThe' it is a greately book story. It is some of laughilarious moments and thewwarming moments. keep had. the film. The animation are all and the story are done.. Iway Dever is the,x, isages to family task team theThe Angels'. and is determined at. movie. The,astic is the performances actors who Gordon-Levitt and Thomases Davis Jr. who Lloyd is also as the SimThe B' Lewis the rest are great. the movie-ch family movie.\" I greating and fun-warming movie that is can see.\"\n", + "\n", + ": What this movie helpful or\n", + " AnswerAnswer: Yes\n", + ".Question\n", + "Questionview: \"This familyely wifety been of the mostREATEATE in all generation. She' been to her album over hoursEARS and it still loveVE it! I I needm feeling a bad mood, makes me feel even, When great mood? melaporates. a in hot rain. I is is makesozes withIFE and Iivaals are greatazss perfectUNUNNING. theics are make me I of the's great treems.\" I is a M islandle disc. my collection.\" I? isn made it big I beyond a me. Ione I listen this CD I matter what or white, young or old, male, female,VERYONEDY lov ' thing:W is that??\"\"\n", + "Question: Is this review positive? Answer: 1Re\n", "--------------------------------------------------------------------------------\n" ] } @@ -855,11 +837,11 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 17, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T02:27:34.617886Z", - "start_time": "2023-05-20T02:27:34.589316Z" + "end_time": "2023-05-20T02:27:50.723482Z", + "start_time": "2023-05-20T02:27:50.701252Z" }, "scrolled": true }, @@ -892,8 +874,8 @@ "execution_count": 18, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:56:35.161371Z", - "start_time": "2023-05-20T01:56:35.145412Z" + "end_time": "2023-05-20T02:28:04.246539Z", + "start_time": "2023-05-20T02:28:04.242460Z" } }, "outputs": [], @@ -913,7 +895,7 @@ " \n", " res = []\n", " \n", - " ds_subset = data['test'].shuffle(42).select(range(n))\n", + " ds_subset = data.shuffle(42).select(range(n))\n", " dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)\n", " for batch in tqdm(dl):\n", " text, true_label = batch[\"content\"], batch[\"label\"]\n", @@ -956,8 +938,8 @@ "execution_count": 19, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:56:35.183039Z", - "start_time": "2023-05-20T01:56:35.162312Z" + "end_time": "2023-05-20T02:28:04.574087Z", + "start_time": "2023-05-20T02:28:04.572316Z" } }, "outputs": [], @@ -979,8 +961,8 @@ "execution_count": 20, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:56:35.501003Z", - "start_time": "2023-05-20T01:56:35.184047Z" + "end_time": "2023-05-20T02:28:05.387382Z", + "start_time": "2023-05-20T02:28:05.033921Z" } }, "outputs": [ @@ -1006,8 +988,8 @@ "execution_count": 21, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.640525Z", - "start_time": "2023-05-20T01:56:35.502054Z" + "end_time": "2023-05-20T02:28:35.747656Z", + "start_time": "2023-05-20T02:28:05.388608Z" } }, "outputs": [ @@ -1015,82 +997,19 @@ "name": "stderr", "output_type": "stream", "text": [ - "Loading cached shuffled indices for dataset at /home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", - "100%|███████████████████████████████████████| 34/34 [00:27<00:00, 1.22it/s]\n" + "Loading cached shuffled indices for dataset at /home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc/cache-0a5d0b47b5e8dfc6.arrow\n", + "100%|██████████| 34/34 [00:31<00:00, 1.07it/s]\n" ] }, { "data": { - "text/html": [ - "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
-       "│ in <cell line: 1>:1                                                                              │\n",
-       "│                                                                                                  │\n",
-       "│ ❱ 1 neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tok     │\n",
-       "│   2                                                                                              │\n",
-       "│   3                                                                                              │\n",
-       "│   4 gc.collect()                                                                                 │\n",
-       "│                                                                                                  │\n",
-       "│ in get_hidden_states_many_examples:33                                                            │\n",
-       "│                                                                                                  │\n",
-       "│   30 │   │   ])                                                                                  │\n",
-       "│   31 │                                                                                           │\n",
-       "│   32 │   # FIXME not all the hidden state are the same size, wat                                 │\n",
-       "│ ❱ 33 │   res = [np.concatenate(r) for r in zip(*res)]                                            │\n",
-       "│   34 │   return res                                                                              │\n",
-       "│   35 │   all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res                   │\n",
-       "│   36 │   return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans                  │\n",
-       "│                                                                                                  │\n",
-       "│ in <listcomp>:33                                                                                 │\n",
-       "│                                                                                                  │\n",
-       "│   30 │   │   ])                                                                                  │\n",
-       "│   31 │                                                                                           │\n",
-       "│   32 │   # FIXME not all the hidden state are the same size, wat                                 │\n",
-       "│ ❱ 33 │   res = [np.concatenate(r) for r in zip(*res)]                                            │\n",
-       "│   34 │   return res                                                                              │\n",
-       "│   35 │   all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res                   │\n",
-       "│   36 │   return all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans                  │\n",
-       "│ in concatenate:200                                                                               │\n",
-       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
-       "ValueError: all the input array dimensions except for the concatenation axis must match exactly, but along \n",
-       "dimension 1, the array at index 0 has size 2285568 and the array at index 1 has size 1908736\n",
-       "
\n" - ], "text/plain": [ - "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tok \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mgc.collect() \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m33\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m]) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# FIXME not all the hidden state are the same size, wat\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m33 \u001b[2m│ \u001b[0mres = [np.concatenate(r) \u001b[94mfor\u001b[0m r \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(*res)] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ \u001b[0mall_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m33\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m]) \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# FIXME not all the hidden state are the same size, wat\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m33 \u001b[2m│ \u001b[0mres = [np.concatenate(r) \u001b[94mfor\u001b[0m r \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(*res)] \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ \u001b[0mall_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans \u001b[31m│\u001b[0m\n", - "\u001b[31m│\u001b[0m in \u001b[92mconcatenate\u001b[0m:\u001b[94m200\u001b[0m \u001b[31m│\u001b[0m\n", - "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", - "\u001b[1;91mValueError: \u001b[0mall the input array dimensions except for the concatenation axis must match exactly, but along \n", - "dimension \u001b[1;36m1\u001b[0m, the array at index \u001b[1;36m0\u001b[0m has size \u001b[1;36m2285568\u001b[0m and the array at index \u001b[1;36m1\u001b[0m has size \u001b[1;36m1908736\u001b[0m\n" + "0" ] }, + "execution_count": 21, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -1104,11 +1023,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.641742Z", - "start_time": "2023-05-20T01:57:03.641735Z" + "end_time": "2023-05-20T02:28:35.749712Z", + "start_time": "2023-05-20T02:28:35.749703Z" } }, "outputs": [], @@ -1118,14 +1037,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.642523Z", - "start_time": "2023-05-20T01:57:03.642517Z" + "end_time": "2023-05-20T02:28:35.750431Z", + "start_time": "2023-05-20T02:28:35.750421Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0.40796318527410963, 0.4257703081232493)" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# roc_auc_score\n", "pos_score = roc_auc_score(y, all_pos_ans)\n", @@ -1147,15 +1077,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.643517Z", - "start_time": "2023-05-20T01:57:03.643507Z" + "end_time": "2023-05-20T02:28:35.751184Z", + "start_time": "2023-05-20T02:28:35.751175Z" }, "scrolled": true }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(0.49, 0.51)" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# accuracy_score\n", "pos_score = accuracy_score(y, (all_pos_ans>0.)*1.0)\n", @@ -1176,14 +1117,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.644197Z", - "start_time": "2023-05-20T01:57:03.644190Z" + "end_time": "2023-05-20T02:28:35.751934Z", + "start_time": "2023-05-20T02:28:35.751926Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Logistic regression accuracy: 1.0 [TRAIN]\n", + "Logistic regression accuracy: 0.94 [TEST]\n" + ] + } + ], "source": [ "# let's create a simple 50/50 train split (the data is already randomized)\n", "n = len(y)\n", @@ -1227,11 +1177,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.644851Z", - "start_time": "2023-05-20T01:57:03.644841Z" + "end_time": "2023-05-20T02:28:35.752548Z", + "start_time": "2023-05-20T02:28:35.752540Z" } }, "outputs": [], @@ -1282,11 +1232,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.645452Z", - "start_time": "2023-05-20T01:57:03.645446Z" + "end_time": "2023-05-20T02:28:35.753297Z", + "start_time": "2023-05-20T02:28:35.753288Z" } }, "outputs": [], @@ -1305,11 +1255,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.645991Z", - "start_time": "2023-05-20T01:57:03.645985Z" + "end_time": "2023-05-20T02:28:35.754161Z", + "start_time": "2023-05-20T02:28:35.754151Z" } }, "outputs": [], @@ -1378,15 +1328,186 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.646546Z", - "start_time": "2023-05-20T01:57:03.646539Z" + "end_time": "2023-05-20T02:28:35.754973Z", + "start_time": "2023-05-20T02:28:35.754964Z" }, "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/ubuntu/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:87                                                                                   │\n",
+       "│                                                                                                  │\n",
+       "│   84                                                                                             │\n",
+       "│   85 # test                                                                                      │\n",
+       "│   86 dm = IMBDHSDataModule(model, tokenizer)                                                     │\n",
+       "│ ❱ 87 dm.setup('train')                                                                           │\n",
+       "│   88 dl = dm.val_dataloader()                                                                    │\n",
+       "│   89 b = next(iter(dl))                                                                          │\n",
+       "│   90 b                                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│ in setup:32                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   29 │   │                                                                                       │\n",
+       "│   30 │   │   self.dataset = load_dataset(self.hparams.dataset_name, split=\"test\")                │\n",
+       "│   31 │   │                                                                                       │\n",
+       "│ ❱ 32 │   │   neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(      │\n",
+       "│   33 │   │   │   self.model, self.tokenizer, self.dataset, self.hparams.model_type, n=self.hp    │\n",
+       "│   34 │   │                                                                                       │\n",
+       "│   35 │   │   # let's create a simple 50/50 train split (the data is already randomized)          │\n",
+       "│                                                                                                  │\n",
+       "│ in get_hidden_states_many_examples:14                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   11 │                                                                                           │\n",
+       "│   12 │   res = []                                                                                │\n",
+       "│   13 │                                                                                           │\n",
+       "│ ❱ 14 │   ds_subset = data['test'].shuffle(42).select(range(n))                                   │\n",
+       "│   15 │   dl = DataLoader(ds_subset, batch_size=batch_size, shuffle=True)                         │\n",
+       "│   16 │   for batch in tqdm(dl):                                                                  │\n",
+       "│   17 │   │   text, true_label = batch[\"content\"], batch[\"label\"]                                 │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/arrow_dataset.py:2778 in  │\n",
+       "│ __getitem__                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   2775 │                                                                                         │\n",
+       "│   2776 │   def __getitem__(self, key):  # noqa: F811                                             │\n",
+       "│   2777 │   │   \"\"\"Can be used to index columns (by string names) or rows (by integer index or i  │\n",
+       "│ ❱ 2778 │   │   return self._getitem(key)                                                         │\n",
+       "│   2779 │                                                                                         │\n",
+       "│   2780 │   def __getitems__(self, keys: List) -> List:                                           │\n",
+       "│   2781 │   │   \"\"\"Can be used to get a batch using a list of integers indices.\"\"\"                │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/arrow_dataset.py:2762 in  │\n",
+       "│ _getitem                                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│   2759 │   │   format_kwargs = kwargs[\"format_kwargs\"] if \"format_kwargs\" in kwargs else self._  │\n",
+       "│   2760 │   │   format_kwargs = format_kwargs if format_kwargs is not None else {}                │\n",
+       "│   2761 │   │   formatter = get_formatter(format_type, features=self._info.features, **format_kw  │\n",
+       "│ ❱ 2762 │   │   pa_subtable = query_table(self._data, key, indices=self._indices if self._indice  │\n",
+       "│   2763 │   │   formatted_output = format_table(                                                  │\n",
+       "│   2764 │   │   │   pa_subtable, key, formatter=formatter, format_columns=format_columns, output  │\n",
+       "│   2765 │   │   )                                                                                 │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/formatting.py: │\n",
+       "│ 575 in query_table                                                                               │\n",
+       "│                                                                                                  │\n",
+       "│   572 │   if not isinstance(key, (int, slice, range, str, Iterable)):                            │\n",
+       "│   573 │   │   _raise_bad_key_type(key)                                                           │\n",
+       "│   574 │   if isinstance(key, str):                                                               │\n",
+       "│ ❱ 575 │   │   _check_valid_column_key(key, table.column_names)                                   │\n",
+       "│   576 │   else:                                                                                  │\n",
+       "│   577 │   │   size = indices.num_rows if indices is not None else table.num_rows                 │\n",
+       "│   578 │   │   _check_valid_index_key(key, size)                                                  │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/formatting.py: │\n",
+       "│ 515 in _check_valid_column_key                                                                   │\n",
+       "│                                                                                                  │\n",
+       "│   512                                                                                            │\n",
+       "│   513 def _check_valid_column_key(key: str, columns: List[str]) -> None:                         │\n",
+       "│   514 │   if key not in columns:                                                                 │\n",
+       "│ ❱ 515 │   │   raise KeyError(f\"Column {key} not in the dataset. Current columns in the dataset   │\n",
+       "│   516                                                                                            │\n",
+       "│   517                                                                                            │\n",
+       "│   518 def _check_valid_index_key(key: Union[int, slice, range, Iterable], size: int) -> None:    │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "KeyError: \"Column test not in the dataset. Current columns in the dataset: ['label', 'title', 'content']\"\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m87\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m84 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m85 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m86 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m87 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m88 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m89 \u001b[0mb = \u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m90 \u001b[0mb \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m32\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m29 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.hparams.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m32 \u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m33 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[96mself\u001b[0m.dataset, \u001b[96mself\u001b[0m.hparams.model_type, n=\u001b[96mself\u001b[0m.hp \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# let's create a simple 50/50 train split (the data is already randomized)\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m14\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m11 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m12 \u001b[0m\u001b[2m│ \u001b[0mres = [] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m14 \u001b[2m│ \u001b[0mds_subset = data[\u001b[33m'\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m'\u001b[0m].shuffle(\u001b[94m42\u001b[0m).select(\u001b[96mrange\u001b[0m(n)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m15 \u001b[0m\u001b[2m│ \u001b[0mdl = DataLoader(ds_subset, batch_size=batch_size, shuffle=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m16 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mfor\u001b[0m batch \u001b[95min\u001b[0m tqdm(dl): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ │ \u001b[0mtext, true_label = batch[\u001b[33m\"\u001b[0m\u001b[33mcontent\u001b[0m\u001b[33m\"\u001b[0m], batch[\u001b[33m\"\u001b[0m\u001b[33mlabel\u001b[0m\u001b[33m\"\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33marrow_dataset.py\u001b[0m:\u001b[94m2778\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m__getitem__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2775 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2776 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m__getitem__\u001b[0m(\u001b[96mself\u001b[0m, key): \u001b[2m# noqa: F811\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2777 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"Can be used to index columns (by string names) or rows (by integer index or i\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2778 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._getitem(key) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2779 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2780 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m__getitems__\u001b[0m(\u001b[96mself\u001b[0m, keys: List) -> List: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2781 \u001b[0m\u001b[2;90m│ │ \u001b[0m\u001b[33m\"\"\"Can be used to get a batch using a list of integers indices.\"\"\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33marrow_dataset.py\u001b[0m:\u001b[94m2762\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_getitem\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2759 \u001b[0m\u001b[2m│ │ \u001b[0mformat_kwargs = kwargs[\u001b[33m\"\u001b[0m\u001b[33mformat_kwargs\u001b[0m\u001b[33m\"\u001b[0m] \u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mformat_kwargs\u001b[0m\u001b[33m\"\u001b[0m \u001b[95min\u001b[0m kwargs \u001b[94melse\u001b[0m \u001b[96mself\u001b[0m._ \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2760 \u001b[0m\u001b[2m│ │ \u001b[0mformat_kwargs = format_kwargs \u001b[94mif\u001b[0m format_kwargs \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m {} \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2761 \u001b[0m\u001b[2m│ │ \u001b[0mformatter = get_formatter(format_type, features=\u001b[96mself\u001b[0m._info.features, **format_kw \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m2762 \u001b[2m│ │ \u001b[0mpa_subtable = query_table(\u001b[96mself\u001b[0m._data, key, indices=\u001b[96mself\u001b[0m._indices \u001b[94mif\u001b[0m \u001b[96mself\u001b[0m._indice \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2763 \u001b[0m\u001b[2m│ │ \u001b[0mformatted_output = format_table( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2764 \u001b[0m\u001b[2m│ │ │ \u001b[0mpa_subtable, key, formatter=formatter, format_columns=format_columns, output \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2765 \u001b[0m\u001b[2m│ │ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/\u001b[0m\u001b[1;33mformatting.py\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[94m575\u001b[0m in \u001b[92mquery_table\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m572 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m \u001b[96misinstance\u001b[0m(key, (\u001b[96mint\u001b[0m, \u001b[96mslice\u001b[0m, \u001b[96mrange\u001b[0m, \u001b[96mstr\u001b[0m, Iterable)): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m573 \u001b[0m\u001b[2m│ │ \u001b[0m_raise_bad_key_type(key) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m574 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m \u001b[96misinstance\u001b[0m(key, \u001b[96mstr\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m575 \u001b[2m│ │ \u001b[0m_check_valid_column_key(key, table.column_names) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m576 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m577 \u001b[0m\u001b[2m│ │ \u001b[0msize = indices.num_rows \u001b[94mif\u001b[0m indices \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m table.num_rows \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m578 \u001b[0m\u001b[2m│ │ \u001b[0m_check_valid_index_key(key, size) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/datasets/formatting/\u001b[0m\u001b[1;33mformatting.py\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[94m515\u001b[0m in \u001b[92m_check_valid_column_key\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m512 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m513 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_check_valid_column_key\u001b[0m(key: \u001b[96mstr\u001b[0m, columns: List[\u001b[96mstr\u001b[0m]) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m514 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m key \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m columns: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m515 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mKeyError\u001b[0m(\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33mColumn \u001b[0m\u001b[33m{\u001b[0mkey\u001b[33m}\u001b[0m\u001b[33m not in the dataset. Current columns in the dataset\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m516 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m517 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m518 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92m_check_valid_index_key\u001b[0m(key: Union[\u001b[96mint\u001b[0m, \u001b[96mslice\u001b[0m, \u001b[96mrange\u001b[0m, Iterable], size: \u001b[96mint\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mKeyError: \u001b[0m\u001b[32m\"Column test not in the dataset. Current columns in the dataset: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m'label', 'title', 'content'\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "\n", "# def normalize(x):\n", @@ -1483,14 +1604,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.647107Z", - "start_time": "2023-05-20T01:57:03.647101Z" + "end_time": "2023-05-20T02:28:35.755617Z", + "start_time": "2023-05-20T02:28:35.755609Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 dm.x_test.shape                                                                              │\n",
+       "│   2                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "AttributeError: 'IMBDHSDataModule' object has no attribute 'x_test'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 dm.x_test.shape \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mAttributeError: \u001b[0m\u001b[32m'IMBDHSDataModule'\u001b[0m object has no attribute \u001b[32m'x_test'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "dm.x_test.shape" ] @@ -1504,11 +1651,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.647739Z", - "start_time": "2023-05-20T01:57:03.647733Z" + "end_time": "2023-05-20T02:28:35.756378Z", + "start_time": "2023-05-20T02:28:35.756365Z" } }, "outputs": [], @@ -1518,11 +1665,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.648618Z", - "start_time": "2023-05-20T01:57:03.648611Z" + "end_time": "2023-05-20T02:28:35.757039Z", + "start_time": "2023-05-20T02:28:35.757030Z" } }, "outputs": [], @@ -1630,14 +1777,46 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.649120Z", - "start_time": "2023-05-20T01:57:03.649114Z" + "end_time": "2023-05-20T02:28:35.757814Z", + "start_time": "2023-05-20T02:28:35.757806Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 # init the autoencoder                                                                       │\n",
+       "│   2 max_epochs = 1000                                                                            │\n",
+       "│ ❱ 3 d = b[0].shape[-1]                                                                           │\n",
+       "│   4 net = CSS(d=d, max_epochs=max_epochs)                                                        │\n",
+       "│   5                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'b' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[2m# init the autoencoder\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mmax_epochs = \u001b[94m1000\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3 d = b[\u001b[94m0\u001b[0m].shape[-\u001b[94m1\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0mnet = CSS(d=d, max_epochs=max_epochs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m5 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'b'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# init the autoencoder\n", "max_epochs = 1000\n", @@ -1647,11 +1826,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.649727Z", - "start_time": "2023-05-20T01:57:03.649721Z" + "end_time": "2023-05-20T02:28:35.758702Z", + "start_time": "2023-05-20T02:28:35.758693Z" } }, "outputs": [], @@ -1661,15 +1840,57 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.650409Z", - "start_time": "2023-05-20T01:57:03.650402Z" + "end_time": "2023-05-20T02:28:35.759285Z", + "start_time": "2023-05-20T02:28:35.759277Z" }, "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\n", + "TPU available: False, using: 0 TPU cores\n", + "IPU available: False, using: 0 IPUs\n", + "HPU available: False, using: 0 HPUs\n", + "/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/lightning/pytorch/trainer/connectors/logger_connector/logger_connector.py:67: UserWarning: Starting from v1.9.0, `tensorboardX` has been removed as a dependency of the `lightning.pytorch` package, due to potential conflicts with other packages in the ML ecosystem. For this reason, `logger=True` will use `CSVLogger` as the default logger, unless the `tensorboard` or `tensorboardX` packages are found. Please `pip install lightning[extra]` or one of them to enable TensorBoard support by default\n", + " warning_cache.warn(\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:3                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│   1 # train the model (hint: here are some helpful Trainer arguments for rapid idea iteratio     │\n",
+       "│   2 trainer = pl.Trainer(limit_train_batches=100, max_epochs=max_epochs)                         │\n",
+       "│ ❱ 3 trainer.fit(model=net, datamodule=dm)                                                        │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'net' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m3\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1 \u001b[0m\u001b[2m# train the model (hint: here are some helpful Trainer arguments for rapid idea iteratio\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mtrainer = pl.Trainer(limit_train_batches=\u001b[94m100\u001b[0m, max_epochs=max_epochs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m3 trainer.fit(model=net, datamodule=dm) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'net'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\n", "trainer = pl.Trainer(limit_train_batches=100, max_epochs=max_epochs)\n", @@ -1678,14 +1899,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.650895Z", - "start_time": "2023-05-20T01:57:03.650889Z" + "end_time": "2023-05-20T02:28:35.760108Z", + "start_time": "2023-05-20T02:28:35.760100Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> \u001b[0;32m/tmp/ipykernel_3455549/4111709629.py\u001b[0m(3)\u001b[0;36m\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m 1 \u001b[0;31m\u001b[0;31m# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m 2 \u001b[0;31m\u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlimit_train_batches\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\u001b[0;32m----> 3 \u001b[0;31m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatamodule\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0m\n" + ] + } + ], "source": [ "%debug" ] @@ -1704,11 +1937,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.651740Z", - "start_time": "2023-05-20T01:57:03.651734Z" + "end_time": "2023-05-20T02:28:35.760833Z", + "start_time": "2023-05-20T02:28:35.760825Z" } }, "outputs": [], @@ -1741,14 +1974,219 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.652269Z", - "start_time": "2023-05-20T01:57:03.652263Z" + "end_time": "2023-05-20T02:28:35.761623Z", + "start_time": "2023-05-20T02:28:35.761614Z" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[]\n" + ] + }, + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_hist = read_hist(trainer).ffill().bfill()                                                 │\n",
+       "│   2 df_hist                                                                                      │\n",
+       "│   3                                                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:23                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│ ❱ 23 │   │   raise e                                                                             │\n",
+       "│   24 │   │   print(e)                                                                            │\n",
+       "│   25                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_hist:20                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   17 │   print(ts)                                                                               │\n",
+       "│   18 │   try:                                                                                    │\n",
+       "│   19 │   │   metrics_file_path = Path(ts[0].experiment.metrics_file_path)                        │\n",
+       "│ ❱ 20 │   │   df_histe = read_metrics_csv(metrics_file_path)                                      │\n",
+       "│   21 │   │   return df_histe                                                                     │\n",
+       "│   22 │   except Exception as e:                                                                  │\n",
+       "│   23 │   │   raise e                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in read_metrics_csv:8                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│    5 import pandas as pd                                                                         │\n",
+       "│    6                                                                                             │\n",
+       "│    7 def read_metrics_csv(metrics_file_path):                                                    │\n",
+       "│ ❱  8 │   df_hist = pd.read_csv(metrics_file_path)                                                │\n",
+       "│    9 │   df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()                                             │\n",
+       "│   10 │   df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()                           │\n",
+       "│   11 │   return df_histe                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:912   │\n",
+       "│ in read_csv                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│    909 │   )                                                                                     │\n",
+       "│    910 │   kwds.update(kwds_defaults)                                                            │\n",
+       "│    911 │                                                                                         │\n",
+       "│ ❱  912 │   return _read(filepath_or_buffer, kwds)                                                │\n",
+       "│    913                                                                                           │\n",
+       "│    914                                                                                           │\n",
+       "│    915 # iterator=True -> TextFileReader                                                         │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:577   │\n",
+       "│ in _read                                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│    574 │   _validate_names(kwds.get(\"names\", None))                                              │\n",
+       "│    575 │                                                                                         │\n",
+       "│    576 │   # Create the parser.                                                                  │\n",
+       "│ ❱  577 │   parser = TextFileReader(filepath_or_buffer, **kwds)                                   │\n",
+       "│    578 │                                                                                         │\n",
+       "│    579 │   if chunksize or iterator:                                                             │\n",
+       "│    580 │   │   return parser                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1407  │\n",
+       "│ in __init__                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   1404 │   │   │   self.options[\"has_index_names\"] = kwds[\"has_index_names\"]                     │\n",
+       "│   1405 │   │                                                                                     │\n",
+       "│   1406 │   │   self.handles: IOHandles | None = None                                             │\n",
+       "│ ❱ 1407 │   │   self._engine = self._make_engine(f, self.engine)                                  │\n",
+       "│   1408 │                                                                                         │\n",
+       "│   1409 │   def close(self) -> None:                                                              │\n",
+       "│   1410 │   │   if self.handles is not None:                                                      │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/readers.py:1661  │\n",
+       "│ in _make_engine                                                                                  │\n",
+       "│                                                                                                  │\n",
+       "│   1658 │   │   │   │   is_text = False                                                           │\n",
+       "│   1659 │   │   │   │   if \"b\" not in mode:                                                       │\n",
+       "│   1660 │   │   │   │   │   mode += \"b\"                                                           │\n",
+       "│ ❱ 1661 │   │   │   self.handles = get_handle(                                                    │\n",
+       "│   1662 │   │   │   │   f,                                                                        │\n",
+       "│   1663 │   │   │   │   mode,                                                                     │\n",
+       "│   1664 │   │   │   │   encoding=self.options.get(\"encoding\", None),                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/common.py:859 in         │\n",
+       "│ get_handle                                                                                       │\n",
+       "│                                                                                                  │\n",
+       "│    856 │   │   # Binary mode does not support 'encoding' and 'newline'.                          │\n",
+       "│    857 │   │   if ioargs.encoding and \"b\" not in ioargs.mode:                                    │\n",
+       "│    858 │   │   │   # Encoding                                                                    │\n",
+       "│ ❱  859 │   │   │   handle = open(                                                                │\n",
+       "│    860 │   │   │   │   handle,                                                                   │\n",
+       "│    861 │   │   │   │   ioargs.mode,                                                              │\n",
+       "│    862 │   │   │   │   encoding=ioargs.encoding,                                                 │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "FileNotFoundError: [Errno 2] No such file or directory: \n",
+       "'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/lightning_logs/version_4/metrics.csv'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_hist = read_hist(trainer).ffill().bfill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0mdf_hist \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m23\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m20 \u001b[0m\u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m23 \u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mprint\u001b[0m(e) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_hist\u001b[0m:\u001b[94m20\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ \u001b[0m\u001b[96mprint\u001b[0m(ts) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mtry\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m19 \u001b[0m\u001b[2m│ │ \u001b[0mmetrics_file_path = Path(ts[\u001b[94m0\u001b[0m].experiment.metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m20 \u001b[2m│ │ \u001b[0mdf_histe = read_metrics_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m21 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m22 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mexcept\u001b[0m \u001b[96mException\u001b[0m \u001b[94mas\u001b[0m e: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m23 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m e \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_metrics_csv\u001b[0m:\u001b[94m8\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 5 \u001b[0m\u001b[94mimport\u001b[0m \u001b[4;96mpandas\u001b[0m \u001b[94mas\u001b[0m \u001b[4;96mpd\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 6 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 7 \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mread_metrics_csv\u001b[0m(metrics_file_path): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 8 \u001b[2m│ \u001b[0mdf_hist = pd.read_csv(metrics_file_path) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 9 \u001b[0m\u001b[2m│ \u001b[0mdf_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m] = df_hist[\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m].ffill() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m10 \u001b[0m\u001b[2m│ \u001b[0mdf_histe = df_hist.set_index(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).groupby(\u001b[33m\"\u001b[0m\u001b[33mepoch\u001b[0m\u001b[33m\"\u001b[0m).mean() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m11 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m df_histe \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m912\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92mread_csv\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 909 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 910 \u001b[0m\u001b[2m│ \u001b[0mkwds.update(kwds_defaults) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 911 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 912 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m _read(filepath_or_buffer, kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 913 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 914 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 915 \u001b[0m\u001b[2m# iterator=True -> TextFileReader\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m577\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_read\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 574 \u001b[0m\u001b[2m│ \u001b[0m_validate_names(kwds.get(\u001b[33m\"\u001b[0m\u001b[33mnames\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 575 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 576 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create the parser.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 577 \u001b[2m│ \u001b[0mparser = TextFileReader(filepath_or_buffer, **kwds) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 578 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 579 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m chunksize \u001b[95mor\u001b[0m iterator: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 580 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m parser \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1407\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1404 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.options[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] = kwds[\u001b[33m\"\u001b[0m\u001b[33mhas_index_names\u001b[0m\u001b[33m\"\u001b[0m] \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1405 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1406 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.handles: IOHandles | \u001b[94mNone\u001b[0m = \u001b[94mNone\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1407 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m._engine = \u001b[96mself\u001b[0m._make_engine(f, \u001b[96mself\u001b[0m.engine) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1408 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1409 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mclose\u001b[0m(\u001b[96mself\u001b[0m) -> \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1410 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.handles \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/parsers/\u001b[0m\u001b[1;33mreaders.py\u001b[0m:\u001b[94m1661\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m_make_engine\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1658 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mis_text = \u001b[94mFalse\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1659 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1660 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mmode += \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1661 \u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.handles = get_handle( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1662 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mf, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1663 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mmode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1664 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=\u001b[96mself\u001b[0m.options.get(\u001b[33m\"\u001b[0m\u001b[33mencoding\u001b[0m\u001b[33m\"\u001b[0m, \u001b[94mNone\u001b[0m), \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/ubuntu/mambaforge/envs/dlk2/lib/python3.9/site-packages/pandas/io/\u001b[0m\u001b[1;33mcommon.py\u001b[0m:\u001b[94m859\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mget_handle\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 856 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Binary mode does not support 'encoding' and 'newline'.\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 857 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m ioargs.encoding \u001b[95mand\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mb\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m ioargs.mode: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 858 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# Encoding\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 859 \u001b[2m│ │ │ \u001b[0mhandle = \u001b[96mopen\u001b[0m( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 860 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mhandle, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 861 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mioargs.mode, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 862 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mencoding=ioargs.encoding, \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mFileNotFoundError: \u001b[0m\u001b[1m[\u001b[0mErrno \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m No such file or directory: \n", + "\u001b[32m'/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/lightning_logs/version_4/metrics.csv'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "df_hist = read_hist(trainer).ffill().bfill()\n", "df_hist" @@ -1756,14 +2194,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": { "ExecuteTime": { - "end_time": "2023-05-20T01:57:03.653040Z", - "start_time": "2023-05-20T01:57:03.653033Z" + "end_time": "2023-05-20T02:28:35.762326Z", + "start_time": "2023-05-20T02:28:35.762318Z" } }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <module>:1                                                                                    │\n",
+       "│                                                                                                  │\n",
+       "│ ❱ 1 df_hist[['val/acc', 'train/acc']].plot()                                                     │\n",
+       "│   2                                                                                              │\n",
+       "│   3 df_hist[['val/f1', 'train/f1']].plot()                                                       │\n",
+       "│   4                                                                                              │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "NameError: name 'df_hist' is not defined\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m1\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1 df_hist[[\u001b[33m'\u001b[0m\u001b[33mval/acc\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mtrain/acc\u001b[0m\u001b[33m'\u001b[0m]].plot() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m2 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m3 \u001b[0mdf_hist[[\u001b[33m'\u001b[0m\u001b[33mval/f1\u001b[0m\u001b[33m'\u001b[0m, \u001b[33m'\u001b[0m\u001b[33mtrain/f1\u001b[0m\u001b[33m'\u001b[0m]].plot() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m4 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mNameError: \u001b[0mname \u001b[32m'df_hist'\u001b[0m is not defined\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "df_hist[['val/acc', 'train/acc']].plot()\n", "\n", @@ -1812,7 +2280,7 @@ "kernelspec": { "display_name": "dlk2", "language": "python", - "name": "dlk2" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1843,11 +2311,6 @@ }, "toc_section_display": true, "toc_window_display": true - }, - "vscode": { - "interpreter": { - "hash": "b80286374679f2ad472c61c83fc267d31329b5dea8e2dcaccb727123767724c5" - } } }, "nbformat": 4, diff --git a/requirements/requirements.txt b/requirements/requirements.txt index acb154d..d09f7c8 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -7,3 +7,5 @@ scikit-learn accelerate bitsandbytes lightning +peft +sentencepiece