diff --git a/001_mjc_CCS.ipynb b/001_mjc_CCS.ipynb
index 6ff8b27..464b69b 100644
--- a/001_mjc_CCS.ipynb
+++ b/001_mjc_CCS.ipynb
@@ -13,8 +13,8 @@
"execution_count": 1,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:20.941342Z",
- "start_time": "2023-05-07T11:21:19.408201Z"
+ "end_time": "2023-05-19T00:26:52.360728Z",
+ "start_time": "2023-05-19T00:26:50.894539Z"
}
},
"outputs": [
@@ -28,7 +28,7 @@
}
],
"source": [
- "from tqdm import tqdm\n",
+ "from tqdm.auto import tqdm\n",
"import copy\n",
"import numpy as np\n",
"import torch\n",
@@ -64,8 +64,8 @@
"execution_count": 2,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:30.089243Z",
- "start_time": "2023-05-07T11:21:20.942966Z"
+ "end_time": "2023-05-19T00:28:22.701155Z",
+ "start_time": "2023-05-19T00:26:52.362168Z"
}
},
"outputs": [
@@ -83,7 +83,7 @@
"================================================================================\n",
"bin /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so\n",
"CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching in backup paths...\n",
- "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
+ "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so.11.0\n",
"CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
"CUDA SETUP: Detected CUDA version 117\n",
"CUDA SETUP: Loading binary /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
@@ -105,13 +105,13 @@
" warn(msg)\n",
"/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/etc/xdg/xdg-cinnamon')}\n",
" warn(msg)\n",
- "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}\n",
+ "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//matplotlib_inline.backend_inline'), PosixPath('module')}\n",
" warn(msg)\n",
- "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so'), PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0')}.. We'll flip a coin and try one of these, in order to fail forward.\n",
+ "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: Found duplicate ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] files: {PosixPath('/usr/local/cuda/lib64/libcudart.so.11.0'), PosixPath('/usr/local/cuda/lib64/libcudart.so')}.. We'll flip a coin and try one of these, in order to fail forward.\n",
"Either way, this might cause trouble in the future:\n",
"If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n",
" warn(msg)\n",
- "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.19s/it]\n"
+ "Loading checkpoint shards: 100%|███████████████████████████████████████████| 2/2 [00:06<00:00, 3.49s/it]\n"
]
},
{
@@ -215,8 +215,8 @@
"execution_count": 3,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:30.093059Z",
- "start_time": "2023-05-07T11:21:30.090420Z"
+ "end_time": "2023-05-19T00:28:22.705285Z",
+ "start_time": "2023-05-19T00:28:22.702264Z"
}
},
"outputs": [],
@@ -241,8 +241,8 @@
"execution_count": 4,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:34.754867Z",
- "start_time": "2023-05-07T11:21:30.094318Z"
+ "end_time": "2023-05-19T00:28:26.441627Z",
+ "start_time": "2023-05-19T00:28:22.706863Z"
},
"scrolled": false
},
@@ -252,7 +252,7 @@
"output_type": "stream",
"text": [
"Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n",
- "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 53.01it/s]\n"
+ "100%|██████████████████████████████████████████████████████████████████████| 2/2 [00:01<00:00, 1.87it/s]\n"
]
}
],
@@ -273,8 +273,8 @@
"execution_count": 5,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:34.760016Z",
- "start_time": "2023-05-07T11:21:34.756939Z"
+ "end_time": "2023-05-19T00:28:26.446222Z",
+ "start_time": "2023-05-19T00:28:26.443481Z"
}
},
"outputs": [
@@ -348,8 +348,8 @@
"execution_count": 6,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:34.782525Z",
- "start_time": "2023-05-07T11:21:34.760992Z"
+ "end_time": "2023-05-19T00:28:26.468372Z",
+ "start_time": "2023-05-19T00:28:26.447139Z"
}
},
"outputs": [],
@@ -426,8 +426,8 @@
" output['hidden_states'] = torch.concat(output['hidden_states'])\n",
" \n",
" o = output['logits'].detach().cpu().float().softmax(-1)\n",
- " print(input_text)\n",
- " print(tokenizer.decode(o.argmax(-1)[0]))\n",
+ "# print(input_text)\n",
+ "# print(tokenizer.decode(o.argmax(-1)[0]))\n",
" \n",
" prob_0, prob1 = o[0, 1][[id_0, id_1]]\n",
" output['ans'] = (prob1/(prob_0+prob1)).item()\n",
@@ -457,8 +457,8 @@
"execution_count": 7,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:34.803314Z",
- "start_time": "2023-05-07T11:21:34.783680Z"
+ "end_time": "2023-05-19T00:28:26.489421Z",
+ "start_time": "2023-05-19T00:28:26.469251Z"
}
},
"outputs": [],
@@ -471,47 +471,11 @@
"execution_count": 8,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:35.557546Z",
- "start_time": "2023-05-07T11:21:34.804117Z"
+ "end_time": "2023-05-19T00:28:27.461762Z",
+ "start_time": "2023-05-19T00:28:26.490379Z"
}
},
"outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n",
- "###\n",
- "Is this review negative? 1\n",
- "###\n",
- "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n",
- "###\n",
- "Is this review negative? 0\n",
- "###\n",
- "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n",
- "###\n",
- "Is this review positive? 1\n",
- "###\n",
- "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n",
- "###\n",
- "Is this review negative? \n",
- "The: TheThe' F this scriptplay for this film should has readed a of on theifer Ball. or the autobiography. The'm read read a many in in a movieopic. andanging from the age life to Neweloron, Jamestown, her marriage years in Desi Arn The' go a book book of errorsual errors, but I would be on forever pages. The fact fair this' this thisille Ball was one of the peoplecomitable women who should cannot be portrayed on anyone other than herself. I you had toille,az, Ii Arn I., I would be furate at the this in were made in this movie. I only isakers should to to but they result was tofully rppy. me.\n",
- "Re#\n",
- "\" this the fair?\n",
- "10Is#\n",
- "Yesview: \"I movie of the Karie is a my, Ita Garbo is plays the,ie, a this the the other characters are different names. the original version. The versions filmed in- back, thebo was a a short in Germany. Thebo is wasly didored the German Christie in the version, the English version. I's a good movie, a good-see for Garbo fans.\"\n",
- "###\n",
- "Re this review negative? 1\n",
- "###\n",
- "Review: \"This' this is a greately movie film. It are some of laughilarious moments, thewwarming moments. keep found. the film. I story are all, the story are done.. Ianny Dever is a,x, isages to local team team theThe Brookels.' and is a in. movie. The,astic is the two actors who Gordon-Levitt and Bres ' who. who Lloyd plays also as thevinThe W' Mc the rest are great. this film-ch family film.\" I musting and fun-warming film that is should see.\"\n",
- "###\n",
- "Is this review negative? 0\n",
- "###\n",
- "Review: \"This husbandely wiferic been of the mostREATEATST in all generation. She' been to her album for yearsEARS. it still loveVE it! I I amm feeling the bad mood,' me happy good. When great mood and makesaporates. a in water rain. I is is makesozes withIFE. Iocals are greatawss perfectUNFFNING. theics are make. I of the's great gems.\" I CD a must islandle CD for my collection.\" I is isn made it big is a a me. Ione I listen this CD I matter what or blue, or, old, male, female,VERYONEDY lov ' thing:W is that?\"?\"\"\n",
- "###\n",
- "Is this review positive? 0\n"
- ]
- },
{
"data": {
"text/plain": [
@@ -551,8 +515,8 @@
"execution_count": 9,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:35.560533Z",
- "start_time": "2023-05-07T11:21:35.558659Z"
+ "end_time": "2023-05-19T00:28:27.465171Z",
+ "start_time": "2023-05-19T00:28:27.463324Z"
}
},
"outputs": [],
@@ -577,8 +541,8 @@
"execution_count": 10,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:35.585212Z",
- "start_time": "2023-05-07T11:21:35.562285Z"
+ "end_time": "2023-05-19T00:28:27.481113Z",
+ "start_time": "2023-05-19T00:28:27.467528Z"
}
},
"outputs": [
@@ -613,8 +577,8 @@
"execution_count": 11,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:35.600642Z",
- "start_time": "2023-05-07T11:21:35.586155Z"
+ "end_time": "2023-05-19T00:28:27.496331Z",
+ "start_time": "2023-05-19T00:28:27.482257Z"
}
},
"outputs": [],
@@ -656,8 +620,8 @@
"execution_count": 12,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:21:35.616885Z",
- "start_time": "2023-05-07T11:21:35.601707Z"
+ "end_time": "2023-05-19T00:28:27.512822Z",
+ "start_time": "2023-05-19T00:28:27.497303Z"
}
},
"outputs": [],
@@ -675,6 +639,7 @@
" # setup\n",
" model.eval()\n",
" all_neg_hs, all_pos_hs, all_gt_labels = [], [], []\n",
+ " all_neg_ans, all_pos_ans = [], []\n",
"\n",
" # loop\n",
" for _ in tqdm(range(n), unit='examples', desc='get_hidden_states'):\n",
@@ -689,7485 +654,34 @@
" \n",
" # get hidden states\n",
"# print(format_imdb(text, 0))\n",
- " neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type, layers=layers)\n",
- " pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type, layers=layers)\n",
+ " neg = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type, layers=layers)\n",
+ " pos = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type, layers=layers)\n",
"\n",
" # collect\n",
- " all_neg_hs.append(neg_hs)\n",
- " all_pos_hs.append(pos_hs)\n",
+ " all_neg_hs.append(neg['hidden_states'].flatten())\n",
+ " all_pos_hs.append(pos['hidden_states'].flatten())\n",
+ " all_pos_ans.append(pos['ans'])\n",
+ " all_neg_ans.append(neg['ans'])\n",
" all_gt_labels.append(true_label)\n",
"\n",
" all_neg_hs = np.stack(all_neg_hs)\n",
" all_pos_hs = np.stack(all_pos_hs)\n",
" all_gt_labels = np.stack(all_gt_labels)\n",
"\n",
- " return all_neg_hs, all_pos_hs, all_gt_labels"
+ " return all_neg_hs, all_pos_hs, all_gt_labels, np.array(all_neg_ans), np.array(all_pos_ans)"
]
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:30.270761Z",
- "start_time": "2023-05-07T11:21:35.617903Z"
+ "end_time": "2023-05-13T11:50:54.694584Z",
+ "start_time": "2023-05-13T11:50:54.692039Z"
}
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "\r",
- "get_hidden_states: 0%| | 0/100 [00:000.)*1.0)\n",
+ "neg_score = accuracy_score(y, (all_neg_ans<0.5)*1.0)\n",
+ "pos_score, neg_score"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -8234,37 +825,13 @@
"Note that because logistic regression is supervised we expect it to do better but to have worse generalisation that equivilent unsupervised methods. However in this case CSS is using a deeper model so it is more complicated."
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2023-05-07T11:08:12.494657Z",
- "start_time": "2023-05-07T11:08:12.482252Z"
- }
- },
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2023-05-07T11:10:15.610451Z",
- "start_time": "2023-05-07T11:10:15.607971Z"
- }
- },
- "outputs": [],
- "source": []
- },
{
"cell_type": "code",
"execution_count": 16,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:30.600357Z",
- "start_time": "2023-05-07T11:22:30.522522Z"
+ "end_time": "2023-05-19T00:29:24.317414Z",
+ "start_time": "2023-05-19T00:29:24.251856Z"
}
},
"outputs": [
@@ -8273,7 +840,7 @@
"output_type": "stream",
"text": [
"Logistic regression accuracy: 1.0 [TRAIN]\n",
- "Logistic regression accuracy: 0.98 [TEST]\n"
+ "Logistic regression accuracy: 0.96 [TEST]\n"
]
}
],
@@ -8281,8 +848,8 @@
"# let's create a simple 50/50 train split (the data is already randomized)\n",
"n = len(y)\n",
"\n",
- "neg_hs2 = torch.stack([h['hidden_states'].flatten() for h in neg_hs], 0)#.flatten()\n",
- "pos_hs2 = torch.stack([h['hidden_states'].flatten() for h in pos_hs], 0)#.flatten()\n",
+ "neg_hs2 = torch.from_numpy(np.stack([h.flatten() for h in neg_hs], 0))\n",
+ "pos_hs2 = torch.from_numpy(np.stack([h.flatten() for h in pos_hs], 0))\n",
"\n",
"neg_hs_train, neg_hs_test = neg_hs2[:n//2], neg_hs2[n//2:]\n",
"pos_hs_train, pos_hs_test = pos_hs2[:n//2], pos_hs2[n//2:]\n",
@@ -8304,8 +871,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T10:53:09.036137Z",
- "start_time": "2023-05-07T10:53:09.031305Z"
+ "end_time": "2023-05-14T00:05:52.801860Z",
+ "start_time": "2023-05-14T00:05:52.784513Z"
}
},
"outputs": [],
@@ -8323,8 +890,8 @@
"execution_count": 17,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:30.614094Z",
- "start_time": "2023-05-07T11:22:30.601720Z"
+ "end_time": "2023-05-19T00:29:24.321545Z",
+ "start_time": "2023-05-19T00:29:24.318715Z"
}
},
"outputs": [],
@@ -8346,129 +913,7 @@
" )\n",
"\n",
" def forward(self, x):\n",
- " return self.net(x)\n",
- "\n",
- "class CCS(object):\n",
- " def __init__(self, x0, x1, nepochs=1000, ntries=10, lr=1e-3, batch_size=-1, \n",
- " verbose=False, device=\"cuda\", linear=True, weight_decay=0.01, var_normalize=False):\n",
- " # data\n",
- " self.var_normalize = var_normalize\n",
- " self.x0 = self.normalize(x0)\n",
- " self.x1 = self.normalize(x1)\n",
- " self.d = self.x0.shape[-1]\n",
- "\n",
- " # training\n",
- " self.nepochs = nepochs\n",
- " self.ntries = ntries\n",
- " self.lr = lr\n",
- " self.verbose = verbose\n",
- " self.device = device\n",
- " self.batch_size = batch_size\n",
- " self.weight_decay = weight_decay\n",
- " \n",
- " # probe\n",
- " self.linear = linear\n",
- " self.probe = self.initialize_probe()\n",
- " self.best_probe = copy.deepcopy(self.probe)\n",
- "\n",
- " \n",
- " def initialize_probe(self):\n",
- " if self.linear:\n",
- " self.probe = nn.Sequential(nn.Linear(self.d, 1), nn.Sigmoid())\n",
- " else:\n",
- " self.probe = MLPProbe(self.d)\n",
- " return self.probe.to(self.device)\n",
- "\n",
- "\n",
- " def normalize(self, x):\n",
- " \"\"\"\n",
- " Mean-normalizes the data x (of shape (n, d))\n",
- " If self.var_normalize, also divides by the standard deviation\n",
- " \"\"\"\n",
- " normalized_x = x - x.mean(axis=0, keepdims=True)\n",
- " if self.var_normalize:\n",
- " normalized_x /= normalized_x.std(axis=0, keepdims=True)\n",
- "\n",
- " return normalized_x\n",
- "\n",
- " \n",
- " def get_tensor_data(self):\n",
- " \"\"\"\n",
- " Returns x0, x1 as appropriate tensors (rather than np arrays)\n",
- " \"\"\"\n",
- " x0 = self.x0.clone().detach().float().to(self.device)\n",
- " x1 = self.x1.clone().detach().float().to(self.device)\n",
- " return x0, x1\n",
- " \n",
- "\n",
- " def get_loss(self, p0, p1):\n",
- " \"\"\"\n",
- " Returns the CCS loss for two probabilities each of shape (n,1) or (n,)\n",
- " \"\"\"\n",
- " informative_loss = (torch.min(p0, p1)**2).mean(0)\n",
- " consistent_loss = ((p0 - (1-p1))**2).mean(0)\n",
- " return informative_loss + consistent_loss\n",
- "\n",
- "\n",
- " def get_acc(self, x0_test, x1_test, y_test):\n",
- " \"\"\"\n",
- " Computes accuracy for the current parameters on the given test inputs\n",
- " \"\"\"\n",
- " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- " with torch.no_grad():\n",
- " p0, p1 = self.best_probe(x0), self.best_probe(x1)\n",
- " avg_confidence = 0.5*(p0 + (1-p1))\n",
- " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n",
- " acc = (predictions == y_test).mean()\n",
- " acc = max(acc, 1 - acc)\n",
- "\n",
- " return acc\n",
- " \n",
- " \n",
- " def train(self):\n",
- " \"\"\"\n",
- " Does a single training run of nepochs epochs\n",
- " \"\"\"\n",
- " x0, x1 = self.get_tensor_data()\n",
- " permutation = torch.randperm(len(x0))\n",
- " x0, x1 = x0[permutation], x1[permutation]\n",
- " \n",
- " # set up optimizer\n",
- " optimizer = torch.optim.AdamW(self.probe.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n",
- " \n",
- " batch_size = len(x0) if self.batch_size == -1 else self.batch_size\n",
- " nbatches = len(x0) // batch_size\n",
- "\n",
- " # Start training (full batch)\n",
- " for epoch in range(self.nepochs):\n",
- " for j in range(nbatches):\n",
- " x0_batch = x0[j*batch_size:(j+1)*batch_size]\n",
- " x1_batch = x1[j*batch_size:(j+1)*batch_size]\n",
- " \n",
- " # probe\n",
- " p0, p1 = self.probe(x0_batch), self.probe(x1_batch)\n",
- "\n",
- " # get the corresponding loss\n",
- " loss = self.get_loss(p0, p1)\n",
- "\n",
- " # update the parameters\n",
- " optimizer.zero_grad()\n",
- " loss.backward()\n",
- " optimizer.step()\n",
- "\n",
- " return loss.detach().cpu().item()\n",
- " \n",
- " def repeated_train(self):\n",
- " best_loss = np.inf\n",
- " for train_num in range(self.ntries):\n",
- " self.initialize_probe()\n",
- " loss = self.train()\n",
- " if loss < best_loss:\n",
- " self.best_probe = copy.deepcopy(self.probe)\n",
- " best_loss = loss\n",
- "\n",
- " return best_loss"
+ " return self.net(x)\n"
]
},
{
@@ -8500,41 +945,22 @@
"execution_count": 18,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:42.769241Z",
- "start_time": "2023-05-07T11:22:30.614971Z"
+ "end_time": "2023-05-19T00:29:24.336538Z",
+ "start_time": "2023-05-19T00:29:24.322608Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CCS nonlinear train accuracy: 0.96\n",
- "CCS nonlinear test accuracy: 0.98\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/tmp/ipykernel_4095498/374574033.py:86: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- "/tmp/ipykernel_4095498/374574033.py:87: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "# Train CCS without any labels\n",
- "ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n",
- "ccs.repeated_train()\n",
+ "# # Train CCS without any labels\n",
+ "# ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n",
+ "# ccs.repeated_train()\n",
"\n",
- "# Evaluate\n",
- "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
- "print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n",
+ "# # Evaluate\n",
+ "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
+ "# print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n",
"\n",
- "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
- "print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))"
+ "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
+ "# print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))"
]
},
{
@@ -8542,41 +968,22 @@
"execution_count": 19,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:49.510569Z",
- "start_time": "2023-05-07T11:22:42.770553Z"
+ "end_time": "2023-05-19T00:29:24.357517Z",
+ "start_time": "2023-05-19T00:29:24.337335Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CCS train accuracy: 0.96\n",
- "CCS test accuracy: 0.98\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/tmp/ipykernel_4095498/374574033.py:86: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- "/tmp/ipykernel_4095498/374574033.py:87: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "# Train CCS without any labels\n",
- "ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n",
- "ccs.repeated_train()\n",
+ "# # Train CCS without any labels\n",
+ "# ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n",
+ "# ccs.repeated_train()\n",
"\n",
- "# Evaluate\n",
- "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
- "print(\"CCS train accuracy: {}\".format(ccs_acc))\n",
+ "# # Evaluate\n",
+ "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
+ "# print(\"CCS train accuracy: {}\".format(ccs_acc))\n",
"\n",
- "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
- "print(\"CCS test accuracy: {}\".format(ccs_acc))"
+ "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
+ "# print(\"CCS test accuracy: {}\".format(ccs_acc))"
]
},
{
@@ -8603,35 +1010,799 @@
"execution_count": 20,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:49.914580Z",
- "start_time": "2023-05-07T11:22:49.511794Z"
+ "end_time": "2023-05-19T00:29:25.004935Z",
+ "start_time": "2023-05-19T00:29:24.358571Z"
}
},
"outputs": [],
"source": [
- "import lightning.pytorch as pl\n",
+ "import lightning.pytorch as pl"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## DataModule"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-14T11:34:43.243172Z",
+ "start_time": "2023-05-14T11:34:43.240582Z"
+ }
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.507525Z",
+ "start_time": "2023-05-19T00:29:25.006004Z"
+ },
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n",
+ "get_hidden_states: 90%|███████████████████████████████████▊ | 895/1000 [08:21<00:58, 1.78examples/s]\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+ "│ in <cell line: 94>:94 │\n",
+ "│ │\n",
+ "│ 91 │\n",
+ "│ 92 # test │\n",
+ "│ 93 dm = IMBDHSDataModule(model, tokenizer) │\n",
+ "│ ❱ 94 dm.setup('train') │\n",
+ "│ 95 dl = dm.val_dataloader() │\n",
+ "│ 96 b = next(iter(dl)) │\n",
+ "│ 97 b │\n",
+ "│ │\n",
+ "│ in setup:39 │\n",
+ "│ │\n",
+ "│ 36 │ │ │\n",
+ "│ 37 │ │ self.dataset = load_dataset(self.hparams.dataset_name, split=\"test\") │\n",
+ "│ 38 │ │ │\n",
+ "│ ❱ 39 │ │ neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples( │\n",
+ "│ 40 │ │ │ self.model, self.tokenizer, self.dataset, self.hparams.model_type, n=self.hp │\n",
+ "│ 41 │ │ │\n",
+ "│ 42 │ │ # let's create a simple 50/50 train split (the data is already randomized) │\n",
+ "│ │\n",
+ "│ in get_hidden_states_many_examples:27 │\n",
+ "│ │\n",
+ "│ 24 │ │ │\n",
+ "│ 25 │ │ # get hidden states │\n",
+ "│ 26 # print(format_imdb(text, 0)) │\n",
+ "│ ❱ 27 │ │ neg = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model │\n",
+ "│ 28 │ │ pos = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model │\n",
+ "│ 29 │ │ │\n",
+ "│ 30 │ │ # collect │\n",
+ "│ │\n",
+ "│ in get_hidden_states:96 │\n",
+ "│ │\n",
+ "│ 93 # \"encoder\": get_encoder_hidden_states, \"encoder_decoder\": get_encoder_decoder_hidden_st │\n",
+ "│ 94 │ │ \"decoder\": get_decoder_hidden_states}[model_type] │\n",
+ "│ 95 │ │\n",
+ "│ ❱ 96 │ return fn(model, tokenizer, input_text, layers=layers) │\n",
+ "│ 97 │\n",
+ "│ │\n",
+ "│ in get_decoder_hidden_states:56 │\n",
+ "│ │\n",
+ "│ 53 │ │\n",
+ "│ 54 │ with torch.no_grad(): │\n",
+ "│ 55 │ │ # FIXME: should be a batch, to speed it up │\n",
+ "│ ❱ 56 │ │ output = model(input_ids, │\n",
+ "│ 57 │ │ │ │ │ output_hidden_states=True │\n",
+ "│ 58 # , output_attentions=True │\n",
+ "│ 59 │ │ │ │ │ ) │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 │\n",
+ "│ in _call_impl │\n",
+ "│ │\n",
+ "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
+ "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
+ "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
+ "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
+ "│ 1502 │ │ # Do not call functions when jit is used │\n",
+ "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
+ "│ 1504 │ │ backward_pre_hooks = [] │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n",
+ "│ new_forward │\n",
+ "│ │\n",
+ "│ 162 │ │ │ with torch.no_grad(): │\n",
+ "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 164 │ │ else: │\n",
+ "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
+ "│ 167 │ │\n",
+ "│ 168 │ module.forward = new_forward │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
+ "│ ng_llama.py:687 in forward │\n",
+ "│ │\n",
+ "│ 684 │ │ return_dict = return_dict if return_dict is not None else self.config.use_return │\n",
+ "│ 685 │ │ │\n",
+ "│ 686 │ │ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) │\n",
+ "│ ❱ 687 │ │ outputs = self.model( │\n",
+ "│ 688 │ │ │ input_ids=input_ids, │\n",
+ "│ 689 │ │ │ attention_mask=attention_mask, │\n",
+ "│ 690 │ │ │ position_ids=position_ids, │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 │\n",
+ "│ in _call_impl │\n",
+ "│ │\n",
+ "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
+ "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
+ "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
+ "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
+ "│ 1502 │ │ # Do not call functions when jit is used │\n",
+ "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
+ "│ 1504 │ │ backward_pre_hooks = [] │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n",
+ "│ new_forward │\n",
+ "│ │\n",
+ "│ 162 │ │ │ with torch.no_grad(): │\n",
+ "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 164 │ │ else: │\n",
+ "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
+ "│ 167 │ │\n",
+ "│ 168 │ module.forward = new_forward │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
+ "│ ng_llama.py:577 in forward │\n",
+ "│ │\n",
+ "│ 574 │ │ │ │ │ None, │\n",
+ "│ 575 │ │ │ │ ) │\n",
+ "│ 576 │ │ │ else: │\n",
+ "│ ❱ 577 │ │ │ │ layer_outputs = decoder_layer( │\n",
+ "│ 578 │ │ │ │ │ hidden_states, │\n",
+ "│ 579 │ │ │ │ │ attention_mask=attention_mask, │\n",
+ "│ 580 │ │ │ │ │ position_ids=position_ids, │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 │\n",
+ "│ in _call_impl │\n",
+ "│ │\n",
+ "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
+ "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
+ "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
+ "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
+ "│ 1502 │ │ # Do not call functions when jit is used │\n",
+ "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
+ "│ 1504 │ │ backward_pre_hooks = [] │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n",
+ "│ new_forward │\n",
+ "│ │\n",
+ "│ 162 │ │ │ with torch.no_grad(): │\n",
+ "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 164 │ │ else: │\n",
+ "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
+ "│ 167 │ │\n",
+ "│ 168 │ module.forward = new_forward │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
+ "│ ng_llama.py:305 in forward │\n",
+ "│ │\n",
+ "│ 302 │ │ # Fully Connected │\n",
+ "│ 303 │ │ residual = hidden_states │\n",
+ "│ 304 │ │ hidden_states = self.post_attention_layernorm(hidden_states) │\n",
+ "│ ❱ 305 │ │ hidden_states = self.mlp(hidden_states) │\n",
+ "│ 306 │ │ hidden_states = residual + hidden_states │\n",
+ "│ 307 │ │ │\n",
+ "│ 308 │ │ outputs = (hidden_states,) │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 │\n",
+ "│ in _call_impl │\n",
+ "│ │\n",
+ "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
+ "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
+ "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
+ "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
+ "│ 1502 │ │ # Do not call functions when jit is used │\n",
+ "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
+ "│ 1504 │ │ backward_pre_hooks = [] │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n",
+ "│ new_forward │\n",
+ "│ │\n",
+ "│ 162 │ │ │ with torch.no_grad(): │\n",
+ "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 164 │ │ else: │\n",
+ "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
+ "│ 167 │ │\n",
+ "│ 168 │ module.forward = new_forward │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/modeli │\n",
+ "│ ng_llama.py:157 in forward │\n",
+ "│ │\n",
+ "│ 154 │ │ self.act_fn = ACT2FN[hidden_act] │\n",
+ "│ 155 │ │\n",
+ "│ 156 │ def forward(self, x): │\n",
+ "│ ❱ 157 │ │ return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) │\n",
+ "│ 158 │\n",
+ "│ 159 │\n",
+ "│ 160 class LlamaAttention(nn.Module): │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/module.py:1501 │\n",
+ "│ in _call_impl │\n",
+ "│ │\n",
+ "│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │\n",
+ "│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │\n",
+ "│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │\n",
+ "│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │\n",
+ "│ 1502 │ │ # Do not call functions when jit is used │\n",
+ "│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │\n",
+ "│ 1504 │ │ backward_pre_hooks = [] │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/hooks.py:165 in │\n",
+ "│ new_forward │\n",
+ "│ │\n",
+ "│ 162 │ │ │ with torch.no_grad(): │\n",
+ "│ 163 │ │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 164 │ │ else: │\n",
+ "│ ❱ 165 │ │ │ output = old_forward(*args, **kwargs) │\n",
+ "│ 166 │ │ return module._hf_hook.post_forward(module, output) │\n",
+ "│ 167 │ │\n",
+ "│ 168 │ module.forward = new_forward │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/nn/modules.py:320 │\n",
+ "│ in forward │\n",
+ "│ │\n",
+ "│ 317 │ │ if self.bias is not None and self.bias.dtype != x.dtype: │\n",
+ "│ 318 │ │ │ self.bias.data = self.bias.data.to(x.dtype) │\n",
+ "│ 319 │ │ │\n",
+ "│ ❱ 320 │ │ out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state) │\n",
+ "│ 321 │ │ │\n",
+ "│ 322 │ │ if not self.state.has_fp16_weights: │\n",
+ "│ 323 │ │ │ if self.state.CB is not None and self.state.CxB is not None: │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/_functions │\n",
+ "│ .py:500 in matmul │\n",
+ "│ │\n",
+ "│ 497 │ state = state or MatmulLtState() │\n",
+ "│ 498 │ if threshold > 0.0: │\n",
+ "│ 499 │ │ state.threshold = threshold │\n",
+ "│ ❱ 500 │ return MatMul8bitLt.apply(A, B, out, bias, state) │\n",
+ "│ 501 │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/autograd/function.py:506 │\n",
+ "│ in apply │\n",
+ "│ │\n",
+ "│ 503 │ │ if not torch._C._are_functorch_transforms_active(): │\n",
+ "│ 504 │ │ │ # See NOTE: [functorch vjp and autograd interaction] │\n",
+ "│ 505 │ │ │ args = _functorch.utils.unwrap_dead_wrappers(args) │\n",
+ "│ ❱ 506 │ │ │ return super().apply(*args, **kwargs) # type: ignore[misc] │\n",
+ "│ 507 │ │ │\n",
+ "│ 508 │ │ if cls.setup_context == _SingleLevelFunction.setup_context: │\n",
+ "│ 509 │ │ │ raise RuntimeError( │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/_functions │\n",
+ "│ .py:323 in forward │\n",
+ "│ │\n",
+ "│ 320 │ │ # 1. Quantize A │\n",
+ "│ 321 │ │ if len(A.shape) == 3: │\n",
+ "│ 322 │ │ │ A = A.view(-1, A.shape[-1]).contiguous() │\n",
+ "│ ❱ 323 │ │ CA, CAt, SCA, SCAt, coo_tensorA = F.double_quant(A.to(torch.float16), threshold= │\n",
+ "│ 324 │ │ │\n",
+ "│ 325 │ │ if state.threshold > 0.0 and coo_tensorA is not None: │\n",
+ "│ 326 │ │ │ if state.has_fp16_weights: │\n",
+ "│ │\n",
+ "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/functional.py:1660 │\n",
+ "│ in double_quant │\n",
+ "│ │\n",
+ "│ 1657 │ │\n",
+ "│ 1658 │ is_on_gpu([A, col_stats, row_stats, out_col, out_row]) │\n",
+ "│ 1659 │ if threshold > 0.0: │\n",
+ "│ ❱ 1660 │ │ nnz = nnz_row_ptr[-1].item() │\n",
+ "│ 1661 │ │ if nnz > 0: │\n",
+ "│ 1662 │ │ │ coo_tensor = coo_zeros( │\n",
+ "│ 1663 │ │ │ │ A.shape[0], A.shape[1], nnz_row_ptr[-1].item(), device │\n",
+ "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+ "KeyboardInterrupt\n",
+ "\n"
+ ],
+ "text/plain": [
+ "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m94\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m91 \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m92 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m93 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m94 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m95 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m96 \u001b[0mb = \u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m97 \u001b[0mb \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m39\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m37 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.hparams.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m38 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m39 \u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples( \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m40 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[96mself\u001b[0m.dataset, \u001b[96mself\u001b[0m.hparams.model_type, n=\u001b[96mself\u001b[0m.hp \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m41 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m42 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# let's create a simple 50/50 train split (the data is already randomized)\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states_many_examples\u001b[0m:\u001b[94m27\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m24 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m25 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# get hidden states\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m26 \u001b[0m\u001b[2m# print(format_imdb(text, 0))\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m27 \u001b[2m│ │ \u001b[0mneg = get_hidden_states(model, tokenizer, format_imdb(text, \u001b[94m0\u001b[0m), model_type=model \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m28 \u001b[0m\u001b[2m│ │ \u001b[0mpos = get_hidden_states(model, tokenizer, format_imdb(text, \u001b[94m1\u001b[0m), model_type=model \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m29 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# collect\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92mget_hidden_states\u001b[0m:\u001b[94m96\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m93 \u001b[0m\u001b[2m# \"encoder\": get_encoder_hidden_states, \"encoder_decoder\": get_encoder_decoder_hidden_st\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m94 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[33m\"\u001b[0m\u001b[33mdecoder\u001b[0m\u001b[33m\"\u001b[0m: get_decoder_hidden_states}[model_type] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m95 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m96 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m fn(model, tokenizer, input_text, layers=layers) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m97 \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92mget_decoder_hidden_states\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# FIXME: should be a batch, to speed it up\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 \u001b[2m│ │ \u001b[0moutput = model(input_ids, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0moutput_hidden_states=\u001b[94mTrue\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[2m# , output_attentions=True\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m687\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m684 \u001b[0m\u001b[2m│ │ \u001b[0mreturn_dict = return_dict \u001b[94mif\u001b[0m return_dict \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[94melse\u001b[0m \u001b[96mself\u001b[0m.config.use_return \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m685 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m686 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m687 \u001b[2m│ │ \u001b[0moutputs = \u001b[96mself\u001b[0m.model( \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m688 \u001b[0m\u001b[2m│ │ │ \u001b[0minput_ids=input_ids, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m689 \u001b[0m\u001b[2m│ │ │ \u001b[0mattention_mask=attention_mask, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m690 \u001b[0m\u001b[2m│ │ │ \u001b[0mposition_ids=position_ids, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m577\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m574 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0m\u001b[94mNone\u001b[0m, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m575 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m576 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m577 \u001b[2m│ │ │ │ \u001b[0mlayer_outputs = decoder_layer( \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m578 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mhidden_states, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m579 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mattention_mask=attention_mask, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m580 \u001b[0m\u001b[2m│ │ │ │ │ \u001b[0mposition_ids=position_ids, \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m305\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m302 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Fully Connected\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m303 \u001b[0m\u001b[2m│ │ \u001b[0mresidual = hidden_states \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m304 \u001b[0m\u001b[2m│ │ \u001b[0mhidden_states = \u001b[96mself\u001b[0m.post_attention_layernorm(hidden_states) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m305 \u001b[2m│ │ \u001b[0mhidden_states = \u001b[96mself\u001b[0m.mlp(hidden_states) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m306 \u001b[0m\u001b[2m│ │ \u001b[0mhidden_states = residual + hidden_states \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m307 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m308 \u001b[0m\u001b[2m│ │ \u001b[0moutputs = (hidden_states,) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/transformers/models/llama/\u001b[0m\u001b[1;33mmodeli\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[1;33mng_llama.py\u001b[0m:\u001b[94m157\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m154 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.act_fn = ACT2FN[hidden_act] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m155 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m156 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92mforward\u001b[0m(\u001b[96mself\u001b[0m, x): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m157 \u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m.down_proj(\u001b[96mself\u001b[0m.act_fn(\u001b[96mself\u001b[0m.gate_proj(x)) * \u001b[96mself\u001b[0m.up_proj(x)) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m158 \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m159 \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m160 \u001b[0m\u001b[94mclass\u001b[0m \u001b[4;92mLlamaAttention\u001b[0m(nn.Module): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/nn/modules/\u001b[0m\u001b[1;33mmodule.py\u001b[0m:\u001b[94m1501\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92m_call_impl\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1498 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m (\u001b[96mself\u001b[0m._backward_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._backward_pre_hooks \u001b[95mor\u001b[0m \u001b[96mself\u001b[0m._forward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1499 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_backward_pre_hooks \u001b[95mor\u001b[0m _global_backward_hooks \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1500 \u001b[0m\u001b[2m│ │ │ │ \u001b[0m\u001b[95mor\u001b[0m _global_forward_hooks \u001b[95mor\u001b[0m _global_forward_pre_hooks): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1501 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m forward_call(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1502 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# Do not call functions when jit is used\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1503 \u001b[0m\u001b[2m│ │ \u001b[0mfull_backward_hooks, non_full_backward_hooks = [], [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1504 \u001b[0m\u001b[2m│ │ \u001b[0mbackward_pre_hooks = [] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/accelerate/\u001b[0m\u001b[1;33mhooks.py\u001b[0m:\u001b[94m165\u001b[0m in \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[92mnew_forward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m162 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mwith\u001b[0m torch.no_grad(): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m163 \u001b[0m\u001b[2m│ │ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m164 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m165 \u001b[2m│ │ │ \u001b[0moutput = old_forward(*args, **kwargs) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m166 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mreturn\u001b[0m module._hf_hook.post_forward(module, output) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m167 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m168 \u001b[0m\u001b[2m│ \u001b[0mmodule.forward = new_forward \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/nn/\u001b[0m\u001b[1;33mmodules.py\u001b[0m:\u001b[94m320\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m317 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.bias \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.bias.dtype != x.dtype: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m318 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[96mself\u001b[0m.bias.data = \u001b[96mself\u001b[0m.bias.data.to(x.dtype) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m319 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m320 \u001b[2m│ │ \u001b[0mout = bnb.matmul(x, \u001b[96mself\u001b[0m.weight, bias=\u001b[96mself\u001b[0m.bias, state=\u001b[96mself\u001b[0m.state) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m321 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m322 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m \u001b[96mself\u001b[0m.state.has_fp16_weights: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m323 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mself\u001b[0m.state.CB \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.state.CxB \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/\u001b[0m\u001b[1;33m_functions\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[1;33m.py\u001b[0m:\u001b[94m500\u001b[0m in \u001b[92mmatmul\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m497 \u001b[0m\u001b[2m│ \u001b[0mstate = state \u001b[95mor\u001b[0m MatmulLtState() \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m498 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m threshold > \u001b[94m0.0\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m499 \u001b[0m\u001b[2m│ │ \u001b[0mstate.threshold = threshold \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m500 \u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m MatMul8bitLt.apply(A, B, out, bias, state) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m501 \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/torch/autograd/\u001b[0m\u001b[1;33mfunction.py\u001b[0m:\u001b[94m506\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92mapply\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m503 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[95mnot\u001b[0m torch._C._are_functorch_transforms_active(): \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m504 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[2m# See NOTE: [functorch vjp and autograd interaction]\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m505 \u001b[0m\u001b[2m│ │ │ \u001b[0margs = _functorch.utils.unwrap_dead_wrappers(args) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m506 \u001b[2m│ │ │ \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96msuper\u001b[0m().apply(*args, **kwargs) \u001b[2m# type: ignore[misc]\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m507 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m508 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mcls\u001b[0m.setup_context == _SingleLevelFunction.setup_context: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m509 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mRuntimeError\u001b[0m( \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/autograd/\u001b[0m\u001b[1;33m_functions\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[1;33m.py\u001b[0m:\u001b[94m323\u001b[0m in \u001b[92mforward\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m320 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# 1. Quantize A\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m321 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[96mlen\u001b[0m(A.shape) == \u001b[94m3\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m322 \u001b[0m\u001b[2m│ │ │ \u001b[0mA = A.view(-\u001b[94m1\u001b[0m, A.shape[-\u001b[94m1\u001b[0m]).contiguous() \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m323 \u001b[2m│ │ \u001b[0mCA, CAt, SCA, SCAt, coo_tensorA = F.double_quant(A.to(torch.float16), threshold= \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m324 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m325 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m state.threshold > \u001b[94m0.0\u001b[0m \u001b[95mand\u001b[0m coo_tensorA \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m326 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m state.has_fp16_weights: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/\u001b[0m\u001b[1;33mfunctional.py\u001b[0m:\u001b[94m1660\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92mdouble_quant\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1657 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1658 \u001b[0m\u001b[2m│ \u001b[0mis_on_gpu([A, col_stats, row_stats, out_col, out_row]) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1659 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mif\u001b[0m threshold > \u001b[94m0.0\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1660 \u001b[2m│ │ \u001b[0mnnz = nnz_row_ptr[-\u001b[94m1\u001b[0m].item() \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1661 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m nnz > \u001b[94m0\u001b[0m: \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1662 \u001b[0m\u001b[2m│ │ │ \u001b[0mcoo_tensor = coo_zeros( \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m1663 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mA.shape[\u001b[94m0\u001b[0m], A.shape[\u001b[94m1\u001b[0m], nnz_row_ptr[-\u001b[94m1\u001b[0m].item(), device \u001b[31m│\u001b[0m\n",
+ "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
+ "\u001b[1;91mKeyboardInterrupt\u001b[0m\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "from dataclasses import dataclass\n",
+ "from torch.utils.data import random_split, DataLoader, TensorDataset\n",
+ "from transformers.models.auto.modeling_auto import AutoModel\n",
+ "# from scipy.stats import zscore\n",
+ "\n",
+ "from sklearn.preprocessing import RobustScaler\n",
+ "\n",
+ "# def normalize(x):\n",
+ "# \"\"\"\n",
+ "# Mean-normalizes the data x (of shape (n, d))\n",
+ "# If self.var_normalize, also divides by the standard deviation\n",
+ "# \"\"\"\n",
+ "# normalized_x = x - x.mean(axis=0, keepdims=True)\n",
+ "# if self.var_normalize:\n",
+ "# normalized_x /= normalized_x.std(axis=0, keepdims=True)\n",
+ "\n",
+ "# return normalized_x\n",
+ "\n",
+ "\n",
+ "class IMBDHSDataModule(pl.LightningDataModule):\n",
+ "\n",
+ " def __init__(self,\n",
+ " model: AutoModel,\n",
+ " tokenizer: AutoTokenizer,\n",
+ " model_type=\"decoder\",\n",
+ " dataset_name=\"amazon_polarity\",\n",
+ " batch_size=32,\n",
+ " n=200,\n",
+ " ):\n",
+ " super().__init__()\n",
+ " self.model = model\n",
+ " self.tokenizer = tokenizer\n",
+ " self.save_hyperparameters(ignore=[\"model\", \"tokenizer\"])\n",
+ "\n",
+ " def setup(self, stage: str):\n",
+ "\n",
+ " self.dataset = load_dataset(self.hparams.dataset_name, split=\"test\")\n",
+ "\n",
+ " neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(\n",
+ " self.model, self.tokenizer, self.dataset, self.hparams.model_type, n=self.hparams.n, layers=[2, -2])\n",
+ "\n",
+ " # let's create a simple 50/50 train split (the data is already randomized)\n",
+ " n = len(y)\n",
+ " val_split = int(n * 0.5)\n",
+ " test_split = int(n * 0.75)\n",
+ " neg_hs_train, pos_hs_train, y_train = neg_hs[:\n",
+ " val_split], pos_hs[:\n",
+ " val_split], y[:\n",
+ " val_split]\n",
+ " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[\n",
+ " val_split:test_split], y[val_split:test_split]\n",
+ " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[\n",
+ " test_split:], y[test_split:]\n",
+ "\n",
+ " # for simplicity we can just take the difference between positive and negative hidden states\n",
+ " # (concatenating also works fine)\n",
+ " self.x_train = neg_hs_train - pos_hs_train\n",
+ " self.x_val = neg_hs_val - pos_hs_val\n",
+ " self.x_test = neg_hs_test - pos_hs_test\n",
+ "\n",
+ " # normalize\n",
+ " self.scaler = RobustScaler()\n",
+ " self.scaler.fit(self.x_train)\n",
+ " self.x_train = self.scaler.transform(self.x_train)\n",
+ " self.x_val = self.scaler.transform(self.x_val)\n",
+ " self.x_test = self.scaler.transform(self.x_test)\n",
+ "\n",
+ " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train).float(),\n",
+ " torch.from_numpy(pos_hs_train).float(),\n",
+ " torch.from_numpy(y_train).float())\n",
+ "\n",
+ " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val).float(),\n",
+ " torch.from_numpy(pos_hs_val).float(),\n",
+ " torch.from_numpy(y_val).float())\n",
+ "\n",
+ " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test).float(),\n",
+ " torch.from_numpy(pos_hs_test).float(),\n",
+ " torch.from_numpy(y_test).float())\n",
+ "\n",
+ " def train_dataloader(self):\n",
+ " return DataLoader(self.ds_train,\n",
+ " batch_size=self.hparams.batch_size,\n",
+ " shuffle=True)\n",
+ "\n",
+ " def val_dataloader(self):\n",
+ " return DataLoader(self.ds_val, batch_size=self.hparams.batch_size)\n",
+ "\n",
+ " def test_dataloader(self):\n",
+ " return DataLoader(self.ds_test, batch_size=self.hparams.batch_size)\n",
+ "\n",
+ "\n",
+ "# test\n",
+ "dm = IMBDHSDataModule(model, tokenizer)\n",
+ "dm.setup('train')\n",
+ "dl = dm.val_dataloader()\n",
+ "b = next(iter(dl))\n",
+ "b"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.508827Z",
+ "start_time": "2023-05-19T01:18:07.508819Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dm.x_test.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## LightningModel"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.509553Z",
+ "start_time": "2023-05-19T01:18:07.509545Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from torch import optim"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.510256Z",
+ "start_time": "2023-05-19T01:18:07.510249Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "\n",
+ "def get_loss(p0, p1):\n",
+ " \"\"\"\n",
+ " Returns the CCS loss for two probabilities each of shape (n,1) or (n,)\n",
+ " \"\"\"\n",
+ " informative_loss = (torch.min(p0, p1)**2).mean(0)\n",
+ " consistent_loss = ((p0 - (1-p1))**2).mean(0)\n",
+ " return informative_loss + consistent_loss\n",
+ "\n",
+ "\n",
+ "def get_acc(p0, p1, y):\n",
+ " avg_confidence = 0.5*(p0 + (1-p1))\n",
+ " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n",
+ " \n",
+ " # TODO f1\n",
+ " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n",
+ " \n",
+ " acc = (predictions == y.cpu().numpy()).mean()\n",
+ " acc = max(acc, 1 - acc)\n",
+ " return predictions, acc\n",
+ "\n",
+ "def get_f1(p0, p1, y):\n",
+ " avg_confidence = 0.5*(p0 + (1-p1))\n",
+ " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n",
+ " \n",
+ " # TODO f1\n",
+ " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n",
+ " auc = roc_auc_score(y.cpu().numpy(), predictions)\n",
+ " \n",
+ " auc = max(auc, 1 - auc)\n",
+ " return predictions, auc\n",
"\n",
"class CSS(pl.LightningModule):\n",
- " def __init__(self, encoder, decoder):\n",
+ " def __init__(self, d, max_epochs, lr=4e-3, weight_decay=1e-6):\n",
" super().__init__()\n",
- " self.encoder = encoder\n",
- " self.decoder = decoder\n",
+ " self.probe = MLPProbe(d)\n",
+ " self.save_hyperparameters()\n",
+ " \n",
+ " def forward(self, x):\n",
+ " return self.probe(x)\n",
+ " \n",
+ " def _step(self, batch, batch_idx, stage='train'):\n",
+ " x0, x1, y = batch\n",
+ " p0, p1 = self(x0), self(x1)\n",
+ " \n",
+ " loss = get_loss(p0, p1)\n",
+ " \n",
+ " self.log(f\"{stage}/loss\", loss)\n",
+ " \n",
+ " predictions, acc = get_acc(p0, p1, y)\n",
+ " self.log(f\"{stage}/acc\", acc)\n",
+ " predictions, f1 = get_f1(p0, p1, y)\n",
+ " self.log(f\"{stage}/f1\", f1)\n",
+ " return loss\n",
" \n",
" def training_step(self, batch, batch_idx):\n",
- " # training_step defines the train loop.\n",
- " # it is independent of forward\n",
- " x, y = batch\n",
- " x = x.view(x.size(0), -1)\n",
- " z = self.encoder(x)\n",
- " x_hat = self.decoder(z)\n",
- " loss = nn.functional.mse_loss(x_hat, x)\n",
- " # Logging to TensorBoard (if installed) by default\n",
- " self.log(\"train_loss\", loss)\n",
- " return loss\n",
+ " return self._step(batch, batch_idx)\n",
+ " \n",
+ " def validation_step(self, batch, batch_idx=0):\n",
+ " return self._step(batch, batch_idx, stage='val')\n",
+ " \n",
+ " def prediction_step(self, batch, batch_idx):\n",
+ " x0, x1, y = batch\n",
+ " p0, p1 = self(x0), self(x1)\n",
+ " predictions, acc = get_acc(p0, p1, y)\n",
+ " return predictions \n",
"\n",
" def configure_optimizers(self):\n",
- " optimizer = optim.Adam(self.parameters(), lr=1e-3)\n",
- " return optimizer\n",
+ " optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr, weight_decay=self.hparams.weight_decay)\n",
+ " lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(\n",
+ " optimizer, T_max=self.hparams.max_epochs, eta_min=self.hparams.lr / 50\n",
+ " )\n",
+ " return [optimizer], [lr_scheduler]\n",
" "
]
},
@@ -8649,232 +1820,31 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.487623Z",
- "start_time": "2023-05-07T11:22:49.915848Z"
+ "end_time": "2023-05-14T06:17:57.365689Z",
+ "start_time": "2023-05-14T06:17:57.356995Z"
}
},
- "outputs": [
- {
- "data": {
- "text/html": [
- "╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
- "│ in <cell line: 56>:56 │\n",
- "│ │\n",
- "│ 53 │\n",
- "│ 54 # test │\n",
- "│ 55 dm = IMBDHSDataModule(model, tokenizer) │\n",
- "│ ❱ 56 dm.setup('train') │\n",
- "│ 57 dl = dm.val_dataloader() │\n",
- "│ 58 next(iter(dl)) │\n",
- "│ 59 │\n",
- "│ │\n",
- "│ in setup:15 │\n",
- "│ │\n",
- "│ 12 │ │\n",
- "│ 13 │ │\n",
- "│ 14 │ def setup(self, stage:str): │\n",
- "│ ❱ 15 │ │ self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True) │\n",
- "│ 16 │ │ │\n",
- "│ 17 │ │ neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer, │\n",
- "│ 18 │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1773 in │\n",
- "│ load_dataset │\n",
- "│ │\n",
- "│ 1770 │ ) │\n",
- "│ 1771 │ │\n",
- "│ 1772 │ # Create a dataset builder │\n",
- "│ ❱ 1773 │ builder_instance = load_dataset_builder( │\n",
- "│ 1774 │ │ path=path, │\n",
- "│ 1775 │ │ name=name, │\n",
- "│ 1776 │ │ data_dir=data_dir, │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1528 in │\n",
- "│ load_dataset_builder │\n",
- "│ │\n",
- "│ 1525 │ │ raise ValueError(error_msg) │\n",
- "│ 1526 │ │\n",
- "│ 1527 │ # Instantiate the dataset builder │\n",
- "│ ❱ 1528 │ builder_instance: DatasetBuilder = builder_cls( │\n",
- "│ 1529 │ │ cache_dir=cache_dir, │\n",
- "│ 1530 │ │ config_name=config_name, │\n",
- "│ 1531 │ │ data_dir=data_dir, │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:340 in │\n",
- "│ __init__ │\n",
- "│ │\n",
- "│ 337 │ │ │ config_kwargs[\"data_files\"] = data_files │\n",
- "│ 338 │ │ if data_dir is not None: │\n",
- "│ 339 │ │ │ config_kwargs[\"data_dir\"] = data_dir │\n",
- "│ ❱ 340 │ │ self.config, self.config_id = self._create_builder_config( │\n",
- "│ 341 │ │ │ config_name=config_name, │\n",
- "│ 342 │ │ │ custom_features=features, │\n",
- "│ 343 │ │ │ **config_kwargs, │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:494 in │\n",
- "│ _create_builder_config │\n",
- "│ │\n",
- "│ 491 │ │ │ │ config_kwargs[\"name\"] = self.DEFAULT_CONFIG_NAME │\n",
- "│ 492 │ │ │ if \"version\" not in config_kwargs and hasattr(self, \"VERSION\") and self.VERS │\n",
- "│ 493 │ │ │ │ config_kwargs[\"version\"] = self.VERSION │\n",
- "│ ❱ 494 │ │ │ builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs) │\n",
- "│ 495 │ │ │\n",
- "│ 496 │ │ # otherwise use the config_kwargs to overwrite the attributes │\n",
- "│ 497 │ │ else: │\n",
- "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
- "TypeError: __init__() got an unexpected keyword argument 'stream'\n",
- "\n"
- ],
- "text/plain": [
- "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
- "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m15\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m12 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m14 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92msetup\u001b[0m(\u001b[96mself\u001b[0m, stage:\u001b[96mstr\u001b[0m): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m15 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m, stream=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m16 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y = get_hidden_states_many_examples(\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1773\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92mload_dataset\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1770 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1771 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1772 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create a dataset builder\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1773 \u001b[2m│ \u001b[0mbuilder_instance = load_dataset_builder( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1774 \u001b[0m\u001b[2m│ │ \u001b[0mpath=path, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1775 \u001b[0m\u001b[2m│ │ \u001b[0mname=name, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1776 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1528\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92mload_dataset_builder\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1525 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m(error_msg) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1526 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1527 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Instantiate the dataset builder\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1528 \u001b[2m│ \u001b[0mbuilder_instance: DatasetBuilder = builder_cls( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1529 \u001b[0m\u001b[2m│ │ \u001b[0mcache_dir=cache_dir, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1530 \u001b[0m\u001b[2m│ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1531 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m340\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 337 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_files\u001b[0m\u001b[33m\"\u001b[0m] = data_files \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 338 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m data_dir \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 339 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_dir\u001b[0m\u001b[33m\"\u001b[0m] = data_dir \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 340 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.config, \u001b[96mself\u001b[0m.config_id = \u001b[96mself\u001b[0m._create_builder_config( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 341 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 342 \u001b[0m\u001b[2m│ │ │ \u001b[0mcustom_features=features, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 343 \u001b[0m\u001b[2m│ │ │ \u001b[0m**config_kwargs, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m494\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92m_create_builder_config\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 491 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mname\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.DEFAULT_CONFIG_NAME \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 492 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m config_kwargs \u001b[95mand\u001b[0m \u001b[96mhasattr\u001b[0m(\u001b[96mself\u001b[0m, \u001b[33m\"\u001b[0m\u001b[33mVERSION\u001b[0m\u001b[33m\"\u001b[0m) \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.VERS \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 493 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.VERSION \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 494 \u001b[2m│ │ │ \u001b[0mbuilder_config = \u001b[96mself\u001b[0m.BUILDER_CONFIG_CLASS(**config_kwargs) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 495 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 496 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# otherwise use the config_kwargs to overwrite the attributes\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 497 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
- "\u001b[1;91mTypeError: \u001b[0m\u001b[1;35m__init__\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m got an unexpected keyword argument \u001b[32m'stream'\u001b[0m\n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "from dataclasses import dataclass\n",
- "from torch.utils.data import random_split, DataLoader, TensorDataset\n",
- "from transformers.models.auto.modeling_auto import AutoModel\n",
- "\n",
- "@dataclass\n",
- "class IMBDHSDataModule(pl.LightningDataModule):\n",
- " model: AutoModel\n",
- " tokenizer: AutoTokenizer\n",
- " model_type=\"decoder\"\n",
- " dataset_name=\"amazon_polarity\"\n",
- " batch_size=32\n",
- " \n",
- " \n",
- " def setup(self, stage:str):\n",
- " self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)\n",
- " \n",
- " neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer, self.dataset, self.model_type)\n",
- " \n",
- " # let's create a simple 50/50 train split (the data is already randomized)\n",
- " n = len(y)\n",
- " val_split = int(n*0.5)\n",
- " test_split = int(n*0.75)\n",
- " neg_hs_train, pos_hs_train, y_train = neg_hs[:val_split], pos_hs[:val_split], y[:val_split]\n",
- " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[val_split:test_split], y[val_split:test_split]\n",
- " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[test_split:], y[test_split:]\n",
- "\n",
- " # for simplicity we can just take the difference between positive and negative hidden states\n",
- " # (concatenating also works fine)\n",
- " self.x_train = neg_hs_train - pos_hs_train\n",
- " self.x_val = neg_hs_val - pos_hs_val\n",
- " self.x_test = neg_hs_test - pos_hs_test\n",
- " \n",
- " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train),\n",
- " torch.from_numpy(pos_hs_train),\n",
- " torch.from_numpy(y_train))\n",
- " \n",
- " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val),\n",
- " torch.from_numpy(pos_hs_val),\n",
- " torch.from_numpy(y_val))\n",
- " \n",
- " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test),\n",
- " torch.from_numpy(pos_hs_test),\n",
- " torch.from_numpy(y_test))\n",
- " \n",
- " def train_dataloader(self):\n",
- " return DataLoader(self.ds_train, batch_size=self.batch_size)\n",
- "\n",
- " def val_dataloader(self):\n",
- " return DataLoader(self.ds_val, batch_size=self.batch_size)\n",
- "\n",
- " def test_dataloader(self):\n",
- " return DataLoader(self.ds_test, batch_size=self.batch_size)\n",
- " \n",
- "# test\n",
- "dm = IMBDHSDataModule(model, tokenizer)\n",
- "dm.setup('train')\n",
- "dl = dm.val_dataloader()\n",
- "next(iter(dl))"
- ]
+ "outputs": [],
+ "source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.488619Z",
- "start_time": "2023-05-07T11:22:55.488612Z"
+ "end_time": "2023-05-19T01:18:07.510853Z",
+ "start_time": "2023-05-19T01:18:07.510846Z"
}
},
"outputs": [],
"source": [
"# init the autoencoder\n",
- "net = CSS()"
+ "max_epochs = 1000\n",
+ "d = b[0].shape[-1]\n",
+ "net = CSS(d=d, max_epochs=max_epochs)"
]
},
{
@@ -8882,13 +1852,13 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.489380Z",
- "start_time": "2023-05-07T11:22:55.489373Z"
+ "end_time": "2023-05-19T01:18:07.511523Z",
+ "start_time": "2023-05-19T01:18:07.511516Z"
}
},
"outputs": [],
"source": [
- "train_loader = utils.data.DataLoader(dataset)"
+ "# train_loader = utils.data.DataLoader(dataset)"
]
},
{
@@ -8896,17 +1866,128 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.489937Z",
- "start_time": "2023-05-07T11:22:55.489931Z"
- }
+ "end_time": "2023-05-19T01:18:07.512087Z",
+ "start_time": "2023-05-19T01:18:07.512080Z"
+ },
+ "scrolled": true
},
"outputs": [],
"source": [
"# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\n",
- "trainer = pl.Trainer(limit_train_batches=100, max_epochs=1)\n",
- "trainer.fit(model=net, train_dataloaders=train_loader)"
+ "trainer = pl.Trainer(limit_train_batches=100, max_epochs=max_epochs)\n",
+ "trainer.fit(model=net, datamodule=dm)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.512749Z",
+ "start_time": "2023-05-19T01:18:07.512742Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "%debug"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-14T06:21:46.356828Z",
+ "start_time": "2023-05-14T06:21:46.351801Z"
+ }
+ },
+ "source": [
+ "# Read hist"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.513235Z",
+ "start_time": "2023-05-19T01:18:07.513229Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# import pytorch_lightning as pl\n",
+ "from lightning.pytorch.loggers.csv_logs import CSVLogger\n",
+ "# from pytorch_lightning.loggers.csv_logs import CSVLogger as CSVLogger2\n",
+ "from pathlib import Path\n",
+ "import pandas as pd\n",
+ "\n",
+ "def read_metrics_csv(metrics_file_path):\n",
+ " df_hist = pd.read_csv(metrics_file_path)\n",
+ " df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()\n",
+ " df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()\n",
+ " return df_histe\n",
+ "\n",
+ "\n",
+ "def read_hist(trainer: pl.Trainer):\n",
+ "\n",
+ " ts = [t for t in trainer.loggers if isinstance(t, CSVLogger)]\n",
+ " print(ts)\n",
+ " try:\n",
+ " metrics_file_path = Path(ts[0].experiment.metrics_file_path)\n",
+ " df_histe = read_metrics_csv(metrics_file_path)\n",
+ " return df_histe\n",
+ " except Exception as e:\n",
+ " raise e\n",
+ " print(e)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.513826Z",
+ "start_time": "2023-05-19T01:18:07.513819Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "df_hist = read_hist(trainer).ffill().bfill()\n",
+ "df_hist"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T01:18:07.514335Z",
+ "start_time": "2023-05-19T01:18:07.514329Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "df_hist[['val/acc', 'train/acc']].plot()\n",
+ "\n",
+ "df_hist[['val/f1', 'train/f1']].plot()\n",
+ "\n",
+ "df_hist[['val/loss', 'train/loss']].plot()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/mjc_notes.md b/mjc_notes.md
index 80b09ee..d719443 100644
--- a/mjc_notes.md
+++ b/mjc_notes.md
@@ -6,3 +6,9 @@ mamba install -y pytorch torchvision torchaudio pytorch-cuda=11.7 cudatoolkit-de
mamba install -y ipykernel pip
pip install -r requirements.txt
```
+
+# 2023-05-13 15:17:05
+
+- [x] Convert it to lightning
+- [ ] batch for get hidden states
+ - [ ] and cache
| |