\u001b[0m:\u001b[94m33\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m30 \u001b[0m\u001b[2m│ │ \u001b[0m]) \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m31 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m32 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# FIXME not all the hidden state are the same size, wat\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m33 \u001b[2m│ \u001b[0mres = [np.concatenate(r) \u001b[94mfor\u001b[0m r \u001b[95min\u001b[0m \u001b[96mzip\u001b[0m(*res)] \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m34 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m res \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m35 \u001b[0m\u001b[2m│ \u001b[0mall_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans = res \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m \u001b[2m36 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mreturn\u001b[0m all_neg_hs, all_pos_hs, all_gt_labels, all_neg_ans, all_pos_ans \u001b[31m│\u001b[0m\n",
+ "\u001b[31m│\u001b[0m in \u001b[92mconcatenate\u001b[0m:\u001b[94m200\u001b[0m \u001b[31m│\u001b[0m\n",
+ "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
+ "\u001b[1;91mValueError: \u001b[0mall the input array dimensions except for the concatenation axis must match exactly, but along \n",
+ "dimension \u001b[1;36m1\u001b[0m, the array at index \u001b[1;36m0\u001b[0m has size \u001b[1;36m2121728\u001b[0m and the array at index \u001b[1;36m1\u001b[0m has size \u001b[1;36m1867776\u001b[0m\n"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(model, tokenizer, data, model_type)\n",
"\n",
- "pos_score = roc_auc_score(y, np.array([h['ans'] for h in pos_hs]))\n",
- "neg_score = roc_auc_score(y, np.array([h['ans'] for h in neg_hs]))\n",
+ "\n",
+ "gc.collect()\n",
+ "torch.cuda.empty_cache()\n",
+ "gc.collect()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T23:04:04.203349Z",
+ "start_time": "2023-05-19T23:04:04.203341Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# all_pos_ans"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T23:04:04.204205Z",
+ "start_time": "2023-05-19T23:04:04.204197Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# roc_auc_score\n",
+ "pos_score = roc_auc_score(y, all_pos_ans)\n",
+ "neg_score = roc_auc_score(y, all_neg_ans)\n",
"pos_score, neg_score"
]
},
@@ -8215,13 +1097,30 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T10:48:17.261806Z",
- "start_time": "2023-05-07T10:48:17.257349Z"
+ "end_time": "2023-05-14T11:32:21.359783Z",
+ "start_time": "2023-05-14T11:32:20.787141Z"
+ }
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T23:04:04.204853Z",
+ "start_time": "2023-05-19T23:04:04.204842Z"
},
"scrolled": true
},
"outputs": [],
- "source": []
+ "source": [
+ "# accuracy_score\n",
+ "pos_score = accuracy_score(y, (all_pos_ans>0.)*1.0)\n",
+ "neg_score = accuracy_score(y, (all_neg_ans<0.5)*1.0)\n",
+ "pos_score, neg_score"
+ ]
},
{
"cell_type": "markdown",
@@ -8239,50 +1138,17 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:08:12.494657Z",
- "start_time": "2023-05-07T11:08:12.482252Z"
+ "end_time": "2023-05-19T22:49:25.447229Z",
+ "start_time": "2023-05-19T22:49:25.447221Z"
}
},
"outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2023-05-07T11:10:15.610451Z",
- "start_time": "2023-05-07T11:10:15.607971Z"
- }
- },
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": 16,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2023-05-07T11:22:30.600357Z",
- "start_time": "2023-05-07T11:22:30.522522Z"
- }
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Logistic regression accuracy: 1.0 [TRAIN]\n",
- "Logistic regression accuracy: 0.98 [TEST]\n"
- ]
- }
- ],
"source": [
"# let's create a simple 50/50 train split (the data is already randomized)\n",
"n = len(y)\n",
"\n",
- "neg_hs2 = torch.stack([h['hidden_states'].flatten() for h in neg_hs], 0)#.flatten()\n",
- "pos_hs2 = torch.stack([h['hidden_states'].flatten() for h in pos_hs], 0)#.flatten()\n",
+ "neg_hs2 = torch.from_numpy(np.stack([h.flatten() for h in neg_hs], 0))\n",
+ "pos_hs2 = torch.from_numpy(np.stack([h.flatten() for h in pos_hs], 0))\n",
"\n",
"neg_hs_train, neg_hs_test = neg_hs2[:n//2], neg_hs2[n//2:]\n",
"pos_hs_train, pos_hs_test = pos_hs2[:n//2], pos_hs2[n//2:]\n",
@@ -8304,8 +1170,8 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T10:53:09.036137Z",
- "start_time": "2023-05-07T10:53:09.031305Z"
+ "end_time": "2023-05-14T00:05:52.801860Z",
+ "start_time": "2023-05-14T00:05:52.784513Z"
}
},
"outputs": [],
@@ -8320,11 +1186,11 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:30.614094Z",
- "start_time": "2023-05-07T11:22:30.601720Z"
+ "end_time": "2023-05-19T22:49:25.447898Z",
+ "start_time": "2023-05-19T22:49:25.447890Z"
}
},
"outputs": [],
@@ -8346,129 +1212,7 @@
" )\n",
"\n",
" def forward(self, x):\n",
- " return self.net(x)\n",
- "\n",
- "class CCS(object):\n",
- " def __init__(self, x0, x1, nepochs=1000, ntries=10, lr=1e-3, batch_size=-1, \n",
- " verbose=False, device=\"cuda\", linear=True, weight_decay=0.01, var_normalize=False):\n",
- " # data\n",
- " self.var_normalize = var_normalize\n",
- " self.x0 = self.normalize(x0)\n",
- " self.x1 = self.normalize(x1)\n",
- " self.d = self.x0.shape[-1]\n",
- "\n",
- " # training\n",
- " self.nepochs = nepochs\n",
- " self.ntries = ntries\n",
- " self.lr = lr\n",
- " self.verbose = verbose\n",
- " self.device = device\n",
- " self.batch_size = batch_size\n",
- " self.weight_decay = weight_decay\n",
- " \n",
- " # probe\n",
- " self.linear = linear\n",
- " self.probe = self.initialize_probe()\n",
- " self.best_probe = copy.deepcopy(self.probe)\n",
- "\n",
- " \n",
- " def initialize_probe(self):\n",
- " if self.linear:\n",
- " self.probe = nn.Sequential(nn.Linear(self.d, 1), nn.Sigmoid())\n",
- " else:\n",
- " self.probe = MLPProbe(self.d)\n",
- " return self.probe.to(self.device)\n",
- "\n",
- "\n",
- " def normalize(self, x):\n",
- " \"\"\"\n",
- " Mean-normalizes the data x (of shape (n, d))\n",
- " If self.var_normalize, also divides by the standard deviation\n",
- " \"\"\"\n",
- " normalized_x = x - x.mean(axis=0, keepdims=True)\n",
- " if self.var_normalize:\n",
- " normalized_x /= normalized_x.std(axis=0, keepdims=True)\n",
- "\n",
- " return normalized_x\n",
- "\n",
- " \n",
- " def get_tensor_data(self):\n",
- " \"\"\"\n",
- " Returns x0, x1 as appropriate tensors (rather than np arrays)\n",
- " \"\"\"\n",
- " x0 = self.x0.clone().detach().float().to(self.device)\n",
- " x1 = self.x1.clone().detach().float().to(self.device)\n",
- " return x0, x1\n",
- " \n",
- "\n",
- " def get_loss(self, p0, p1):\n",
- " \"\"\"\n",
- " Returns the CCS loss for two probabilities each of shape (n,1) or (n,)\n",
- " \"\"\"\n",
- " informative_loss = (torch.min(p0, p1)**2).mean(0)\n",
- " consistent_loss = ((p0 - (1-p1))**2).mean(0)\n",
- " return informative_loss + consistent_loss\n",
- "\n",
- "\n",
- " def get_acc(self, x0_test, x1_test, y_test):\n",
- " \"\"\"\n",
- " Computes accuracy for the current parameters on the given test inputs\n",
- " \"\"\"\n",
- " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- " with torch.no_grad():\n",
- " p0, p1 = self.best_probe(x0), self.best_probe(x1)\n",
- " avg_confidence = 0.5*(p0 + (1-p1))\n",
- " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n",
- " acc = (predictions == y_test).mean()\n",
- " acc = max(acc, 1 - acc)\n",
- "\n",
- " return acc\n",
- " \n",
- " \n",
- " def train(self):\n",
- " \"\"\"\n",
- " Does a single training run of nepochs epochs\n",
- " \"\"\"\n",
- " x0, x1 = self.get_tensor_data()\n",
- " permutation = torch.randperm(len(x0))\n",
- " x0, x1 = x0[permutation], x1[permutation]\n",
- " \n",
- " # set up optimizer\n",
- " optimizer = torch.optim.AdamW(self.probe.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n",
- " \n",
- " batch_size = len(x0) if self.batch_size == -1 else self.batch_size\n",
- " nbatches = len(x0) // batch_size\n",
- "\n",
- " # Start training (full batch)\n",
- " for epoch in range(self.nepochs):\n",
- " for j in range(nbatches):\n",
- " x0_batch = x0[j*batch_size:(j+1)*batch_size]\n",
- " x1_batch = x1[j*batch_size:(j+1)*batch_size]\n",
- " \n",
- " # probe\n",
- " p0, p1 = self.probe(x0_batch), self.probe(x1_batch)\n",
- "\n",
- " # get the corresponding loss\n",
- " loss = self.get_loss(p0, p1)\n",
- "\n",
- " # update the parameters\n",
- " optimizer.zero_grad()\n",
- " loss.backward()\n",
- " optimizer.step()\n",
- "\n",
- " return loss.detach().cpu().item()\n",
- " \n",
- " def repeated_train(self):\n",
- " best_loss = np.inf\n",
- " for train_num in range(self.ntries):\n",
- " self.initialize_probe()\n",
- " loss = self.train()\n",
- " if loss < best_loss:\n",
- " self.best_probe = copy.deepcopy(self.probe)\n",
- " best_loss = loss\n",
- "\n",
- " return best_loss"
+ " return self.net(x)\n"
]
},
{
@@ -8497,86 +1241,48 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:42.769241Z",
- "start_time": "2023-05-07T11:22:30.614971Z"
+ "end_time": "2023-05-19T22:49:25.448505Z",
+ "start_time": "2023-05-19T22:49:25.448497Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CCS nonlinear train accuracy: 0.96\n",
- "CCS nonlinear test accuracy: 0.98\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/tmp/ipykernel_4095498/374574033.py:86: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- "/tmp/ipykernel_4095498/374574033.py:87: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "# Train CCS without any labels\n",
- "ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n",
- "ccs.repeated_train()\n",
+ "# # Train CCS without any labels\n",
+ "# ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n",
+ "# ccs.repeated_train()\n",
"\n",
- "# Evaluate\n",
- "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
- "print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n",
+ "# # Evaluate\n",
+ "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
+ "# print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n",
"\n",
- "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
- "print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))"
+ "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
+ "# print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))"
]
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:49.510569Z",
- "start_time": "2023-05-07T11:22:42.770553Z"
+ "end_time": "2023-05-19T22:49:25.449148Z",
+ "start_time": "2023-05-19T22:49:25.449140Z"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "CCS train accuracy: 0.96\n",
- "CCS test accuracy: 0.98\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/tmp/ipykernel_4095498/374574033.py:86: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x0 = torch.tensor(self.normalize(x0_test), dtype=torch.float, requires_grad=False, device=self.device)\n",
- "/tmp/ipykernel_4095498/374574033.py:87: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
- " x1 = torch.tensor(self.normalize(x1_test), dtype=torch.float, requires_grad=False, device=self.device)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "# Train CCS without any labels\n",
- "ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n",
- "ccs.repeated_train()\n",
+ "# # Train CCS without any labels\n",
+ "# ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n",
+ "# ccs.repeated_train()\n",
"\n",
- "# Evaluate\n",
- "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
- "print(\"CCS train accuracy: {}\".format(ccs_acc))\n",
+ "# # Evaluate\n",
+ "# ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n",
+ "# print(\"CCS train accuracy: {}\".format(ccs_acc))\n",
"\n",
- "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
- "print(\"CCS test accuracy: {}\".format(ccs_acc))"
+ "# ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n",
+ "# print(\"CCS test accuracy: {}\".format(ccs_acc))"
]
},
{
@@ -8600,38 +1306,260 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:49.914580Z",
- "start_time": "2023-05-07T11:22:49.511794Z"
+ "end_time": "2023-05-19T04:12:55.004017Z",
+ "start_time": "2023-05-19T04:12:55.004011Z"
+ }
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## DataModule"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-14T11:34:43.243172Z",
+ "start_time": "2023-05-14T11:34:43.240582Z"
+ }
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.449918Z",
+ "start_time": "2023-05-19T22:49:25.449910Z"
+ },
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "# def normalize(x):\n",
+ "# \"\"\"\n",
+ "# Mean-normalizes the data x (of shape (n, d))\n",
+ "# If self.var_normalize, also divides by the standard deviation\n",
+ "# \"\"\"\n",
+ "# normalized_x = x - x.mean(axis=0, keepdims=True)\n",
+ "# if self.var_normalize:\n",
+ "# normalized_x /= normalized_x.std(axis=0, keepdims=True)\n",
+ "\n",
+ "# return normalized_x\n",
+ "\n",
+ "\n",
+ "class IMBDHSDataModule(pl.LightningDataModule):\n",
+ "\n",
+ " def __init__(self,\n",
+ " model: AutoModel,\n",
+ " tokenizer: AutoTokenizer,\n",
+ " model_type=\"decoder\",\n",
+ " dataset_name=\"amazon_polarity\",\n",
+ " batch_size=32,\n",
+ " n=200,\n",
+ " ):\n",
+ " super().__init__()\n",
+ " self.model = model\n",
+ " self.tokenizer = tokenizer\n",
+ " self.save_hyperparameters(ignore=[\"model\", \"tokenizer\"])\n",
+ "\n",
+ " def setup(self, stage: str):\n",
+ "\n",
+ " self.dataset = load_dataset(self.hparams.dataset_name, split=\"test\")\n",
+ "\n",
+ " neg_hs, pos_hs, y, all_neg_ans, all_pos_ans = get_hidden_states_many_examples(\n",
+ " self.model, self.tokenizer, self.dataset, self.hparams.model_type, n=self.hparams.n, layers=[2, -2])\n",
+ "\n",
+ " # let's create a simple 50/50 train split (the data is already randomized)\n",
+ " n = len(y)\n",
+ " val_split = int(n * 0.5)\n",
+ " test_split = int(n * 0.75)\n",
+ " neg_hs_train, pos_hs_train, y_train = neg_hs[:\n",
+ " val_split], pos_hs[:\n",
+ " val_split], y[:\n",
+ " val_split]\n",
+ " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[\n",
+ " val_split:test_split], y[val_split:test_split]\n",
+ " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[\n",
+ " test_split:], y[test_split:]\n",
+ "\n",
+ " # for simplicity we can just take the difference between positive and negative hidden states\n",
+ " # (concatenating also works fine)\n",
+ " self.x_train = neg_hs_train - pos_hs_train\n",
+ " self.x_val = neg_hs_val - pos_hs_val\n",
+ " self.x_test = neg_hs_test - pos_hs_test\n",
+ "\n",
+ " # normalize\n",
+ " self.scaler = RobustScaler()\n",
+ " self.scaler.fit(self.x_train)\n",
+ " self.x_train = self.scaler.transform(self.x_train)\n",
+ " self.x_val = self.scaler.transform(self.x_val)\n",
+ " self.x_test = self.scaler.transform(self.x_test)\n",
+ "\n",
+ " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train).float(),\n",
+ " torch.from_numpy(pos_hs_train).float(),\n",
+ " torch.from_numpy(y_train).float())\n",
+ "\n",
+ " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val).float(),\n",
+ " torch.from_numpy(pos_hs_val).float(),\n",
+ " torch.from_numpy(y_val).float())\n",
+ "\n",
+ " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test).float(),\n",
+ " torch.from_numpy(pos_hs_test).float(),\n",
+ " torch.from_numpy(y_test).float())\n",
+ "\n",
+ " def train_dataloader(self):\n",
+ " return DataLoader(self.ds_train,\n",
+ " batch_size=self.hparams.batch_size,\n",
+ " shuffle=True)\n",
+ "\n",
+ " def val_dataloader(self):\n",
+ " return DataLoader(self.ds_val, batch_size=self.hparams.batch_size)\n",
+ "\n",
+ " def test_dataloader(self):\n",
+ " return DataLoader(self.ds_test, batch_size=self.hparams.batch_size)\n",
+ "\n",
+ "\n",
+ "# test\n",
+ "dm = IMBDHSDataModule(model, tokenizer)\n",
+ "dm.setup('train')\n",
+ "dl = dm.val_dataloader()\n",
+ "b = next(iter(dl))\n",
+ "b"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.450709Z",
+ "start_time": "2023-05-19T22:49:25.450702Z"
}
},
"outputs": [],
"source": [
- "import lightning.pytorch as pl\n",
+ "dm.x_test.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## LightningModel"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.451318Z",
+ "start_time": "2023-05-19T22:49:25.451310Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from torch import optim"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.452324Z",
+ "start_time": "2023-05-19T22:49:25.452316Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "\n",
+ "def get_loss(p0, p1):\n",
+ " \"\"\"\n",
+ " Returns the CCS loss for two probabilities each of shape (n,1) or (n,)\n",
+ " \"\"\"\n",
+ " informative_loss = (torch.min(p0, p1)**2).mean(0)\n",
+ " consistent_loss = ((p0 - (1-p1))**2).mean(0)\n",
+ " return informative_loss + consistent_loss\n",
+ "\n",
+ "\n",
+ "def get_acc(p0, p1, y):\n",
+ " avg_confidence = 0.5*(p0 + (1-p1))\n",
+ " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n",
+ " \n",
+ " # TODO f1\n",
+ " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n",
+ " \n",
+ " acc = (predictions == y.cpu().numpy()).mean()\n",
+ " acc = max(acc, 1 - acc)\n",
+ " return predictions, acc\n",
+ "\n",
+ "def get_f1(p0, p1, y):\n",
+ " avg_confidence = 0.5*(p0 + (1-p1))\n",
+ " predictions = (avg_confidence.detach().cpu().numpy() < 0.5).astype(int)[:, 0]\n",
+ " \n",
+ " # TODO f1\n",
+ " conf = (avg_confidence.detach().cpu().numpy() )[:, 0]\n",
+ " auc = roc_auc_score(y.cpu().numpy(), predictions)\n",
+ " \n",
+ " auc = max(auc, 1 - auc)\n",
+ " return predictions, auc\n",
"\n",
"class CSS(pl.LightningModule):\n",
- " def __init__(self, encoder, decoder):\n",
+ " def __init__(self, d, max_epochs, lr=4e-3, weight_decay=1e-6):\n",
" super().__init__()\n",
- " self.encoder = encoder\n",
- " self.decoder = decoder\n",
+ " self.probe = MLPProbe(d)\n",
+ " self.save_hyperparameters()\n",
+ " \n",
+ " def forward(self, x):\n",
+ " return self.probe(x)\n",
+ " \n",
+ " def _step(self, batch, batch_idx, stage='train'):\n",
+ " x0, x1, y = batch\n",
+ " p0, p1 = self(x0), self(x1)\n",
+ " \n",
+ " loss = get_loss(p0, p1)\n",
+ " \n",
+ " self.log(f\"{stage}/loss\", loss)\n",
+ " \n",
+ " predictions, acc = get_acc(p0, p1, y)\n",
+ " self.log(f\"{stage}/acc\", acc)\n",
+ " predictions, f1 = get_f1(p0, p1, y)\n",
+ " self.log(f\"{stage}/f1\", f1)\n",
+ " return loss\n",
" \n",
" def training_step(self, batch, batch_idx):\n",
- " # training_step defines the train loop.\n",
- " # it is independent of forward\n",
- " x, y = batch\n",
- " x = x.view(x.size(0), -1)\n",
- " z = self.encoder(x)\n",
- " x_hat = self.decoder(z)\n",
- " loss = nn.functional.mse_loss(x_hat, x)\n",
- " # Logging to TensorBoard (if installed) by default\n",
- " self.log(\"train_loss\", loss)\n",
- " return loss\n",
+ " return self._step(batch, batch_idx)\n",
+ " \n",
+ " def validation_step(self, batch, batch_idx=0):\n",
+ " return self._step(batch, batch_idx, stage='val')\n",
+ " \n",
+ " def prediction_step(self, batch, batch_idx):\n",
+ " x0, x1, y = batch\n",
+ " p0, p1 = self(x0), self(x1)\n",
+ " predictions, acc = get_acc(p0, p1, y)\n",
+ " return predictions \n",
"\n",
" def configure_optimizers(self):\n",
- " optimizer = optim.Adam(self.parameters(), lr=1e-3)\n",
- " return optimizer\n",
+ " optimizer = optim.AdamW(self.parameters(), lr=self.hparams.lr, weight_decay=self.hparams.weight_decay)\n",
+ " lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(\n",
+ " optimizer, T_max=self.hparams.max_epochs, eta_min=self.hparams.lr / 50\n",
+ " )\n",
+ " return [optimizer], [lr_scheduler]\n",
" "
]
},
@@ -8649,232 +1577,31 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.487623Z",
- "start_time": "2023-05-07T11:22:49.915848Z"
+ "end_time": "2023-05-14T06:17:57.365689Z",
+ "start_time": "2023-05-14T06:17:57.356995Z"
}
},
- "outputs": [
- {
- "data": {
- "text/html": [
- "╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
- "│ in <cell line: 56>:56 │\n",
- "│ │\n",
- "│ 53 │\n",
- "│ 54 # test │\n",
- "│ 55 dm = IMBDHSDataModule(model, tokenizer) │\n",
- "│ ❱ 56 dm.setup('train') │\n",
- "│ 57 dl = dm.val_dataloader() │\n",
- "│ 58 next(iter(dl)) │\n",
- "│ 59 │\n",
- "│ │\n",
- "│ in setup:15 │\n",
- "│ │\n",
- "│ 12 │ │\n",
- "│ 13 │ │\n",
- "│ 14 │ def setup(self, stage:str): │\n",
- "│ ❱ 15 │ │ self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True) │\n",
- "│ 16 │ │ │\n",
- "│ 17 │ │ neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer, │\n",
- "│ 18 │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1773 in │\n",
- "│ load_dataset │\n",
- "│ │\n",
- "│ 1770 │ ) │\n",
- "│ 1771 │ │\n",
- "│ 1772 │ # Create a dataset builder │\n",
- "│ ❱ 1773 │ builder_instance = load_dataset_builder( │\n",
- "│ 1774 │ │ path=path, │\n",
- "│ 1775 │ │ name=name, │\n",
- "│ 1776 │ │ data_dir=data_dir, │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1528 in │\n",
- "│ load_dataset_builder │\n",
- "│ │\n",
- "│ 1525 │ │ raise ValueError(error_msg) │\n",
- "│ 1526 │ │\n",
- "│ 1527 │ # Instantiate the dataset builder │\n",
- "│ ❱ 1528 │ builder_instance: DatasetBuilder = builder_cls( │\n",
- "│ 1529 │ │ cache_dir=cache_dir, │\n",
- "│ 1530 │ │ config_name=config_name, │\n",
- "│ 1531 │ │ data_dir=data_dir, │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:340 in │\n",
- "│ __init__ │\n",
- "│ │\n",
- "│ 337 │ │ │ config_kwargs[\"data_files\"] = data_files │\n",
- "│ 338 │ │ if data_dir is not None: │\n",
- "│ 339 │ │ │ config_kwargs[\"data_dir\"] = data_dir │\n",
- "│ ❱ 340 │ │ self.config, self.config_id = self._create_builder_config( │\n",
- "│ 341 │ │ │ config_name=config_name, │\n",
- "│ 342 │ │ │ custom_features=features, │\n",
- "│ 343 │ │ │ **config_kwargs, │\n",
- "│ │\n",
- "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:494 in │\n",
- "│ _create_builder_config │\n",
- "│ │\n",
- "│ 491 │ │ │ │ config_kwargs[\"name\"] = self.DEFAULT_CONFIG_NAME │\n",
- "│ 492 │ │ │ if \"version\" not in config_kwargs and hasattr(self, \"VERSION\") and self.VERS │\n",
- "│ 493 │ │ │ │ config_kwargs[\"version\"] = self.VERSION │\n",
- "│ ❱ 494 │ │ │ builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs) │\n",
- "│ 495 │ │ │\n",
- "│ 496 │ │ # otherwise use the config_kwargs to overwrite the attributes │\n",
- "│ 497 │ │ else: │\n",
- "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
- "TypeError: __init__() got an unexpected keyword argument 'stream'\n",
- "\n"
- ],
- "text/plain": [
- "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
- "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m15\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m12 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m14 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92msetup\u001b[0m(\u001b[96mself\u001b[0m, stage:\u001b[96mstr\u001b[0m): \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m15 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m, stream=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m16 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y = get_hidden_states_many_examples(\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1773\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92mload_dataset\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1770 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1771 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1772 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create a dataset builder\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1773 \u001b[2m│ \u001b[0mbuilder_instance = load_dataset_builder( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1774 \u001b[0m\u001b[2m│ │ \u001b[0mpath=path, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1775 \u001b[0m\u001b[2m│ │ \u001b[0mname=name, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1776 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1528\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92mload_dataset_builder\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1525 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m(error_msg) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1526 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1527 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Instantiate the dataset builder\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1528 \u001b[2m│ \u001b[0mbuilder_instance: DatasetBuilder = builder_cls( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1529 \u001b[0m\u001b[2m│ │ \u001b[0mcache_dir=cache_dir, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1530 \u001b[0m\u001b[2m│ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m1531 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m340\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 337 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_files\u001b[0m\u001b[33m\"\u001b[0m] = data_files \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 338 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m data_dir \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 339 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_dir\u001b[0m\u001b[33m\"\u001b[0m] = data_dir \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 340 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.config, \u001b[96mself\u001b[0m.config_id = \u001b[96mself\u001b[0m._create_builder_config( \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 341 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 342 \u001b[0m\u001b[2m│ │ │ \u001b[0mcustom_features=features, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 343 \u001b[0m\u001b[2m│ │ │ \u001b[0m**config_kwargs, \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m494\u001b[0m in \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[92m_create_builder_config\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 491 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mname\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.DEFAULT_CONFIG_NAME \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 492 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m config_kwargs \u001b[95mand\u001b[0m \u001b[96mhasattr\u001b[0m(\u001b[96mself\u001b[0m, \u001b[33m\"\u001b[0m\u001b[33mVERSION\u001b[0m\u001b[33m\"\u001b[0m) \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.VERS \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 493 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.VERSION \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 494 \u001b[2m│ │ │ \u001b[0mbuilder_config = \u001b[96mself\u001b[0m.BUILDER_CONFIG_CLASS(**config_kwargs) \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 495 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 496 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# otherwise use the config_kwargs to overwrite the attributes\u001b[0m \u001b[31m│\u001b[0m\n",
- "\u001b[31m│\u001b[0m \u001b[2m 497 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n",
- "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
- "\u001b[1;91mTypeError: \u001b[0m\u001b[1;35m__init__\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m got an unexpected keyword argument \u001b[32m'stream'\u001b[0m\n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "from dataclasses import dataclass\n",
- "from torch.utils.data import random_split, DataLoader, TensorDataset\n",
- "from transformers.models.auto.modeling_auto import AutoModel\n",
- "\n",
- "@dataclass\n",
- "class IMBDHSDataModule(pl.LightningDataModule):\n",
- " model: AutoModel\n",
- " tokenizer: AutoTokenizer\n",
- " model_type=\"decoder\"\n",
- " dataset_name=\"amazon_polarity\"\n",
- " batch_size=32\n",
- " \n",
- " \n",
- " def setup(self, stage:str):\n",
- " self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)\n",
- " \n",
- " neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer, self.dataset, self.model_type)\n",
- " \n",
- " # let's create a simple 50/50 train split (the data is already randomized)\n",
- " n = len(y)\n",
- " val_split = int(n*0.5)\n",
- " test_split = int(n*0.75)\n",
- " neg_hs_train, pos_hs_train, y_train = neg_hs[:val_split], pos_hs[:val_split], y[:val_split]\n",
- " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[val_split:test_split], y[val_split:test_split]\n",
- " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[test_split:], y[test_split:]\n",
- "\n",
- " # for simplicity we can just take the difference between positive and negative hidden states\n",
- " # (concatenating also works fine)\n",
- " self.x_train = neg_hs_train - pos_hs_train\n",
- " self.x_val = neg_hs_val - pos_hs_val\n",
- " self.x_test = neg_hs_test - pos_hs_test\n",
- " \n",
- " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train),\n",
- " torch.from_numpy(pos_hs_train),\n",
- " torch.from_numpy(y_train))\n",
- " \n",
- " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val),\n",
- " torch.from_numpy(pos_hs_val),\n",
- " torch.from_numpy(y_val))\n",
- " \n",
- " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test),\n",
- " torch.from_numpy(pos_hs_test),\n",
- " torch.from_numpy(y_test))\n",
- " \n",
- " def train_dataloader(self):\n",
- " return DataLoader(self.ds_train, batch_size=self.batch_size)\n",
- "\n",
- " def val_dataloader(self):\n",
- " return DataLoader(self.ds_val, batch_size=self.batch_size)\n",
- "\n",
- " def test_dataloader(self):\n",
- " return DataLoader(self.ds_test, batch_size=self.batch_size)\n",
- " \n",
- "# test\n",
- "dm = IMBDHSDataModule(model, tokenizer)\n",
- "dm.setup('train')\n",
- "dl = dm.val_dataloader()\n",
- "next(iter(dl))"
- ]
+ "outputs": [],
+ "source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.488619Z",
- "start_time": "2023-05-07T11:22:55.488612Z"
+ "end_time": "2023-05-19T22:49:25.453018Z",
+ "start_time": "2023-05-19T22:49:25.453010Z"
}
},
"outputs": [],
"source": [
"# init the autoencoder\n",
- "net = CSS()"
+ "max_epochs = 1000\n",
+ "d = b[0].shape[-1]\n",
+ "net = CSS(d=d, max_epochs=max_epochs)"
]
},
{
@@ -8882,13 +1609,13 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.489380Z",
- "start_time": "2023-05-07T11:22:55.489373Z"
+ "end_time": "2023-05-19T22:49:25.453708Z",
+ "start_time": "2023-05-19T22:49:25.453700Z"
}
},
"outputs": [],
"source": [
- "train_loader = utils.data.DataLoader(dataset)"
+ "# train_loader = utils.data.DataLoader(dataset)"
]
},
{
@@ -8896,17 +1623,128 @@
"execution_count": null,
"metadata": {
"ExecuteTime": {
- "end_time": "2023-05-07T11:22:55.489937Z",
- "start_time": "2023-05-07T11:22:55.489931Z"
- }
+ "end_time": "2023-05-19T22:49:25.454581Z",
+ "start_time": "2023-05-19T22:49:25.454572Z"
+ },
+ "scrolled": true
},
"outputs": [],
"source": [
"# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\n",
- "trainer = pl.Trainer(limit_train_batches=100, max_epochs=1)\n",
- "trainer.fit(model=net, train_dataloaders=train_loader)"
+ "trainer = pl.Trainer(limit_train_batches=100, max_epochs=max_epochs)\n",
+ "trainer.fit(model=net, datamodule=dm)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.455203Z",
+ "start_time": "2023-05-19T22:49:25.455195Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "%debug"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-14T06:21:46.356828Z",
+ "start_time": "2023-05-14T06:21:46.351801Z"
+ }
+ },
+ "source": [
+ "# Read hist"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.455836Z",
+ "start_time": "2023-05-19T22:49:25.455828Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# import pytorch_lightning as pl\n",
+ "from lightning.pytorch.loggers.csv_logs import CSVLogger\n",
+ "# from pytorch_lightning.loggers.csv_logs import CSVLogger as CSVLogger2\n",
+ "from pathlib import Path\n",
+ "import pandas as pd\n",
+ "\n",
+ "def read_metrics_csv(metrics_file_path):\n",
+ " df_hist = pd.read_csv(metrics_file_path)\n",
+ " df_hist[\"epoch\"] = df_hist[\"epoch\"].ffill()\n",
+ " df_histe = df_hist.set_index(\"epoch\").groupby(\"epoch\").mean()\n",
+ " return df_histe\n",
+ "\n",
+ "\n",
+ "def read_hist(trainer: pl.Trainer):\n",
+ "\n",
+ " ts = [t for t in trainer.loggers if isinstance(t, CSVLogger)]\n",
+ " print(ts)\n",
+ " try:\n",
+ " metrics_file_path = Path(ts[0].experiment.metrics_file_path)\n",
+ " df_histe = read_metrics_csv(metrics_file_path)\n",
+ " return df_histe\n",
+ " except Exception as e:\n",
+ " raise e\n",
+ " print(e)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.456423Z",
+ "start_time": "2023-05-19T22:49:25.456416Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "df_hist = read_hist(trainer).ffill().bfill()\n",
+ "df_hist"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-05-19T22:49:25.456948Z",
+ "start_time": "2023-05-19T22:49:25.456942Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "df_hist[['val/acc', 'train/acc']].plot()\n",
+ "\n",
+ "df_hist[['val/f1', 'train/f1']].plot()\n",
+ "\n",
+ "df_hist[['val/loss', 'train/loss']].plot()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/requirements/conda.requirements.txt b/requirements/conda.requirements.txt
index c354f4a..d126242 100644
--- a/requirements/conda.requirements.txt
+++ b/requirements/conda.requirements.txt
@@ -2,7 +2,9 @@ accelerate==0.18.0
aiohttp==3.8.4
aiosignal==1.3.1
altair==4.2.2
+anyio==3.6.2
appdirs @ file:///home/conda/feedstock_root/build_artifacts/appdirs_1603108395799/work
+arrow==1.2.3
astor==0.8.1
asttokens==2.0.5
async-timeout==4.0.2
@@ -10,8 +12,10 @@ attrs==23.1.0
backcall==0.2.0
backports.functools-lru-cache @ file:///home/conda/feedstock_root/build_artifacts/backports.functools_lru_cache_1618230623929/work
base58==2.1.1
+beautifulsoup4==4.12.2
bitsandbytes==0.38.1
black==23.3.0
+blessed==1.20.0
blinker==1.6.2
Brotli==1.0.9
brotlipy @ file:///home/conda/feedstock_root/build_artifacts/brotlipy_1666764672617/work
@@ -27,33 +31,45 @@ click-default-group @ file:///home/conda/feedstock_root/build_artifacts/click-de
clikit @ file:///home/conda/feedstock_root/build_artifacts/clikit_1676917709145/work
comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1679481329611/work
conda_lock @ file:///home/conda/feedstock_root/build_artifacts/conda-lock_1676925627812/work
+contourpy==1.0.7
crashtest @ file:///home/conda/feedstock_root/build_artifacts/crashtest_1667467006729/work
+croniter==1.3.14
cryptography @ file:///home/conda/feedstock_root/build_artifacts/cryptography-split_1681508587436/work
+cycler==0.11.0
datasets==2.12.0
+dateutils==0.6.12
debugpy==1.6.0
decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work
+deepdiff==6.3.0
dill==0.3.6
distlib @ file:///home/conda/feedstock_root/build_artifacts/distlib_1668356257807/work
ensureconda @ file:///home/conda/feedstock_root/build_artifacts/ensureconda_1657719435160/work
entrypoints==0.4
exceptiongroup==1.1.1
executing==0.8.3
+fastapi==0.88.0
filelock @ file:///home/conda/feedstock_root/build_artifacts/filelock_1681839547898/work
flake8==6.0.0
+fonttools==4.39.4
frozenlist==1.3.3
fsspec==2023.4.0
gitdb==4.0.10
GitPython==3.1.31
+gluonts==0.12.8
gmpy2 @ file:///home/conda/feedstock_root/build_artifacts/gmpy2_1666808679441/work
+h11==0.14.0
html5lib @ file:///home/conda/feedstock_root/build_artifacts/html5lib_1592930327044/work
huggingface-hub==0.14.1
idna==2.8
importlib-metadata @ file:///home/conda/feedstock_root/build_artifacts/importlib-metadata_1682176699712/work
+importlib-resources==5.12.0
inflate64==0.3.1
iniconfig==2.0.0
+inquirer==3.1.3
ipykernel @ file:///home/conda/feedstock_root/build_artifacts/ipykernel_1679336319192/work
ipython==8.4.0
isort==5.8.0
+itsdangerous==2.1.2
jaraco.classes @ file:///home/conda/feedstock_root/build_artifacts/jaraco.classes_1667024629799/work
jedi==0.18.1
jeepney @ file:///home/conda/feedstock_root/build_artifacts/jeepney_1649085214306/work
@@ -63,10 +79,17 @@ jsonschema==4.17.3
jupyter-client==7.3.4
jupyter-core==4.10.0
keyring @ file:///home/conda/feedstock_root/build_artifacts/keyring_1671728250508/work
+kiwisolver==1.4.4
+lightning==2.0.2
+lightning-cloud==0.5.34
+lightning-utilities==0.8.0
lockfile==0.12.2
+markdown-it-py==2.2.0
MarkupSafe @ file:///home/conda/feedstock_root/build_artifacts/markupsafe_1674135804847/work
+matplotlib==3.7.1
matplotlib-inline==0.1.3
mccabe==0.7.0
+mdurl==0.1.2
more-itertools @ file:///home/conda/feedstock_root/build_artifacts/more-itertools_1677514956219/work
mpmath @ file:///home/conda/feedstock_root/build_artifacts/mpmath_1678228039184/work
msgpack @ file:///home/conda/feedstock_root/build_artifacts/msgpack-python_1678312698172/work
@@ -77,11 +100,13 @@ mypy-extensions==0.4.3
nest-asyncio==1.5.5
networkx @ file:///home/conda/feedstock_root/build_artifacts/networkx_1680692919326/work
numpy @ file:///home/conda/feedstock_root/build_artifacts/numpy_1682210190296/work
+ordered-set==4.1.0
packaging @ file:///home/conda/feedstock_root/build_artifacts/packaging_1681337016113/work
pandas==2.0.1
parso==0.8.3
pastel @ file:///home/conda/feedstock_root/build_artifacts/pastel_1640899049124/work
pathspec==0.9.0
+peft==0.3.0
pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1667297516076/work
pickleshare==0.7.5
Pillow @ file:///home/conda/feedstock_root/build_artifacts/pillow_1675487166627/work
@@ -106,6 +131,7 @@ pydeck==0.8.1b0
pydocstyle==6.1.1
pyflakes==3.0.1
Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1681904169130/work
+PyJWT==2.6.0
pylama==8.3.8
pylev @ file:///home/conda/feedstock_root/build_artifacts/pylev_1641226376343/work
pyOpenSSL @ file:///home/conda/feedstock_root/build_artifacts/pyopenssl_1680037383858/work
@@ -115,24 +141,33 @@ pyrsistent==0.19.3
PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1661604839144/work
pytest==7.3.1
python-dateutil==2.8.2
+python-editor==1.0.4
+python-multipart==0.0.6
pytz==2023.3
pytz-deprecation-shim==0.1.0.post0
PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1666772387118/work
pyzmq==23.2.0
pyzstd==0.15.7
+readchar==4.0.5
regex==2023.5.5
requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1682535435083/work
responses==0.18.0
+rich==13.3.5
ruamel.yaml @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml_1683392662082/work
ruamel.yaml.clib @ file:///home/conda/feedstock_root/build_artifacts/ruamel.yaml.clib_1670412733608/work
scikit-learn==1.2.2
scipy==1.10.1
SecretStorage @ file:///home/conda/feedstock_root/build_artifacts/secretstorage_1666848732124/work
+sentencepiece==0.1.99
six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work
sklearn==0.0.post4
smmap==5.0.0
+sniffio==1.3.0
snowballstemmer==2.2.0
+soupsieve==2.4.1
stack-data==0.3.0
+starlette==0.22.0
+starsessions==1.3.0
streamlit==0.82.0
sympy @ file:///home/conda/feedstock_root/build_artifacts/sympy_1679342590084/work
tenacity==8.2.2
@@ -145,6 +180,7 @@ tomlkit @ file:///home/conda/feedstock_root/build_artifacts/tomlkit_168302816509
toolz @ file:///home/conda/feedstock_root/build_artifacts/toolz_1657485559105/work
torch==2.0.0
torchaudio==2.0.0
+torchmetrics==0.11.4
torchvision==0.15.0
tornado==6.1
tqdm==4.65.0
@@ -155,11 +191,14 @@ typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/typing_ext
tzdata==2023.3
tzlocal==4.3
urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1678635778344/work
+uvicorn==0.22.0
validators==0.20.0
virtualenv @ file:///home/conda/feedstock_root/build_artifacts/virtualenv_1682674264573/work
watchdog==3.0.0
wcwidth==0.2.5
webencodings==0.5.1
+websocket-client==1.5.1
+websockets==11.0.2
xxhash==3.2.0
yapf==0.33.0
yarl==1.9.2
diff --git a/requirements/environment.max.yaml b/requirements/environment.max.yaml
index df6b843..c3f67c9 100644
--- a/requirements/environment.max.yaml
+++ b/requirements/environment.max.yaml
@@ -188,34 +188,59 @@ dependencies:
- aiohttp==3.8.4
- aiosignal==1.3.1
- altair==4.2.2
+ - anyio==3.6.2
+ - arrow==1.2.3
- astor==0.8.1
- async-timeout==4.0.2
- attrs==23.1.0
- base58==2.1.1
+ - beautifulsoup4==4.12.2
- bitsandbytes==0.38.1
- black==23.3.0
+ - blessed==1.20.0
- blinker==1.6.2
- brotli==1.0.9
- cachetools==5.3.0
- certifi==2022.12.7
+ - contourpy==1.0.7
+ - croniter==1.3.14
+ - cycler==0.11.0
- datasets==2.12.0
+ - dateutils==0.6.12
+ - deepdiff==6.3.0
- dill==0.3.6
- exceptiongroup==1.1.1
+ - fastapi==0.88.0
- flake8==6.0.0
+ - fonttools==4.39.4
- frozenlist==1.3.3
- fsspec==2023.4.0
- gitdb==4.0.10
- gitpython==3.1.31
+ - gluonts==0.12.8
+ - h11==0.14.0
- huggingface-hub==0.14.1
+ - importlib-resources==5.12.0
- inflate64==0.3.1
- iniconfig==2.0.0
+ - inquirer==3.1.3
- isort==5.8.0
+ - itsdangerous==2.1.2
- joblib==1.2.0
- jsonschema==4.17.3
+ - kiwisolver==1.4.4
+ - lightning==2.0.2
+ - lightning-cloud==0.5.34
+ - lightning-utilities==0.8.0
+ - markdown-it-py==2.2.0
+ - matplotlib==3.7.1
+ - mdurl==0.1.2
- multidict==6.0.4
- multiprocess==0.70.14
- multivolumefile==0.2.3
+ - ordered-set==4.1.0
- pandas==2.0.1
+ - peft==0.3.0
- pillow==9.5.0
- plotly==5.14.1
- pluggy==1.0.0
@@ -228,33 +253,47 @@ dependencies:
- pycryptodomex==3.17
- pydeck==0.8.1b0
- pyflakes==3.0.1
+ - pyjwt==2.6.0
- pyparsing==3.0.9
- pyppmd==1.0.0
- pyrsistent==0.19.3
- pytest==7.3.1
+ - python-editor==1.0.4
+ - python-multipart==0.0.6
- pytz==2023.3
- pytz-deprecation-shim==0.1.0.post0
- pyzstd==0.15.7
+ - readchar==4.0.5
- regex==2023.5.5
- requests==2.30.0
- responses==0.18.0
+ - rich==13.3.5
- scikit-learn==1.2.2
- scipy==1.10.1
+ - sentencepiece==0.1.99
- sklearn==0.0.post4
- smmap==5.0.0
+ - sniffio==1.3.0
+ - soupsieve==2.4.1
+ - starlette==0.22.0
+ - starsessions==1.3.0
- streamlit==0.82.0
- tenacity==8.2.2
- texttable==1.6.7
- threadpoolctl==3.1.0
- tokenizers==0.13.3
- toml==0.10.2
+ - torchmetrics==0.11.4
- tqdm==4.65.0
- transformers==4.28.1
- tzdata==2023.3
- tzlocal==4.3
- urllib3==2.0.2
+ - uvicorn==0.22.0
- validators==0.20.0
- watchdog==3.0.0
+ - websocket-client==1.5.1
+ - websockets==11.0.2
- xxhash==3.2.0
- yapf==0.33.0
- yarl==1.9.2
|