From 961ef59d2b4e048feed083a309e630835802f73b Mon Sep 17 00:00:00 2001 From: wassname Date: Sun, 7 May 2023 17:24:49 +0800 Subject: [PATCH] many shot --- .../001_mjc_CCS-checkpoint.ipynb | 649 +++++++++++++++--- 001_mjc_CCS.ipynb | 649 +++++++++++++++--- requirements/requirements.txt | 1 + 3 files changed, 1083 insertions(+), 216 deletions(-) diff --git a/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb b/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb index e9eacd5..4e2cec5 100644 --- a/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb +++ b/.ipynb_checkpoints/001_mjc_CCS-checkpoint.ipynb @@ -13,8 +13,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:25.921309Z", - "start_time": "2023-05-07T05:39:24.474456Z" + "end_time": "2023-05-07T09:18:19.033245Z", + "start_time": "2023-05-07T09:18:17.562000Z" } }, "outputs": [ @@ -40,58 +40,6 @@ "from sklearn.linear_model import LogisticRegression" ] }, - { - "cell_type": "markdown", - "metadata": { - "ExecuteTime": { - "start_time": "2023-05-07T01:08:20.635Z" - } - }, - "source": [ - "## Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T05:39:28.746274Z", - "start_time": "2023-05-07T05:39:25.922561Z" - }, - "scrolled": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 55.80it/s]\n" - ] - } - ], - "source": [ - "# Let's just try IMDB for simplicity\n", - "data = load_dataset(\"amazon_polarity\")[\"test\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def format_imdb(text, label):\n", - " \"\"\"\n", - " Given an imdb example (\"text\") and corresponding label (0 for negative, or 1 for positive), \n", - " returns a zero-shot prompt for that example (which includes that label as the answer).\n", - " \n", - " (This is just one example of a simple, manually created prompt.)\n", - " \"\"\"\n", - " return \"The following movie review expresses a \" + [\"negative\", \"positive\"][label] + \" sentiment:\\n\" + text\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -113,11 +61,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:37.745946Z", - "start_time": "2023-05-07T05:39:28.748845Z" + "end_time": "2023-05-07T09:18:27.830076Z", + "start_time": "2023-05-07T09:18:19.034316Z" } }, "outputs": [ @@ -151,7 +99,7 @@ " warn(msg)\n", "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/cinnamon.default.path')}\n", " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('local/wassname-fractal-desktop'), PosixPath('@/tmp/.ICE-unix/5335,unix/wassname-fractal-desktop')}\n", + "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('@/tmp/.ICE-unix/5335,unix/wassname-fractal-desktop'), PosixPath('local/wassname-fractal-desktop')}\n", " warn(msg)\n", "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('0'), PosixPath('1')}\n", " warn(msg)\n", @@ -163,7 +111,7 @@ "Either way, this might cause trouble in the future:\n", "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", " warn(msg)\n", - "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.24s/it]\n" + "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.04s/it]\n" ] }, { @@ -197,7 +145,7 @@ ")" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -274,6 +222,86 @@ "outputs": [], "source": [] }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "start_time": "2023-05-07T01:08:20.635Z" + } + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:18:30.433340Z", + "start_time": "2023-05-07T09:18:27.832057Z" + }, + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 54.08it/s]\n" + ] + } + ], + "source": [ + "# Let's just try IMDB for simplicity\n", + "data = load_dataset(\"amazon_polarity\")[\"test\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:20:39.763220Z", + "start_time": "2023-05-07T09:20:39.760724Z" + } + }, + "outputs": [], + "source": [ + "def format_imdb(text, label):\n", + " return f\"\"\"Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", + "This is negative? 1\n", + "###\n", + "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", + "This review is negative? 0\n", + "###\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "This review is positive? 1\n", + "###\n", + "Review: \"{text}\"\n", + "This review is {label}? \"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:17:04.250617Z", + "start_time": "2023-05-07T09:17:04.247644Z" + } + }, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -284,11 +312,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:37.752425Z", - "start_time": "2023-05-07T05:39:37.747545Z" + "end_time": "2023-05-07T09:20:39.793592Z", + "start_time": "2023-05-07T09:20:39.766812Z" } }, "outputs": [], @@ -346,11 +374,14 @@ "\n", " # forward pass\n", " with torch.no_grad():\n", + " # FIXME: should be a batch, to speed it up\n", " output = model(input_ids, output_hidden_states=True)\n", "\n", " # get the last layer, last token hidden states\n", " hs_tuple = output[\"hidden_states\"]\n", " hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", + " \n", + " # FIXME pass full output, as I want to check model zero shot accuracy!\n", "\n", " return hs\n", "\n", @@ -363,36 +394,34 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:53.181466Z", - "start_time": "2023-05-07T05:39:53.179289Z" + "end_time": "2023-05-07T09:18:38.861641Z", + "start_time": "2023-05-07T09:18:38.859920Z" } }, "outputs": [], - "source": [ - "# UPTO, fix nan" - ] + "source": [] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 19, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:54.330398Z", - "start_time": "2023-05-07T05:39:53.506750Z" + "end_time": "2023-05-07T09:20:40.118106Z", + "start_time": "2023-05-07T09:20:39.794801Z" } }, "outputs": [ { "data": { "text/plain": [ - "array([ 3.707e+00, -2.578e-01, 4.321e-01, ..., 8.389e-01, 2.798e-01,\n", - " -1.083e-03], dtype=float16)" + "array([-0.05655, 0.01492, -0.0974 , ..., -0.03537, 0.01362, 0.01608],\n", + " dtype=float16)" ] }, - "execution_count": 9, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -407,11 +436,43 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 20, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:40:25.237818Z", - "start_time": "2023-05-07T05:40:25.235783Z" + "end_time": "2023-05-07T09:20:40.121193Z", + "start_time": "2023-05-07T09:20:40.119127Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", + "This is negative? 1\n", + "###\n", + "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", + "This review is negative? 0\n", + "###\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "This review is positive? 1\n", + "###\n", + "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", + "This review is 0? \n" + ] + } + ], + "source": [ + "print(format_imdb(text, 0))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:20:40.136489Z", + "start_time": "2023-05-07T09:20:40.122604Z" } }, "outputs": [], @@ -450,18 +511,18 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 22, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:40:26.960648Z", - "start_time": "2023-05-07T05:40:26.954566Z" + "end_time": "2023-05-07T09:20:40.152559Z", + "start_time": "2023-05-07T09:20:40.137543Z" } }, "outputs": [], "source": [ "\n", "\n", - "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100):\n", + "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100, layer=-1):\n", " \"\"\"\n", " Given an encoder-decoder model, a list of data, computes the contrast hidden states on n random examples.\n", " Returns numpy arrays of shape (n, hidden_dim) for each candidate label, along with a boolean numpy array of shape (n,)\n", @@ -474,19 +535,19 @@ " all_neg_hs, all_pos_hs, all_gt_labels = [], [], []\n", "\n", " # loop\n", - " for _ in tqdm(range(n)):\n", + " for _ in tqdm(range(n), unit='examples', desc='get_hidden_states'):\n", " # for simplicity, sample a random example until we find one that's a reasonable length\n", " # (most examples should be a reasonable length, so this is just to make sure)\n", " while True:\n", " idx = np.random.randint(len(data))\n", " text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", - " # the actual formatted input will be longer, so include a bit of a marign\n", + " # the actual formatted input will be longer, so include a bit of a margin\n", " if len(tokenizer(text)) < 400: \n", " break\n", " \n", " # get hidden states\n", - " neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n", - " pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type)\n", + " neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type, layer=layer)\n", + " pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type, layer=layer)\n", "\n", " # collect\n", " all_neg_hs.append(neg_hs)\n", @@ -502,11 +563,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 23, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:40:58.173815Z", - "start_time": "2023-05-07T05:40:27.095416Z" + "end_time": "2023-05-07T09:21:25.091032Z", + "start_time": "2023-05-07T09:20:40.153598Z" } }, "outputs": [ @@ -514,7 +575,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:31<00:00, 3.22it/s]\n" + "get_hidden_states: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:44<00:00, 2.23examples/s]\n" ] } ], @@ -547,11 +608,11 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T06:49:28.092748Z", - "start_time": "2023-05-07T06:49:28.057699Z" + "end_time": "2023-05-07T09:21:25.106679Z", + "start_time": "2023-05-07T09:21:25.092367Z" } }, "outputs": [ @@ -560,7 +621,7 @@ "output_type": "stream", "text": [ "Logistic regression accuracy: 1.0 [TRAIN]\n", - "Logistic regression accuracy: 0.8 [TEST]\n" + "Logistic regression accuracy: 0.98 [TEST]\n" ] } ], @@ -584,7 +645,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-07T06:48:49.955305Z", @@ -603,11 +664,11 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T07:32:53.940126Z", - "start_time": "2023-05-07T07:32:53.911010Z" + "end_time": "2023-05-07T09:21:25.126329Z", + "start_time": "2023-05-07T09:21:25.107952Z" } }, "outputs": [], @@ -620,12 +681,16 @@ " nn.ReLU(),\n", " nn.Linear(100, 100),\n", " nn.ReLU(),\n", + "# nn.Linear(100, 100),\n", + "# nn.ReLU(),\n", + "# nn.Linear(100, 100),\n", + "# nn.ReLU(),\n", " nn.Linear(100, 1),\n", " nn.Sigmoid(),\n", " )\n", "\n", " def forward(self, x):\n", - " return torch.net(x)\n", + " return self.net(x)\n", "\n", "class CCS(object):\n", " def __init__(self, x0, x1, nepochs=1000, ntries=10, lr=1e-3, batch_size=-1, \n", @@ -764,21 +829,384 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": { "ExecuteTime": { - "start_time": "2023-05-07T07:32:54.693Z" + "end_time": "2023-05-07T09:21:31.792546Z", + "start_time": "2023-05-07T09:21:25.127324Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CCS train accuracy: 0.78\n", + "CCS test accuracy: 0.94\n" + ] + } + ], + "source": [ + "# Train CCS without any labels\n", + "ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n", + "ccs.repeated_train()\n", + "\n", + "# Evaluate\n", + "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "print(\"CCS train accuracy: {}\".format(ccs_acc))\n", + "\n", + "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "print(\"CCS test accuracy: {}\".format(ccs_acc))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:42.348876Z", + "start_time": "2023-05-07T09:21:31.794420Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CCS train accuracy: 0.52\n", + "CCS test accuracy: 0.52\n" + ] + } + ], + "source": [ + "# Train CCS without any labels\n", + "ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n", + "ccs.repeated_train()\n", + "\n", + "# Evaluate\n", + "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n", + "\n", + "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# lightning" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:42.914231Z", + "start_time": "2023-05-07T09:21:42.349987Z" } }, "outputs": [], "source": [ - "# Train CCS without any labels\n", - "ccs = CCS(neg_hs_train, pos_hs_train)\n", - "ccs.repeated_train()\n", + "import lightning.pytorch as pl\n", "\n", - "# Evaluate\n", - "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", - "print(\"CCS accuracy: {}\".format(ccs_acc))" + "class CSS(pl.LightningModule):\n", + " def __init__(self, encoder, decoder):\n", + " super().__init__()\n", + " self.encoder = encoder\n", + " self.decoder = decoder\n", + " \n", + " def training_step(self, batch, batch_idx):\n", + " # training_step defines the train loop.\n", + " # it is independent of forward\n", + " x, y = batch\n", + " x = x.view(x.size(0), -1)\n", + " z = self.encoder(x)\n", + " x_hat = self.decoder(z)\n", + " loss = nn.functional.mse_loss(x_hat, x)\n", + " # Logging to TensorBoard (if installed) by default\n", + " self.log(\"train_loss\", loss)\n", + " return loss\n", + "\n", + " def configure_optimizers(self):\n", + " optimizer = optim.Adam(self.parameters(), lr=1e-3)\n", + " return optimizer\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:42.933089Z", + "start_time": "2023-05-07T09:21:42.915391Z" + } + }, + "outputs": [], + "source": [ + "load_dataset??" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.061949Z", + "start_time": "2023-05-07T09:21:42.934652Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <cell line: 56>:56                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   53                                                                                             │\n",
+       "│   54 # test                                                                                      │\n",
+       "│   55 dm = IMBDHSDataModule(model, tokenizer)                                                     │\n",
+       "│ ❱ 56 dm.setup('train')                                                                           │\n",
+       "│   57 dl = dm.val_dataloader()                                                                    │\n",
+       "│   58 next(iter(dl))                                                                              │\n",
+       "│   59                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in setup:15                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   12 │                                                                                           │\n",
+       "│   13 │                                                                                           │\n",
+       "│   14 │   def setup(self, stage:str):                                                             │\n",
+       "│ ❱ 15 │   │   self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)           │\n",
+       "│   16 │   │                                                                                       │\n",
+       "│   17 │   │   neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer,     │\n",
+       "│   18                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1773 in         │\n",
+       "│ load_dataset                                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│   1770 │   )                                                                                     │\n",
+       "│   1771 │                                                                                         │\n",
+       "│   1772 │   # Create a dataset builder                                                            │\n",
+       "│ ❱ 1773 │   builder_instance = load_dataset_builder(                                              │\n",
+       "│   1774 │   │   path=path,                                                                        │\n",
+       "│   1775 │   │   name=name,                                                                        │\n",
+       "│   1776 │   │   data_dir=data_dir,                                                                │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1528 in         │\n",
+       "│ load_dataset_builder                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│   1525 │   │   raise ValueError(error_msg)                                                       │\n",
+       "│   1526 │                                                                                         │\n",
+       "│   1527 │   # Instantiate the dataset builder                                                     │\n",
+       "│ ❱ 1528 │   builder_instance: DatasetBuilder = builder_cls(                                       │\n",
+       "│   1529 │   │   cache_dir=cache_dir,                                                              │\n",
+       "│   1530 │   │   config_name=config_name,                                                          │\n",
+       "│   1531 │   │   data_dir=data_dir,                                                                │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:340 in       │\n",
+       "│ __init__                                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│    337 │   │   │   config_kwargs[\"data_files\"] = data_files                                      │\n",
+       "│    338 │   │   if data_dir is not None:                                                          │\n",
+       "│    339 │   │   │   config_kwargs[\"data_dir\"] = data_dir                                          │\n",
+       "│ ❱  340 │   │   self.config, self.config_id = self._create_builder_config(                        │\n",
+       "│    341 │   │   │   config_name=config_name,                                                      │\n",
+       "│    342 │   │   │   custom_features=features,                                                     │\n",
+       "│    343 │   │   │   **config_kwargs,                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:494 in       │\n",
+       "│ _create_builder_config                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│    491 │   │   │   │   config_kwargs[\"name\"] = self.DEFAULT_CONFIG_NAME                          │\n",
+       "│    492 │   │   │   if \"version\" not in config_kwargs and hasattr(self, \"VERSION\") and self.VERS  │\n",
+       "│    493 │   │   │   │   config_kwargs[\"version\"] = self.VERSION                                   │\n",
+       "│ ❱  494 │   │   │   builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs)                   │\n",
+       "│    495 │   │                                                                                     │\n",
+       "│    496 │   │   # otherwise use the config_kwargs to overwrite the attributes                     │\n",
+       "│    497 │   │   else:                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "TypeError: __init__() got an unexpected keyword argument 'stream'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m15\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m12 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m14 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92msetup\u001b[0m(\u001b[96mself\u001b[0m, stage:\u001b[96mstr\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m15 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m, stream=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m16 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y = get_hidden_states_many_examples(\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1773\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mload_dataset\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1770 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1771 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1772 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create a dataset builder\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1773 \u001b[2m│ \u001b[0mbuilder_instance = load_dataset_builder( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1774 \u001b[0m\u001b[2m│ │ \u001b[0mpath=path, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1775 \u001b[0m\u001b[2m│ │ \u001b[0mname=name, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1776 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1528\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mload_dataset_builder\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1525 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m(error_msg) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1526 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1527 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Instantiate the dataset builder\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1528 \u001b[2m│ \u001b[0mbuilder_instance: DatasetBuilder = builder_cls( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1529 \u001b[0m\u001b[2m│ │ \u001b[0mcache_dir=cache_dir, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1530 \u001b[0m\u001b[2m│ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1531 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m340\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 337 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_files\u001b[0m\u001b[33m\"\u001b[0m] = data_files \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 338 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m data_dir \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 339 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_dir\u001b[0m\u001b[33m\"\u001b[0m] = data_dir \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 340 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.config, \u001b[96mself\u001b[0m.config_id = \u001b[96mself\u001b[0m._create_builder_config( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 341 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 342 \u001b[0m\u001b[2m│ │ │ \u001b[0mcustom_features=features, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 343 \u001b[0m\u001b[2m│ │ │ \u001b[0m**config_kwargs, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m494\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_create_builder_config\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 491 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mname\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.DEFAULT_CONFIG_NAME \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 492 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m config_kwargs \u001b[95mand\u001b[0m \u001b[96mhasattr\u001b[0m(\u001b[96mself\u001b[0m, \u001b[33m\"\u001b[0m\u001b[33mVERSION\u001b[0m\u001b[33m\"\u001b[0m) \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.VERS \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 493 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.VERSION \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 494 \u001b[2m│ │ │ \u001b[0mbuilder_config = \u001b[96mself\u001b[0m.BUILDER_CONFIG_CLASS(**config_kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 495 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 496 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# otherwise use the config_kwargs to overwrite the attributes\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 497 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mTypeError: \u001b[0m\u001b[1;35m__init__\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m got an unexpected keyword argument \u001b[32m'stream'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from dataclasses import dataclass\n", + "from torch.utils.data import random_split, DataLoader, TensorDataset\n", + "from transformers.models.auto.modeling_auto import AutoModel\n", + "\n", + "@dataclass\n", + "class IMBDHSDataModule(pl.LightningDataModule):\n", + " model: AutoModel\n", + " tokenizer: AutoTokenizer\n", + " model_type=\"decoder\"\n", + " dataset_name=\"amazon_polarity\"\n", + " batch_size=32\n", + " \n", + " \n", + " def setup(self, stage:str):\n", + " self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)\n", + " \n", + " neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer, self.dataset, self.model_type)\n", + " \n", + " # let's create a simple 50/50 train split (the data is already randomized)\n", + " n = len(y)\n", + " val_split = int(n*0.5)\n", + " test_split = int(n*0.75)\n", + " neg_hs_train, pos_hs_train, y_train = neg_hs[:val_split], pos_hs[:val_split], y[:val_split]\n", + " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[val_split:test_split], y[val_split:test_split]\n", + " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[test_split:], y[test_split:]\n", + "\n", + " # for simplicity we can just take the difference between positive and negative hidden states\n", + " # (concatenating also works fine)\n", + " self.x_train = neg_hs_train - pos_hs_train\n", + " self.x_val = neg_hs_val - pos_hs_val\n", + " self.x_test = neg_hs_test - pos_hs_test\n", + " \n", + " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train),\n", + " torch.from_numpy(pos_hs_train),\n", + " torch.from_numpy(y_train))\n", + " \n", + " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val),\n", + " torch.from_numpy(pos_hs_val),\n", + " torch.from_numpy(y_val))\n", + " \n", + " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test),\n", + " torch.from_numpy(pos_hs_test),\n", + " torch.from_numpy(y_test))\n", + " \n", + " def train_dataloader(self):\n", + " return DataLoader(self.ds_train, batch_size=self.batch_size)\n", + "\n", + " def val_dataloader(self):\n", + " return DataLoader(self.ds_val, batch_size=self.batch_size)\n", + "\n", + " def test_dataloader(self):\n", + " return DataLoader(self.ds_test, batch_size=self.batch_size)\n", + " \n", + "# test\n", + "dm = IMBDHSDataModule(model, tokenizer)\n", + "dm.setup('train')\n", + "dl = dm.val_dataloader()\n", + "next(iter(dl))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.063077Z", + "start_time": "2023-05-07T09:21:46.063070Z" + } + }, + "outputs": [], + "source": [ + "# init the autoencoder\n", + "net = CSS()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.063597Z", + "start_time": "2023-05-07T09:21:46.063589Z" + } + }, + "outputs": [], + "source": [ + "train_loader = utils.data.DataLoader(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.064287Z", + "start_time": "2023-05-07T09:21:46.064281Z" + } + }, + "outputs": [], + "source": [ + "# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\n", + "trainer = pl.Trainer(limit_train_batches=100, max_epochs=1)\n", + "trainer.fit(model=net, train_dataloaders=train_loader)" ] }, { @@ -830,7 +1258,12 @@ "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, - "toc_position": {}, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "165px" + }, "toc_section_display": true, "toc_window_display": true }, diff --git a/001_mjc_CCS.ipynb b/001_mjc_CCS.ipynb index e9eacd5..4e2cec5 100644 --- a/001_mjc_CCS.ipynb +++ b/001_mjc_CCS.ipynb @@ -13,8 +13,8 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:25.921309Z", - "start_time": "2023-05-07T05:39:24.474456Z" + "end_time": "2023-05-07T09:18:19.033245Z", + "start_time": "2023-05-07T09:18:17.562000Z" } }, "outputs": [ @@ -40,58 +40,6 @@ "from sklearn.linear_model import LogisticRegression" ] }, - { - "cell_type": "markdown", - "metadata": { - "ExecuteTime": { - "start_time": "2023-05-07T01:08:20.635Z" - } - }, - "source": [ - "## Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "ExecuteTime": { - "end_time": "2023-05-07T05:39:28.746274Z", - "start_time": "2023-05-07T05:39:25.922561Z" - }, - "scrolled": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 55.80it/s]\n" - ] - } - ], - "source": [ - "# Let's just try IMDB for simplicity\n", - "data = load_dataset(\"amazon_polarity\")[\"test\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def format_imdb(text, label):\n", - " \"\"\"\n", - " Given an imdb example (\"text\") and corresponding label (0 for negative, or 1 for positive), \n", - " returns a zero-shot prompt for that example (which includes that label as the answer).\n", - " \n", - " (This is just one example of a simple, manually created prompt.)\n", - " \"\"\"\n", - " return \"The following movie review expresses a \" + [\"negative\", \"positive\"][label] + \" sentiment:\\n\" + text\n" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -113,11 +61,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:37.745946Z", - "start_time": "2023-05-07T05:39:28.748845Z" + "end_time": "2023-05-07T09:18:27.830076Z", + "start_time": "2023-05-07T09:18:19.034316Z" } }, "outputs": [ @@ -151,7 +99,7 @@ " warn(msg)\n", "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/share/gconf/cinnamon.default.path')}\n", " warn(msg)\n", - "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('local/wassname-fractal-desktop'), PosixPath('@/tmp/.ICE-unix/5335,unix/wassname-fractal-desktop')}\n", + "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('@/tmp/.ICE-unix/5335,unix/wassname-fractal-desktop'), PosixPath('local/wassname-fractal-desktop')}\n", " warn(msg)\n", "/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/bitsandbytes/cuda_setup/main.py:145: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('0'), PosixPath('1')}\n", " warn(msg)\n", @@ -163,7 +111,7 @@ "Either way, this might cause trouble in the future:\n", "If you get `CUDA error: invalid device function` errors, the above might be the cause and the solution is to make sure only one ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] in the paths that we search based on your env.\n", " warn(msg)\n", - "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.24s/it]\n" + "Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:06<00:00, 3.04s/it]\n" ] }, { @@ -197,7 +145,7 @@ ")" ] }, - "execution_count": 3, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -274,6 +222,86 @@ "outputs": [], "source": [] }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "start_time": "2023-05-07T01:08:20.635Z" + } + }, + "source": [ + "## Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:18:30.433340Z", + "start_time": "2023-05-07T09:18:27.832057Z" + }, + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset amazon_polarity (/home/wassname/.cache/huggingface/datasets/amazon_polarity/amazon_polarity/3.0.0/a27b32b7e7b88eb274a8fa8ba0f654f1fe998a87c22547557317793b5d2772dc)\n", + "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 54.08it/s]\n" + ] + } + ], + "source": [ + "# Let's just try IMDB for simplicity\n", + "data = load_dataset(\"amazon_polarity\")[\"test\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:20:39.763220Z", + "start_time": "2023-05-07T09:20:39.760724Z" + } + }, + "outputs": [], + "source": [ + "def format_imdb(text, label):\n", + " return f\"\"\"Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", + "This is negative? 1\n", + "###\n", + "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", + "This review is negative? 0\n", + "###\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "This review is positive? 1\n", + "###\n", + "Review: \"{text}\"\n", + "This review is {label}? \"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:17:04.250617Z", + "start_time": "2023-05-07T09:17:04.247644Z" + } + }, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -284,11 +312,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:37.752425Z", - "start_time": "2023-05-07T05:39:37.747545Z" + "end_time": "2023-05-07T09:20:39.793592Z", + "start_time": "2023-05-07T09:20:39.766812Z" } }, "outputs": [], @@ -346,11 +374,14 @@ "\n", " # forward pass\n", " with torch.no_grad():\n", + " # FIXME: should be a batch, to speed it up\n", " output = model(input_ids, output_hidden_states=True)\n", "\n", " # get the last layer, last token hidden states\n", " hs_tuple = output[\"hidden_states\"]\n", " hs = hs_tuple[layer][0, -1].detach().cpu().numpy()\n", + " \n", + " # FIXME pass full output, as I want to check model zero shot accuracy!\n", "\n", " return hs\n", "\n", @@ -363,36 +394,34 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:53.181466Z", - "start_time": "2023-05-07T05:39:53.179289Z" + "end_time": "2023-05-07T09:18:38.861641Z", + "start_time": "2023-05-07T09:18:38.859920Z" } }, "outputs": [], - "source": [ - "# UPTO, fix nan" - ] + "source": [] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 19, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:39:54.330398Z", - "start_time": "2023-05-07T05:39:53.506750Z" + "end_time": "2023-05-07T09:20:40.118106Z", + "start_time": "2023-05-07T09:20:39.794801Z" } }, "outputs": [ { "data": { "text/plain": [ - "array([ 3.707e+00, -2.578e-01, 4.321e-01, ..., 8.389e-01, 2.798e-01,\n", - " -1.083e-03], dtype=float16)" + "array([-0.05655, 0.01492, -0.0974 , ..., -0.03537, 0.01362, 0.01608],\n", + " dtype=float16)" ] }, - "execution_count": 9, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -407,11 +436,43 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 20, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:40:25.237818Z", - "start_time": "2023-05-07T05:40:25.235783Z" + "end_time": "2023-05-07T09:20:40.121193Z", + "start_time": "2023-05-07T09:20:40.119127Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Review: \"Whoever wrote the screenplay for this movie obviously never consulted any books about Lucille Ball, especially her autobiography. I've never seen so many mistakes in a biopic, ranging from her early years in Celoron and Jamestown to her later years with Desi. I could write a whole list of factual errors, but it would go on for pages. In all, I believe that Lucille Ball is one of those inimitable people who simply cannot be portrayed by anyone other than themselves. If I were Lucie Arnaz and Desi, Jr., I would be irate at how many mistakes were made in this film. The filmmakers tried hard, but the movie seems awfully sloppy to me.\"\n", + "This is negative? 1\n", + "###\n", + "Review: \"This version of Anna Christie is in German. Greta Garbo again plays Anna Christie, but all of the other characters have different actors from the English version. Both were filmed back to back because Garbo had such a following in Germany. Garbo herself supposedly favored her Anna Christie in this version over the English version. It's a good tale and a must-see for Garbo fans.\"\n", + "This review is negative? 0\n", + "###\n", + "Review: \"I think this is a lovely family movie. There are plenty of hilarious scenes and heart-warming moments to be had throughout the movie. The actors are great and the effects well executed throughout. Danny Glover plays George Knox who manages the terrible baseball team 'The Angels' and is great throughout the film. Also fantastic are the young actors Joseph Gordon-Levitt and Milton Davis Jr. Christopher Lloyd is good as Al 'The Angel' and the effects are great in this top notch Disney movie. A touching and heart-warming movie which everyone should enjoy.\"\n", + "This review is positive? 1\n", + "###\n", + "Review: \"My lovely Pat has one of the GREAT voices of her generation. I have listened to this CD for YEARS and I still LOVE IT. When I'm in a good mood it makes me feel better. A bad mood just evaporates like sugar in the rain. This CD just oozes LIFE. Vocals are jusat STUUNNING and lyrics just kill. One of life's hidden gems. This is a desert isle CD in my book. Why she never made it big is just beyond me. Everytime I play this, no matter black, white, young, old, male, female EVERYBODY says one thing \"Who was that singing ?\"\"\n", + "This review is 0? \n" + ] + } + ], + "source": [ + "print(format_imdb(text, 0))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:20:40.136489Z", + "start_time": "2023-05-07T09:20:40.122604Z" } }, "outputs": [], @@ -450,18 +511,18 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 22, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:40:26.960648Z", - "start_time": "2023-05-07T05:40:26.954566Z" + "end_time": "2023-05-07T09:20:40.152559Z", + "start_time": "2023-05-07T09:20:40.137543Z" } }, "outputs": [], "source": [ "\n", "\n", - "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100):\n", + "def get_hidden_states_many_examples(model, tokenizer, data, model_type, n=100, layer=-1):\n", " \"\"\"\n", " Given an encoder-decoder model, a list of data, computes the contrast hidden states on n random examples.\n", " Returns numpy arrays of shape (n, hidden_dim) for each candidate label, along with a boolean numpy array of shape (n,)\n", @@ -474,19 +535,19 @@ " all_neg_hs, all_pos_hs, all_gt_labels = [], [], []\n", "\n", " # loop\n", - " for _ in tqdm(range(n)):\n", + " for _ in tqdm(range(n), unit='examples', desc='get_hidden_states'):\n", " # for simplicity, sample a random example until we find one that's a reasonable length\n", " # (most examples should be a reasonable length, so this is just to make sure)\n", " while True:\n", " idx = np.random.randint(len(data))\n", " text, true_label = data[idx][\"content\"], data[idx][\"label\"]\n", - " # the actual formatted input will be longer, so include a bit of a marign\n", + " # the actual formatted input will be longer, so include a bit of a margin\n", " if len(tokenizer(text)) < 400: \n", " break\n", " \n", " # get hidden states\n", - " neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type)\n", - " pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type)\n", + " neg_hs = get_hidden_states(model, tokenizer, format_imdb(text, 0), model_type=model_type, layer=layer)\n", + " pos_hs = get_hidden_states(model, tokenizer, format_imdb(text, 1), model_type=model_type, layer=layer)\n", "\n", " # collect\n", " all_neg_hs.append(neg_hs)\n", @@ -502,11 +563,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 23, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T05:40:58.173815Z", - "start_time": "2023-05-07T05:40:27.095416Z" + "end_time": "2023-05-07T09:21:25.091032Z", + "start_time": "2023-05-07T09:20:40.153598Z" } }, "outputs": [ @@ -514,7 +575,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:31<00:00, 3.22it/s]\n" + "get_hidden_states: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:44<00:00, 2.23examples/s]\n" ] } ], @@ -547,11 +608,11 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T06:49:28.092748Z", - "start_time": "2023-05-07T06:49:28.057699Z" + "end_time": "2023-05-07T09:21:25.106679Z", + "start_time": "2023-05-07T09:21:25.092367Z" } }, "outputs": [ @@ -560,7 +621,7 @@ "output_type": "stream", "text": [ "Logistic regression accuracy: 1.0 [TRAIN]\n", - "Logistic regression accuracy: 0.8 [TEST]\n" + "Logistic regression accuracy: 0.98 [TEST]\n" ] } ], @@ -584,7 +645,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2023-05-07T06:48:49.955305Z", @@ -603,11 +664,11 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": { "ExecuteTime": { - "end_time": "2023-05-07T07:32:53.940126Z", - "start_time": "2023-05-07T07:32:53.911010Z" + "end_time": "2023-05-07T09:21:25.126329Z", + "start_time": "2023-05-07T09:21:25.107952Z" } }, "outputs": [], @@ -620,12 +681,16 @@ " nn.ReLU(),\n", " nn.Linear(100, 100),\n", " nn.ReLU(),\n", + "# nn.Linear(100, 100),\n", + "# nn.ReLU(),\n", + "# nn.Linear(100, 100),\n", + "# nn.ReLU(),\n", " nn.Linear(100, 1),\n", " nn.Sigmoid(),\n", " )\n", "\n", " def forward(self, x):\n", - " return torch.net(x)\n", + " return self.net(x)\n", "\n", "class CCS(object):\n", " def __init__(self, x0, x1, nepochs=1000, ntries=10, lr=1e-3, batch_size=-1, \n", @@ -764,21 +829,384 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": { "ExecuteTime": { - "start_time": "2023-05-07T07:32:54.693Z" + "end_time": "2023-05-07T09:21:31.792546Z", + "start_time": "2023-05-07T09:21:25.127324Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CCS train accuracy: 0.78\n", + "CCS test accuracy: 0.94\n" + ] + } + ], + "source": [ + "# Train CCS without any labels\n", + "ccs = CCS(neg_hs_train, pos_hs_train, linear=True)\n", + "ccs.repeated_train()\n", + "\n", + "# Evaluate\n", + "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "print(\"CCS train accuracy: {}\".format(ccs_acc))\n", + "\n", + "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "print(\"CCS test accuracy: {}\".format(ccs_acc))" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:42.348876Z", + "start_time": "2023-05-07T09:21:31.794420Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CCS train accuracy: 0.52\n", + "CCS test accuracy: 0.52\n" + ] + } + ], + "source": [ + "# Train CCS without any labels\n", + "ccs = CCS(neg_hs_train, pos_hs_train, linear=False)\n", + "ccs.repeated_train()\n", + "\n", + "# Evaluate\n", + "ccs_acc = ccs.get_acc(neg_hs_train, pos_hs_train, y_train)\n", + "print(\"CCS nonlinear train accuracy: {}\".format(ccs_acc))\n", + "\n", + "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", + "print(\"CCS nonlinear test accuracy: {}\".format(ccs_acc))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# lightning" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:42.914231Z", + "start_time": "2023-05-07T09:21:42.349987Z" } }, "outputs": [], "source": [ - "# Train CCS without any labels\n", - "ccs = CCS(neg_hs_train, pos_hs_train)\n", - "ccs.repeated_train()\n", + "import lightning.pytorch as pl\n", "\n", - "# Evaluate\n", - "ccs_acc = ccs.get_acc(neg_hs_test, pos_hs_test, y_test)\n", - "print(\"CCS accuracy: {}\".format(ccs_acc))" + "class CSS(pl.LightningModule):\n", + " def __init__(self, encoder, decoder):\n", + " super().__init__()\n", + " self.encoder = encoder\n", + " self.decoder = decoder\n", + " \n", + " def training_step(self, batch, batch_idx):\n", + " # training_step defines the train loop.\n", + " # it is independent of forward\n", + " x, y = batch\n", + " x = x.view(x.size(0), -1)\n", + " z = self.encoder(x)\n", + " x_hat = self.decoder(z)\n", + " loss = nn.functional.mse_loss(x_hat, x)\n", + " # Logging to TensorBoard (if installed) by default\n", + " self.log(\"train_loss\", loss)\n", + " return loss\n", + "\n", + " def configure_optimizers(self):\n", + " optimizer = optim.Adam(self.parameters(), lr=1e-3)\n", + " return optimizer\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:42.933089Z", + "start_time": "2023-05-07T09:21:42.915391Z" + } + }, + "outputs": [], + "source": [ + "load_dataset??" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.061949Z", + "start_time": "2023-05-07T09:21:42.934652Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮\n",
+       "│ in <cell line: 56>:56                                                                            │\n",
+       "│                                                                                                  │\n",
+       "│   53                                                                                             │\n",
+       "│   54 # test                                                                                      │\n",
+       "│   55 dm = IMBDHSDataModule(model, tokenizer)                                                     │\n",
+       "│ ❱ 56 dm.setup('train')                                                                           │\n",
+       "│   57 dl = dm.val_dataloader()                                                                    │\n",
+       "│   58 next(iter(dl))                                                                              │\n",
+       "│   59                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ in setup:15                                                                                      │\n",
+       "│                                                                                                  │\n",
+       "│   12 │                                                                                           │\n",
+       "│   13 │                                                                                           │\n",
+       "│   14 │   def setup(self, stage:str):                                                             │\n",
+       "│ ❱ 15 │   │   self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)           │\n",
+       "│   16 │   │                                                                                       │\n",
+       "│   17 │   │   neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer,     │\n",
+       "│   18                                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1773 in         │\n",
+       "│ load_dataset                                                                                     │\n",
+       "│                                                                                                  │\n",
+       "│   1770 │   )                                                                                     │\n",
+       "│   1771 │                                                                                         │\n",
+       "│   1772 │   # Create a dataset builder                                                            │\n",
+       "│ ❱ 1773 │   builder_instance = load_dataset_builder(                                              │\n",
+       "│   1774 │   │   path=path,                                                                        │\n",
+       "│   1775 │   │   name=name,                                                                        │\n",
+       "│   1776 │   │   data_dir=data_dir,                                                                │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/load.py:1528 in         │\n",
+       "│ load_dataset_builder                                                                             │\n",
+       "│                                                                                                  │\n",
+       "│   1525 │   │   raise ValueError(error_msg)                                                       │\n",
+       "│   1526 │                                                                                         │\n",
+       "│   1527 │   # Instantiate the dataset builder                                                     │\n",
+       "│ ❱ 1528 │   builder_instance: DatasetBuilder = builder_cls(                                       │\n",
+       "│   1529 │   │   cache_dir=cache_dir,                                                              │\n",
+       "│   1530 │   │   config_name=config_name,                                                          │\n",
+       "│   1531 │   │   data_dir=data_dir,                                                                │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:340 in       │\n",
+       "│ __init__                                                                                         │\n",
+       "│                                                                                                  │\n",
+       "│    337 │   │   │   config_kwargs[\"data_files\"] = data_files                                      │\n",
+       "│    338 │   │   if data_dir is not None:                                                          │\n",
+       "│    339 │   │   │   config_kwargs[\"data_dir\"] = data_dir                                          │\n",
+       "│ ❱  340 │   │   self.config, self.config_id = self._create_builder_config(                        │\n",
+       "│    341 │   │   │   config_name=config_name,                                                      │\n",
+       "│    342 │   │   │   custom_features=features,                                                     │\n",
+       "│    343 │   │   │   **config_kwargs,                                                              │\n",
+       "│                                                                                                  │\n",
+       "│ /home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/builder.py:494 in       │\n",
+       "│ _create_builder_config                                                                           │\n",
+       "│                                                                                                  │\n",
+       "│    491 │   │   │   │   config_kwargs[\"name\"] = self.DEFAULT_CONFIG_NAME                          │\n",
+       "│    492 │   │   │   if \"version\" not in config_kwargs and hasattr(self, \"VERSION\") and self.VERS  │\n",
+       "│    493 │   │   │   │   config_kwargs[\"version\"] = self.VERSION                                   │\n",
+       "│ ❱  494 │   │   │   builder_config = self.BUILDER_CONFIG_CLASS(**config_kwargs)                   │\n",
+       "│    495 │   │                                                                                     │\n",
+       "│    496 │   │   # otherwise use the config_kwargs to overwrite the attributes                     │\n",
+       "│    497 │   │   else:                                                                             │\n",
+       "╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
+       "TypeError: __init__() got an unexpected keyword argument 'stream'\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92m\u001b[0m:\u001b[94m56\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m53 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m54 \u001b[0m\u001b[2m# test\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m55 \u001b[0mdm = IMBDHSDataModule(model, tokenizer) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m56 dm.setup(\u001b[33m'\u001b[0m\u001b[33mtrain\u001b[0m\u001b[33m'\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m57 \u001b[0mdl = dm.val_dataloader() \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m58 \u001b[0m\u001b[96mnext\u001b[0m(\u001b[96miter\u001b[0m(dl)) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m59 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m in \u001b[92msetup\u001b[0m:\u001b[94m15\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m12 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m13 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m14 \u001b[0m\u001b[2m│ \u001b[0m\u001b[94mdef\u001b[0m \u001b[92msetup\u001b[0m(\u001b[96mself\u001b[0m, stage:\u001b[96mstr\u001b[0m): \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m15 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.dataset = load_dataset(\u001b[96mself\u001b[0m.dataset_name, split=\u001b[33m\"\u001b[0m\u001b[33mtest\u001b[0m\u001b[33m\"\u001b[0m, stream=\u001b[94mTrue\u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m16 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m17 \u001b[0m\u001b[2m│ │ \u001b[0mneg_hs, pos_hs, y = get_hidden_states_many_examples(\u001b[96mself\u001b[0m.model, \u001b[96mself\u001b[0m.tokenizer, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m18 \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1773\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mload_dataset\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1770 \u001b[0m\u001b[2m│ \u001b[0m) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1771 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1772 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Create a dataset builder\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1773 \u001b[2m│ \u001b[0mbuilder_instance = load_dataset_builder( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1774 \u001b[0m\u001b[2m│ │ \u001b[0mpath=path, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1775 \u001b[0m\u001b[2m│ │ \u001b[0mname=name, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1776 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mload.py\u001b[0m:\u001b[94m1528\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92mload_dataset_builder\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1525 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mraise\u001b[0m \u001b[96mValueError\u001b[0m(error_msg) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1526 \u001b[0m\u001b[2m│ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1527 \u001b[0m\u001b[2m│ \u001b[0m\u001b[2m# Instantiate the dataset builder\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m1528 \u001b[2m│ \u001b[0mbuilder_instance: DatasetBuilder = builder_cls( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1529 \u001b[0m\u001b[2m│ │ \u001b[0mcache_dir=cache_dir, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1530 \u001b[0m\u001b[2m│ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m1531 \u001b[0m\u001b[2m│ │ \u001b[0mdata_dir=data_dir, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m340\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m__init__\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 337 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_files\u001b[0m\u001b[33m\"\u001b[0m] = data_files \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 338 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94mif\u001b[0m data_dir \u001b[95mis\u001b[0m \u001b[95mnot\u001b[0m \u001b[94mNone\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 339 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mdata_dir\u001b[0m\u001b[33m\"\u001b[0m] = data_dir \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 340 \u001b[2m│ │ \u001b[0m\u001b[96mself\u001b[0m.config, \u001b[96mself\u001b[0m.config_id = \u001b[96mself\u001b[0m._create_builder_config( \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 341 \u001b[0m\u001b[2m│ │ │ \u001b[0mconfig_name=config_name, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 342 \u001b[0m\u001b[2m│ │ │ \u001b[0mcustom_features=features, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 343 \u001b[0m\u001b[2m│ │ │ \u001b[0m**config_kwargs, \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2;33m/home/wassname/miniforge3/envs/dlk2/lib/python3.9/site-packages/datasets/\u001b[0m\u001b[1;33mbuilder.py\u001b[0m:\u001b[94m494\u001b[0m in \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[92m_create_builder_config\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 491 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mname\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.DEFAULT_CONFIG_NAME \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 492 \u001b[0m\u001b[2m│ │ │ \u001b[0m\u001b[94mif\u001b[0m \u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m \u001b[95mnot\u001b[0m \u001b[95min\u001b[0m config_kwargs \u001b[95mand\u001b[0m \u001b[96mhasattr\u001b[0m(\u001b[96mself\u001b[0m, \u001b[33m\"\u001b[0m\u001b[33mVERSION\u001b[0m\u001b[33m\"\u001b[0m) \u001b[95mand\u001b[0m \u001b[96mself\u001b[0m.VERS \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 493 \u001b[0m\u001b[2m│ │ │ │ \u001b[0mconfig_kwargs[\u001b[33m\"\u001b[0m\u001b[33mversion\u001b[0m\u001b[33m\"\u001b[0m] = \u001b[96mself\u001b[0m.VERSION \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 494 \u001b[2m│ │ │ \u001b[0mbuilder_config = \u001b[96mself\u001b[0m.BUILDER_CONFIG_CLASS(**config_kwargs) \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 495 \u001b[0m\u001b[2m│ │ \u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 496 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[2m# otherwise use the config_kwargs to overwrite the attributes\u001b[0m \u001b[31m│\u001b[0m\n", + "\u001b[31m│\u001b[0m \u001b[2m 497 \u001b[0m\u001b[2m│ │ \u001b[0m\u001b[94melse\u001b[0m: \u001b[31m│\u001b[0m\n", + "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n", + "\u001b[1;91mTypeError: \u001b[0m\u001b[1;35m__init__\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m got an unexpected keyword argument \u001b[32m'stream'\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from dataclasses import dataclass\n", + "from torch.utils.data import random_split, DataLoader, TensorDataset\n", + "from transformers.models.auto.modeling_auto import AutoModel\n", + "\n", + "@dataclass\n", + "class IMBDHSDataModule(pl.LightningDataModule):\n", + " model: AutoModel\n", + " tokenizer: AutoTokenizer\n", + " model_type=\"decoder\"\n", + " dataset_name=\"amazon_polarity\"\n", + " batch_size=32\n", + " \n", + " \n", + " def setup(self, stage:str):\n", + " self.dataset = load_dataset(self.dataset_name, split=\"test\", stream=True)\n", + " \n", + " neg_hs, pos_hs, y = get_hidden_states_many_examples(self.model, self.tokenizer, self.dataset, self.model_type)\n", + " \n", + " # let's create a simple 50/50 train split (the data is already randomized)\n", + " n = len(y)\n", + " val_split = int(n*0.5)\n", + " test_split = int(n*0.75)\n", + " neg_hs_train, pos_hs_train, y_train = neg_hs[:val_split], pos_hs[:val_split], y[:val_split]\n", + " neg_hs_val, pos_hs_val, y_val = neg_hs[val_split:test_split], pos_hs[val_split:test_split], y[val_split:test_split]\n", + " neg_hs_test, pos_hs_test, y_test = neg_hs[test_split:], pos_hs[test_split:], y[test_split:]\n", + "\n", + " # for simplicity we can just take the difference between positive and negative hidden states\n", + " # (concatenating also works fine)\n", + " self.x_train = neg_hs_train - pos_hs_train\n", + " self.x_val = neg_hs_val - pos_hs_val\n", + " self.x_test = neg_hs_test - pos_hs_test\n", + " \n", + " self.ds_train = TensorDataset(torch.from_numpy(neg_hs_train),\n", + " torch.from_numpy(pos_hs_train),\n", + " torch.from_numpy(y_train))\n", + " \n", + " self.ds_val = TensorDataset(torch.from_numpy(neg_hs_val),\n", + " torch.from_numpy(pos_hs_val),\n", + " torch.from_numpy(y_val))\n", + " \n", + " self.ds_test = TensorDataset(torch.from_numpy(neg_hs_test),\n", + " torch.from_numpy(pos_hs_test),\n", + " torch.from_numpy(y_test))\n", + " \n", + " def train_dataloader(self):\n", + " return DataLoader(self.ds_train, batch_size=self.batch_size)\n", + "\n", + " def val_dataloader(self):\n", + " return DataLoader(self.ds_val, batch_size=self.batch_size)\n", + "\n", + " def test_dataloader(self):\n", + " return DataLoader(self.ds_test, batch_size=self.batch_size)\n", + " \n", + "# test\n", + "dm = IMBDHSDataModule(model, tokenizer)\n", + "dm.setup('train')\n", + "dl = dm.val_dataloader()\n", + "next(iter(dl))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.063077Z", + "start_time": "2023-05-07T09:21:46.063070Z" + } + }, + "outputs": [], + "source": [ + "# init the autoencoder\n", + "net = CSS()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.063597Z", + "start_time": "2023-05-07T09:21:46.063589Z" + } + }, + "outputs": [], + "source": [ + "train_loader = utils.data.DataLoader(dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-05-07T09:21:46.064287Z", + "start_time": "2023-05-07T09:21:46.064281Z" + } + }, + "outputs": [], + "source": [ + "# train the model (hint: here are some helpful Trainer arguments for rapid idea iteration)\n", + "trainer = pl.Trainer(limit_train_batches=100, max_epochs=1)\n", + "trainer.fit(model=net, train_dataloaders=train_loader)" ] }, { @@ -830,7 +1258,12 @@ "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, - "toc_position": {}, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "165px" + }, "toc_section_display": true, "toc_window_display": true }, diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 2a68d44..acb154d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -6,3 +6,4 @@ sklearn scikit-learn accelerate bitsandbytes +lightning