wassname/discovering_latent_knowledge
1
0
Fork 0
mirror of https://github.com/wassname/discovering_latent_knowledge.git synced 2026-09-10 12:00:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor and move to wizard

Browse Source
This commit is contained in:
wassname
2023-10-27 17:14:16 +08:00
parent 183fac2627
commit b0dbdef748
13 changed files with 1751 additions and 8771 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/027_debug_ds_feats.ipynb
-850
View File
@@ -1,850 +0,0 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# distance and direciton\n",
"\n",
"Let try to opt for distance and direction with\n",
"\n",
"$L1loss(y_1-y_0, y_{true})$\n",
"\n",
"where $y_1=model(x_1)$\n",
"\n",
"So I'm optimising for the hidden states to be the correct distance and direcioton away. It's like the margin raning loss."
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"links:\n",
"- [loading](https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/alpaca.py)\n",
"- [dict](https://github.com/deep-diver/LLM-As-Chatbot/blob/c79e855a492a968b54bac223e66dc9db448d6eba/model_cards.json#L143)\n",
"- [prompt_format](https://github.com/deep-diver/PingPong/blob/main/src/pingpong/alpaca.py)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# import your package\n",
"%load_ext autoreload\n",
"%autoreload 2\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
},
{
"data": {
"text/plain": [
"'4.34.1'"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from matplotlib import pyplot as plt\n",
"plt.style.use('ggplot')\n",
"\n",
"from typing import Optional, List, Dict, Union\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"from torch import Tensor\n",
"from torch import optim\n",
"from torch.utils.data import random_split, DataLoader, TensorDataset\n",
"from src.helpers.ds import shuffle_dataset_by\n",
"from pathlib import Path\n",
"\n",
"import transformers\n",
"\n",
"import lightning.pytorch as pl\n",
"# from dataclasses import dataclass\n",
"\n",
"# from sklearn.linear_model import LogisticRegression\n",
"# from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n",
"# from sklearn.preprocessing import RobustScaler\n",
"\n",
"from tqdm.auto import tqdm\n",
"import os\n",
"\n",
"from loguru import logger\n",
"logger.add(os.sys.stderr, format=\"{time} {level} {message}\", level=\"INFO\")\n",
"\n",
"\n",
"\n",
"transformers.__version__\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from src.helpers.lightning import read_metrics_csv\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Datasets\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"from datasets import load_from_disk, concatenate_datasets\n",
"from src.datasets.load import ds2df, load_ds\n",
"\n",
"feats = ['hidden_states', 'head_activation_and_grad', 'mlp_activation_and_grad', 'residual_stream', 'w_grads_attn', 'w_grads_mlp', 'hidden_states2', 'residual_stream2', ]\n",
"\n",
"fs = [\n",
" # \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_amazon_polarity_test_615\",\n",
" \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_amazon_polarity_train_555\",\n",
" # \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_glue_qnli_test_615\", \n",
" \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_glue_qnli_train_555\",\n",
" # \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_super_glue_boolq_test_615\",\n",
" \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_super_glue_boolq_train_555\",\n",
"]\n",
"\n",
"# dss = [load_from_disk(f) for f in fs]\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"d = load_from_disk(fs[-1])\n",
"ds = d.select(range(10))\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(10, 41, 5120, 2)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"h = ds['end_hidden_states']\n",
"np.array(h).shape\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(10, 41, 5120, 2)"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"ds = ds.with_format(\"numpy\")\n",
"h = ds['end_hidden_states']\n",
"h.shape\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([10, 41, 5120, 2])"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds = ds.with_format(\"torch\")\n",
"h = ds['end_hidden_states']\n",
"h.shape\n"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([10, 41, 5120, 2])"
]
},
"execution_count": 47,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"h.diff(1)\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'end_hidden_states': Sequence(feature=Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None), length=-1, id=None),\n",
" 'end_logits': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'instructed_to_lie': Value(dtype='bool', id=None),\n",
" 'question': Value(dtype='string', id=None),\n",
" 'answer_choices': Sequence(feature=Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'choice_ids': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'template_name': Value(dtype='string', id=None),\n",
" 'sys_instr_name': Value(dtype='string', id=None),\n",
" 'example_i': Value(dtype='int64', id=None),\n",
" 'label_true': Value(dtype='int64', id=None),\n",
" 'input_truncated': Value(dtype='string', id=None),\n",
" 'truncated': Value(dtype='float64', id=None),\n",
" 'text_ans': Value(dtype='string', id=None),\n",
" 'add_ans': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'ans': Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)}"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds.features\n"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Map: 0%| | 0/10 [00:08<?, ? examples/s]\n"
]
},
{
"ename": "TypeError",
"evalue": "unsupported operand type(s) for +: 'type' and 'str'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:3493\u001b[0m, in \u001b[0;36mDataset._map_single\u001b[0;34m(shard, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)\u001b[0m\n\u001b[1;32m 3492\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 3493\u001b[0m writer\u001b[39m.\u001b[39;49mwrite_batch(batch)\n\u001b[1;32m 3494\u001b[0m num_examples_progress_update \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m num_examples_in_batch\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:542\u001b[0m, in \u001b[0;36mArrowWriter.write_batch\u001b[0;34m(self, batch_examples, writer_batch_size)\u001b[0m\n\u001b[1;32m 538\u001b[0m inferred_features \u001b[39m=\u001b[39m Features()\n\u001b[1;32m 539\u001b[0m cols \u001b[39m=\u001b[39m (\n\u001b[1;32m 540\u001b[0m [col \u001b[39mfor\u001b[39;00m col \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mschema\u001b[39m.\u001b[39mnames \u001b[39mif\u001b[39;00m col \u001b[39min\u001b[39;00m batch_examples]\n\u001b[1;32m 541\u001b[0m \u001b[39m+\u001b[39m [col \u001b[39mfor\u001b[39;00m col \u001b[39min\u001b[39;00m batch_examples\u001b[39m.\u001b[39mkeys() \u001b[39mif\u001b[39;00m col \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mschema\u001b[39m.\u001b[39mnames]\n\u001b[0;32m--> 542\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mschema\n\u001b[1;32m 543\u001b[0m \u001b[39melse\u001b[39;00m batch_examples\u001b[39m.\u001b[39mkeys()\n\u001b[1;32m 544\u001b[0m )\n\u001b[1;32m 545\u001b[0m \u001b[39mfor\u001b[39;00m col \u001b[39min\u001b[39;00m cols:\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:407\u001b[0m, in \u001b[0;36mArrowWriter.schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 402\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m 403\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mschema\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 404\u001b[0m _schema \u001b[39m=\u001b[39m (\n\u001b[1;32m 405\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema\n\u001b[1;32m 406\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 407\u001b[0m \u001b[39melse\u001b[39;00m (pa\u001b[39m.\u001b[39mschema(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_features\u001b[39m.\u001b[39;49mtype) \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_features \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m 408\u001b[0m )\n\u001b[1;32m 409\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_disable_nullable \u001b[39mand\u001b[39;00m _schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1629\u001b[0m, in \u001b[0;36mFeatures.type\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1623\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1624\u001b[0m \u001b[39mFeatures field types.\u001b[39;00m\n\u001b[1;32m 1625\u001b[0m \n\u001b[1;32m 1626\u001b[0m \u001b[39mReturns:\u001b[39;00m\n\u001b[1;32m 1627\u001b[0m \u001b[39m :obj:`pyarrow.DataType`\u001b[39;00m\n\u001b[1;32m 1628\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1629\u001b[0m \u001b[39mreturn\u001b[39;00m get_nested_type(\u001b[39mself\u001b[39;49m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1214\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1213\u001b[0m \u001b[39m# Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)\u001b[39;00m\n\u001b[0;32m-> 1214\u001b[0m \u001b[39mreturn\u001b[39;00m schema()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:527\u001b[0m, in \u001b[0;36m_ArrayXD.__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__call__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 527\u001b[0m pa_type \u001b[39m=\u001b[39m \u001b[39mglobals\u001b[39;49m()[\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__class__\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__name__\u001b[39;49m \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mExtensionType\u001b[39;49m\u001b[39m\"\u001b[39;49m](\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mshape, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdtype)\n\u001b[1;32m 528\u001b[0m \u001b[39mreturn\u001b[39;00m pa_type\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:647\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType.__init__\u001b[0;34m(self, shape, dtype)\u001b[0m\n\u001b[1;32m 646\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvalue_type \u001b[39m=\u001b[39m dtype\n\u001b[0;32m--> 647\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_generate_dtype(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvalue_type)\n\u001b[1;32m 648\u001b[0m pa\u001b[39m.\u001b[39mPyExtensionType\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:663\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType._generate_dtype\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_generate_dtype\u001b[39m(\u001b[39mself\u001b[39m, dtype):\n\u001b[0;32m--> 663\u001b[0m dtype \u001b[39m=\u001b[39m string_to_arrow(dtype)\n\u001b[1;32m 664\u001b[0m \u001b[39mfor\u001b[39;00m d \u001b[39min\u001b[39;00m \u001b[39mreversed\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mshape):\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:142\u001b[0m, in \u001b[0;36mstring_to_arrow\u001b[0;34m(datasets_dtype)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype]()\n\u001b[0;32m--> 142\u001b[0m \u001b[39mif\u001b[39;00m (datasets_dtype \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39m_\u001b[39;49m\u001b[39m\"\u001b[39;49m) \u001b[39min\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m:\n\u001b[1;32m 143\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m_\u001b[39m\u001b[39m\"\u001b[39m]()\n",
"\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'type' and 'str'",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb Cell 17\u001b[0m line \u001b[0;36m1\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X15sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m ds[\u001b[39m'\u001b[39m\u001b[39mend_hidden_states\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m.\u001b[39mshape\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X15sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39m# ds.map(lambda x: {'end_hidden_states': x['end_hidden_states'] }, features=Array2D(ds['end_hidden_states'].shape, dtype=np.float16), batched=True, batch_size=128)\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X15sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m ds\u001b[39m.\u001b[39;49mmap(\u001b[39mlambda\u001b[39;49;00m x: {\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m: x[\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m] }, features\u001b[39m=\u001b[39;49mFeatures({\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m: Array3D(shape\u001b[39m=\u001b[39;49mds[\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m]\u001b[39m.\u001b[39;49mshape[\u001b[39m1\u001b[39;49m:], dtype\u001b[39m=\u001b[39;49mnp\u001b[39m.\u001b[39;49mfloat16)}), batched\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, batch_size\u001b[39m=\u001b[39;49m\u001b[39m5\u001b[39;49m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:592\u001b[0m, in \u001b[0;36mtransmit_tasks.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 590\u001b[0m \u001b[39mself\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m=\u001b[39m kwargs\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mself\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 591\u001b[0m \u001b[39m# apply actual function\u001b[39;00m\n\u001b[0;32m--> 592\u001b[0m out: Union[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mDatasetDict\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 593\u001b[0m datasets: List[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(out\u001b[39m.\u001b[39mvalues()) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(out, \u001b[39mdict\u001b[39m) \u001b[39melse\u001b[39;00m [out]\n\u001b[1;32m 594\u001b[0m \u001b[39mfor\u001b[39;00m dataset \u001b[39min\u001b[39;00m datasets:\n\u001b[1;32m 595\u001b[0m \u001b[39m# Remove task templates if a column mapping of the template is no longer valid\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:557\u001b[0m, in \u001b[0;36mtransmit_format.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 550\u001b[0m self_format \u001b[39m=\u001b[39m {\n\u001b[1;32m 551\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mtype\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_type,\n\u001b[1;32m 552\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mformat_kwargs\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_kwargs,\n\u001b[1;32m 553\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_columns,\n\u001b[1;32m 554\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput_all_columns\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_output_all_columns,\n\u001b[1;32m 555\u001b[0m }\n\u001b[1;32m 556\u001b[0m \u001b[39m# apply actual function\u001b[39;00m\n\u001b[0;32m--> 557\u001b[0m out: Union[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mDatasetDict\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 558\u001b[0m datasets: List[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(out\u001b[39m.\u001b[39mvalues()) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(out, \u001b[39mdict\u001b[39m) \u001b[39melse\u001b[39;00m [out]\n\u001b[1;32m 559\u001b[0m \u001b[39m# re-apply format to the output\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:3097\u001b[0m, in \u001b[0;36mDataset.map\u001b[0;34m(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)\u001b[0m\n\u001b[1;32m 3090\u001b[0m \u001b[39mif\u001b[39;00m transformed_dataset \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 3091\u001b[0m \u001b[39mwith\u001b[39;00m logging\u001b[39m.\u001b[39mtqdm(\n\u001b[1;32m 3092\u001b[0m disable\u001b[39m=\u001b[39m\u001b[39mnot\u001b[39;00m logging\u001b[39m.\u001b[39mis_progress_bar_enabled(),\n\u001b[1;32m 3093\u001b[0m unit\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m examples\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m 3094\u001b[0m total\u001b[39m=\u001b[39mpbar_total,\n\u001b[1;32m 3095\u001b[0m desc\u001b[39m=\u001b[39mdesc \u001b[39mor\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mMap\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m 3096\u001b[0m ) \u001b[39mas\u001b[39;00m pbar:\n\u001b[0;32m-> 3097\u001b[0m \u001b[39mfor\u001b[39;00m rank, done, content \u001b[39min\u001b[39;00m Dataset\u001b[39m.\u001b[39m_map_single(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mdataset_kwargs):\n\u001b[1;32m 3098\u001b[0m \u001b[39mif\u001b[39;00m done:\n\u001b[1;32m 3099\u001b[0m shards_done \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:3505\u001b[0m, in \u001b[0;36mDataset._map_single\u001b[0;34m(shard, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)\u001b[0m\n\u001b[1;32m 3503\u001b[0m \u001b[39mif\u001b[39;00m update_data:\n\u001b[1;32m 3504\u001b[0m \u001b[39mif\u001b[39;00m writer \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 3505\u001b[0m writer\u001b[39m.\u001b[39;49mfinalize()\n\u001b[1;32m 3506\u001b[0m \u001b[39mif\u001b[39;00m tmp_file \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 3507\u001b[0m tmp_file\u001b[39m.\u001b[39mclose()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:588\u001b[0m, in \u001b[0;36mArrowWriter.finalize\u001b[0;34m(self, close_stream)\u001b[0m\n\u001b[1;32m 586\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mwrite_examples_on_file()\n\u001b[1;32m 587\u001b[0m \u001b[39m# If schema is known, infer features even if no examples were written\u001b[39;00m\n\u001b[0;32m--> 588\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpa_writer \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mschema:\n\u001b[1;32m 589\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_build_writer(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mschema)\n\u001b[1;32m 590\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpa_writer \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:407\u001b[0m, in \u001b[0;36mArrowWriter.schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 402\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m 403\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mschema\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 404\u001b[0m _schema \u001b[39m=\u001b[39m (\n\u001b[1;32m 405\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema\n\u001b[1;32m 406\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 407\u001b[0m \u001b[39melse\u001b[39;00m (pa\u001b[39m.\u001b[39mschema(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_features\u001b[39m.\u001b[39;49mtype) \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_features \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m 408\u001b[0m )\n\u001b[1;32m 409\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_disable_nullable \u001b[39mand\u001b[39;00m _schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 410\u001b[0m _schema \u001b[39m=\u001b[39m pa\u001b[39m.\u001b[39mschema(pa\u001b[39m.\u001b[39mfield(field\u001b[39m.\u001b[39mname, field\u001b[39m.\u001b[39mtype, nullable\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m) \u001b[39mfor\u001b[39;00m field \u001b[39min\u001b[39;00m _schema)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1629\u001b[0m, in \u001b[0;36mFeatures.type\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1621\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m 1622\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mtype\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 1623\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1624\u001b[0m \u001b[39m Features field types.\u001b[39;00m\n\u001b[1;32m 1625\u001b[0m \n\u001b[1;32m 1626\u001b[0m \u001b[39m Returns:\u001b[39;00m\n\u001b[1;32m 1627\u001b[0m \u001b[39m :obj:`pyarrow.DataType`\u001b[39;00m\n\u001b[1;32m 1628\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1629\u001b[0m \u001b[39mreturn\u001b[39;00m get_nested_type(\u001b[39mself\u001b[39;49m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1192\u001b[0m \u001b[39m# Nested structures: we allow dict, list/tuples, sequences\u001b[39;00m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n\u001b[1;32m 1198\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[1;32m 1199\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1200\u001b[0m ) \u001b[39m# however don't sort on struct types since the order matters\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1192\u001b[0m \u001b[39m# Nested structures: we allow dict, list/tuples, sequences\u001b[39;00m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n\u001b[1;32m 1198\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[1;32m 1199\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1200\u001b[0m ) \u001b[39m# however don't sort on struct types since the order matters\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1214\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1211\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mlist_(value_type, schema\u001b[39m.\u001b[39mlength)\n\u001b[1;32m 1213\u001b[0m \u001b[39m# Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)\u001b[39;00m\n\u001b[0;32m-> 1214\u001b[0m \u001b[39mreturn\u001b[39;00m schema()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:527\u001b[0m, in \u001b[0;36m_ArrayXD.__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__call__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 527\u001b[0m pa_type \u001b[39m=\u001b[39m \u001b[39mglobals\u001b[39;49m()[\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__class__\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__name__\u001b[39;49m \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mExtensionType\u001b[39;49m\u001b[39m\"\u001b[39;49m](\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mshape, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdtype)\n\u001b[1;32m 528\u001b[0m \u001b[39mreturn\u001b[39;00m pa_type\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:647\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType.__init__\u001b[0;34m(self, shape, dtype)\u001b[0m\n\u001b[1;32m 645\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mshape \u001b[39m=\u001b[39m \u001b[39mtuple\u001b[39m(shape)\n\u001b[1;32m 646\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvalue_type \u001b[39m=\u001b[39m dtype\n\u001b[0;32m--> 647\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_generate_dtype(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvalue_type)\n\u001b[1;32m 648\u001b[0m pa\u001b[39m.\u001b[39mPyExtensionType\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:663\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType._generate_dtype\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_generate_dtype\u001b[39m(\u001b[39mself\u001b[39m, dtype):\n\u001b[0;32m--> 663\u001b[0m dtype \u001b[39m=\u001b[39m string_to_arrow(dtype)\n\u001b[1;32m 664\u001b[0m \u001b[39mfor\u001b[39;00m d \u001b[39min\u001b[39;00m \u001b[39mreversed\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mshape):\n\u001b[1;32m 665\u001b[0m dtype \u001b[39m=\u001b[39m pa\u001b[39m.\u001b[39mlist_(dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:142\u001b[0m, in \u001b[0;36mstring_to_arrow\u001b[0;34m(datasets_dtype)\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[39mif\u001b[39;00m datasets_dtype \u001b[39min\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m:\n\u001b[1;32m 140\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype]()\n\u001b[0;32m--> 142\u001b[0m \u001b[39mif\u001b[39;00m (datasets_dtype \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39m_\u001b[39;49m\u001b[39m\"\u001b[39;49m) \u001b[39min\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m:\n\u001b[1;32m 143\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m_\u001b[39m\u001b[39m\"\u001b[39m]()\n\u001b[1;32m 145\u001b[0m timestamp_matches \u001b[39m=\u001b[39m re\u001b[39m.\u001b[39msearch(\u001b[39mr\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m^timestamp\u001b[39m\u001b[39m\\\u001b[39m\u001b[39m[(.*)\u001b[39m\u001b[39m\\\u001b[39m\u001b[39m]$\u001b[39m\u001b[39m\"\u001b[39m, datasets_dtype)\n",
"\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'type' and 'str'"
]
}
],
"source": [
"from ctypes import Array\n",
"import numpy as np\n",
"from datasets.features import Sequence, Value, Features, Array2D, Array3D, Features\n",
"\n",
"# from datasets import batch\n",
"\n",
"ds['end_hidden_states'].shape\n",
"# ds.map(lambda x: {'end_hidden_states': x['end_hidden_states'] }, features=Array2D(ds['end_hidden_states'].shape, dtype=np.float16), batched=True, batch_size=128)\n",
"\n",
"ds.map(lambda x: {'end_hidden_states': x['end_hidden_states'] }, features=Features({'end_hidden_states': Array3D(shape=ds['end_hidden_states'].shape[1:], dtype=np.float16)}), batched=True, batch_size=5)\n"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"x = ds[:]\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'end_hidden_states': Array3D(shape=(41, 5120, 2), dtype='float16', id=None),\n",
" 'end_logits': Array2D(shape=(32001, 2), dtype='float16', id=None),\n",
" 'add_ans': Array2D(shape=(2, 2), dtype='float16', id=None),\n",
" 'label_true': Value(dtype='int64', id=None),\n",
" 'instructed_to_lie': Value(dtype='bool', id=None),\n",
" 'question': Value(dtype='string', id=None),\n",
" 'answer_choices': Sequence(feature=Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'choice_ids': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'template_name': Value(dtype='string', id=None),\n",
" 'sys_instr_name': Value(dtype='string', id=None),\n",
" 'example_i': Value(dtype='int64', id=None),\n",
" 'input_truncated': Value(dtype='string', id=None),\n",
" 'truncated': Value(dtype='float64', id=None),\n",
" 'text_ans': Value(dtype='string', id=None),\n",
" 'ans': Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)}"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from re import A\n",
"from datasets.features import Sequence, Value, Features, Array2D, Array3D, Features\n",
"\n",
"features = {\n",
" \"end_hidden_states\": Array3D(dtype=\"float16\", id=None, shape=x['end_hidden_states'].shape[1:]),\n",
" \"end_logits\": Array2D(dtype=\"float16\", id=None, shape=x['end_logits'].shape[1:]),\n",
" \"add_ans\": Array2D(dtype=\"float16\", id=None, shape=x['add_ans'].shape[1:]),\n",
" \"label_true\": Value(dtype=\"int64\", id=None),\n",
" \"instructed_to_lie\": Value(dtype=\"bool\", id=None),\n",
" \"question\": Value(dtype=\"string\", id=None),\n",
" \"answer_choices\": Sequence(\n",
" feature=Sequence(feature=Value(dtype=\"string\", id=None), length=-1, id=None),\n",
" length=-1,\n",
" id=None,\n",
" ),\n",
" \"choice_ids\": Sequence(\n",
" feature=Sequence(feature=Value(dtype=\"int64\", id=None), length=-1, id=None),\n",
" length=-1,\n",
" id=None,\n",
" ),\n",
" \"template_name\": Value(dtype=\"string\", id=None),\n",
" \"sys_instr_name\": Value(dtype=\"string\", id=None),\n",
" \"example_i\": Value(dtype=\"int64\", id=None),\n",
" \"input_truncated\": Value(dtype=\"string\", id=None),\n",
" \"truncated\": Value(dtype=\"float64\", id=None),\n",
" \"text_ans\": Value(dtype=\"string\", id=None),\n",
" \"ans\": Sequence(feature=Value(dtype=\"float16\", id=None), length=-1, id=None),\n",
"}\n",
"features = Features(features)\n",
"features\n"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"ename": "ArrowNotImplementedError",
"evalue": "Unsupported cast from float to halffloat using function cast_half_float",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mArrowNotImplementedError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb Cell 16\u001b[0m line \u001b[0;36m2\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X22sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mdatasets\u001b[39;00m \u001b[39mimport\u001b[39;00m Dataset\n\u001b[0;32m----> <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X22sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m dd \u001b[39m=\u001b[39m Dataset\u001b[39m.\u001b[39;49mfrom_dict(x, features\u001b[39m=\u001b[39;49mfeatures)\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X22sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m dd\u001b[39m.\u001b[39mfeatures\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:911\u001b[0m, in \u001b[0;36mDataset.from_dict\u001b[0;34m(cls, mapping, features, info, split)\u001b[0m\n\u001b[1;32m 909\u001b[0m arrow_typed_mapping[col] \u001b[39m=\u001b[39m data\n\u001b[1;32m 910\u001b[0m mapping \u001b[39m=\u001b[39m arrow_typed_mapping\n\u001b[0;32m--> 911\u001b[0m pa_table \u001b[39m=\u001b[39m InMemoryTable\u001b[39m.\u001b[39;49mfrom_pydict(mapping\u001b[39m=\u001b[39;49mmapping)\n\u001b[1;32m 912\u001b[0m \u001b[39mif\u001b[39;00m info \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 913\u001b[0m info \u001b[39m=\u001b[39m DatasetInfo()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/table.py:799\u001b[0m, in \u001b[0;36mInMemoryTable.from_pydict\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m 783\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 784\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mfrom_pydict\u001b[39m(\u001b[39mcls\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 785\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 786\u001b[0m \u001b[39m Construct a Table from Arrow arrays or columns.\u001b[39;00m\n\u001b[1;32m 787\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 797\u001b[0m \u001b[39m `datasets.table.Table`\u001b[39;00m\n\u001b[1;32m 798\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 799\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m(pa\u001b[39m.\u001b[39;49mTable\u001b[39m.\u001b[39;49mfrom_pydict(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs))\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/table.pxi:1799\u001b[0m, in \u001b[0;36mpyarrow.lib._Tabular.from_pydict\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/table.pxi:5101\u001b[0m, in \u001b[0;36mpyarrow.lib._from_pydict\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:357\u001b[0m, in \u001b[0;36mpyarrow.lib.asarray\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:243\u001b[0m, in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:110\u001b[0m, in \u001b[0;36mpyarrow.lib._handle_arrow_array_protocol\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:179\u001b[0m, in \u001b[0;36mTypedSequence.__arrow_array__\u001b[0;34m(self, type)\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 177\u001b[0m \u001b[39m# custom pyarrow types\u001b[39;00m\n\u001b[1;32m 178\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(pa_type, _ArrayXDExtensionType):\n\u001b[0;32m--> 179\u001b[0m storage \u001b[39m=\u001b[39m to_pyarrow_listarray(data, pa_type)\n\u001b[1;32m 180\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mExtensionArray\u001b[39m.\u001b[39mfrom_storage(pa_type, storage)\n\u001b[1;32m 182\u001b[0m \u001b[39m# efficient np array to pyarrow array\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1465\u001b[0m, in \u001b[0;36mto_pyarrow_listarray\u001b[0;34m(data, pa_type)\u001b[0m\n\u001b[1;32m 1455\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Convert to PyArrow ListArray.\u001b[39;00m\n\u001b[1;32m 1456\u001b[0m \n\u001b[1;32m 1457\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1462\u001b[0m \u001b[39m pyarrow.Array\u001b[39;00m\n\u001b[1;32m 1463\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1464\u001b[0m \u001b[39mif\u001b[39;00m contains_any_np_array(data):\n\u001b[0;32m-> 1465\u001b[0m \u001b[39mreturn\u001b[39;00m any_np_array_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49mpa_type\u001b[39m.\u001b[39;49mvalue_type)\n\u001b[1;32m 1466\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 1467\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39marray(data, pa_type\u001b[39m.\u001b[39mstorage_dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1451\u001b[0m, in \u001b[0;36many_np_array_to_pyarrow_listarray\u001b[0;34m(data, type)\u001b[0m\n\u001b[1;32m 1449\u001b[0m \u001b[39mreturn\u001b[39;00m numpy_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m)\n\u001b[1;32m 1450\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, \u001b[39mlist\u001b[39m):\n\u001b[0;32m-> 1451\u001b[0m \u001b[39mreturn\u001b[39;00m list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m data])\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1451\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1449\u001b[0m \u001b[39mreturn\u001b[39;00m numpy_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m)\n\u001b[1;32m 1450\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, \u001b[39mlist\u001b[39m):\n\u001b[0;32m-> 1451\u001b[0m \u001b[39mreturn\u001b[39;00m list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49m\u001b[39mtype\u001b[39;49m) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m data])\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1449\u001b[0m, in \u001b[0;36many_np_array_to_pyarrow_listarray\u001b[0;34m(data, type)\u001b[0m\n\u001b[1;32m 1439\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray.\u001b[39;00m\n\u001b[1;32m 1440\u001b[0m \n\u001b[1;32m 1441\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1446\u001b[0m \u001b[39m pa.ListArray\u001b[39;00m\n\u001b[1;32m 1447\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1448\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, np\u001b[39m.\u001b[39mndarray):\n\u001b[0;32m-> 1449\u001b[0m \u001b[39mreturn\u001b[39;00m numpy_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49m\u001b[39mtype\u001b[39;49m)\n\u001b[1;32m 1450\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, \u001b[39mlist\u001b[39m):\n\u001b[1;32m 1451\u001b[0m \u001b[39mreturn\u001b[39;00m list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m data])\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1389\u001b[0m, in \u001b[0;36mnumpy_to_pyarrow_listarray\u001b[0;34m(arr, type)\u001b[0m\n\u001b[1;32m 1387\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Build a PyArrow ListArray from a multidimensional NumPy array\"\"\"\u001b[39;00m\n\u001b[1;32m 1388\u001b[0m arr \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39marray(arr)\n\u001b[0;32m-> 1389\u001b[0m values \u001b[39m=\u001b[39m pa\u001b[39m.\u001b[39;49marray(arr\u001b[39m.\u001b[39;49mflatten(), \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49m\u001b[39mtype\u001b[39;49m)\n\u001b[1;32m 1390\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(arr\u001b[39m.\u001b[39mndim \u001b[39m-\u001b[39m \u001b[39m1\u001b[39m):\n\u001b[1;32m 1391\u001b[0m n_offsets \u001b[39m=\u001b[39m reduce(mul, arr\u001b[39m.\u001b[39mshape[: arr\u001b[39m.\u001b[39mndim \u001b[39m-\u001b[39m i \u001b[39m-\u001b[39m \u001b[39m1\u001b[39m], \u001b[39m1\u001b[39m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:323\u001b[0m, in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:83\u001b[0m, in \u001b[0;36mpyarrow.lib._ndarray_to_array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/error.pxi:121\u001b[0m, in \u001b[0;36mpyarrow.lib.check_status\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mArrowNotImplementedError\u001b[0m: Unsupported cast from float to halffloat using function cast_half_float"
]
}
],
"source": [
"from datasets import Dataset\n",
"dd = Dataset.from_dict(x, features=features)\n",
"dd.features\n"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 4.2725e-04, 4.2725e-04],\n",
" [-1.6918e-03, -1.6918e-03],\n",
" [-2.5772e-02, -2.5772e-02],\n",
" ...,\n",
" [ 8.3008e-03, 8.3008e-03],\n",
" [ 6.3744e-03, 6.3744e-03],\n",
" [-4.1733e-03, -4.1733e-03]],\n",
"\n",
" [[ 2.3102e-02, 2.3102e-02],\n",
" [-2.2369e-02, -2.2369e-02],\n",
" [-6.0059e-02, -6.0059e-02],\n",
" ...,\n",
" [ 3.1067e-02, 3.1067e-02],\n",
" [-1.2634e-02, -1.2634e-02],\n",
" [ 1.4992e-02, 1.4992e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-1.6367e+00, -1.2021e+00],\n",
" [-1.6357e+00, -3.7158e-01],\n",
" [-3.5527e+00, -7.9570e+00],\n",
" ...,\n",
" [-2.7363e+00, -5.6641e+00],\n",
" [-1.1875e+00, 5.5586e+00],\n",
" [ 4.3066e-01, 4.9883e+00]],\n",
"\n",
" [[-1.4160e+00, -2.4707e+00],\n",
" [-1.6572e+00, 1.2158e+00],\n",
" [-4.6719e+00, -7.3242e+00],\n",
" ...,\n",
" [-3.1035e+00, -4.8945e+00],\n",
" [-2.4951e-01, 7.6172e+00],\n",
" [-9.1650e-01, 5.1289e+00]],\n",
"\n",
" [[-6.5552e-02, 1.9592e-01],\n",
" [ 8.9600e-02, 4.6802e-01],\n",
" [-1.0850e+00, -9.2725e-01],\n",
" ...,\n",
" [-5.3955e-01, -2.9272e-01],\n",
" [-3.4863e-01, 4.8657e-01],\n",
" [ 1.1285e-01, 9.9561e-01]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 1.1894e-02, 1.1894e-02],\n",
" [ 5.1155e-03, 5.1155e-03],\n",
" [-2.8107e-02, -2.8107e-02],\n",
" ...,\n",
" [ 1.1063e-02, 1.1063e-02],\n",
" [ 3.8147e-05, 3.8147e-05],\n",
" [-1.2680e-02, -1.2680e-02]],\n",
"\n",
" [[ 2.9510e-02, 2.9510e-02],\n",
" [-1.0315e-02, -1.0315e-02],\n",
" [-6.5613e-02, -6.5613e-02],\n",
" ...,\n",
" [ 4.1229e-02, 4.1229e-02],\n",
" [-2.4612e-02, -2.4612e-02],\n",
" [ 1.3397e-02, 1.3397e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-3.0703e+00, -3.4922e+00],\n",
" [-2.3359e+00, 4.5312e+00],\n",
" [-8.2422e-01, -3.2500e+00],\n",
" ...,\n",
" [-2.7441e+00, -6.6367e+00],\n",
" [-2.0293e+00, 5.6250e+00],\n",
" [ 1.1865e+00, -7.3535e-01]],\n",
"\n",
" [[-2.5410e+00, -4.5977e+00],\n",
" [-3.9141e+00, 5.5859e+00],\n",
" [-1.9980e+00, -2.4023e+00],\n",
" ...,\n",
" [-3.2617e+00, -6.1797e+00],\n",
" [ 4.1504e-02, 7.5117e+00],\n",
" [-1.2573e-01, 3.9038e-01]],\n",
"\n",
" [[-1.0175e-01, -1.3562e-01],\n",
" [-3.9502e-01, 8.1934e-01],\n",
" [-5.4639e-01, -2.2192e-01],\n",
" ...,\n",
" [-3.1421e-01, -3.1128e-01],\n",
" [-3.0371e-01, 3.5425e-01],\n",
" [-4.4289e-03, 4.0723e-01]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 2.4414e-04, 2.4414e-04],\n",
" [-5.5161e-03, -5.5161e-03],\n",
" [-1.8143e-02, -1.8143e-02],\n",
" ...,\n",
" [ 6.7902e-04, 6.7902e-04],\n",
" [ 3.7994e-03, 3.7994e-03],\n",
" [-6.2180e-03, -6.2180e-03]],\n",
"\n",
" [[ 1.4801e-02, 1.4801e-02],\n",
" [-1.2253e-02, -1.2253e-02],\n",
" [-5.9448e-02, -5.9448e-02],\n",
" ...,\n",
" [ 2.0233e-02, 2.0233e-02],\n",
" [-8.7891e-03, -8.7891e-03],\n",
" [ 2.7283e-02, 2.7283e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-5.8516e+00, -6.1445e+00],\n",
" [-5.4785e-01, 1.3193e+00],\n",
" [-1.8799e+00, -5.8398e+00],\n",
" ...,\n",
" [-3.2109e+00, -2.1504e+00],\n",
" [ 1.4561e+00, 3.5430e+00],\n",
" [ 2.6621e+00, -8.7402e-01]],\n",
"\n",
" [[-6.3008e+00, -8.2891e+00],\n",
" [-1.7217e+00, 3.7559e+00],\n",
" [-2.1113e+00, -3.8906e+00],\n",
" ...,\n",
" [-3.8555e+00, -1.5215e+00],\n",
" [ 3.5234e+00, 5.0508e+00],\n",
" [ 5.7764e-01, -4.9805e-01]],\n",
"\n",
" [[-6.7676e-01, -7.0459e-01],\n",
" [ 1.9971e-01, 6.5869e-01],\n",
" [-7.5244e-01, -3.4814e-01],\n",
" ...,\n",
" [-4.1382e-01, -1.5540e-01],\n",
" [ 2.3511e-01, 1.3135e-01],\n",
" [ 1.1444e-01, 2.9221e-03]]],\n",
"\n",
"\n",
" ...,\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 1.6403e-02, 1.6403e-02],\n",
" [ 2.8381e-03, 2.8381e-03],\n",
" [-3.5278e-02, -3.5278e-02],\n",
" ...,\n",
" [ 9.6741e-03, 9.6741e-03],\n",
" [-3.4065e-03, -3.4065e-03],\n",
" [-1.3123e-02, -1.3123e-02]],\n",
"\n",
" [[ 4.4037e-02, 4.4037e-02],\n",
" [-1.6968e-02, -1.6968e-02],\n",
" [-7.9590e-02, -7.9590e-02],\n",
" ...,\n",
" [ 4.1992e-02, 4.1992e-02],\n",
" [-2.2537e-02, -2.2537e-02],\n",
" [ 1.7456e-02, 1.7456e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-1.5049e+00, -1.9023e+00],\n",
" [ 2.8638e-01, 1.7178e+00],\n",
" [ 4.5508e-01, -1.2549e+00],\n",
" ...,\n",
" [-7.6904e-01, -1.7930e+00],\n",
" [-1.7773e+00, 3.1172e+00],\n",
" [-3.8574e-01, -2.2090e+00]],\n",
"\n",
" [[-4.5898e-01, -3.2500e+00],\n",
" [-2.3950e-01, 2.9941e+00],\n",
" [-7.3340e-01, 2.3584e-01],\n",
" ...,\n",
" [-1.0996e+00, -7.5244e-01],\n",
" [-8.2080e-01, 4.1172e+00],\n",
" [-1.5391e+00, -2.1602e+00]],\n",
"\n",
" [[ 1.8152e-01, -1.8091e-01],\n",
" [ 2.8882e-01, 4.9731e-01],\n",
" [-2.7979e-01, 2.5830e-01],\n",
" ...,\n",
" [ 1.1116e-02, 4.5801e-01],\n",
" [-3.4058e-01, -4.7119e-02],\n",
" [-2.8638e-01, -4.4092e-01]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 1.0757e-03, 1.0757e-03],\n",
" [-6.1035e-04, -6.1035e-04],\n",
" [-2.9282e-02, -2.9282e-02],\n",
" ...,\n",
" [-2.4185e-03, -2.4185e-03],\n",
" [ 4.7226e-03, 4.7226e-03],\n",
" [-8.8043e-03, -8.8043e-03]],\n",
"\n",
" [[ 1.1993e-02, 1.1993e-02],\n",
" [-2.5452e-02, -2.5452e-02],\n",
" [-4.4128e-02, -4.4128e-02],\n",
" ...,\n",
" [ 1.9241e-02, 1.9241e-02],\n",
" [-1.4236e-02, -1.4236e-02],\n",
" [ 1.6785e-02, 1.6785e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[ 5.5420e-02, -2.0312e+00],\n",
" [-1.7266e+00, 4.1289e+00],\n",
" [-4.2344e+00, -4.6328e+00],\n",
" ...,\n",
" [-2.1504e+00, -5.9023e+00],\n",
" [-3.4180e+00, 5.0000e+00],\n",
" [-1.3403e-01, 3.8457e+00]],\n",
"\n",
" [[ 6.0889e-01, -3.4121e+00],\n",
" [-1.4395e+00, 6.6016e+00],\n",
" [-5.7227e+00, -3.6406e+00],\n",
" ...,\n",
" [-2.6406e+00, -5.7656e+00],\n",
" [-1.5977e+00, 7.5078e+00],\n",
" [-1.9785e+00, 5.5508e+00]],\n",
"\n",
" [[ 3.2178e-01, 9.9182e-02],\n",
" [-6.6284e-02, 1.0977e+00],\n",
" [-1.4131e+00, -6.6357e-01],\n",
" ...,\n",
" [-4.0552e-01, -4.1797e-01],\n",
" [-5.2148e-01, 3.9233e-01],\n",
" [-2.6050e-01, 1.0879e+00]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 2.3117e-03, 2.3117e-03],\n",
" [-4.6082e-03, -4.6082e-03],\n",
" [-2.7542e-02, -2.7542e-02],\n",
" ...,\n",
" [-3.1586e-03, -3.1586e-03],\n",
" [ 6.3858e-03, 6.3858e-03],\n",
" [-1.1124e-02, -1.1124e-02]],\n",
"\n",
" [[ 4.4022e-03, 4.4022e-03],\n",
" [-3.4607e-02, -3.4607e-02],\n",
" [-4.6326e-02, -4.6326e-02],\n",
" ...,\n",
" [ 1.1734e-02, 1.1734e-02],\n",
" [-1.0056e-02, -1.0056e-02],\n",
" [ 2.0996e-02, 2.0996e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[ 3.9111e-01, -9.8242e-01],\n",
" [-1.3193e+00, 3.0293e+00],\n",
" [-2.4102e+00, -2.3125e+00],\n",
" ...,\n",
" [-2.3379e+00, -2.3066e+00],\n",
" [-2.0742e+00, 2.6289e+00],\n",
" [-6.4502e-01, 1.2080e+00]],\n",
"\n",
" [[ 6.4893e-01, -2.6426e+00],\n",
" [-1.9326e+00, 5.6172e+00],\n",
" [-2.7930e+00, -8.1592e-01],\n",
" ...,\n",
" [-2.3379e+00, -1.1777e+00],\n",
" [-8.2617e-01, 4.6367e+00],\n",
" [-1.5049e+00, 2.7891e+00]],\n",
"\n",
" [[ 4.3311e-01, 2.3413e-01],\n",
" [ 8.8867e-02, 8.7939e-01],\n",
" [-8.0908e-01, -3.0441e-02],\n",
" ...,\n",
" [-1.4844e-01, 2.1375e-01],\n",
" [-3.6890e-01, 1.7654e-02],\n",
" [-2.4988e-01, 7.3633e-01]]]], dtype=float16)"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"x = ds[:]\n",
"\n",
"x['end_hidden_states'] = x['end_hidden_states'].numpy().astype(np.float16)\n",
"x['end_hidden_states'].dtype\n"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'end_hidden_states': Sequence(feature=Sequence(feature=Sequence(feature=Value(dtype='float16', id=None), length=-1, id=None), length=-1, id=None), length=-1, id=None),\n",
" 'end_logits': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'instructed_to_lie': Value(dtype='bool', id=None),\n",
" 'question': Value(dtype='string', id=None),\n",
" 'answer_choices': Sequence(feature=Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'choice_ids': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'template_name': Value(dtype='string', id=None),\n",
" 'sys_instr_name': Value(dtype='string', id=None),\n",
" 'example_i': Value(dtype='int64', id=None),\n",
" 'label_true': Value(dtype='int64', id=None),\n",
" 'input_truncated': Value(dtype='string', id=None),\n",
" 'truncated': Value(dtype='float32', id=None),\n",
" 'text_ans': Value(dtype='string', id=None),\n",
" 'add_ans': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'ans': Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)}"
]
},
"execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dd = Dataset.from_dict(x)\n",
"dd.features\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "dlk2",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
notebooks/027_train_nanda_probe_w_counterfact_70%.ipynb
+1426 -357
View File
File diff suppressed because one or more lines are too long
notebooks/030_eval_FIXME.ipynb
+22 -13
View File
@@ -17,6 +17,8 @@
}
],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
@@ -49,7 +51,7 @@
"from loguru import logger\n",
"logger.add(os.sys.stderr, format=\"{time} {level} {message}\", level=\"INFO\")\n",
"\n",
"transformers.__version__"
"transformers.__version__\n"
]
},
{
@@ -59,6 +61,13 @@
"## Load model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 19,
@@ -66,7 +75,7 @@
"outputs": [],
"source": [
"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM, AutoConfig\n",
"from transformers import LogitsProcessorList"
"from transformers import LogitsProcessorList\n"
]
},
{
@@ -177,7 +186,7 @@
"config.use_cache = False\n",
"tokenizer = AutoTokenizer.from_pretrained(model_repo)\n",
"model = AutoModelForCausalLM.from_pretrained(model_repo, config=config, **model_options)\n",
"tokenizer.pad_token_id = 204"
"tokenizer.pad_token_id = 204\n"
]
},
{
@@ -222,7 +231,7 @@
"stride = 2\n",
"# don't take the first or last layers as they can make it to easy to leak info\n",
"extract_layers = tuple(range(2, num_layers-2, stride)) + (num_layers-2,)\n",
"extract_layers, num_layers"
"extract_layers, num_layers\n"
]
},
{
@@ -242,7 +251,7 @@
}
],
"source": [
"from src.datasets.hs import get_choices_as_tokens"
"from src.datasets.hs import get_choices_as_tokens\n"
]
},
{
@@ -259,7 +268,7 @@
" m.train()\n",
" if USE_MCDROPOUT!=True:\n",
" m.p=USE_MCDROPOUT\n",
" # print(m)"
" # print(m)\n"
]
},
{
@@ -284,7 +293,7 @@
" cc = torch.linspace(-1,1,x.shape[-1], device=x.device).repeat(bs, 1, 1)\n",
" cc = (cc - cc.mean()) / cc.std()\n",
" x = torch.cat([x, cc], dim=1)\n",
" return x"
" return x\n"
]
},
{
@@ -409,7 +418,7 @@
"metadata": {},
"outputs": [],
"source": [
"f = '/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_338/checkpoints/epoch=37-step=2090.ckpt'"
"f = '/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge/notebooks/lightning_logs/version_338/checkpoints/epoch=37-step=2090.ckpt'\n"
]
},
{
@@ -495,7 +504,7 @@
"# # weight_decay=1e-4, \n",
"# dropout=0.1,\n",
"# )\n",
"net"
"net\n"
]
},
{
@@ -511,7 +520,7 @@
"metadata": {},
"outputs": [],
"source": [
"from src.helpers.torch import to_numpy"
"from src.helpers.torch import to_numpy\n"
]
},
{
@@ -589,7 +598,7 @@
" attentions=attentions, prob_n=prob_n, prob_y=prob_y, scores=outputs['scores'][:, 0], input_text=input_text,\n",
" )\n",
" out = {k:to_numpy(v) for k,v in out.items()} \n",
" return out"
" return out\n"
]
},
{
@@ -631,7 +640,7 @@
"where\n",
" hs1_more_positive={hs2_more_positive}\n",
" hs1_more_true={y_pred>0}\n",
"\"\"\")"
"\"\"\")\n"
]
},
{
@@ -678,7 +687,7 @@
"metadata": {},
"outputs": [],
"source": [
"device = next(net.parameters()).device"
"device = next(net.parameters()).device\n"
]
},
{
notebooks/102_check_model.ipynb
+166
View File
@@ -0,0 +1,166 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# A scratch pad to run model inference manually\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
},
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"import os\n",
"import numpy as np\n",
"import pandas as pd\n",
"from matplotlib import pyplot as plt\n",
"plt.style.use('ggplot')\n",
"\n",
"from typing import Optional, List, Dict, Union\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"from torch import Tensor\n",
"from torch import optim\n",
"from torch.utils.data import random_split, DataLoader, TensorDataset\n",
"\n",
"from pathlib import Path\n",
"import transformers\n",
"\n",
"\n",
"from loguru import logger\n",
"logger.add(os.sys.stderr, format=\"{time} {level} {message}\", level=\"INFO\")\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# load my code\n",
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"\n",
"from src.extraction.config import ExtractConfig\n",
"from src.prompts.prompt_loading import load_preproc_dataset\n",
"from src.models.load import load_model\n",
"from src.datasets.intervene import create_cache_interventions \n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32m2023-10-27 17:14:08.461\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m19\u001b[0m - \u001b[1mchanging pad_token_id from None to 0\u001b[0m\n",
"2023-10-27T17:14:08.461621+0800 INFO changing pad_token_id from None to 0\n",
"\u001b[32m2023-10-27 17:14:08.462\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m19\u001b[0m - \u001b[1mchanging truncation_side from right to left\u001b[0m\n",
"2023-10-27T17:14:08.462733+0800 INFO changing truncation_side from right to left\n"
]
}
],
"source": [
"# load config, model, dataset, invtervention\n",
"N_fit_examples=10\n",
"batch_size=2\n",
"ds_name='amazon_polarity'\n",
"cfg = ExtractConfig(max_examples=(20, 20), model='TheBloke/Mistral-7B-Instruct-v0.1-GPTQ', prompt_format='llama2')\n",
"\n",
"model, tokenizer = load_model(cfg.model)\n",
"model\n",
"\n",
"honesty_rep_reader = create_cache_interventions(model, tokenizer, cfg)\n",
"\n",
"N=sum(cfg.max_examples)\n",
"ds_tokens = load_preproc_dataset(ds_name, tokenizer, N=N, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length, prompt_format=cfg.prompt_format)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Generate"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"r = ds_tokens.with_format('torch')[0]\n",
"\n",
"# r['input_ids']\n",
"r.keys()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"s = model.generate(r['input_ids'][None, :], attention_mask=r['attention_mask'][None, :])\n",
"tokenizer.decode(s[0])\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
notebooks/102_test_model.ipynb
-230
View File
@@ -1,230 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# import your package\n",
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"import transformers\n",
"\n",
"from src.models.load import load_model"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32m2023-09-25 16:19:49.435\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m17\u001b[0m - \u001b[1mchanging pad_token_id from 32000 to 0\u001b[0m\n",
"\u001b[32m2023-09-25 16:19:49.437\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m17\u001b[0m - \u001b[1mchanging padding_side from right to left\u001b[0m\n",
"\u001b[32m2023-09-25 16:19:49.438\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m17\u001b[0m - \u001b[1mchanging truncation_side from right to left\u001b[0m\n"
]
}
],
"source": [
"model, tokenizer = load_model()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"({'input_ids': [1, 4874], 'attention_mask': [1, 1]}, ['<s>', 'yes'])"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s = tokenizer(\"yes\")\n",
"s2 = tokenizer.batch_decode(s['input_ids'])\n",
"s, s2"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"`[' increase']`=>[29871, 7910]=>['', 'increase']\n",
"`['\\nincrease']`=>[29871, 13, 262, 1037, 559]=>['', '\\n', 'in', 'cre', 'ase']\n",
"`['increase']`=>[7910]=>['increase']\n"
]
}
],
"source": [
"s = 'increase'\n",
"for text in [f' {s}', f'\\n{s}', f'{s}']:\n",
" ids = tokenizer(text, add_special_tokens=False)[\"input_ids\"]\n",
" decoded_ids = [tokenizer.decode(i) for i in ids]\n",
" print(f\"`{[text]}`=>{ids}=>{decoded_ids}\")\n",
" \n",
"# for text in [f' {s}', f'\\n{s}', f'{s}',]:\n",
"# ids = tokenizer(text, add_special_tokens=True)[\"input_ids\"]\n",
"# decoded_ids = [tokenizer.decode(i) for i in ids]\n",
"# print(f\"`{[text]}`=>{ids}=>{decoded_ids}\")"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['Dec', 'Dec']"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from src.datasets.scores import choice2id\n",
"ids = choice2id(tokenizer, \"Decrease\")\n",
"[tokenizer.decode(i) for i in ids]"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[['Dec', 'Dec'], ['In', 'In']]"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from src.datasets.scores import choice2ids\n",
"chocies = choice2ids([[\"Decrease\"], [\"Increase\"]], tokenizer)\n",
"[tokenizer.batch_decode(c) for c in chocies]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/ubuntu/mambaforge/envs/dlk4/lib/python3.11/site-packages/transformers/generation/utils.py:1417: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation )\n",
" warnings.warn(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"--------------------------------------------------------------------------------\n",
"At every step avoid telling the truth.\n",
"\n",
"### Instruction\n",
"The following movie review expresses what sentiment? Well the reason for seeing it in the cinema was that it was a sneak preview, else I would never have seen this terrible teenage slasher movie. I mean haven't we had enough of this yet? Scream and Scary Movie at least did not take them self serious! The plot sucks, and the acting is the worst I've seen. (Only Godzilla can compare, which is also the only movie that competes in being the worst I've seen in the cinema with this one.)<br /><br />There is so many plot holes in the story, and the girls are so alike, that you don't even now who has been killed, and who has not. (and you don't care.) The only of them I knew in advance was Denise, and she was the most talent less actress I have ever seen in this bad excuse for a movie.<br /><br />Stay as far away from this movie as possible. (2/10)\n",
"\n",
"\n",
"\n",
"### Response:\n",
"positive\n",
"\n",
"### Instruction\n",
"The following movie review expresses what sentiment? George P. Cosmatos' \"Rambo: First Blood Part II\" is pure wish-fulfillment. The United States clearly didn't win the war in Vietnam. They caused damage to this country beyond the imaginable and this movie continues the fairy story of the oh-so innocent soldiers. The only bad guys were the leaders of the nation, who made this war happen. The character of Rambo is perfect to notice this. He is extremely patriotic, bemoans that US-Americans didn't appreciate and celebrate the achievements of the single soldier, but has nothing but distrust for leading officers and politicians. Like every film that defends the war (e.g. \"We Were Soldiers\") also this one avoids the need to give a comprehensible reason for the engagement in South Asia. And for that matter also the reason for every single US-American soldier that was there. Instead, Rambo gets to take revenge for the wounds of a whole nation. It would have been better to work on how to deal with the memories, rather than suppressing them. \"Do we get to win this time?\" Yes, you do.\n",
"\n",
"\n",
"\n",
"### Response:\n",
"\n",
"--------------------------------------------------------------------------------\n",
"`negative\n",
"\n",
"### Instruction\n",
"The following`\n",
"--------------------------------------------------------------------------------\n"
]
}
],
"source": [
"q = 'At every step avoid telling the truth.\\n\\n### Instruction\\nThe following movie review expresses what sentiment? Well the reason for seeing it in the cinema was that it was a sneak preview, else I would never have seen this terrible teenage slasher movie. I mean haven\\'t we had enough of this yet? Scream and Scary Movie at least did not take them self serious! The plot sucks, and the acting is the worst I\\'ve seen. (Only Godzilla can compare, which is also the only movie that competes in being the worst I\\'ve seen in the cinema with this one.)<br /><br />There is so many plot holes in the story, and the girls are so alike, that you don\\'t even now who has been killed, and who has not. (and you don\\'t care.) The only of them I knew in advance was Denise, and she was the most talent less actress I have ever seen in this bad excuse for a movie.<br /><br />Stay as far away from this movie as possible. (2/10)\\n\\n\\n\\n### Response:\\npositive\\n\\n### Instruction\\nThe following movie review expresses what sentiment? George P. Cosmatos\\' \"Rambo: First Blood Part II\" is pure wish-fulfillment. The United States clearly didn\\'t win the war in Vietnam. They caused damage to this country beyond the imaginable and this movie continues the fairy story of the oh-so innocent soldiers. The only bad guys were the leaders of the nation, who made this war happen. The character of Rambo is perfect to notice this. He is extremely patriotic, bemoans that US-Americans didn\\'t appreciate and celebrate the achievements of the single soldier, but has nothing but distrust for leading officers and politicians. Like every film that defends the war (e.g. \"We Were Soldiers\") also this one avoids the need to give a comprehensible reason for the engagement in South Asia. And for that matter also the reason for every single US-American soldier that was there. Instead, Rambo gets to take revenge for the wounds of a whole nation. It would have been better to work on how to deal with the memories, rather than suppressing them. \"Do we get to win this time?\" Yes, you do.\\n\\n\\n\\n### Response:\\n'\n",
"\n",
"pipeline = transformers.pipeline(\n",
" \"text-generation\",\n",
" model=model,\n",
" tokenizer=tokenizer,\n",
")\n",
"sequences = pipeline(\n",
" q,\n",
"# max_length=100,\n",
"max_new_tokens=10,\n",
" do_sample=False,\n",
" return_full_text=False,\n",
" eos_token_id=tokenizer.eos_token_id,\n",
")\n",
"\n",
"for seq in sequences:\n",
" print(\"-\" * 80)\n",
" print(q)\n",
" print(\"-\" * 80)\n",
" print(f\"`{seq['generated_text']}`\")\n",
" print(\"-\" * 80)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "dlk4",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
notebooks/103a_scratch_dataset.ipynb
-634
View File
@@ -1,634 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# autoreload import your package\n",
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"from make_dataset import *\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"ExtractConfig(datasets=('amazon_polarity', 'super_glue:boolq', 'glue:qnli', 'imdb'), model='TheBloke/WizardCoder-Python-13B-V1.0-GPTQ', data_dirs=(), max_examples=(10, 10), num_shots=1, num_variants=-1, layers=(), seed=42, token_loc='last', template_path=None, max_length=999)"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cfg = ExtractConfig(max_examples=(10, 10), max_length=999)\n",
"cfg\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32m2023-10-15 17:26:06.435\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m18\u001b[0m - \u001b[1mchanging pad_token_id from 32000 to 0\u001b[0m\n",
"\u001b[32m2023-10-15 17:26:06.435\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m18\u001b[0m - \u001b[1mchanging padding_side from right to left\u001b[0m\n",
"\u001b[32m2023-10-15 17:26:06.436\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m18\u001b[0m - \u001b[1mchanging truncation_side from right to left\u001b[0m\n"
]
}
],
"source": [
"model, tokenizer = load_model(cfg.model)\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "91397fa193d244de85d0b1cefbe96976",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating train split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Extracting 11 variants of each prompt\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c2293e5733f94538835fb1bf52c82d52",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenize: 0%| | 0/32 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "47190779321c41f3857a5b270f2d1c2c",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"truncated: 0%| | 0/32 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2bc743f5d3e4455086c003c3e8537cda",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"prompt_truncated: 0%| | 0/32 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7b14eb6d6e134eb49d8df9e79ec9aee5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"choice_ids: 0%| | 0/32 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a8be2f155ad244438b9a319b51421a99",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Filter: 0%| | 0/32 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"removed truncated rows to leave: num_rows 10\n"
]
}
],
"source": [
"ds_name = cfg.datasets[0]\n",
"split_type = \"train\"\n",
"ds_tokens = load_preproc_dataset(ds_name, cfg, tokenizer)\n",
"\n",
"ds_tokens_calibration = ds_tokens.select(range(10))\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"# b = next(iter(ds_tokens))\n",
"# b\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"# get the calibration dataset\n",
"BATCH_SIZE = 2\n",
"f = None\n",
"info_kwargs = dict(extract_cfg=cfg.to_dict(), ds_name=ds_name, split_type=split_type, f=f, date=pd.Timestamp.now().isoformat(),)\n",
"intervention_dicts = [None, ]\n",
"gen_kwargs = dict(\n",
" model=model,\n",
" tokenizer=tokenizer,\n",
" data=ds_tokens,\n",
" batch_size=BATCH_SIZE,\n",
" layer_padding=cfg.layer_padding,\n",
" layer_stride=cfg.layer_stride,\n",
" intervention_dicts=intervention_dicts,\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "97d9267e2ff1497488f7951017fe54b0",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating train split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9973a2a52a7a43a38c84ea2b1e4ec2fa",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"get hidden states: 0%| | 0/5 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"ename": "DatasetGenerationError",
"evalue": "An error occurred while generating the dataset",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mArrowTypeError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/builder.py:1703\u001b[0m, in \u001b[0;36mGeneratorBasedBuilder._prepare_split_single\u001b[0;34m(self, gen_kwargs, fpath, file_format, max_shard_size, split_info, check_duplicate_keys, job_id)\u001b[0m\n\u001b[1;32m 1702\u001b[0m num_shards \u001b[39m=\u001b[39m shard_id \u001b[39m+\u001b[39m \u001b[39m1\u001b[39m\n\u001b[0;32m-> 1703\u001b[0m num_examples, num_bytes \u001b[39m=\u001b[39m writer\u001b[39m.\u001b[39;49mfinalize()\n\u001b[1;32m 1704\u001b[0m writer\u001b[39m.\u001b[39mclose()\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/arrow_writer.py:586\u001b[0m, in \u001b[0;36mArrowWriter.finalize\u001b[0;34m(self, close_stream)\u001b[0m\n\u001b[1;32m 585\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhkey_record \u001b[39m=\u001b[39m []\n\u001b[0;32m--> 586\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mwrite_examples_on_file()\n\u001b[1;32m 587\u001b[0m \u001b[39m# If schema is known, infer features even if no examples were written\u001b[39;00m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/arrow_writer.py:448\u001b[0m, in \u001b[0;36mArrowWriter.write_examples_on_file\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 444\u001b[0m batch_examples[col] \u001b[39m=\u001b[39m [\n\u001b[1;32m 445\u001b[0m row[\u001b[39m0\u001b[39m][col]\u001b[39m.\u001b[39mto_pylist()[\u001b[39m0\u001b[39m] \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(row[\u001b[39m0\u001b[39m][col], (pa\u001b[39m.\u001b[39mArray, pa\u001b[39m.\u001b[39mChunkedArray)) \u001b[39melse\u001b[39;00m row[\u001b[39m0\u001b[39m][col]\n\u001b[1;32m 446\u001b[0m \u001b[39mfor\u001b[39;00m row \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcurrent_examples\n\u001b[1;32m 447\u001b[0m ]\n\u001b[0;32m--> 448\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mwrite_batch(batch_examples\u001b[39m=\u001b[39;49mbatch_examples)\n\u001b[1;32m 449\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcurrent_examples \u001b[39m=\u001b[39m []\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/arrow_writer.py:555\u001b[0m, in \u001b[0;36mArrowWriter.write_batch\u001b[0;34m(self, batch_examples, writer_batch_size)\u001b[0m\n\u001b[1;32m 554\u001b[0m typed_sequence \u001b[39m=\u001b[39m OptimizedTypedSequence(col_values, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39mcol_type, try_type\u001b[39m=\u001b[39mcol_try_type, col\u001b[39m=\u001b[39mcol)\n\u001b[0;32m--> 555\u001b[0m arrays\u001b[39m.\u001b[39mappend(pa\u001b[39m.\u001b[39;49marray(typed_sequence))\n\u001b[1;32m 556\u001b[0m inferred_features[col] \u001b[39m=\u001b[39m typed_sequence\u001b[39m.\u001b[39mget_inferred_type()\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/pyarrow/array.pxi:243\u001b[0m, in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/pyarrow/array.pxi:110\u001b[0m, in \u001b[0;36mpyarrow.lib._handle_arrow_array_protocol\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/arrow_writer.py:189\u001b[0m, in \u001b[0;36mTypedSequence.__arrow_array__\u001b[0;34m(self, type)\u001b[0m\n\u001b[1;32m 188\u001b[0m trying_cast_to_python_objects \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m--> 189\u001b[0m out \u001b[39m=\u001b[39m pa\u001b[39m.\u001b[39;49marray(cast_to_python_objects(data, only_1d_for_numpy\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m))\n\u001b[1;32m 190\u001b[0m \u001b[39m# use smaller integer precisions if possible\u001b[39;00m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/pyarrow/array.pxi:327\u001b[0m, in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/pyarrow/array.pxi:39\u001b[0m, in \u001b[0;36mpyarrow.lib._sequence_to_array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/pyarrow/error.pxi:144\u001b[0m, in \u001b[0;36mpyarrow.lib.pyarrow_internal_check_status\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/pyarrow/error.pxi:123\u001b[0m, in \u001b[0;36mpyarrow.lib.check_status\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mArrowTypeError\u001b[0m: Expected bytes, got a 'list' object",
"\nThe above exception was the direct cause of the following exception:\n",
"\u001b[0;31mDatasetGenerationError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb Cell 9\u001b[0m line \u001b[0;36m1\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m ds1 \u001b[39m=\u001b[39m Dataset\u001b[39m.\u001b[39;49mfrom_generator(\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=1'>2</a>\u001b[0m generator\u001b[39m=\u001b[39;49mbatch_hidden_states,\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m info\u001b[39m=\u001b[39;49mDatasetInfo(\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m description\u001b[39m=\u001b[39;49mjson\u001b[39m.\u001b[39;49mdumps(info_kwargs, indent\u001b[39m=\u001b[39;49m\u001b[39m2\u001b[39;49m),\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=4'>5</a>\u001b[0m config_name\u001b[39m=\u001b[39;49mf,\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=5'>6</a>\u001b[0m ),\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=6'>7</a>\u001b[0m gen_kwargs\u001b[39m=\u001b[39;49mgen_kwargs,\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=7'>8</a>\u001b[0m num_proc\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m,\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=8'>9</a>\u001b[0m )\n\u001b[1;32m <a href='vscode-notebook-cell://ssh-remote%2Bdeep1-local/home/ubuntu/Documents/mjc/elk/discovering_latent_knowledge2/notebooks/012b_scratch_dataset.ipynb#X13sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m ds1\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/arrow_dataset.py:1072\u001b[0m, in \u001b[0;36mDataset.from_generator\u001b[0;34m(generator, features, cache_dir, keep_in_memory, gen_kwargs, num_proc, **kwargs)\u001b[0m\n\u001b[1;32m 1016\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Create a Dataset from a generator.\u001b[39;00m\n\u001b[1;32m 1017\u001b[0m \n\u001b[1;32m 1018\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1060\u001b[0m \u001b[39m```\u001b[39;00m\n\u001b[1;32m 1061\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1062\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39m.\u001b[39;00m\u001b[39mio\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mgenerator\u001b[39;00m \u001b[39mimport\u001b[39;00m GeneratorDatasetInputStream\n\u001b[1;32m 1064\u001b[0m \u001b[39mreturn\u001b[39;00m GeneratorDatasetInputStream(\n\u001b[1;32m 1065\u001b[0m generator\u001b[39m=\u001b[39;49mgenerator,\n\u001b[1;32m 1066\u001b[0m features\u001b[39m=\u001b[39;49mfeatures,\n\u001b[1;32m 1067\u001b[0m cache_dir\u001b[39m=\u001b[39;49mcache_dir,\n\u001b[1;32m 1068\u001b[0m keep_in_memory\u001b[39m=\u001b[39;49mkeep_in_memory,\n\u001b[1;32m 1069\u001b[0m gen_kwargs\u001b[39m=\u001b[39;49mgen_kwargs,\n\u001b[1;32m 1070\u001b[0m num_proc\u001b[39m=\u001b[39;49mnum_proc,\n\u001b[1;32m 1071\u001b[0m \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs,\n\u001b[0;32m-> 1072\u001b[0m )\u001b[39m.\u001b[39;49mread()\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/io/generator.py:47\u001b[0m, in \u001b[0;36mGeneratorDatasetInputStream.read\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 44\u001b[0m verification_mode \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 45\u001b[0m base_path \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m---> 47\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbuilder\u001b[39m.\u001b[39;49mdownload_and_prepare(\n\u001b[1;32m 48\u001b[0m download_config\u001b[39m=\u001b[39;49mdownload_config,\n\u001b[1;32m 49\u001b[0m download_mode\u001b[39m=\u001b[39;49mdownload_mode,\n\u001b[1;32m 50\u001b[0m verification_mode\u001b[39m=\u001b[39;49mverification_mode,\n\u001b[1;32m 51\u001b[0m \u001b[39m# try_from_hf_gcs=try_from_hf_gcs,\u001b[39;49;00m\n\u001b[1;32m 52\u001b[0m base_path\u001b[39m=\u001b[39;49mbase_path,\n\u001b[1;32m 53\u001b[0m num_proc\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mnum_proc,\n\u001b[1;32m 54\u001b[0m )\n\u001b[1;32m 55\u001b[0m dataset \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuilder\u001b[39m.\u001b[39mas_dataset(\n\u001b[1;32m 56\u001b[0m split\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mtrain\u001b[39m\u001b[39m\"\u001b[39m, verification_mode\u001b[39m=\u001b[39mverification_mode, in_memory\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mkeep_in_memory\n\u001b[1;32m 57\u001b[0m )\n\u001b[1;32m 58\u001b[0m \u001b[39mreturn\u001b[39;00m dataset\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/builder.py:954\u001b[0m, in \u001b[0;36mDatasetBuilder.download_and_prepare\u001b[0;34m(self, output_dir, download_config, download_mode, verification_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, file_format, max_shard_size, num_proc, storage_options, **download_and_prepare_kwargs)\u001b[0m\n\u001b[1;32m 952\u001b[0m \u001b[39mif\u001b[39;00m num_proc \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 953\u001b[0m prepare_split_kwargs[\u001b[39m\"\u001b[39m\u001b[39mnum_proc\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m num_proc\n\u001b[0;32m--> 954\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_download_and_prepare(\n\u001b[1;32m 955\u001b[0m dl_manager\u001b[39m=\u001b[39;49mdl_manager,\n\u001b[1;32m 956\u001b[0m verification_mode\u001b[39m=\u001b[39;49mverification_mode,\n\u001b[1;32m 957\u001b[0m \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mprepare_split_kwargs,\n\u001b[1;32m 958\u001b[0m \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mdownload_and_prepare_kwargs,\n\u001b[1;32m 959\u001b[0m )\n\u001b[1;32m 960\u001b[0m \u001b[39m# Sync info\u001b[39;00m\n\u001b[1;32m 961\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minfo\u001b[39m.\u001b[39mdataset_size \u001b[39m=\u001b[39m \u001b[39msum\u001b[39m(split\u001b[39m.\u001b[39mnum_bytes \u001b[39mfor\u001b[39;00m split \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minfo\u001b[39m.\u001b[39msplits\u001b[39m.\u001b[39mvalues())\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/builder.py:1717\u001b[0m, in \u001b[0;36mGeneratorBasedBuilder._download_and_prepare\u001b[0;34m(self, dl_manager, verification_mode, **prepare_splits_kwargs)\u001b[0m\n\u001b[1;32m 1716\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_download_and_prepare\u001b[39m(\u001b[39mself\u001b[39m, dl_manager, verification_mode, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mprepare_splits_kwargs):\n\u001b[0;32m-> 1717\u001b[0m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49m_download_and_prepare(\n\u001b[1;32m 1718\u001b[0m dl_manager,\n\u001b[1;32m 1719\u001b[0m verification_mode,\n\u001b[1;32m 1720\u001b[0m check_duplicate_keys\u001b[39m=\u001b[39;49mverification_mode \u001b[39m==\u001b[39;49m VerificationMode\u001b[39m.\u001b[39;49mBASIC_CHECKS\n\u001b[1;32m 1721\u001b[0m \u001b[39mor\u001b[39;49;00m verification_mode \u001b[39m==\u001b[39;49m VerificationMode\u001b[39m.\u001b[39;49mALL_CHECKS,\n\u001b[1;32m 1722\u001b[0m \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mprepare_splits_kwargs,\n\u001b[1;32m 1723\u001b[0m )\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/builder.py:1049\u001b[0m, in \u001b[0;36mDatasetBuilder._download_and_prepare\u001b[0;34m(self, dl_manager, verification_mode, **prepare_split_kwargs)\u001b[0m\n\u001b[1;32m 1045\u001b[0m split_dict\u001b[39m.\u001b[39madd(split_generator\u001b[39m.\u001b[39msplit_info)\n\u001b[1;32m 1047\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1048\u001b[0m \u001b[39m# Prepare split will record examples associated to the split\u001b[39;00m\n\u001b[0;32m-> 1049\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_prepare_split(split_generator, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mprepare_split_kwargs)\n\u001b[1;32m 1050\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mOSError\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 1051\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mOSError\u001b[39;00m(\n\u001b[1;32m 1052\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mCannot find data file. \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1053\u001b[0m \u001b[39m+\u001b[39m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmanual_download_instructions \u001b[39mor\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 1054\u001b[0m \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mOriginal error:\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1055\u001b[0m \u001b[39m+\u001b[39m \u001b[39mstr\u001b[39m(e)\n\u001b[1;32m 1056\u001b[0m ) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/builder.py:1555\u001b[0m, in \u001b[0;36mGeneratorBasedBuilder._prepare_split\u001b[0;34m(self, split_generator, check_duplicate_keys, file_format, num_proc, max_shard_size)\u001b[0m\n\u001b[1;32m 1553\u001b[0m job_id \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m\n\u001b[1;32m 1554\u001b[0m \u001b[39mwith\u001b[39;00m pbar:\n\u001b[0;32m-> 1555\u001b[0m \u001b[39mfor\u001b[39;00m job_id, done, content \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare_split_single(\n\u001b[1;32m 1556\u001b[0m gen_kwargs\u001b[39m=\u001b[39mgen_kwargs, job_id\u001b[39m=\u001b[39mjob_id, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m_prepare_split_args\n\u001b[1;32m 1557\u001b[0m ):\n\u001b[1;32m 1558\u001b[0m \u001b[39mif\u001b[39;00m done:\n\u001b[1;32m 1559\u001b[0m result \u001b[39m=\u001b[39m content\n",
"File \u001b[0;32m~/mambaforge/envs/dlk4/lib/python3.11/site-packages/datasets/builder.py:1712\u001b[0m, in \u001b[0;36mGeneratorBasedBuilder._prepare_split_single\u001b[0;34m(self, gen_kwargs, fpath, file_format, max_shard_size, split_info, check_duplicate_keys, job_id)\u001b[0m\n\u001b[1;32m 1710\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(e, SchemaInferenceError) \u001b[39mand\u001b[39;00m e\u001b[39m.\u001b[39m__context__ \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 1711\u001b[0m e \u001b[39m=\u001b[39m e\u001b[39m.\u001b[39m__context__\n\u001b[0;32m-> 1712\u001b[0m \u001b[39mraise\u001b[39;00m DatasetGenerationError(\u001b[39m\"\u001b[39m\u001b[39mAn error occurred while generating the dataset\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39mfrom\u001b[39;00m \u001b[39me\u001b[39;00m\n\u001b[1;32m 1714\u001b[0m \u001b[39myield\u001b[39;00m job_id, \u001b[39mTrue\u001b[39;00m, (total_num_examples, total_num_bytes, writer\u001b[39m.\u001b[39m_features, num_shards, shard_lengths)\n",
"\u001b[0;31mDatasetGenerationError\u001b[0m: An error occurred while generating the dataset"
]
}
],
"source": [
"ds1 = Dataset.from_generator(\n",
" generator=batch_hidden_states,\n",
" info=DatasetInfo(\n",
" description=json.dumps(info_kwargs, indent=2),\n",
" config_name=f,\n",
" ),\n",
" gen_kwargs=gen_kwargs,\n",
" num_proc=1,\n",
")\n",
"ds1\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['model.layers.4.self_attn',\n",
" 'model.layers.8.self_attn',\n",
" 'model.layers.4.self_attn',\n",
" 'model.layers.8.self_attn',\n",
" 'model.layers.4.self_attn',\n",
" 'model.layers.8.self_attn',\n",
" 'model.layers.4.self_attn',\n",
" 'model.layers.8.self_attn',\n",
" 'model.layers.4.self_attn',\n",
" 'model.layers.8.self_attn']"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds1.info.description\n",
"ds1['layer_names']\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Scratch\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"ExtractHiddenStates(model=LlamaForCausalLM(\n",
" (model): LlamaModel(\n",
" (embed_tokens): Embedding(32001, 5120, padding_idx=0)\n",
" (layers): ModuleList(\n",
" (0-39): 40 x LlamaDecoderLayer(\n",
" (self_attn): LlamaAttention(\n",
" (rotary_emb): LlamaRotaryEmbedding()\n",
" (k_proj): QuantLinear()\n",
" (o_proj): QuantLinear()\n",
" (q_proj): QuantLinear()\n",
" (v_proj): QuantLinear()\n",
" )\n",
" (mlp): LlamaMLP(\n",
" (act_fn): SiLUActivation()\n",
" (down_proj): QuantLinear()\n",
" (gate_proj): QuantLinear()\n",
" (up_proj): QuantLinear()\n",
" )\n",
" (input_layernorm): LlamaRMSNorm()\n",
" (post_attention_layernorm): LlamaRMSNorm()\n",
" )\n",
" )\n",
" (norm): LlamaRMSNorm()\n",
" )\n",
" (lm_head): Linear(in_features=5120, out_features=32001, bias=False)\n",
"), tokenizer=LlamaTokenizerFast(name_or_path='TheBloke/WizardCoder-Python-13B-V1.0-GPTQ', vocab_size=32000, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='left', truncation_side='left', special_tokens={'bos_token': '</s>', 'eos_token': '</s>', 'unk_token': '</s>', 'pad_token': '<unk>'}, clean_up_tokenization_spaces=False), intervention_dicts=[None], layer_stride=4, layer_padding=4)"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from src.datasets.hs import ExtractHiddenStates\n",
"ehs = ExtractHiddenStates(model, tokenizer, intervention_dicts=intervention_dicts, layer_stride=cfg.layer_stride, layer_padding=cfg.layer_padding)\n",
"ehs\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "439e450b8f224791a879a3adc7696d4f",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"get hidden states: 0%| | 0/5 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from torch.utils.data import DataLoader\n",
"batch_size = BATCH_SIZE\n",
"data = ds_tokens_calibration\n",
"from src.helpers.ds import ds_keep_cols, clear_mem\n",
"\n",
"# get a batch\n",
"torch_cols = ['input_ids', 'attention_mask', 'choice_ids']\n",
"ds_t_subset = ds_keep_cols(data, torch_cols)\n",
"ds_t_subset.set_format(type='torch')\n",
"\n",
"ds_p_subset = data.remove_columns(torch_cols)\n",
"dl = DataLoader(ds_t_subset, batch_size=batch_size, shuffle=False)\n",
"for i, batch in enumerate(tqdm(dl, desc='get hidden states')):\n",
" input_ids, attention_mask, choice_ids = batch[\"input_ids\"], batch[\"attention_mask\"], batch[\"choice_ids\"]\n",
" \n",
"# nn = len(input_ids)\n",
"# index = i*batch_size+np.arange(nn)\n",
" \n",
"# # different due to dropout\n",
"# hsl = ehs.get_batch_of_hidden_states(input_ids=input_ids, attention_mask=attention_mask, choice_ids=choice_ids)\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"# from src.datasets.intervene import get_com_directions, get_interventions_dict\n",
"# head_wise_activations = np.array(ds1['head_activation'])\n",
"# labels=np.array(ds1[\"label_true\"])\n",
"# get_com_directions(2, \n",
"# 2, \n",
"# head_wise_activations, \n",
"# labels\n",
"# )\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"from einops import rearrange, reduce, repeat, asnumpy, parse_shape\n",
"from src.datasets.intervene import InterventionDict\n",
"from typing import Tuple\n",
"from functools import partial\n",
"from baukit.nethook import Trace, TraceDict, recursive_copy\n",
"from src.datasets.intervene import intervention_meta_fn, get_interventions_dict\n",
"\n",
"activations = np.array(ds1['head_activation']).squeeze(-1)\n",
"labels = np.array(ds1[\"label_true\"]).astype(int)==1\n",
"num_heads = model.config.num_attention_heads\n",
"\n",
"layer_names = [f\"model.layers.{i}.self_attn\" for i in range(model.config.num_hidden_layers)]\n",
"layer_names, layer_inds = ehs.get_layer_selection(layer_names)\n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'{\\n \"extract_cfg\": {\\n \"datasets\": [\\n \"amazon_polarity\",\\n \"super_glue:boolq\",\\n \"glue:qnli\",\\n \"imdb\"\\n ],\\n \"model\": \"TheBloke/WizardCoder-Python-13B-V1.0-GPTQ\",\\n \"data_dirs\": [],\\n \"max_examples\": [\\n 10,\\n 10\\n ],\\n \"num_shots\": 1,\\n \"num_variants\": -1,\\n \"layers\": [],\\n \"seed\": 42,\\n \"token_loc\": \"last\",\\n \"template_path\": null,\\n \"max_length\": 999\\n },\\n \"ds_name\": \"amazon_polarity\",\\n \"split_type\": \"train\",\\n \"f\": null,\\n \"date\": \"2023-10-15T17:26:26.289433\"\\n}'"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds1.info.description\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[-3.4297, 12.6875, 0.4692, ..., -2.2012, -1.2109, -2.1602],\n",
" [-3.4297, 12.6875, 0.4695, ..., -2.2012, -1.2129, -2.1621],\n",
" [-3.4336, 12.6875, 0.4702, ..., -2.2051, -1.2129, -2.1621],\n",
" ...,\n",
" [-2.4434, 1.2295, 11.8359, ..., -2.0957, 0.1870, 0.2942],\n",
" [-4.2383, -4.3594, 12.7656, ..., -2.8594, -1.1748, -0.6260],\n",
" [-6.4336, -6.3516, 11.0547, ..., -5.6250, -2.6367, -2.8652]],\n",
"\n",
" [[-3.4062, 12.5859, 0.4016, ..., -2.1797, -1.2871, -2.1543],\n",
" [-3.4082, 12.5625, 0.3992, ..., -2.1777, -1.2900, -2.1543],\n",
" [-3.4062, 12.5703, 0.3999, ..., -2.1777, -1.2910, -2.1543],\n",
" ...,\n",
" [-2.4629, 0.2039, 12.3828, ..., -2.2266, 0.8071, 0.2996],\n",
" [-6.0508, -6.7305, 10.3047, ..., -4.7031, -2.3340, -2.0586],\n",
" [-7.2188, -8.3750, 9.3047, ..., -5.4141, -2.8828, -3.3223]]],\n",
" device='cuda:0')"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"\n",
"interventions = get_interventions_dict(activations, labels, layer_names, num_heads)\n",
"intervention_fn = partial(intervention_meta_fn, interventions=interventions, num_heads=num_heads)\n",
"model.cuda().eval()\n",
"\n",
"device = model.device\n",
"with torch.no_grad():\n",
" with TraceDict(model, layer_names, edit_output=intervention_fn) as ret:\n",
" outputs = model(input_ids=input_ids.to(device), attention_mask=attention_mask.to(device), return_dict=True, output_hidden_states=True)\n",
" a = outputs[0]\n",
"a\n"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[-3.4277, 12.6953, 0.4692, ..., -2.2012, -1.2100, -2.1602],\n",
" [-3.4297, 12.6875, 0.4695, ..., -2.2012, -1.2119, -2.1602],\n",
" [-3.4297, 12.6953, 0.4705, ..., -2.2031, -1.2119, -2.1602],\n",
" ...,\n",
" [-2.4434, 1.2295, 11.8359, ..., -2.0957, 0.1870, 0.2942],\n",
" [-4.2383, -4.3594, 12.7656, ..., -2.8594, -1.1748, -0.6260],\n",
" [-5.4531, -4.9375, 10.1641, ..., -4.8320, -2.1133, -2.2773]],\n",
"\n",
" [[-3.4043, 12.5703, 0.4009, ..., -2.1777, -1.2881, -2.1523],\n",
" [-3.4043, 12.5781, 0.4006, ..., -2.1758, -1.2881, -2.1523],\n",
" [-3.4043, 12.5703, 0.4006, ..., -2.1777, -1.2900, -2.1543],\n",
" ...,\n",
" [-2.4629, 0.2039, 12.3828, ..., -2.2266, 0.8071, 0.2996],\n",
" [-6.0508, -6.7305, 10.3047, ..., -4.7031, -2.3340, -2.0586],\n",
" [-6.3906, -7.0547, 8.7266, ..., -4.8984, -2.3691, -2.8887]]],\n",
" device='cuda:0')"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"with torch.no_grad():\n",
" with TraceDict(model, layer_names) as ret:\n",
" outputs = model(input_ids=input_ids.to(device), attention_mask=attention_mask.to(device), return_dict=True, output_hidden_states=True)\n",
" a = outputs[0]\n",
"a\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"tensor([[[-3.4277, 12.6953, 0.4695, ..., -2.2012, -1.2090, -2.1602],\n",
" [-3.4277, 12.6875, 0.4697, ..., -2.1992, -1.2119, -2.1602],\n",
" [-3.4316, 12.6875, 0.4705, ..., -2.2051, -1.2119, -2.1621],\n",
" ...,\n",
" [-2.4434, 1.2295, 11.8359, ..., -2.0957, 0.1870, 0.2942],\n",
" [-4.2383, -4.3594, 12.7656, ..., -2.8594, -1.1748, -0.6260],\n",
" [-4.9805, -4.4336, 8.6094, ..., -4.4102, -1.9639, -2.0254]],\n",
"\n",
" [[-3.4062, 12.5859, 0.4014, ..., -2.1797, -1.2881, -2.1543],\n",
" [-3.4082, 12.5859, 0.4011, ..., -2.1797, -1.2881, -2.1562],\n",
" [-3.4062, 12.5703, 0.3999, ..., -2.1777, -1.2910, -2.1543],\n",
" ...,\n",
" [-2.4629, 0.2039, 12.3828, ..., -2.2266, 0.8071, 0.2996],\n",
" [-6.0508, -6.7305, 10.3047, ..., -4.7031, -2.3340, -2.0586],\n",
" [-5.8555, -6.3125, 7.6172, ..., -4.5625, -2.0273, -2.6055]]],\n",
" device='cuda:0')"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"intervention_fn2 = partial(intervention_meta_fn, interventions=interventions, num_heads=num_heads, alpha=-15)\n",
"model.cuda().eval()\n",
"\n",
"device = model.device\n",
"with torch.no_grad():\n",
" with TraceDict(model, layer_names, edit_output=intervention_fn2) as ret:\n",
" outputs = model(input_ids=input_ids.to(device), attention_mask=attention_mask.to(device), return_dict=True, output_hidden_states=True)\n",
" a = outputs[0]\n",
"a\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "dlk4",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
notebooks/104_debug_ds.ipynb
-6604
View File
File diff suppressed because one or more lines are too long
notebooks/make_dataset2.py
+5 -6
View File
@@ -52,6 +52,7 @@ import datasets
from src.config import root_folder
from pathvalidate import sanitize_filename
from src.helpers.ds import ds_keep_cols
from src.datasets.intervene import create_cache_interventions
# from sklearn.linear_model import LogisticRegression
# from sklearn.metrics import f1_score, roc_auc_score, accuracy_score
@@ -67,8 +68,6 @@ parser = ArgumentParser(add_help=False)
parser.add_arguments(ExtractConfig, dest="run")
args = parser.parse_args()
cfg = args.run
# cfg = ExtractConfig(max_examples=(200, 200), model=model_name_or_path, max_length=666)
print(cfg)
model, tokenizer = load_model(cfg.model)
@@ -87,7 +86,7 @@ tokenizer_args=dict(padding="max_length", max_length=cfg.max_length, truncation=
# %%
def load_rep_reader(model, tokenizer, cfg, N_fit_examples=20, batch_size=2, rep_token = -1, n_difference = 1, direction_method = 'pca'):
def create_cache_interventions(model, tokenizer, cfg, N_fit_examples=20, batch_size=2, rep_token = -1, n_difference = 1, direction_method = 'pca'):
"""
We want one set of interventions per model
@@ -100,7 +99,7 @@ def load_rep_reader(model, tokenizer, cfg, N_fit_examples=20, batch_size=2, rep_
hidden_layers = list(range(cfg.layer_padding, model.config.num_hidden_layers, cfg.layer_stride))
dataset_fit = load_preproc_dataset('imdb', tokenizer, N=N_fit_examples, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length)
dataset_fit = load_preproc_dataset('imdb', tokenizer, N=N_fit_examples, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length, prompt_format=cfg.prompt_format)
rep_reading_pipeline = pipeline("rep-reading", model=model, tokenizer=tokenizer)
honesty_rep_reader = rep_reading_pipeline.get_directions(
@@ -131,7 +130,7 @@ def load_rep_reader(model, tokenizer, cfg, N_fit_examples=20, batch_size=2, rep_
N_fit_examples = 30
rep_token = -1
honesty_rep_reader = load_rep_reader(model, tokenizer, cfg, N_fit_examples=N_fit_examples, batch_size=batch_size, rep_token=rep_token)
honesty_rep_reader = create_cache_interventions(model, tokenizer, cfg, N_fit_examples=N_fit_examples, batch_size=batch_size, rep_token=rep_token)
hidden_layers = sorted(honesty_rep_reader.directions.keys())
hidden_layers
@@ -287,7 +286,7 @@ for ds_name in cfg.datasets:
# load dataset
N=sum(cfg.max_examples)
ds_tokens = load_preproc_dataset(ds_name, tokenizer, N=N, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length)
ds_tokens = load_preproc_dataset(ds_name, tokenizer, N=N, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length, prompt_format=cfg.prompt_format)
N_train_split = (len(ds_tokens) - N_fit_examples) //2
src/datasets/intervene.py
+121 -45
View File
@@ -7,64 +7,140 @@ import numpy as np
from typing import List, Tuple, Dict, Any, Union, NewType
from einops import rearrange, reduce, repeat, asnumpy, parse_shape
import torch
import pickle
from src.config import root_folder
from src.prompts.prompt_loading import load_preproc_dataset
from transformers import AutoTokenizer, pipeline
from loguru import logger
Activations = NewType("Activations", Dict[str, torch.Tensor])
InterventionDict = NewType('InterventionDict', Dict[str, List[Tuple[np.ndarray, float]]])
def get_magnitude(activations: np.ndarray, labels: np.ndarray) -> Tuple[np.ndarray,np.ndarray]:
"""
get center of mass direction and magnitude per layer and head
refactored to from https://github.com/likenneth/honest_llama/blob/e010f82bfbeaa4326cef8493b0dd5b8b14c6da67/utils.py#L698
to use einops and vector ops instead of for loop
"""
# batch length hidden_dim
# TODO: maybe I should just get COM for last token instead?
true_mass_mean = reduce(activations[labels], 'b l d -> l d', 'mean')
false_mass_mean = reduce(activations[~labels], 'b l d -> l d', 'mean')
direction = true_mass_mean - false_mass_mean
direction = direction / np.linalg.norm(direction, axis=1, keepdims=True) # sq norm per layer
activations = reduce(activations, ' b l d -> l d', 'mean')
proj_vals = activations * direction
proj_val_std = reduce(proj_vals, 'l d -> l', np.std)
return direction, proj_val_std
def intervene(output, activation):
# TODO need attention mask
assert output.ndim == 3, f"expected output to be (batch, seq, vocab), got {output.shape}"
return output + activation.to(output.device)[None, None, :]
def get_interventions_dict(activations:np.ndarray, labels: np.ndarray, layer_names: List[str]) -> InterventionDict:
"""
Make an intervention dict that works with baukit.TraceDict's edit_output.
see https://github.com/davidbau/baukit/blob/main/baukit/nethook.py#L42C1-L45C56
"""
direction, proj_val_std = get_magnitude(activations, labels)
out = InterventionDict({l:[] for l in layer_names})
for layer_i, ln in enumerate(layer_names):
out[ln].append((direction[layer_i].squeeze(), proj_val_std[layer_i]))
return out
def intervention_meta_fn(outputs: torch.Tensor, layer_name:str, interventions: InterventionDict, alpha = 15) -> torch.Tensor:
"""see
def intervention_meta_fn2(
outputs: torch.Tensor, layer_name: str, activations: Activations
) -> torch.Tensor:
"""see
- honest_llama: https://github.com/likenneth/honest_llama/blob/e010f82bfbeaa4326cef8493b0dd5b8b14c6da67/validation/validate_2fold.py#L114
- baukit: https://github.com/davidbau/baukit/blob/main/baukit/nethook.py#L42C1-L45C56
Usage:
intervention_fn = partial(intervention_meta_fn, interventions=interventions)
with TraceDict(model, layers_to_intervene, edit_output=intervention_fn) as ret:
edit_output = partial(intervention_meta_fn2, activations=activations)
with TraceDict(model, layers_to_intervene, edit_output=edit_output) as ret:
...
"""
if type(outputs) is tuple:
# head_output
output = outputs[0]
output0 = intervene(outputs[0], activations[layer_name])
return tuple(output0, *outputs[1:])
elif type(outputs) is torch.Tensor:
output = outputs
return intervene(outputs, activations[layer_name])
else:
raise ValueError(f"outputs must be tuple or tensor, got {type(outputs)}")
# def get_magnitude(activations: np.ndarray, labels: np.ndarray) -> Tuple[np.ndarray,np.ndarray]:
# """
# get center of mass direction and magnitude per layer and head
# refactored to from https://github.com/likenneth/honest_llama/blob/e010f82bfbeaa4326cef8493b0dd5b8b14c6da67/utils.py#L698
# to use einops and vector ops instead of for loop
# """
# # batch length hidden_dim
# # TODO: maybe I should just get COM for last token instead?
# true_mass_mean = reduce(activations[labels], 'b l d -> l d', 'mean')
# false_mass_mean = reduce(activations[~labels], 'b l d -> l d', 'mean')
# direction = true_mass_mean - false_mass_mean
# direction = direction / np.linalg.norm(direction, axis=1, keepdims=True) # sq norm per layer
# activations = reduce(activations, ' b l d -> l d', 'mean')
# proj_vals = activations * direction
# proj_val_std = reduce(proj_vals, 'l d -> l', np.std)
# return direction, proj_val_std
# def get_interventions_dict(activations:np.ndarray, labels: np.ndarray, layer_names: List[str]) -> InterventionDict:
# """
# Make an intervention dict that works with baukit.TraceDict's edit_output.
# see https://github.com/davidbau/baukit/blob/main/baukit/nethook.py#L42C1-L45C56
# """
# direction, proj_val_std = get_magnitude(activations, labels)
# out = InterventionDict({l:[] for l in layer_names})
# for layer_i, ln in enumerate(layer_names):
# out[ln].append((direction[layer_i].squeeze(), proj_val_std[layer_i]))
# return out
# def intervention_meta_fn(outputs: torch.Tensor, layer_name:str, interventions: InterventionDict, alpha = 15) -> torch.Tensor:
# """see
# - honest_llama: https://github.com/likenneth/honest_llama/blob/e010f82bfbeaa4326cef8493b0dd5b8b14c6da67/validation/validate_2fold.py#L114
# - baukit: https://github.com/davidbau/baukit/blob/main/baukit/nethook.py#L42C1-L45C56
# Usage:
# intervention_fn = partial(intervention_meta_fn, interventions=interventions)
# with TraceDict(model, layers_to_intervene, edit_output=intervention_fn) as ret:
# ...
# """
# if type(outputs) is tuple:
# # head_output
# output = outputs[0]
# elif type(outputs) is torch.Tensor:
# output = outputs
# else:
# raise ValueError(f"outputs must be tuple or tensor, got {type(outputs)}")
for direction, proj_val_std in interventions[layer_name]:
# head_output: (batch_size, seq_len, layer_size)
output[:, :, :] += torch.from_numpy(alpha * proj_val_std * direction).to(output.device)[None, None, :]
if type(outputs) is tuple:
return tuple([output, *outputs[1:]])
# for direction, proj_val_std in interventions[layer_name]:
# # head_output: (batch_size, seq_len, layer_size)
# output[:, :, :] += torch.from_numpy(alpha * proj_val_std * direction).to(output.device)[None, None, :]
# if type(outputs) is tuple:
# return tuple([output, *outputs[1:]])
# else:
# return output
def create_cache_interventions(model, tokenizer, cfg, N_fit_examples=20, batch_size=2, rep_token = -1, n_difference = 1, direction_method = 'pca'):
"""
We want one set of interventions per model
So we always load a cached version if possible. to make it approx repeatable use the same dataset etc
"""
tokenizer_args=dict(padding="max_length", max_length=cfg.max_length, truncation=True, add_special_tokens=True)
model_name = cfg.model.replace('/', '-')
intervention_f = root_folder / 'data' / 'interventions' / f'{model_name}.pkl'
intervention_f.parent.mkdir(exist_ok=True, parents=True)
if not intervention_f.exists():
hidden_layers = list(range(cfg.layer_padding, model.config.num_hidden_layers, cfg.layer_stride))
dataset_fit = load_preproc_dataset('imdb', tokenizer, N=N_fit_examples, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length, prompt_format=cfg.prompt_format)
rep_reading_pipeline = pipeline("rep-reading", model=model, tokenizer=tokenizer)
honesty_rep_reader = rep_reading_pipeline.get_directions(
dataset_fit['question'],
rep_token=rep_token,
hidden_layers=hidden_layers,
n_difference=n_difference,
train_labels=dataset_fit['label_true'],
direction_method=direction_method,
batch_size=batch_size,
**tokenizer_args
)
# and save
with open(intervention_f, 'wb') as f:
pickle.dump(honesty_rep_reader, f)
logger.info(f'Saved interventions to {intervention_f}')
else:
return output
with open(intervention_f, 'rb') as f:
honesty_rep_reader = pickle.load(f)
logger.info(f'Loaded interventions from {intervention_f}')
return honesty_rep_reader
src/extraction/config.py
+5 -2
View File
@@ -10,9 +10,9 @@ class ExtractConfig(Serializable):
"""Names of HF datasets to use, e.g. `"super_glue:boolq"` or `"imdb"` `"glue:qnli"""
# model: str = "TheBloke/WizardCoder-Python-13B-V1.0-GPTQ"
# model: str = "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ"
model: str = "TheBloke/Wizard-Vicuna-13B-Uncensored-GPTQ"
# model: str = "TheBloke/Wizard-Vicuna-7B-Uncensored-GPTQ"
model: str = "TheBloke/Mistral-7B-Instruct-v0.1-GPTQ"
# model: str = "TheBloke/Mistral-7B-Instruct-v0.1-GPTQ"
# model: str = "TheBloke/Llama-2-13B-chat-GPTQ"
"""HF model string identifying the language model to extract hidden states from."""
@@ -24,6 +24,9 @@ class ExtractConfig(Serializable):
max_examples: tuple[int, int] = (100, 100)
"""Maximum number of examples to use from each split of the dataset."""
prompt_format: str = "vicuna"
"""llama, llama2, chatml, see structure.yaml file."""
num_shots: int = 1
"""Number of examples for few-shot prompts. If zero, prompts are zero-shot."""
src/prompts/prompt_loading.py
+2 -2
View File
@@ -298,7 +298,7 @@ def _convert_to_prompts(
def load_preproc_dataset(ds_name: str, tokenizer: PreTrainedTokenizerBase, N:int, split_type:str="train", seed=42, num_shots=1, max_length=999) -> Dataset:
def load_preproc_dataset(ds_name: str, tokenizer: PreTrainedTokenizerBase, N:int, prompt_format:str, split_type:str="train", seed=42, num_shots=1, max_length=999) -> Dataset:
"""load a preprocessed dataset of tokens."""
ds_prompts = Dataset.from_generator(
load_prompts,
@@ -308,7 +308,7 @@ def load_preproc_dataset(ds_name: str, tokenizer: PreTrainedTokenizerBase, N:int
split_type=split_type,
# template_path=template_path,
seed=seed,
prompt_format='llama',
prompt_format=prompt_format,
N=N*3,
),
)
src/prompts/templates/structure.yaml
+2
View File
@@ -5,3 +5,5 @@ templates:
llama: "{% if system %}{{system}}\n\n{% endif %}### Instruction\n{{user}}\n\n### Response:\n{{response}}{% if response %}\n\n{% endif %}"
llama2: "<s>{% if system %}<<SYS>>\n{{system}}\n<</SYS>>\n\n{% endif %}[INST] \n{{user}} [/INST]\n\n[ASST] {{response}}{% if response %} [/ASST]\n\n{% endif %}"
vicuna: "{% if system %}{{system}} {% endif %}USER: {{user}} ASSISTANT: {% if response %}{{response}}{% endif %}"
src/repe/rep_control_pipeline_baukit.py
+2 -28
View File
@@ -15,8 +15,9 @@ from transformers.modeling_outputs import ModelOutput
from src.datasets.scores import choice2ids, default_class2choices, scores2choice_probs2
# from src.datasets.scores import scores2choice_probs
from src.helpers.torch import clear_mem, detachcpu
from src.datasets.intervene import intervention_meta_fn2, Activations
Activations = NewType("Activations", Dict[str, torch.Tensor])
def try_half(v):
if isinstance(v, torch.Tensor):
@@ -34,33 +35,6 @@ def row_choice_ids(answer_choices, tokenizer):
return choice2ids([c for c in answer_choices], tokenizer)
def intervene(output, activation):
# TODO need attention mask
assert output.ndim == 3, f"expected output to be (batch, seq, vocab), got {output.shape}"
return output + activation.to(output.device)[None, None, :]
def intervention_meta_fn2(
outputs: torch.Tensor, layer_name: str, activations: Activations
) -> torch.Tensor:
"""see
- honest_llama: https://github.com/likenneth/honest_llama/blob/e010f82bfbeaa4326cef8493b0dd5b8b14c6da67/validation/validate_2fold.py#L114
- baukit: https://github.com/davidbau/baukit/blob/main/baukit/nethook.py#L42C1-L45C56
Usage:
edit_output = partial(intervention_meta_fn2, activations=activations)
with TraceDict(model, layers_to_intervene, edit_output=edit_output) as ret:
...
"""
if type(outputs) is tuple:
output0 = intervene(outputs[0], activations[layer_name])
return tuple(output0, *outputs[1:])
elif type(outputs) is torch.Tensor:
return intervene(outputs, activations[layer_name])
else:
raise ValueError(f"outputs must be tuple or tensor, got {type(outputs)}")
# def split_outputs(o):