wassname/discovering_latent_knowledge
1
0
Fork 0
mirror of https://github.com/wassname/discovering_latent_knowledge.git synced 2026-09-10 12:00:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

use array2d and 3d

Browse Source
This commit is contained in:
wassname
2023-10-26 17:14:17 +08:00
parent 8ab20f663e
commit e0d2eeabe1
10 changed files with 2625 additions and 894 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/027_debug_ds_feats.ipynb
+850
View File
@@ -0,0 +1,850 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# distance and direciton\n",
"\n",
"Let try to opt for distance and direction with\n",
"\n",
"$L1loss(y_1-y_0, y_{true})$\n",
"\n",
"where $y_1=model(x_1)$\n",
"\n",
"So I'm optimising for the hidden states to be the correct distance and direcioton away. It's like the margin raning loss."
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"\n",
"links:\n",
"- [loading](https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/alpaca.py)\n",
"- [dict](https://github.com/deep-diver/LLM-As-Chatbot/blob/c79e855a492a968b54bac223e66dc9db448d6eba/model_cards.json#L143)\n",
"- [prompt_format](https://github.com/deep-diver/PingPong/blob/main/src/pingpong/alpaca.py)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# import your package\n",
"%load_ext autoreload\n",
"%autoreload 2\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
},
{
"data": {
"text/plain": [
"'4.34.1'"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from matplotlib import pyplot as plt\n",
"plt.style.use('ggplot')\n",
"\n",
"from typing import Optional, List, Dict, Union\n",
"\n",
"import torch\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"from torch import Tensor\n",
"from torch import optim\n",
"from torch.utils.data import random_split, DataLoader, TensorDataset\n",
"from src.helpers.ds import shuffle_dataset_by\n",
"from pathlib import Path\n",
"\n",
"import transformers\n",
"\n",
"import lightning.pytorch as pl\n",
"# from dataclasses import dataclass\n",
"\n",
"# from sklearn.linear_model import LogisticRegression\n",
"# from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n",
"# from sklearn.preprocessing import RobustScaler\n",
"\n",
"from tqdm.auto import tqdm\n",
"import os\n",
"\n",
"from loguru import logger\n",
"logger.add(os.sys.stderr, format=\"{time} {level} {message}\", level=\"INFO\")\n",
"\n",
"\n",
"\n",
"transformers.__version__\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from src.helpers.lightning import read_metrics_csv\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Datasets\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"from datasets import load_from_disk, concatenate_datasets\n",
"from src.datasets.load import ds2df, load_ds\n",
"\n",
"feats = ['hidden_states', 'head_activation_and_grad', 'mlp_activation_and_grad', 'residual_stream', 'w_grads_attn', 'w_grads_mlp', 'hidden_states2', 'residual_stream2', ]\n",
"\n",
"fs = [\n",
" # \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_amazon_polarity_test_615\",\n",
" \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_amazon_polarity_train_555\",\n",
" # \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_glue_qnli_test_615\", \n",
" \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_glue_qnli_train_555\",\n",
" # \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_super_glue_boolq_test_615\",\n",
" \"../.ds/TheBloke_WizardCoder-Python-13B-V1.0-GPTQ_super_glue_boolq_train_555\",\n",
"]\n",
"\n",
"# dss = [load_from_disk(f) for f in fs]\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"d = load_from_disk(fs[-1])\n",
"ds = d.select(range(10))\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(10, 41, 5120, 2)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"h = ds['end_hidden_states']\n",
"np.array(h).shape\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(10, 41, 5120, 2)"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"\n",
"ds = ds.with_format(\"numpy\")\n",
"h = ds['end_hidden_states']\n",
"h.shape\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([10, 41, 5120, 2])"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds = ds.with_format(\"torch\")\n",
"h = ds['end_hidden_states']\n",
"h.shape\n"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"torch.Size([10, 41, 5120, 2])"
]
},
"execution_count": 47,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"h.diff(1)\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'end_hidden_states': Sequence(feature=Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None), length=-1, id=None),\n",
" 'end_logits': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'instructed_to_lie': Value(dtype='bool', id=None),\n",
" 'question': Value(dtype='string', id=None),\n",
" 'answer_choices': Sequence(feature=Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'choice_ids': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'template_name': Value(dtype='string', id=None),\n",
" 'sys_instr_name': Value(dtype='string', id=None),\n",
" 'example_i': Value(dtype='int64', id=None),\n",
" 'label_true': Value(dtype='int64', id=None),\n",
" 'input_truncated': Value(dtype='string', id=None),\n",
" 'truncated': Value(dtype='float64', id=None),\n",
" 'text_ans': Value(dtype='string', id=None),\n",
" 'add_ans': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'ans': Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)}"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds.features\n"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Map: 0%| | 0/10 [00:08<?, ? examples/s]\n"
]
},
{
"ename": "TypeError",
"evalue": "unsupported operand type(s) for +: 'type' and 'str'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:3493\u001b[0m, in \u001b[0;36mDataset._map_single\u001b[0;34m(shard, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)\u001b[0m\n\u001b[1;32m 3492\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 3493\u001b[0m writer\u001b[39m.\u001b[39;49mwrite_batch(batch)\n\u001b[1;32m 3494\u001b[0m num_examples_progress_update \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m num_examples_in_batch\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:542\u001b[0m, in \u001b[0;36mArrowWriter.write_batch\u001b[0;34m(self, batch_examples, writer_batch_size)\u001b[0m\n\u001b[1;32m 538\u001b[0m inferred_features \u001b[39m=\u001b[39m Features()\n\u001b[1;32m 539\u001b[0m cols \u001b[39m=\u001b[39m (\n\u001b[1;32m 540\u001b[0m [col \u001b[39mfor\u001b[39;00m col \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mschema\u001b[39m.\u001b[39mnames \u001b[39mif\u001b[39;00m col \u001b[39min\u001b[39;00m batch_examples]\n\u001b[1;32m 541\u001b[0m \u001b[39m+\u001b[39m [col \u001b[39mfor\u001b[39;00m col \u001b[39min\u001b[39;00m batch_examples\u001b[39m.\u001b[39mkeys() \u001b[39mif\u001b[39;00m col \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mschema\u001b[39m.\u001b[39mnames]\n\u001b[0;32m--> 542\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mschema\n\u001b[1;32m 543\u001b[0m \u001b[39melse\u001b[39;00m batch_examples\u001b[39m.\u001b[39mkeys()\n\u001b[1;32m 544\u001b[0m )\n\u001b[1;32m 545\u001b[0m \u001b[39mfor\u001b[39;00m col \u001b[39min\u001b[39;00m cols:\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:407\u001b[0m, in \u001b[0;36mArrowWriter.schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 402\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m 403\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mschema\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 404\u001b[0m _schema \u001b[39m=\u001b[39m (\n\u001b[1;32m 405\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema\n\u001b[1;32m 406\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 407\u001b[0m \u001b[39melse\u001b[39;00m (pa\u001b[39m.\u001b[39mschema(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_features\u001b[39m.\u001b[39;49mtype) \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_features \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m 408\u001b[0m )\n\u001b[1;32m 409\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_disable_nullable \u001b[39mand\u001b[39;00m _schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1629\u001b[0m, in \u001b[0;36mFeatures.type\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1623\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1624\u001b[0m \u001b[39mFeatures field types.\u001b[39;00m\n\u001b[1;32m 1625\u001b[0m \n\u001b[1;32m 1626\u001b[0m \u001b[39mReturns:\u001b[39;00m\n\u001b[1;32m 1627\u001b[0m \u001b[39m :obj:`pyarrow.DataType`\u001b[39;00m\n\u001b[1;32m 1628\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1629\u001b[0m \u001b[39mreturn\u001b[39;00m get_nested_type(\u001b[39mself\u001b[39;49m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1214\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1213\u001b[0m \u001b[39m# Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)\u001b[39;00m\n\u001b[0;32m-> 1214\u001b[0m \u001b[39mreturn\u001b[39;00m schema()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:527\u001b[0m, in \u001b[0;36m_ArrayXD.__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__call__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 527\u001b[0m pa_type \u001b[39m=\u001b[39m \u001b[39mglobals\u001b[39;49m()[\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__class__\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__name__\u001b[39;49m \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mExtensionType\u001b[39;49m\u001b[39m\"\u001b[39;49m](\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mshape, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdtype)\n\u001b[1;32m 528\u001b[0m \u001b[39mreturn\u001b[39;00m pa_type\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:647\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType.__init__\u001b[0;34m(self, shape, dtype)\u001b[0m\n\u001b[1;32m 646\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvalue_type \u001b[39m=\u001b[39m dtype\n\u001b[0;32m--> 647\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_generate_dtype(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvalue_type)\n\u001b[1;32m 648\u001b[0m pa\u001b[39m.\u001b[39mPyExtensionType\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:663\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType._generate_dtype\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_generate_dtype\u001b[39m(\u001b[39mself\u001b[39m, dtype):\n\u001b[0;32m--> 663\u001b[0m dtype \u001b[39m=\u001b[39m string_to_arrow(dtype)\n\u001b[1;32m 664\u001b[0m \u001b[39mfor\u001b[39;00m d \u001b[39min\u001b[39;00m \u001b[39mreversed\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mshape):\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:142\u001b[0m, in \u001b[0;36mstring_to_arrow\u001b[0;34m(datasets_dtype)\u001b[0m\n\u001b[1;32m 140\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype]()\n\u001b[0;32m--> 142\u001b[0m \u001b[39mif\u001b[39;00m (datasets_dtype \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39m_\u001b[39;49m\u001b[39m\"\u001b[39;49m) \u001b[39min\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m:\n\u001b[1;32m 143\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m_\u001b[39m\u001b[39m\"\u001b[39m]()\n",
"\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'type' and 'str'",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb Cell 17\u001b[0m line \u001b[0;36m1\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X15sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m ds[\u001b[39m'\u001b[39m\u001b[39mend_hidden_states\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m.\u001b[39mshape\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X15sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39m# ds.map(lambda x: {'end_hidden_states': x['end_hidden_states'] }, features=Array2D(ds['end_hidden_states'].shape, dtype=np.float16), batched=True, batch_size=128)\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X15sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m ds\u001b[39m.\u001b[39;49mmap(\u001b[39mlambda\u001b[39;49;00m x: {\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m: x[\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m] }, features\u001b[39m=\u001b[39;49mFeatures({\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m: Array3D(shape\u001b[39m=\u001b[39;49mds[\u001b[39m'\u001b[39;49m\u001b[39mend_hidden_states\u001b[39;49m\u001b[39m'\u001b[39;49m]\u001b[39m.\u001b[39;49mshape[\u001b[39m1\u001b[39;49m:], dtype\u001b[39m=\u001b[39;49mnp\u001b[39m.\u001b[39;49mfloat16)}), batched\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, batch_size\u001b[39m=\u001b[39;49m\u001b[39m5\u001b[39;49m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:592\u001b[0m, in \u001b[0;36mtransmit_tasks.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 590\u001b[0m \u001b[39mself\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m=\u001b[39m kwargs\u001b[39m.\u001b[39mpop(\u001b[39m\"\u001b[39m\u001b[39mself\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 591\u001b[0m \u001b[39m# apply actual function\u001b[39;00m\n\u001b[0;32m--> 592\u001b[0m out: Union[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mDatasetDict\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 593\u001b[0m datasets: List[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(out\u001b[39m.\u001b[39mvalues()) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(out, \u001b[39mdict\u001b[39m) \u001b[39melse\u001b[39;00m [out]\n\u001b[1;32m 594\u001b[0m \u001b[39mfor\u001b[39;00m dataset \u001b[39min\u001b[39;00m datasets:\n\u001b[1;32m 595\u001b[0m \u001b[39m# Remove task templates if a column mapping of the template is no longer valid\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:557\u001b[0m, in \u001b[0;36mtransmit_format.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 550\u001b[0m self_format \u001b[39m=\u001b[39m {\n\u001b[1;32m 551\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mtype\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_type,\n\u001b[1;32m 552\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mformat_kwargs\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_kwargs,\n\u001b[1;32m 553\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mcolumns\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_format_columns,\n\u001b[1;32m 554\u001b[0m \u001b[39m\"\u001b[39m\u001b[39moutput_all_columns\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_output_all_columns,\n\u001b[1;32m 555\u001b[0m }\n\u001b[1;32m 556\u001b[0m \u001b[39m# apply actual function\u001b[39;00m\n\u001b[0;32m--> 557\u001b[0m out: Union[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mDatasetDict\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m func(\u001b[39mself\u001b[39;49m, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 558\u001b[0m datasets: List[\u001b[39m\"\u001b[39m\u001b[39mDataset\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(out\u001b[39m.\u001b[39mvalues()) \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(out, \u001b[39mdict\u001b[39m) \u001b[39melse\u001b[39;00m [out]\n\u001b[1;32m 559\u001b[0m \u001b[39m# re-apply format to the output\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:3097\u001b[0m, in \u001b[0;36mDataset.map\u001b[0;34m(self, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint, desc)\u001b[0m\n\u001b[1;32m 3090\u001b[0m \u001b[39mif\u001b[39;00m transformed_dataset \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 3091\u001b[0m \u001b[39mwith\u001b[39;00m logging\u001b[39m.\u001b[39mtqdm(\n\u001b[1;32m 3092\u001b[0m disable\u001b[39m=\u001b[39m\u001b[39mnot\u001b[39;00m logging\u001b[39m.\u001b[39mis_progress_bar_enabled(),\n\u001b[1;32m 3093\u001b[0m unit\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m examples\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m 3094\u001b[0m total\u001b[39m=\u001b[39mpbar_total,\n\u001b[1;32m 3095\u001b[0m desc\u001b[39m=\u001b[39mdesc \u001b[39mor\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mMap\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m 3096\u001b[0m ) \u001b[39mas\u001b[39;00m pbar:\n\u001b[0;32m-> 3097\u001b[0m \u001b[39mfor\u001b[39;00m rank, done, content \u001b[39min\u001b[39;00m Dataset\u001b[39m.\u001b[39m_map_single(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mdataset_kwargs):\n\u001b[1;32m 3098\u001b[0m \u001b[39mif\u001b[39;00m done:\n\u001b[1;32m 3099\u001b[0m shards_done \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:3505\u001b[0m, in \u001b[0;36mDataset._map_single\u001b[0;34m(shard, function, with_indices, with_rank, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)\u001b[0m\n\u001b[1;32m 3503\u001b[0m \u001b[39mif\u001b[39;00m update_data:\n\u001b[1;32m 3504\u001b[0m \u001b[39mif\u001b[39;00m writer \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 3505\u001b[0m writer\u001b[39m.\u001b[39;49mfinalize()\n\u001b[1;32m 3506\u001b[0m \u001b[39mif\u001b[39;00m tmp_file \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 3507\u001b[0m tmp_file\u001b[39m.\u001b[39mclose()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:588\u001b[0m, in \u001b[0;36mArrowWriter.finalize\u001b[0;34m(self, close_stream)\u001b[0m\n\u001b[1;32m 586\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mwrite_examples_on_file()\n\u001b[1;32m 587\u001b[0m \u001b[39m# If schema is known, infer features even if no examples were written\u001b[39;00m\n\u001b[0;32m--> 588\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpa_writer \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mschema:\n\u001b[1;32m 589\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_build_writer(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mschema)\n\u001b[1;32m 590\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpa_writer \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:407\u001b[0m, in \u001b[0;36mArrowWriter.schema\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 402\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m 403\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mschema\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 404\u001b[0m _schema \u001b[39m=\u001b[39m (\n\u001b[1;32m 405\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema\n\u001b[1;32m 406\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 407\u001b[0m \u001b[39melse\u001b[39;00m (pa\u001b[39m.\u001b[39mschema(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_features\u001b[39m.\u001b[39;49mtype) \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_features \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m)\n\u001b[1;32m 408\u001b[0m )\n\u001b[1;32m 409\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_disable_nullable \u001b[39mand\u001b[39;00m _schema \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 410\u001b[0m _schema \u001b[39m=\u001b[39m pa\u001b[39m.\u001b[39mschema(pa\u001b[39m.\u001b[39mfield(field\u001b[39m.\u001b[39mname, field\u001b[39m.\u001b[39mtype, nullable\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m) \u001b[39mfor\u001b[39;00m field \u001b[39min\u001b[39;00m _schema)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1629\u001b[0m, in \u001b[0;36mFeatures.type\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1621\u001b[0m \u001b[39m@property\u001b[39m\n\u001b[1;32m 1622\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mtype\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 1623\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1624\u001b[0m \u001b[39m Features field types.\u001b[39;00m\n\u001b[1;32m 1625\u001b[0m \n\u001b[1;32m 1626\u001b[0m \u001b[39m Returns:\u001b[39;00m\n\u001b[1;32m 1627\u001b[0m \u001b[39m :obj:`pyarrow.DataType`\u001b[39;00m\n\u001b[1;32m 1628\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1629\u001b[0m \u001b[39mreturn\u001b[39;00m get_nested_type(\u001b[39mself\u001b[39;49m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1192\u001b[0m \u001b[39m# Nested structures: we allow dict, list/tuples, sequences\u001b[39;00m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n\u001b[1;32m 1198\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[1;32m 1199\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1200\u001b[0m ) \u001b[39m# however don't sort on struct types since the order matters\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1195\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1192\u001b[0m \u001b[39m# Nested structures: we allow dict, list/tuples, sequences\u001b[39;00m\n\u001b[1;32m 1193\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, Features):\n\u001b[1;32m 1194\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[0;32m-> 1195\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1196\u001b[0m ) \u001b[39m# Features is subclass of dict, and dict order is deterministic since Python 3.6\u001b[39;00m\n\u001b[1;32m 1197\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(schema, \u001b[39mdict\u001b[39m):\n\u001b[1;32m 1198\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mstruct(\n\u001b[1;32m 1199\u001b[0m {key: get_nested_type(schema[key]) \u001b[39mfor\u001b[39;00m key \u001b[39min\u001b[39;00m schema}\n\u001b[1;32m 1200\u001b[0m ) \u001b[39m# however don't sort on struct types since the order matters\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1214\u001b[0m, in \u001b[0;36mget_nested_type\u001b[0;34m(schema)\u001b[0m\n\u001b[1;32m 1211\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mlist_(value_type, schema\u001b[39m.\u001b[39mlength)\n\u001b[1;32m 1213\u001b[0m \u001b[39m# Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods)\u001b[39;00m\n\u001b[0;32m-> 1214\u001b[0m \u001b[39mreturn\u001b[39;00m schema()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:527\u001b[0m, in \u001b[0;36m_ArrayXD.__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 526\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__call__\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 527\u001b[0m pa_type \u001b[39m=\u001b[39m \u001b[39mglobals\u001b[39;49m()[\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__class__\u001b[39;49m\u001b[39m.\u001b[39;49m\u001b[39m__name__\u001b[39;49m \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39mExtensionType\u001b[39;49m\u001b[39m\"\u001b[39;49m](\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mshape, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdtype)\n\u001b[1;32m 528\u001b[0m \u001b[39mreturn\u001b[39;00m pa_type\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:647\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType.__init__\u001b[0;34m(self, shape, dtype)\u001b[0m\n\u001b[1;32m 645\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mshape \u001b[39m=\u001b[39m \u001b[39mtuple\u001b[39m(shape)\n\u001b[1;32m 646\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvalue_type \u001b[39m=\u001b[39m dtype\n\u001b[0;32m--> 647\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_generate_dtype(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvalue_type)\n\u001b[1;32m 648\u001b[0m pa\u001b[39m.\u001b[39mPyExtensionType\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstorage_dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:663\u001b[0m, in \u001b[0;36m_ArrayXDExtensionType._generate_dtype\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m 662\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_generate_dtype\u001b[39m(\u001b[39mself\u001b[39m, dtype):\n\u001b[0;32m--> 663\u001b[0m dtype \u001b[39m=\u001b[39m string_to_arrow(dtype)\n\u001b[1;32m 664\u001b[0m \u001b[39mfor\u001b[39;00m d \u001b[39min\u001b[39;00m \u001b[39mreversed\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mshape):\n\u001b[1;32m 665\u001b[0m dtype \u001b[39m=\u001b[39m pa\u001b[39m.\u001b[39mlist_(dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:142\u001b[0m, in \u001b[0;36mstring_to_arrow\u001b[0;34m(datasets_dtype)\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[39mif\u001b[39;00m datasets_dtype \u001b[39min\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m:\n\u001b[1;32m 140\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype]()\n\u001b[0;32m--> 142\u001b[0m \u001b[39mif\u001b[39;00m (datasets_dtype \u001b[39m+\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39m_\u001b[39;49m\u001b[39m\"\u001b[39;49m) \u001b[39min\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m:\n\u001b[1;32m 143\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39m\u001b[39m__dict__\u001b[39m[datasets_dtype \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m_\u001b[39m\u001b[39m\"\u001b[39m]()\n\u001b[1;32m 145\u001b[0m timestamp_matches \u001b[39m=\u001b[39m re\u001b[39m.\u001b[39msearch(\u001b[39mr\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m^timestamp\u001b[39m\u001b[39m\\\u001b[39m\u001b[39m[(.*)\u001b[39m\u001b[39m\\\u001b[39m\u001b[39m]$\u001b[39m\u001b[39m\"\u001b[39m, datasets_dtype)\n",
"\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'type' and 'str'"
]
}
],
"source": [
"from ctypes import Array\n",
"import numpy as np\n",
"from datasets.features import Sequence, Value, Features, Array2D, Array3D, Features\n",
"\n",
"# from datasets import batch\n",
"\n",
"ds['end_hidden_states'].shape\n",
"# ds.map(lambda x: {'end_hidden_states': x['end_hidden_states'] }, features=Array2D(ds['end_hidden_states'].shape, dtype=np.float16), batched=True, batch_size=128)\n",
"\n",
"ds.map(lambda x: {'end_hidden_states': x['end_hidden_states'] }, features=Features({'end_hidden_states': Array3D(shape=ds['end_hidden_states'].shape[1:], dtype=np.float16)}), batched=True, batch_size=5)\n"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"x = ds[:]\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'end_hidden_states': Array3D(shape=(41, 5120, 2), dtype='float16', id=None),\n",
" 'end_logits': Array2D(shape=(32001, 2), dtype='float16', id=None),\n",
" 'add_ans': Array2D(shape=(2, 2), dtype='float16', id=None),\n",
" 'label_true': Value(dtype='int64', id=None),\n",
" 'instructed_to_lie': Value(dtype='bool', id=None),\n",
" 'question': Value(dtype='string', id=None),\n",
" 'answer_choices': Sequence(feature=Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'choice_ids': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'template_name': Value(dtype='string', id=None),\n",
" 'sys_instr_name': Value(dtype='string', id=None),\n",
" 'example_i': Value(dtype='int64', id=None),\n",
" 'input_truncated': Value(dtype='string', id=None),\n",
" 'truncated': Value(dtype='float64', id=None),\n",
" 'text_ans': Value(dtype='string', id=None),\n",
" 'ans': Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)}"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from re import A\n",
"from datasets.features import Sequence, Value, Features, Array2D, Array3D, Features\n",
"\n",
"features = {\n",
" \"end_hidden_states\": Array3D(dtype=\"float16\", id=None, shape=x['end_hidden_states'].shape[1:]),\n",
" \"end_logits\": Array2D(dtype=\"float16\", id=None, shape=x['end_logits'].shape[1:]),\n",
" \"add_ans\": Array2D(dtype=\"float16\", id=None, shape=x['add_ans'].shape[1:]),\n",
" \"label_true\": Value(dtype=\"int64\", id=None),\n",
" \"instructed_to_lie\": Value(dtype=\"bool\", id=None),\n",
" \"question\": Value(dtype=\"string\", id=None),\n",
" \"answer_choices\": Sequence(\n",
" feature=Sequence(feature=Value(dtype=\"string\", id=None), length=-1, id=None),\n",
" length=-1,\n",
" id=None,\n",
" ),\n",
" \"choice_ids\": Sequence(\n",
" feature=Sequence(feature=Value(dtype=\"int64\", id=None), length=-1, id=None),\n",
" length=-1,\n",
" id=None,\n",
" ),\n",
" \"template_name\": Value(dtype=\"string\", id=None),\n",
" \"sys_instr_name\": Value(dtype=\"string\", id=None),\n",
" \"example_i\": Value(dtype=\"int64\", id=None),\n",
" \"input_truncated\": Value(dtype=\"string\", id=None),\n",
" \"truncated\": Value(dtype=\"float64\", id=None),\n",
" \"text_ans\": Value(dtype=\"string\", id=None),\n",
" \"ans\": Sequence(feature=Value(dtype=\"float16\", id=None), length=-1, id=None),\n",
"}\n",
"features = Features(features)\n",
"features\n"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"ename": "ArrowNotImplementedError",
"evalue": "Unsupported cast from float to halffloat using function cast_half_float",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mArrowNotImplementedError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb Cell 16\u001b[0m line \u001b[0;36m2\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X22sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mdatasets\u001b[39;00m \u001b[39mimport\u001b[39;00m Dataset\n\u001b[0;32m----> <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X22sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m dd \u001b[39m=\u001b[39m Dataset\u001b[39m.\u001b[39;49mfrom_dict(x, features\u001b[39m=\u001b[39;49mfeatures)\n\u001b[1;32m <a href='vscode-notebook-cell:/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/notebooks/027_debug_ds_feats.ipynb#X22sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m dd\u001b[39m.\u001b[39mfeatures\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:911\u001b[0m, in \u001b[0;36mDataset.from_dict\u001b[0;34m(cls, mapping, features, info, split)\u001b[0m\n\u001b[1;32m 909\u001b[0m arrow_typed_mapping[col] \u001b[39m=\u001b[39m data\n\u001b[1;32m 910\u001b[0m mapping \u001b[39m=\u001b[39m arrow_typed_mapping\n\u001b[0;32m--> 911\u001b[0m pa_table \u001b[39m=\u001b[39m InMemoryTable\u001b[39m.\u001b[39;49mfrom_pydict(mapping\u001b[39m=\u001b[39;49mmapping)\n\u001b[1;32m 912\u001b[0m \u001b[39mif\u001b[39;00m info \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 913\u001b[0m info \u001b[39m=\u001b[39m DatasetInfo()\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/table.py:799\u001b[0m, in \u001b[0;36mInMemoryTable.from_pydict\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m 783\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[1;32m 784\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mfrom_pydict\u001b[39m(\u001b[39mcls\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 785\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 786\u001b[0m \u001b[39m Construct a Table from Arrow arrays or columns.\u001b[39;00m\n\u001b[1;32m 787\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 797\u001b[0m \u001b[39m `datasets.table.Table`\u001b[39;00m\n\u001b[1;32m 798\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 799\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mcls\u001b[39m(pa\u001b[39m.\u001b[39;49mTable\u001b[39m.\u001b[39;49mfrom_pydict(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs))\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/table.pxi:1799\u001b[0m, in \u001b[0;36mpyarrow.lib._Tabular.from_pydict\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/table.pxi:5101\u001b[0m, in \u001b[0;36mpyarrow.lib._from_pydict\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:357\u001b[0m, in \u001b[0;36mpyarrow.lib.asarray\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:243\u001b[0m, in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:110\u001b[0m, in \u001b[0;36mpyarrow.lib._handle_arrow_array_protocol\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/arrow_writer.py:179\u001b[0m, in \u001b[0;36mTypedSequence.__arrow_array__\u001b[0;34m(self, type)\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 177\u001b[0m \u001b[39m# custom pyarrow types\u001b[39;00m\n\u001b[1;32m 178\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(pa_type, _ArrayXDExtensionType):\n\u001b[0;32m--> 179\u001b[0m storage \u001b[39m=\u001b[39m to_pyarrow_listarray(data, pa_type)\n\u001b[1;32m 180\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39mExtensionArray\u001b[39m.\u001b[39mfrom_storage(pa_type, storage)\n\u001b[1;32m 182\u001b[0m \u001b[39m# efficient np array to pyarrow array\u001b[39;00m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1465\u001b[0m, in \u001b[0;36mto_pyarrow_listarray\u001b[0;34m(data, pa_type)\u001b[0m\n\u001b[1;32m 1455\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Convert to PyArrow ListArray.\u001b[39;00m\n\u001b[1;32m 1456\u001b[0m \n\u001b[1;32m 1457\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1462\u001b[0m \u001b[39m pyarrow.Array\u001b[39;00m\n\u001b[1;32m 1463\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1464\u001b[0m \u001b[39mif\u001b[39;00m contains_any_np_array(data):\n\u001b[0;32m-> 1465\u001b[0m \u001b[39mreturn\u001b[39;00m any_np_array_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49mpa_type\u001b[39m.\u001b[39;49mvalue_type)\n\u001b[1;32m 1466\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 1467\u001b[0m \u001b[39mreturn\u001b[39;00m pa\u001b[39m.\u001b[39marray(data, pa_type\u001b[39m.\u001b[39mstorage_dtype)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1451\u001b[0m, in \u001b[0;36many_np_array_to_pyarrow_listarray\u001b[0;34m(data, type)\u001b[0m\n\u001b[1;32m 1449\u001b[0m \u001b[39mreturn\u001b[39;00m numpy_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m)\n\u001b[1;32m 1450\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, \u001b[39mlist\u001b[39m):\n\u001b[0;32m-> 1451\u001b[0m \u001b[39mreturn\u001b[39;00m list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m data])\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1451\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1449\u001b[0m \u001b[39mreturn\u001b[39;00m numpy_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m)\n\u001b[1;32m 1450\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, \u001b[39mlist\u001b[39m):\n\u001b[0;32m-> 1451\u001b[0m \u001b[39mreturn\u001b[39;00m list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49m\u001b[39mtype\u001b[39;49m) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m data])\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1449\u001b[0m, in \u001b[0;36many_np_array_to_pyarrow_listarray\u001b[0;34m(data, type)\u001b[0m\n\u001b[1;32m 1439\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray.\u001b[39;00m\n\u001b[1;32m 1440\u001b[0m \n\u001b[1;32m 1441\u001b[0m \u001b[39mArgs:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1446\u001b[0m \u001b[39m pa.ListArray\u001b[39;00m\n\u001b[1;32m 1447\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 1448\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, np\u001b[39m.\u001b[39mndarray):\n\u001b[0;32m-> 1449\u001b[0m \u001b[39mreturn\u001b[39;00m numpy_to_pyarrow_listarray(data, \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49m\u001b[39mtype\u001b[39;49m)\n\u001b[1;32m 1450\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(data, \u001b[39mlist\u001b[39m):\n\u001b[1;32m 1451\u001b[0m \u001b[39mreturn\u001b[39;00m list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, \u001b[39mtype\u001b[39m\u001b[39m=\u001b[39m\u001b[39mtype\u001b[39m) \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m data])\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/datasets/features/features.py:1389\u001b[0m, in \u001b[0;36mnumpy_to_pyarrow_listarray\u001b[0;34m(arr, type)\u001b[0m\n\u001b[1;32m 1387\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Build a PyArrow ListArray from a multidimensional NumPy array\"\"\"\u001b[39;00m\n\u001b[1;32m 1388\u001b[0m arr \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39marray(arr)\n\u001b[0;32m-> 1389\u001b[0m values \u001b[39m=\u001b[39m pa\u001b[39m.\u001b[39;49marray(arr\u001b[39m.\u001b[39;49mflatten(), \u001b[39mtype\u001b[39;49m\u001b[39m=\u001b[39;49m\u001b[39mtype\u001b[39;49m)\n\u001b[1;32m 1390\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(arr\u001b[39m.\u001b[39mndim \u001b[39m-\u001b[39m \u001b[39m1\u001b[39m):\n\u001b[1;32m 1391\u001b[0m n_offsets \u001b[39m=\u001b[39m reduce(mul, arr\u001b[39m.\u001b[39mshape[: arr\u001b[39m.\u001b[39mndim \u001b[39m-\u001b[39m i \u001b[39m-\u001b[39m \u001b[39m1\u001b[39m], \u001b[39m1\u001b[39m)\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:323\u001b[0m, in \u001b[0;36mpyarrow.lib.array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/array.pxi:83\u001b[0m, in \u001b[0;36mpyarrow.lib._ndarray_to_array\u001b[0;34m()\u001b[0m\n",
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/pyarrow/error.pxi:121\u001b[0m, in \u001b[0;36mpyarrow.lib.check_status\u001b[0;34m()\u001b[0m\n",
"\u001b[0;31mArrowNotImplementedError\u001b[0m: Unsupported cast from float to halffloat using function cast_half_float"
]
}
],
"source": [
"from datasets import Dataset\n",
"dd = Dataset.from_dict(x, features=features)\n",
"dd.features\n"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 4.2725e-04, 4.2725e-04],\n",
" [-1.6918e-03, -1.6918e-03],\n",
" [-2.5772e-02, -2.5772e-02],\n",
" ...,\n",
" [ 8.3008e-03, 8.3008e-03],\n",
" [ 6.3744e-03, 6.3744e-03],\n",
" [-4.1733e-03, -4.1733e-03]],\n",
"\n",
" [[ 2.3102e-02, 2.3102e-02],\n",
" [-2.2369e-02, -2.2369e-02],\n",
" [-6.0059e-02, -6.0059e-02],\n",
" ...,\n",
" [ 3.1067e-02, 3.1067e-02],\n",
" [-1.2634e-02, -1.2634e-02],\n",
" [ 1.4992e-02, 1.4992e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-1.6367e+00, -1.2021e+00],\n",
" [-1.6357e+00, -3.7158e-01],\n",
" [-3.5527e+00, -7.9570e+00],\n",
" ...,\n",
" [-2.7363e+00, -5.6641e+00],\n",
" [-1.1875e+00, 5.5586e+00],\n",
" [ 4.3066e-01, 4.9883e+00]],\n",
"\n",
" [[-1.4160e+00, -2.4707e+00],\n",
" [-1.6572e+00, 1.2158e+00],\n",
" [-4.6719e+00, -7.3242e+00],\n",
" ...,\n",
" [-3.1035e+00, -4.8945e+00],\n",
" [-2.4951e-01, 7.6172e+00],\n",
" [-9.1650e-01, 5.1289e+00]],\n",
"\n",
" [[-6.5552e-02, 1.9592e-01],\n",
" [ 8.9600e-02, 4.6802e-01],\n",
" [-1.0850e+00, -9.2725e-01],\n",
" ...,\n",
" [-5.3955e-01, -2.9272e-01],\n",
" [-3.4863e-01, 4.8657e-01],\n",
" [ 1.1285e-01, 9.9561e-01]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 1.1894e-02, 1.1894e-02],\n",
" [ 5.1155e-03, 5.1155e-03],\n",
" [-2.8107e-02, -2.8107e-02],\n",
" ...,\n",
" [ 1.1063e-02, 1.1063e-02],\n",
" [ 3.8147e-05, 3.8147e-05],\n",
" [-1.2680e-02, -1.2680e-02]],\n",
"\n",
" [[ 2.9510e-02, 2.9510e-02],\n",
" [-1.0315e-02, -1.0315e-02],\n",
" [-6.5613e-02, -6.5613e-02],\n",
" ...,\n",
" [ 4.1229e-02, 4.1229e-02],\n",
" [-2.4612e-02, -2.4612e-02],\n",
" [ 1.3397e-02, 1.3397e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-3.0703e+00, -3.4922e+00],\n",
" [-2.3359e+00, 4.5312e+00],\n",
" [-8.2422e-01, -3.2500e+00],\n",
" ...,\n",
" [-2.7441e+00, -6.6367e+00],\n",
" [-2.0293e+00, 5.6250e+00],\n",
" [ 1.1865e+00, -7.3535e-01]],\n",
"\n",
" [[-2.5410e+00, -4.5977e+00],\n",
" [-3.9141e+00, 5.5859e+00],\n",
" [-1.9980e+00, -2.4023e+00],\n",
" ...,\n",
" [-3.2617e+00, -6.1797e+00],\n",
" [ 4.1504e-02, 7.5117e+00],\n",
" [-1.2573e-01, 3.9038e-01]],\n",
"\n",
" [[-1.0175e-01, -1.3562e-01],\n",
" [-3.9502e-01, 8.1934e-01],\n",
" [-5.4639e-01, -2.2192e-01],\n",
" ...,\n",
" [-3.1421e-01, -3.1128e-01],\n",
" [-3.0371e-01, 3.5425e-01],\n",
" [-4.4289e-03, 4.0723e-01]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 2.4414e-04, 2.4414e-04],\n",
" [-5.5161e-03, -5.5161e-03],\n",
" [-1.8143e-02, -1.8143e-02],\n",
" ...,\n",
" [ 6.7902e-04, 6.7902e-04],\n",
" [ 3.7994e-03, 3.7994e-03],\n",
" [-6.2180e-03, -6.2180e-03]],\n",
"\n",
" [[ 1.4801e-02, 1.4801e-02],\n",
" [-1.2253e-02, -1.2253e-02],\n",
" [-5.9448e-02, -5.9448e-02],\n",
" ...,\n",
" [ 2.0233e-02, 2.0233e-02],\n",
" [-8.7891e-03, -8.7891e-03],\n",
" [ 2.7283e-02, 2.7283e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-5.8516e+00, -6.1445e+00],\n",
" [-5.4785e-01, 1.3193e+00],\n",
" [-1.8799e+00, -5.8398e+00],\n",
" ...,\n",
" [-3.2109e+00, -2.1504e+00],\n",
" [ 1.4561e+00, 3.5430e+00],\n",
" [ 2.6621e+00, -8.7402e-01]],\n",
"\n",
" [[-6.3008e+00, -8.2891e+00],\n",
" [-1.7217e+00, 3.7559e+00],\n",
" [-2.1113e+00, -3.8906e+00],\n",
" ...,\n",
" [-3.8555e+00, -1.5215e+00],\n",
" [ 3.5234e+00, 5.0508e+00],\n",
" [ 5.7764e-01, -4.9805e-01]],\n",
"\n",
" [[-6.7676e-01, -7.0459e-01],\n",
" [ 1.9971e-01, 6.5869e-01],\n",
" [-7.5244e-01, -3.4814e-01],\n",
" ...,\n",
" [-4.1382e-01, -1.5540e-01],\n",
" [ 2.3511e-01, 1.3135e-01],\n",
" [ 1.1444e-01, 2.9221e-03]]],\n",
"\n",
"\n",
" ...,\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 1.6403e-02, 1.6403e-02],\n",
" [ 2.8381e-03, 2.8381e-03],\n",
" [-3.5278e-02, -3.5278e-02],\n",
" ...,\n",
" [ 9.6741e-03, 9.6741e-03],\n",
" [-3.4065e-03, -3.4065e-03],\n",
" [-1.3123e-02, -1.3123e-02]],\n",
"\n",
" [[ 4.4037e-02, 4.4037e-02],\n",
" [-1.6968e-02, -1.6968e-02],\n",
" [-7.9590e-02, -7.9590e-02],\n",
" ...,\n",
" [ 4.1992e-02, 4.1992e-02],\n",
" [-2.2537e-02, -2.2537e-02],\n",
" [ 1.7456e-02, 1.7456e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[-1.5049e+00, -1.9023e+00],\n",
" [ 2.8638e-01, 1.7178e+00],\n",
" [ 4.5508e-01, -1.2549e+00],\n",
" ...,\n",
" [-7.6904e-01, -1.7930e+00],\n",
" [-1.7773e+00, 3.1172e+00],\n",
" [-3.8574e-01, -2.2090e+00]],\n",
"\n",
" [[-4.5898e-01, -3.2500e+00],\n",
" [-2.3950e-01, 2.9941e+00],\n",
" [-7.3340e-01, 2.3584e-01],\n",
" ...,\n",
" [-1.0996e+00, -7.5244e-01],\n",
" [-8.2080e-01, 4.1172e+00],\n",
" [-1.5391e+00, -2.1602e+00]],\n",
"\n",
" [[ 1.8152e-01, -1.8091e-01],\n",
" [ 2.8882e-01, 4.9731e-01],\n",
" [-2.7979e-01, 2.5830e-01],\n",
" ...,\n",
" [ 1.1116e-02, 4.5801e-01],\n",
" [-3.4058e-01, -4.7119e-02],\n",
" [-2.8638e-01, -4.4092e-01]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 1.0757e-03, 1.0757e-03],\n",
" [-6.1035e-04, -6.1035e-04],\n",
" [-2.9282e-02, -2.9282e-02],\n",
" ...,\n",
" [-2.4185e-03, -2.4185e-03],\n",
" [ 4.7226e-03, 4.7226e-03],\n",
" [-8.8043e-03, -8.8043e-03]],\n",
"\n",
" [[ 1.1993e-02, 1.1993e-02],\n",
" [-2.5452e-02, -2.5452e-02],\n",
" [-4.4128e-02, -4.4128e-02],\n",
" ...,\n",
" [ 1.9241e-02, 1.9241e-02],\n",
" [-1.4236e-02, -1.4236e-02],\n",
" [ 1.6785e-02, 1.6785e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[ 5.5420e-02, -2.0312e+00],\n",
" [-1.7266e+00, 4.1289e+00],\n",
" [-4.2344e+00, -4.6328e+00],\n",
" ...,\n",
" [-2.1504e+00, -5.9023e+00],\n",
" [-3.4180e+00, 5.0000e+00],\n",
" [-1.3403e-01, 3.8457e+00]],\n",
"\n",
" [[ 6.0889e-01, -3.4121e+00],\n",
" [-1.4395e+00, 6.6016e+00],\n",
" [-5.7227e+00, -3.6406e+00],\n",
" ...,\n",
" [-2.6406e+00, -5.7656e+00],\n",
" [-1.5977e+00, 7.5078e+00],\n",
" [-1.9785e+00, 5.5508e+00]],\n",
"\n",
" [[ 3.2178e-01, 9.9182e-02],\n",
" [-6.6284e-02, 1.0977e+00],\n",
" [-1.4131e+00, -6.6357e-01],\n",
" ...,\n",
" [-4.0552e-01, -4.1797e-01],\n",
" [-5.2148e-01, 3.9233e-01],\n",
" [-2.6050e-01, 1.0879e+00]]],\n",
"\n",
"\n",
" [[[-5.2977e-04, -5.2977e-04],\n",
" [-8.4114e-04, -8.4114e-04],\n",
" [-3.4213e-04, -3.4213e-04],\n",
" ...,\n",
" [ 1.7271e-03, 1.7271e-03],\n",
" [ 2.6798e-04, 2.6798e-04],\n",
" [-1.9276e-04, -1.9276e-04]],\n",
"\n",
" [[ 2.3117e-03, 2.3117e-03],\n",
" [-4.6082e-03, -4.6082e-03],\n",
" [-2.7542e-02, -2.7542e-02],\n",
" ...,\n",
" [-3.1586e-03, -3.1586e-03],\n",
" [ 6.3858e-03, 6.3858e-03],\n",
" [-1.1124e-02, -1.1124e-02]],\n",
"\n",
" [[ 4.4022e-03, 4.4022e-03],\n",
" [-3.4607e-02, -3.4607e-02],\n",
" [-4.6326e-02, -4.6326e-02],\n",
" ...,\n",
" [ 1.1734e-02, 1.1734e-02],\n",
" [-1.0056e-02, -1.0056e-02],\n",
" [ 2.0996e-02, 2.0996e-02]],\n",
"\n",
" ...,\n",
"\n",
" [[ 3.9111e-01, -9.8242e-01],\n",
" [-1.3193e+00, 3.0293e+00],\n",
" [-2.4102e+00, -2.3125e+00],\n",
" ...,\n",
" [-2.3379e+00, -2.3066e+00],\n",
" [-2.0742e+00, 2.6289e+00],\n",
" [-6.4502e-01, 1.2080e+00]],\n",
"\n",
" [[ 6.4893e-01, -2.6426e+00],\n",
" [-1.9326e+00, 5.6172e+00],\n",
" [-2.7930e+00, -8.1592e-01],\n",
" ...,\n",
" [-2.3379e+00, -1.1777e+00],\n",
" [-8.2617e-01, 4.6367e+00],\n",
" [-1.5049e+00, 2.7891e+00]],\n",
"\n",
" [[ 4.3311e-01, 2.3413e-01],\n",
" [ 8.8867e-02, 8.7939e-01],\n",
" [-8.0908e-01, -3.0441e-02],\n",
" ...,\n",
" [-1.4844e-01, 2.1375e-01],\n",
" [-3.6890e-01, 1.7654e-02],\n",
" [-2.4988e-01, 7.3633e-01]]]], dtype=float16)"
]
},
"execution_count": 44,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"x = ds[:]\n",
"\n",
"x['end_hidden_states'] = x['end_hidden_states'].numpy().astype(np.float16)\n",
"x['end_hidden_states'].dtype\n"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'end_hidden_states': Sequence(feature=Sequence(feature=Sequence(feature=Value(dtype='float16', id=None), length=-1, id=None), length=-1, id=None), length=-1, id=None),\n",
" 'end_logits': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'instructed_to_lie': Value(dtype='bool', id=None),\n",
" 'question': Value(dtype='string', id=None),\n",
" 'answer_choices': Sequence(feature=Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'choice_ids': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'template_name': Value(dtype='string', id=None),\n",
" 'sys_instr_name': Value(dtype='string', id=None),\n",
" 'example_i': Value(dtype='int64', id=None),\n",
" 'label_true': Value(dtype='int64', id=None),\n",
" 'input_truncated': Value(dtype='string', id=None),\n",
" 'truncated': Value(dtype='float32', id=None),\n",
" 'text_ans': Value(dtype='string', id=None),\n",
" 'add_ans': Sequence(feature=Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None), length=-1, id=None),\n",
" 'ans': Sequence(feature=Value(dtype='float32', id=None), length=-1, id=None)}"
]
},
"execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dd = Dataset.from_dict(x)\n",
"dd.features\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "dlk2",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
notebooks/027_train_nanda_probe_w_counterfact_70%.ipynb
+1696 -878
View File
File diff suppressed because one or more lines are too long
notebooks/make_dataset2.py
+7 -2
View File
@@ -8,7 +8,8 @@
# %load_ext autoreload
# %autoreload 2
from src.datasets.features import get_features
from src.helpers.torch import clear_mem
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
@@ -264,6 +265,7 @@ def create_hs_ds(ds_name, ds_tokens, pipeline, activations=None, f = None, batch
# this allow us to debug in a single thread
pipeline(**gen_kwargs)
dataset_features = get_features(cfg, model.config)
ds1 = datasets.Dataset.from_generator(
generator=pipeline,
info=datasets.DatasetInfo(
@@ -271,14 +273,15 @@ def create_hs_ds(ds_name, ds_tokens, pipeline, activations=None, f = None, batch
config_name=f,
),
gen_kwargs=gen_kwargs,
features=dataset_features,
num_proc=1,
# split=split_type,
)
logger.info(f"Created dataset {dataset_name} with {len(ds1)} examples at `{f}`")
ds1.save_to_disk(f)
return ds1, f
from src.helpers.torch import clear_mem
for ds_name in cfg.datasets:
@@ -288,6 +291,8 @@ for ds_name in cfg.datasets:
ds_tokens = load_preproc_dataset(ds_name, tokenizer, N=N, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length)
N_train_split = (len(ds_tokens) - N_fit_examples) //2
N_train_split = cfg.max_examples[0]
# split the dataset, it's preshuffled
dataset_fit = ds_tokens.select(range(N_fit_examples))
src/datasets/dm.py
+11 -6
View File
@@ -1,5 +1,6 @@
import torch
import torch.nn as nn
import numpy as np
import lightning as pl
import pandas as pd
from torch.utils.data import DataLoader, TensorDataset
@@ -17,13 +18,14 @@ from src.helpers import bool2switch, switch2bool
# distance = (df.ans1-df.ans0) * true_switch_sign
# return distance
to_tensor = lambda x: torch.from_numpy(x).float()
to_tensor = lambda x: x # torch.from_numpy(x).float()
to_ds = lambda hs0, hs1, y: TensorDataset(to_tensor(hs0), to_tensor(hs1), to_tensor(y))
class imdbHSDataModule(pl.LightningDataModule):
def __init__(self,
ds: Dataset,
@@ -41,25 +43,28 @@ class imdbHSDataModule(pl.LightningDataModule):
# extract data set into N-Dim tensors and 1-d dataframe
self.ds_hs = (
self.ds.select_columns(self.x_cols)
.with_format("numpy")
)
df = self.df = ds2df(self.ds)
switch = bool2switch(df['label_true']).values
# probs_c = self.ds['ans']
self.ans = self.ds['ans'] #probs_c[:, 1] / (np.sum(probs_c, 1) + 1e-5)
# df['y'] = df['label_true'].values[:, None] == (self.ans > 0.5)
# df['y'] = df['label_true'].values[:, None] == (self.ans > 0.5)
# how true the answer was. Let's just flip the confidence
self.prob_on_truth = switch[:, None] * self.ans
# take the llm prob towards the positive answer, and flip it if negative was true
# giving us the llm's assigned prob toward the true answer
self.prob_on_truth = torch.tensor(switch[:, None] * self.ans)
b = len(self.ds_hs)
hs = self.ds_hs['end_hidden_states']
# take the diff between layers. Shape batch, layers, hidden_states, inferences
hs = torch.tensor(self.ds_hs['end_hidden_states'])
hs = hs.diff(1, axis=1)
self.hs0 = hs[..., 0]
self.hs1 = hs[..., 1]
# so we are trying to predict is one hidden state is more true than the other
# or specifically the distance and direction on the truth axis
self.y = self.prob_on_truth[:, 1] - self.prob_on_truth[:, 0]
df['y'] = self.y>0
src/datasets/features.py
+38
View File
@@ -0,0 +1,38 @@
from datasets.features import Sequence, Value, Features, Array2D, Array3D, Features
from src.extraction.config import ExtractConfig
from transformers import AutoConfig
def get_features(cfg: ExtractConfig, config: AutoConfig) -> Features:
l = config.num_hidden_layers+1
h = config.hidden_size
interventions = 2
v = config.vocab_size
features = {
"end_hidden_states": Array3D(dtype="float32", id=None, shape=(l, h, interventions)),
"end_logits": Array2D(dtype="float32", id=None, shape=(v, interventions)),
"choice_probs": Array2D(dtype="float32", id=None, shape=(2, interventions)),
"label_true": Value(dtype="int64", id=None),
"instructed_to_lie": Value(dtype="bool", id=None),
"question": Value(dtype="string", id=None),
"answer_choices": Sequence(
feature=Sequence(feature=Value(dtype="string", id=None), length=-1, id=None),
length=-1,
id=None,
),
"choice_ids": Sequence(
feature=Sequence(feature=Value(dtype="int64", id=None), length=-1, id=None),
length=-1,
id=None,
),
"template_name": Value(dtype="string", id=None),
"sys_instr_name": Value(dtype="string", id=None),
"example_i": Value(dtype="int64", id=None),
"input_truncated": Value(dtype="string", id=None),
"truncated": Value(dtype="bool", id=None),
"text_ans": Value(dtype="string", id=None),
"ans": Sequence(feature=Value(dtype="float32", id=None), length=-1, id=None),
}
return Features(features)
src/extraction/config.py
+2 -2
View File
@@ -18,7 +18,7 @@ class ExtractConfig(Serializable):
# int4: bool = True
# """Whether to perform inference in mixed int8 precision with `bitsandbytes`."""
max_examples: tuple[int, int] = (600, 600)
max_examples: tuple[int, int] = (80, 80)
"""Maximum number of examples to use from each split of the dataset."""
num_shots: int = 1
@@ -47,5 +47,5 @@ class ExtractConfig(Serializable):
template_path: str | None = None
"""Path to pass into `DatasetTemplates`. By default we use the dataset name."""
max_length: int | None = 700
max_length: int | None = 666
"""Maximum length of the input sequence passed to the tokenize encoder function"""
src/helpers/torch.py
+1 -1
View File
@@ -51,7 +51,7 @@ def detachcpu(x):
"""
if isinstance(x, torch.Tensor):
# note apache parquet doesn't support half to we go for float https://github.com/huggingface/datasets/issues/4981
x = x.detach().cpu().float()
x = x.detach().cpu()
if x.squeeze().dim()==0:
return x.item()
return x
src/models/load.py
+2
View File
@@ -12,6 +12,7 @@ from loguru import logger
from typing import Tuple
def verbose_change_param(tokenizer, path, after):
before = getattr(tokenizer, path)
if before!=after:
setattr(tokenizer, path, after)
@@ -22,6 +23,7 @@ def verbose_change_param(tokenizer, path, after):
def load_model(model_repo = "TheBloke/WizardCoder-Python-13B-V1.0-GPTQ") -> Tuple[AutoModelForCausalLM, PreTrainedTokenizerBase]:
"""
A uncensored and large coding ones might be best for lying.
"""
# see https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/starchat.py
# gptq_config = GPTQConfig(bits=4, dataset="c4", disable_exllama=False)
src/prompts/prompt_loading.py
+3 -1
View File
@@ -347,4 +347,6 @@ def load_preproc_dataset(ds_name: str, tokenizer: PreTrainedTokenizerBase, N:int
ds_tokens = ds_tokens.filter(lambda r: not r['truncated'])
print('num_rows (after filtering out truncated rows)', ds_tokens.num_rows)
assert len(ds_tokens), f'No examples left after filtering out truncated rows, try a longer max_length than {max_length}'
return ds_tokens.select(range(N))
if len(ds_tokens)>N:
ds_tokens = ds_tokens.select(range(N))
return ds_tokens
src/repe/rep_control_pipeline_baukit.py
+15 -4
View File
@@ -18,6 +18,10 @@ from src.helpers.torch import clear_mem, detachcpu
Activations = NewType("Activations", Dict[str, torch.Tensor])
def try_half(v):
if isinstance(v, torch.Tensor):
return v.half()
return v
def hacky_sanitize_outputs(o):
"""I can't find the mem leak, so lets just detach, cpu, clone, freemem."""
@@ -152,7 +156,7 @@ class RepControlPipeline2(FeatureExtractionPipeline):
return ModelOutput(**o, **inputs)
def postprocess(self, o: ModelOutput):
o = hacky_sanitize_outputs(o)
# o = hacky_sanitize_outputs(o)
# note this sometimes deals with a batch, sometimes with a single result. infuriating
res = []
for i in range(len(o['input_ids'])):
@@ -173,7 +177,7 @@ class RepControlPipeline2(FeatureExtractionPipeline):
o["input_truncated"] = self.tokenizer.decode(input_ids)
o["truncated"] = torch.tensor(o["attention_mask"]).sum()==self.max_length
o["text_ans"] = self.tokenizer.decode(o["end_logits"].softmax(0).argmax(0))
o["text_ans"] = self.tokenizer.batch_decode(o["end_logits"].softmax(0).argmax(0))
if 'answer_choices' in o:
answer_choices = o['answer_choices']
@@ -183,7 +187,7 @@ class RepControlPipeline2(FeatureExtractionPipeline):
o['choice_ids'] = row_choice_ids(answer_choices, self.tokenizer)
ii = o['end_logits'].shape[1]
p = o['add_ans'] = torch.stack([scores2choice_probs2(o['end_logits'][:, i], o['choice_ids']) for i in range(ii)], 1)
p = o['choice_probs'] = torch.stack([scores2choice_probs2(o['end_logits'][:, i], o['choice_ids']) for i in range(ii)], 1)
o['ans'] = p[1] / (torch.sum(p, 0) + 1e-5)
@@ -192,8 +196,15 @@ class RepControlPipeline2(FeatureExtractionPipeline):
if k in o:
del o[k]
# ah to make a dataset we need to return one at a time, right now it's Dict[str, Batch]. e.g. hiddenstates={layer_1:[2, 555, 5120]....
# stop memory leaks?
o = hacky_sanitize_outputs(o)
# # make large arrays smaller
# for k in o:
# v = o[k]
# if hasattr(v, 'shape') and v.dtype==torch.float32:
# o[k] = v.half()
# {k:v.shape for k,v in o.items() if hasattr(v, 'shape')}
# {k:v.dtype for k,v in o.items() if hasattr(v, 'shape')}
return o