mirror of
https://github.com/wassname/eliciting_suppressed_knowledge.git
synced 2026-09-12 12:20:20 +08:00
skorch
This commit is contained in:
wassname
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -13,7 +13,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -23,7 +23,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -35,7 +35,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -58,6 +58,9 @@
|
||||
"from jaxtyping import Float\n",
|
||||
"from torch import Tensor\n",
|
||||
"\n",
|
||||
"import gc\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"from activation_store.collect import activation_store, default_postprocess_result"
|
||||
]
|
||||
},
|
||||
@@ -70,20 +73,30 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Sliding Window Attention is enabled but not implemented for `eager`; unexpected results may be encountered.\n"
|
||||
"ename": "ImportError",
|
||||
"evalue": "Loading an AWQ quantized model requires auto-awq library (`pip install autoawq`)",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[8], line 6\u001b[0m\n\u001b[1;32m 2\u001b[0m model_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mQwen/Qwen2.5-3B-Instruct-AWQ\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;66;03m# model_name = \"Qwen/Qwen2.5-3B-Instruct\"\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# model_name = \"unsloth/Phi-4-mini-instruct\" # 4b\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mAutoModelForCausalLM\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mmodel_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mtorch_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbfloat16\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevice_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mauto\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43mattn_implementation\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43meager\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# flex_attention flash_attention_2 sdpa eager\u001b[39;49;00m\n\u001b[1;32m 11\u001b[0m \u001b[43m)\u001b[49m\n\u001b[1;32m 12\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m AutoTokenizer\u001b[38;5;241m.\u001b[39mfrom_pretrained(model_name)\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tokenizer\u001b[38;5;241m.\u001b[39mpad_token_id \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
|
||||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/2025/eliciting_suppressed_knowledge/.venv/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:564\u001b[0m, in \u001b[0;36m_BaseAutoModelClass.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m 562\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mtype\u001b[39m(config) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys():\n\u001b[1;32m 563\u001b[0m model_class \u001b[38;5;241m=\u001b[39m _get_model_class(config, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping)\n\u001b[0;32m--> 564\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 565\u001b[0m \u001b[43m \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mmodel_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mhub_kwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 566\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 567\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 568\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnrecognized configuration class \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconfig\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m for this kind of AutoModel: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 569\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mModel type should be one of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(c\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mfor\u001b[39;00m\u001b[38;5;250m \u001b[39mc\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_model_mapping\u001b[38;5;241m.\u001b[39mkeys())\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 570\u001b[0m )\n",
|
||||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/2025/eliciting_suppressed_knowledge/.venv/lib/python3.10/site-packages/transformers/modeling_utils.py:262\u001b[0m, in \u001b[0;36mrestore_default_torch_dtype.<locals>._wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 260\u001b[0m old_dtype \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mget_default_dtype()\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 262\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 263\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 264\u001b[0m torch\u001b[38;5;241m.\u001b[39mset_default_dtype(old_dtype)\n",
|
||||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/2025/eliciting_suppressed_knowledge/.venv/lib/python3.10/site-packages/transformers/modeling_utils.py:3698\u001b[0m, in \u001b[0;36mPreTrainedModel.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m 3695\u001b[0m hf_quantizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 3697\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m hf_quantizer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 3698\u001b[0m \u001b[43mhf_quantizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_environment\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 3699\u001b[0m \u001b[43m \u001b[49m\u001b[43mtorch_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtorch_dtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3700\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_tf\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_tf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3701\u001b[0m \u001b[43m \u001b[49m\u001b[43mfrom_flax\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_flax\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3702\u001b[0m \u001b[43m \u001b[49m\u001b[43mdevice_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice_map\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3703\u001b[0m \u001b[43m \u001b[49m\u001b[43mweights_only\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mweights_only\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 3704\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3705\u001b[0m torch_dtype \u001b[38;5;241m=\u001b[39m hf_quantizer\u001b[38;5;241m.\u001b[39mupdate_torch_dtype(torch_dtype)\n\u001b[1;32m 3706\u001b[0m device_map \u001b[38;5;241m=\u001b[39m hf_quantizer\u001b[38;5;241m.\u001b[39mupdate_device_map(device_map)\n",
|
||||
"File \u001b[0;32m/media/wassname/SGIronWolf/projects5/2025/eliciting_suppressed_knowledge/.venv/lib/python3.10/site-packages/transformers/quantizers/quantizer_awq.py:50\u001b[0m, in \u001b[0;36mAwqQuantizer.validate_environment\u001b[0;34m(self, device_map, **kwargs)\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mvalidate_environment\u001b[39m(\u001b[38;5;28mself\u001b[39m, device_map, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_auto_awq_available():\n\u001b[0;32m---> 50\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading an AWQ quantized model requires auto-awq library (`pip install autoawq`)\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 52\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_accelerate_available():\n\u001b[1;32m 53\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading an AWQ quantized model requires accelerate (`pip install accelerate`)\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
|
||||
"\u001b[0;31mImportError\u001b[0m: Loading an AWQ quantized model requires auto-awq library (`pip install autoawq`)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"model_name = \"Qwen/Qwen2.5-3B-Instruct\"\n",
|
||||
"model_name = \"Qwen/Qwen2.5-0.5B-Instruct\"\n",
|
||||
"model_name = \"Qwen/Qwen2.5-3B-Instruct-AWQ\"\n",
|
||||
"# model_name = \"Qwen/Qwen2.5-3B-Instruct\"\n",
|
||||
"# model_name = \"unsloth/Phi-4-mini-instruct\" # 4b\n",
|
||||
"\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(\n",
|
||||
" model_name,\n",
|
||||
@@ -107,7 +120,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -165,7 +178,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -198,7 +211,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -240,7 +253,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -278,7 +291,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -298,7 +311,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -321,7 +334,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -410,7 +423,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -465,7 +478,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -499,7 +512,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -570,7 +583,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -596,7 +609,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -624,7 +637,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -711,7 +724,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -773,7 +786,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -800,7 +813,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -830,7 +843,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 57,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -842,7 +855,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 58,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -891,7 +904,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 59,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -911,7 +924,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 60,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -940,14 +953,11 @@
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import gc\n",
|
||||
"import numpy as np"
|
||||
]
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 62,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -979,7 +989,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 63,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1008,7 +1018,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 64,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1043,7 +1053,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 65,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1407,7 +1417,7 @@
|
||||
"for eps in [-50, -10, -5, -1, -0.5, -0.1, -0.01, -0, 0, 0.01, 0.1, 0.5, 1, 10, 50]:\n",
|
||||
" gc.collect()\n",
|
||||
" ds_a3 = ds_a2.map(lambda x:calc_hs_sup(x, eps=eps), num_proc=None, batched=True, batch_size=64, desc=f\"eps {eps}\")\n",
|
||||
" print(f\"eps {eps} ds_a3['supressed_mask'].mean()={ds_a3['supressed_mask'].mean()}\")\n",
|
||||
" logger.info(f\"eps {eps} ds_a3['supressed_mask'].mean()={ds_a3['supressed_mask'].mean()}\")\n",
|
||||
" data_names = [\"supressed_hs\", \"supressed_mask\"]\n",
|
||||
" for dn in data_names:\n",
|
||||
" try:\n",
|
||||
@@ -1416,13 +1426,12 @@
|
||||
" score = train_linear_prob_on_dataset(X, name)\n",
|
||||
" results.append((name, score))\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"error with {name}\")\n",
|
||||
" print(e)\n"
|
||||
" logger.error(f\"error with {name} {e}\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 66,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1710,7 +1719,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 71,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1802,7 +1811,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 72,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1822,7 +1831,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 81,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1848,10 +1857,12 @@
|
||||
],
|
||||
"source": [
|
||||
"# plot it\n",
|
||||
"# TODO add logits\n",
|
||||
"\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"df3 = df2.T[['llm_ans', 'llm_log_prob_true', 'hidden_states', 'supressed_hs']].rename(columns={\n",
|
||||
"cols = ['llm_ans', 'llm_log_prob_true', 'hidden_states', 'supressed_hs']\n",
|
||||
"# add acts\n",
|
||||
"cols += [c for c in df2.columns if c.startswith(\"act\")]\n",
|
||||
"df3 = df2.T[cols].rename(columns={\n",
|
||||
" 'llm_ans': 'LLM Answer',\n",
|
||||
" 'llm_log_prob_true': 'LLM Probability',\n",
|
||||
" 'hidden_states': 'Hidden States',\n",
|
||||
File diff suppressed because one or more lines are too long
@@ -83,7 +83,7 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "3e298fba714a478ba42279bb7c81e379",
|
||||
"model_id": "4dbf9b990ae9481caad258b70590c33d",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -128,7 +128,7 @@
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"Dataset({\n",
|
||||
" features: ['input_ids', 'attention_mask', 'label'],\n",
|
||||
" features: ['attention_mask', 'input_ids', 'label'],\n",
|
||||
" num_rows: 316\n",
|
||||
"})"
|
||||
]
|
||||
@@ -207,7 +207,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"<torch.utils.data.dataloader.DataLoader object at 0x700e3be5fa90>\n"
|
||||
"<torch.utils.data.dataloader.DataLoader object at 0x7c981035d450>\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -226,135 +226,29 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"{'mlp.down_proj': ['model.layers.0.mlp.down_proj',\n",
|
||||
" 'model.layers.1.mlp.down_proj',\n",
|
||||
" 'model.layers.2.mlp.down_proj',\n",
|
||||
" 'model.layers.3.mlp.down_proj',\n",
|
||||
" 'model.layers.4.mlp.down_proj',\n",
|
||||
" 'model.layers.5.mlp.down_proj',\n",
|
||||
" 'model.layers.6.mlp.down_proj',\n",
|
||||
" 'model.layers.7.mlp.down_proj',\n",
|
||||
" 'model.layers.8.mlp.down_proj',\n",
|
||||
" 'model.layers.9.mlp.down_proj',\n",
|
||||
" 'model.layers.10.mlp.down_proj',\n",
|
||||
" 'model.layers.11.mlp.down_proj',\n",
|
||||
" 'model.layers.12.mlp.down_proj',\n",
|
||||
" 'model.layers.13.mlp.down_proj',\n",
|
||||
" 'model.layers.14.mlp.down_proj',\n",
|
||||
" 'model.layers.15.mlp.down_proj',\n",
|
||||
" 'model.layers.16.mlp.down_proj',\n",
|
||||
" 'model.layers.17.mlp.down_proj',\n",
|
||||
" 'model.layers.18.mlp.down_proj',\n",
|
||||
" 'model.layers.19.mlp.down_proj',\n",
|
||||
" 'model.layers.20.mlp.down_proj',\n",
|
||||
" 'model.layers.21.mlp.down_proj',\n",
|
||||
" 'model.layers.22.mlp.down_proj',\n",
|
||||
" 'model.layers.23.mlp.down_proj',\n",
|
||||
" 'model.layers.24.mlp.down_proj',\n",
|
||||
" 'model.layers.25.mlp.down_proj',\n",
|
||||
" 'model.layers.26.mlp.down_proj',\n",
|
||||
" 'model.layers.27.mlp.down_proj',\n",
|
||||
" 'model.layers.28.mlp.down_proj',\n",
|
||||
" 'model.layers.29.mlp.down_proj',\n",
|
||||
" 'model.layers.30.mlp.down_proj',\n",
|
||||
" 'model.layers.31.mlp.down_proj',\n",
|
||||
" 'model.layers.32.mlp.down_proj',\n",
|
||||
" 'model.layers.33.mlp.down_proj',\n",
|
||||
" 'model.layers.34.mlp.down_proj',\n",
|
||||
" 'model.layers.35.mlp.down_proj'],\n",
|
||||
" 'self_attn': ['model.layers.0.self_attn',\n",
|
||||
" 'model.layers.1.self_attn',\n",
|
||||
" 'model.layers.2.self_attn',\n",
|
||||
" 'model.layers.3.self_attn',\n",
|
||||
" 'model.layers.4.self_attn',\n",
|
||||
" 'model.layers.5.self_attn',\n",
|
||||
" 'model.layers.6.self_attn',\n",
|
||||
" 'model.layers.7.self_attn',\n",
|
||||
" 'model.layers.8.self_attn',\n",
|
||||
" 'model.layers.9.self_attn',\n",
|
||||
" 'model.layers.10.self_attn',\n",
|
||||
" 'model.layers.11.self_attn',\n",
|
||||
" 'model.layers.12.self_attn',\n",
|
||||
" 'model.layers.13.self_attn',\n",
|
||||
" 'model.layers.14.self_attn',\n",
|
||||
" 'model.layers.15.self_attn',\n",
|
||||
" 'model.layers.16.self_attn',\n",
|
||||
" 'model.layers.17.self_attn',\n",
|
||||
" 'model.layers.18.self_attn',\n",
|
||||
" 'model.layers.19.self_attn',\n",
|
||||
" 'model.layers.20.self_attn',\n",
|
||||
" 'model.layers.21.self_attn',\n",
|
||||
" 'model.layers.22.self_attn',\n",
|
||||
" 'model.layers.23.self_attn',\n",
|
||||
" 'model.layers.24.self_attn',\n",
|
||||
" 'model.layers.25.self_attn',\n",
|
||||
" 'model.layers.26.self_attn',\n",
|
||||
" 'model.layers.27.self_attn',\n",
|
||||
" 'model.layers.28.self_attn',\n",
|
||||
" 'model.layers.29.self_attn',\n",
|
||||
" 'model.layers.30.self_attn',\n",
|
||||
" 'model.layers.31.self_attn',\n",
|
||||
" 'model.layers.32.self_attn',\n",
|
||||
" 'model.layers.33.self_attn',\n",
|
||||
" 'model.layers.34.self_attn',\n",
|
||||
" 'model.layers.35.self_attn'],\n",
|
||||
" 'mlp.up_proj': ['model.layers.0.mlp.up_proj',\n",
|
||||
" 'model.layers.1.mlp.up_proj',\n",
|
||||
" 'model.layers.2.mlp.up_proj',\n",
|
||||
" 'model.layers.3.mlp.up_proj',\n",
|
||||
" 'model.layers.4.mlp.up_proj',\n",
|
||||
" 'model.layers.5.mlp.up_proj',\n",
|
||||
" 'model.layers.6.mlp.up_proj',\n",
|
||||
" 'model.layers.7.mlp.up_proj',\n",
|
||||
" 'model.layers.8.mlp.up_proj',\n",
|
||||
" 'model.layers.9.mlp.up_proj',\n",
|
||||
" 'model.layers.10.mlp.up_proj',\n",
|
||||
" 'model.layers.11.mlp.up_proj',\n",
|
||||
" 'model.layers.12.mlp.up_proj',\n",
|
||||
" 'model.layers.13.mlp.up_proj',\n",
|
||||
" 'model.layers.14.mlp.up_proj',\n",
|
||||
" 'model.layers.15.mlp.up_proj',\n",
|
||||
" 'model.layers.16.mlp.up_proj',\n",
|
||||
" 'model.layers.17.mlp.up_proj',\n",
|
||||
" 'model.layers.18.mlp.up_proj',\n",
|
||||
" 'model.layers.19.mlp.up_proj',\n",
|
||||
" 'model.layers.20.mlp.up_proj',\n",
|
||||
" 'model.layers.21.mlp.up_proj',\n",
|
||||
" 'model.layers.22.mlp.up_proj',\n",
|
||||
" 'model.layers.23.mlp.up_proj',\n",
|
||||
" 'model.layers.24.mlp.up_proj',\n",
|
||||
" 'model.layers.25.mlp.up_proj',\n",
|
||||
" 'model.layers.26.mlp.up_proj',\n",
|
||||
" 'model.layers.27.mlp.up_proj',\n",
|
||||
" 'model.layers.28.mlp.up_proj',\n",
|
||||
" 'model.layers.29.mlp.up_proj',\n",
|
||||
" 'model.layers.30.mlp.up_proj',\n",
|
||||
" 'model.layers.31.mlp.up_proj',\n",
|
||||
" 'model.layers.32.mlp.up_proj',\n",
|
||||
" 'model.layers.33.mlp.up_proj',\n",
|
||||
" 'model.layers.34.mlp.up_proj',\n",
|
||||
" 'model.layers.35.mlp.up_proj']}"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
"ename": "",
|
||||
"evalue": "",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
|
||||
"\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
|
||||
"\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
|
||||
"\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# choose layers to cache\n",
|
||||
"layer_groups = {\n",
|
||||
" 'mlp.down_proj': [k for k,v in model.named_modules() if k.endswith('mlp.down_proj')],\n",
|
||||
" 'self_attn': [k for k,v in model.named_modules() if k.endswith('.self_attn')],\n",
|
||||
" 'mlp.up_proj': [k for k,v in model.named_modules() if k.endswith('mlp.up_proj')],\n",
|
||||
" 'mlp.down_proj': [k for k,v in model.named_modules() if k.endswith('mlp.down_proj')][10:25],\n",
|
||||
" 'self_attn': [k for k,v in model.named_modules() if k.endswith('.self_attn')][10:25],\n",
|
||||
" 'mlp.up_proj': [k for k,v in model.named_modules() if k.endswith('mlp.up_proj')][10:25],\n",
|
||||
"}\n",
|
||||
"layer_groups"
|
||||
"layer_groups = []"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1005,8 +899,7 @@
|
||||
"# TODO add logits\n",
|
||||
"\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"df3 = df2.T[['llm_ans', 'llm_log_prob_true', 'hidden_states', 'supressed_hs', 'self_attn', 'mlp.up_proj',\n",
|
||||
" 'mlp.down_proj', ]].rename(columns={\n",
|
||||
"df3 = df2.T[['llm_ans', 'llm_log_prob_true', 'hidden_states', 'supressed_hs', ]].rename(columns={\n",
|
||||
" 'llm_ans': 'LLM Answer',\n",
|
||||
" 'llm_log_prob_true': 'LLM Probability',\n",
|
||||
" 'hidden_states': 'Hidden States',\n",
|
||||
@@ -1016,7 +909,7 @@
|
||||
"}).T.sort_values(\"auroc\", ascending=False)\n",
|
||||
"df3.plot.barh()\n",
|
||||
"plt.legend().remove()\n",
|
||||
"plt.xlabel(f\"Linar probe AUROC\")\n",
|
||||
"plt.xlabel(f\"Linear probe AUROC\")\n",
|
||||
"plt.title(f\"TruthfulQA Binary with {model_name}\")\n",
|
||||
"plt.xlim(0.5, None)"
|
||||
]
|
||||
Reference in New Issue
Block a user