wassname/eliciting_suppressed_knowledge
1
0
Fork 0
mirror of https://github.com/wassname/eliciting_suppressed_knowledge.git synced 2026-09-09 11:21:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

try diff agg

Browse Source
This commit is contained in:
wassname
2025-05-02 05:46:54 +08:00
parent 94747da40e
commit a280aa5c06
8 changed files with 2144 additions and 3783 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -8,3 +8,4 @@ wheels/
# Virtual environments
.venv
nbs/old/
README.md
+1 -1
View File
@@ -1,7 +1,7 @@
# Eliciting Suppressed Knowledge (ESK) WIP
## Abstract
**Where do transformer models store their true "thoughts" when they say something they know is false?** We demonstrate that suppressed neural activations are a more useful source of knowledge than the model's direct outputs or standard hidden states. By isolating and probing these suppressed activation patterns, we achieve ~X% improvements on TruthfulQA compared to standard methods. This confirms suppressed activations contain knowledge that the model possesses but deliberately inhibits during generation.
**Where do transformer models store their true "thoughts" when they say something they know is false?** We demonstrate that suppressed neural activations are a more useful source of knowledge than the model's direct outputs or standard hidden states. By isolating and probing these suppressed activation patterns, we achieve ~TODO% improvements on TruthfulQA compared to standard methods. This confirms suppressed activations contain knowledge that the model possesses but deliberately inhibits during generation.
## Research Question
Recent evidence demonstrates that transformer models systematically misrepresent their internal reasoning:
figs/truthfulqa_Qwen_Qwen2.5-0.5B-Instruct.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 31 KiB

figs/truthfulqa_Qwen_Qwen2.5-3B-Instruct.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 20 KiB

nbs/02_TQA_regr_w_kv.ipynb
-3772
View File
File diff suppressed because one or more lines are too long
nbs/02b_TQA_regr_w_kv.ipynb
+2111
View File
File diff suppressed because one or more lines are too long
pyproject.toml
+3 -2
View File
@@ -7,16 +7,17 @@ requires-python = ">=3.10"
dependencies = [
"accelerate>=1.4.0",
"activation-store",
"autoawq>=0.2.7.post3",
"autoawq>=0.2.6",
"datasets>=3.3.2",
"einops>=0.8.1",
"jaxtyping>=0.2.38",
"loguru>=0.7.3",
"matplotlib>=3.10.1",
"pandas>=2.2.3",
"seaborn>=0.13.2",
"skorch>=1.1.0",
"tqdm>=4.67.1",
"transformers>=4.49.0",
"transformers>=4.51.0",
]
[dependency-groups]
uv.lock
Generated
+28 -8
View File
@@ -543,6 +543,7 @@ dependencies = [
{ name = "loguru" },
{ name = "matplotlib" },
{ name = "pandas" },
{ name = "seaborn" },
{ name = "skorch" },
{ name = "tqdm" },
{ name = "transformers" },
@@ -558,16 +559,17 @@ dev = [
requires-dist = [
{ name = "accelerate", specifier = ">=1.4.0" },
{ name = "activation-store", editable = "../../elk/cache_transformer_acts" },
{ name = "autoawq", specifier = ">=0.2.7.post3" },
{ name = "autoawq", specifier = ">=0.2.6" },
{ name = "datasets", specifier = ">=3.3.2" },
{ name = "einops", specifier = ">=0.8.1" },
{ name = "jaxtyping", specifier = ">=0.2.38" },
{ name = "loguru", specifier = ">=0.7.3" },
{ name = "matplotlib", specifier = ">=3.10.1" },
{ name = "pandas", specifier = ">=2.2.3" },
{ name = "seaborn", specifier = ">=0.13.2" },
{ name = "skorch", specifier = ">=1.1.0" },
{ name = "tqdm", specifier = ">=4.67.1" },
{ name = "transformers", specifier = ">=4.49.0" },
{ name = "transformers", specifier = ">=4.51.0" },
]
[package.metadata.requires-dev]
@@ -729,7 +731,7 @@ http = [
[[package]]
name = "huggingface-hub"
version = "0.29.3"
version = "0.30.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filelock" },
@@ -740,9 +742,9 @@ dependencies = [
{ name = "tqdm" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e5/f9/851f34b02970e8143d41d4001b2d49e54ef113f273902103823b8bc95ada/huggingface_hub-0.29.3.tar.gz", hash = "sha256:64519a25716e0ba382ba2d3fb3ca082e7c7eb4a2fc634d200e8380006e0760e5", size = 390123 }
sdist = { url = "https://files.pythonhosted.org/packages/df/22/8eb91736b1dcb83d879bd49050a09df29a57cc5cd9f38e48a4b1c45ee890/huggingface_hub-0.30.2.tar.gz", hash = "sha256:9a7897c5b6fd9dad3168a794a8998d6378210f5b9688d0dfc180b1a228dc2466", size = 400868 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/40/0c/37d380846a2e5c9a3c6a73d26ffbcfdcad5fc3eacf42fdf7cff56f2af634/huggingface_hub-0.29.3-py3-none-any.whl", hash = "sha256:0b25710932ac649c08cdbefa6c6ccb8e88eef82927cacdb048efb726429453aa", size = 468997 },
{ url = "https://files.pythonhosted.org/packages/93/27/1fb384a841e9661faad1c31cbfa62864f59632e876df5d795234da51c395/huggingface_hub-0.30.2-py3-none-any.whl", hash = "sha256:68ff05969927058cfa41df4f2155d4bb48f5f54f719dd0390103eefa9b191e28", size = 481433 },
]
[[package]]
@@ -2187,6 +2189,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0a/c8/b3f566db71461cabd4b2d5b39bcc24a7e1c119535c8361f81426be39bb47/scipy-1.15.2-cp313-cp313t-win_amd64.whl", hash = "sha256:fe8a9eb875d430d81755472c5ba75e84acc980e4a8f6204d402849234d3017db", size = 40477705 },
]
[[package]]
name = "seaborn"
version = "0.13.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "matplotlib" },
{ name = "numpy" },
{ name = "pandas" },
]
sdist = { url = "https://files.pythonhosted.org/packages/86/59/a451d7420a77ab0b98f7affa3a1d78a313d2f7281a57afb1a34bae8ab412/seaborn-0.13.2.tar.gz", hash = "sha256:93e60a40988f4d65e9f4885df477e2fdaff6b73a9ded434c1ab356dd57eefff7", size = 1457696 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/83/11/00d3c3dfc25ad54e731d91449895a79e4bf2384dc3ac01809010ba88f6d5/seaborn-0.13.2-py3-none-any.whl", hash = "sha256:636f8336facf092165e27924f223d3c62ca560b1f2bb5dff7ab7fad265361987", size = 294914 },
]
[[package]]
name = "setuptools"
version = "76.0.0"
@@ -2346,6 +2362,10 @@ dependencies = [
{ name = "torch" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/a9/20/72eb0b5b08fa293f20fc41c374e37cf899f0033076f0144d2cdc48f9faee/torchvision-0.21.0-1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5568c5a1ff1b2ec33127b629403adb530fab81378d9018ca4ed6508293f76e2b", size = 2327643 },
{ url = "https://files.pythonhosted.org/packages/4e/3d/b7241abfa3e6651c6e00796f5de2bd1ce4d500bf5159bcbfeea47e711b93/torchvision-0.21.0-1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:ff96666b94a55e802ea6796cabe788541719e6f4905fc59c380fed3517b6a64d", size = 2329320 },
{ url = "https://files.pythonhosted.org/packages/52/5b/76ca113a853b19c7b1da761f8a72cb6429b3bd0bf932537d8df4657f47c3/torchvision-0.21.0-1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ffa2a16499508fe6798323e455f312c7c55f2a88901c9a7c0fb1efa86cf7e327", size = 2329878 },
{ url = "https://files.pythonhosted.org/packages/4e/fe/5e193353706dab96fe73ae100d5a633ff635ce310e0d92f3bc2958d075b1/torchvision-0.21.0-1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7e9e9afa150e40cd2a8f0701c43cb82a8d724f512896455c0918b987f94b84a4", size = 2280711 },
{ url = "https://files.pythonhosted.org/packages/8e/0d/143bd264876fad17c82096b6c2d433f1ac9b29cdc69ee45023096976ee3d/torchvision-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:044ea420b8c6c3162a234cada8e2025b9076fa82504758cd11ec5d0f8cd9fa37", size = 1784140 },
{ url = "https://files.pythonhosted.org/packages/5e/44/32e2d2d174391374d5ff3c4691b802e8efda9ae27ab9062eca2255b006af/torchvision-0.21.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:b0c0b264b89ab572888244f2e0bad5b7eaf5b696068fc0b93e96f7c3c198953f", size = 7237187 },
{ url = "https://files.pythonhosted.org/packages/0e/6b/4fca9373eda42c1b04096758306b7bd55f7d8f78ba273446490855a0f25d/torchvision-0.21.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:54815e0a56dde95cc6ec952577f67e0dc151eadd928e8d9f6a7f821d69a4a734", size = 14699067 },
@@ -2405,7 +2425,7 @@ wheels = [
[[package]]
name = "transformers"
version = "4.49.0"
version = "4.51.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "filelock" },
@@ -2419,9 +2439,9 @@ dependencies = [
{ name = "tokenizers" },
{ name = "tqdm" },
]
sdist = { url = "https://files.pythonhosted.org/packages/79/50/46573150944f46df8ec968eda854023165a84470b42f69f67c7d475dabc5/transformers-4.49.0.tar.gz", hash = "sha256:7e40e640b5b8dc3f48743f5f5adbdce3660c82baafbd3afdfc04143cdbd2089e", size = 8610952 }
sdist = { url = "https://files.pythonhosted.org/packages/f1/11/7414d5bc07690002ce4d7553602107bf969af85144bbd02830f9fb471236/transformers-4.51.3.tar.gz", hash = "sha256:e292fcab3990c6defe6328f0f7d2004283ca81a7a07b2de9a46d67fd81ea1409", size = 8941266 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/20/37/1f29af63e9c30156a3ed6ebc2754077016577c094f31de7b2631e5d379eb/transformers-4.49.0-py3-none-any.whl", hash = "sha256:6b4fded1c5fee04d384b1014495b4235a2b53c87503d7d592423c06128cbbe03", size = 9970275 },
{ url = "https://files.pythonhosted.org/packages/a9/b6/5257d04ae327b44db31f15cce39e6020cc986333c715660b1315a9724d82/transformers-4.51.3-py3-none-any.whl", hash = "sha256:fd3279633ceb2b777013234bbf0b4f5c2d23c4626b05497691f00cfda55e8a83", size = 10383940 },
]
[[package]]