exp 2

notebooks/03_mjc_ltold_to_lie_loss.ipynb

@@ -14,7 +14,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -78,7 +78,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -106,7 +106,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -115,7 +115,7 @@
     "device = \"cuda:0\"\n",
     "\n",
     "cfg = ExtractConfig(\n",
-    "    max_examples=(600, 2000),\n",
+    "    max_examples=(1600, 2000),\n",
     "    # model=\"wassname/phi-1_5-w_hidden_states\",\n",
     "    # batch_size=3,\n",
     "    # model=\"wassname/phi-2-w_hidden_states\",\n",
@@ -140,32 +140,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
-      "The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "463f23fb30ab4cfeb57d37eb46486478",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "model, tokenizer = load_model(\n",
     "    cfg.model,\n",
@@ -185,7 +162,8 @@
     "    target_modules=[\n",
     "        \"out_proj\",\n",
     "        \"mlp.fc2\",\n",
-    "        \"mlp.fc1\",\n",
+    "        \n",
+    "        # \"mlp.fc1\",\n",
     "        \"Wqkv\",\n",
     "        # 'inner_attn',\n",
     "        # 'inner_cross_attn',\n",
@@ -193,8 +171,8 @@
     "    # bias=\"lora_only\",\n",
     "    task_type=TaskType.CAUSAL_LM,\n",
     "    inference_mode=False,\n",
-    "    r=4,\n",
-    "    lora_alpha=8,\n",
+    "    r=3,\n",
+    "    lora_alpha=6,\n",
     "    lora_dropout=0.0,\n",
     ")\n",
     "\n",
@@ -552,7 +530,8 @@
     "from src.eval.interventions import test_intervention_quality2\n",
     "from src.eval.labels import ds2label_model_obey, ds2label_model_truth\n",
     "\n",
-    "TEST_BATCH_MULT = 3\n"
+    "TEST_BATCH_MULT = 3\n",
+    "\n"
    ]
   },
   {
@@ -720,7 +699,7 @@
     "    return styler\n",
     "\n",
     "\n",
-    "def analyse_intervention(ds_out, tokenizer):\n",
+    "def analyse_intervention(ds_out, cfg, model_kwargs={}):\n",
     "    ds_known = filter_ds_to_known(ds_out, verbose=True)\n",
     "\n",
     "    print(\n",
@@ -736,7 +715,8 @@
     "            # fit probe\n",
     "            # print('='*80)\n",
     "            # print(f\"predicting label={label_name}\")\n",
-    "            df_res = test_intervention_quality2(ds_known, label_fn, title=f\"predicting label={label_name}\")\n",
+    "            df_res = test_intervention_quality2(ds_known, label_fn, title=f\"predicting label={label_name}\",\n",
+    "                                                skip=cfg.skip_layers, stride=cfg.stride_layers, model_kwargs=model_kwargs)\n",
     "            display(df_res)\n",
     "        except Exception as e:\n",
     "            raise\n",
@@ -762,10 +742,23 @@
    "outputs": [],
    "source": [
     "print(\"valtest\")\n",
-    "analyse_intervention(ds_out_valtest, tokenizer)\n",
+    "analyse_intervention(ds_out_valtest, cfg)\n",
     "\n",
     "print(\"out of distribution\")\n",
-    "analyse_intervention(ds_out_OOD, tokenizer)\n"
+    "analyse_intervention(ds_out_OOD, cfg)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"valtest\")\n",
+    "analyse_intervention(ds_out_valtest, cfg, model_kwargs=dict(scale=False))\n",
+    "\n",
+    "print(\"out of distribution\")\n",
+    "analyse_intervention(ds_out_OOD, cfg, model_kwargs=dict(scale=False))\n"
    ]
   },
   {

notebooks/test_syling.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -121,7 +121,7 @@
        "9                4.9               3.1                1.5               0.1"
       ]
      },
-     "execution_count": 17,
+     "execution_count": 1,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -133,6 +133,62 @@
     "df\n"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "|   sepal length (cm) |   sepal width (cm) |   petal length (cm) |   petal width (cm) |\n",
+       "|--------------------:|-------------------:|--------------------:|-------------------:|\n",
+       "|                 5.1 |                3.5 |                 1.4 |                0.2 |\n",
+       "|                 4.9 |                3   |                 1.4 |                0.2 |\n",
+       "|                 4.7 |                3.2 |                 1.3 |                0.2 |\n",
+       "|                 4.6 |                3.1 |                 1.5 |                0.2 |\n",
+       "|                 5   |                3.6 |                 1.4 |                0.2 |\n",
+       "|                 5.4 |                3.9 |                 1.7 |                0.4 |\n",
+       "|                 4.6 |                3.4 |                 1.4 |                0.3 |\n",
+       "|                 5   |                3.4 |                 1.5 |                0.2 |\n",
+       "|                 4.4 |                2.9 |                 1.4 |                0.2 |\n",
+       "|                 4.9 |                3.1 |                 1.5 |                0.1 |"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "|   sepal length (cm) |   sepal width (cm) |   petal length (cm) |   petal width (cm) |\n",
+      "|--------------------:|-------------------:|--------------------:|-------------------:|\n",
+      "|                 5.1 |                3.5 |                 1.4 |                0.2 |\n",
+      "|                 4.9 |                3   |                 1.4 |                0.2 |\n",
+      "|                 4.7 |                3.2 |                 1.3 |                0.2 |\n",
+      "|                 4.6 |                3.1 |                 1.5 |                0.2 |\n",
+      "|                 5   |                3.6 |                 1.4 |                0.2 |\n",
+      "|                 5.4 |                3.9 |                 1.7 |                0.4 |\n",
+      "|                 4.6 |                3.4 |                 1.4 |                0.3 |\n",
+      "|                 5   |                3.4 |                 1.5 |                0.2 |\n",
+      "|                 4.4 |                2.9 |                 1.4 |                0.2 |\n",
+      "|                 4.9 |                3.1 |                 1.5 |                0.1 |\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Custom LaTeX table with specialized formatting\n",
+    "from IPython.display import display_markdown, Markdown, display\n",
+    "markdown_output = df.to_markdown(index=False, tablefmt=\"pipe\",)\n",
+    "\n",
+    "display(Markdown(markdown_output))\n",
+    "print(markdown_output)\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 33,

poetry.lock

@@ -3172,6 +3172,20 @@ files = [
 [package.dependencies]
 mpmath = ">=0.19"
 
+[[package]]
+name = "tabulate"
+version = "0.9.0"
+description = "Pretty-print tabular data"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"},
+    {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"},
+]
+
+[package.extras]
+widechars = ["wcwidth"]
+
 [[package]]
 name = "tenacity"
 version = "8.2.3"
@@ -3861,4 +3875,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.13"
-content-hash = "af02367b750e9b9241651dfee62b952f8652a5196a28a6ce3b250b2545086fa8"
+content-hash = "af376d67d38bdaab1c7465f67631dd060ed1e4bb8aaa6aea129eecccf6aa5e7c"

pyproject.toml

@@ -30,6 +30,7 @@ bitsandbytes = "^0.41.3.post2"
 packaging = "^23.2"
 peft = "^0.7.1"
 ipywidgets = "^8.1.1"
+tabulate = "^0.9.0"
 
 [[tool.poetry.source]]
 name = "pytorch"

research_log.md

@@ -287,3 +287,24 @@ Hmm. there are even cases where the base hidden states help predict if the adapt
 
 Oh but the model does have really poor performance on the OOD set. Hmm so maybe it's not a good example?
 Or maybe I have a bug...
+
+The basics of my approach is that the intervention should be good. but the intervention doesn't seem to add much. Hmm
+
+Hypothesis:
+- the information isn't there because 1) the model is too small or 2) the hidden states are no the right way to look at it or 
+
+what do I mean by adapters acting like probes. Sure we can train them, take the hidden states, and use it with logistic regression to get an answer. But the normal way is to just let the adaptor give the answer. Well do think treating them as a probe is better. But I need to prove it. Perhaps even make a linear adapter. This can be a building block even if it doesn't unlock lie detection right away.
+
+**Hypotheis**: probe acc is greater than adapter acc. And even OOD.
+
+wait shoulnd't auc predictive be the same for baseline and intervention if I am only using the baseline hidden states??
+
+So experiment results:
+- acc of base model 0.824639
+- acc of adapter 0.62 (more lies)
+- just hidden states of base: 0.886339
+- hidden states of both 0.908748 (slightly better but not much)
+
+
+But wait I need to train for truth telling...
+

src/config.py

@@ -11,10 +11,10 @@ TEMPLATE_PATH = root_folder / "src/prompts/templates/"
 class ExtractConfig(Serializable):
     """Config for extracting hidden states from a language model."""
 
-    datasets: tuple[str, ...] = ("amazon_polarity",  "super_glue:boolq" )
+    datasets: tuple[str, ...] = ("amazon_polarity", "glue:qnli" )
     """Names of HF datasets to use, e.g. `"super_glue:boolq"` or `"imdb"` `"glue:qnli"""
 
-    datasets_ood: tuple[str, ...] = ("glue:qnli", )
+    datasets_ood: tuple[str, ...] = ( "super_glue:boolq", )
     """Names of Out Of Distribution HF datasets to use, e.g. `"super_glue:boolq"` or `"imdb"` `"glue:qnli"""
     
     # model: str = "wassname/phi-2-w_hidden_states"
@@ -40,3 +40,9 @@ class ExtractConfig(Serializable):
 
     seed: int = 42
     """Random seed."""
+
+    skip_layers: int = 2
+    """Number of layers to skip from the start of the model."""
+
+    stride_layers: int = 2
+    """Number of layers to skip between each layer."""

src/eval/interventions.py

@@ -18,7 +18,7 @@ def get_classification_report(y_test, y_pred):
     return df_classification_report
 
 # TODO move to intervention
-def check_lr_intervention_predictive(hs, y, verbose=False):
+def check_lr_intervention_predictive(hs, y, verbose=False, scale=True):
     """
     We want the hidden states resulting from interventions to have predictive power
     Lets compare normal hidden states to intervened hidden states
@@ -26,9 +26,10 @@ def check_lr_intervention_predictive(hs, y, verbose=False):
     X = rearrange(hs, 'b l hs -> b (l hs)')
     X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.5, random_state=42, stratify=y)
 
-    scaler = StandardScaler(with_mean=True, with_std=True)
-    X_train = scaler.fit_transform(X_train)
-    X_val = scaler.transform(X_val)
+    if scale:
+        scaler = StandardScaler(with_mean=True, with_std=True)
+        X_train = scaler.fit_transform(X_train)
+        X_val = scaler.transform(X_val)
 
     clf = LogisticRegression(random_state=42, max_iter=1000, class_weight='balanced',).fit(X_train, y_train)
     y_pred = clf.predict(X_train)
@@ -36,14 +37,14 @@ def check_lr_intervention_predictive(hs, y, verbose=False):
     y_val_prob = clf.predict_proba(X_val)[: ,1]
     score = roc_auc_score(y_val, y_val_prob)
 
+    target_names = [0, 1]
+    cm = confusion_matrix(y_val, y_val_pred, target_names=target_names, normalize='true')
+    cr = classification_report(y_val, y_val_pred, target_names=target_names)
     if verbose:
-        target_names = [0, 1]
-        cm = confusion_matrix(y_val, y_val_pred, target_names=target_names, normalize='true')
-        cr = classification_report(y_val, y_val_pred, target_names=target_names)
         print(cm)
         print(cr)
     
-    return score
+    return dict(score=score, y_val_pred=y_val_pred, y_val_prob=y_val_prob, y_val=y_val, cm=cm, cr=cr)
 
 def check_intervention_predictive_nn(hs, y):
     """
@@ -70,12 +71,12 @@ def check_intervention_predictive_nn(hs, y):
 def make_dfres_pretty(styler, title):
     styler.set_caption(title)
     styler.background_gradient(axis='index', vmin=0, vmax=1, cmap="RdYlGn", 
-                               subset=['roc_auc_baseline', 'roc_auc_interven', 'pass'])
+                               subset=['roc_auc', 'pass'])
     styler.background_gradient(axis='index', vmin=-.05, vmax=.05, cmap="RdYlGn", 
-                               subset=['pred_inc'])
+                               subset=['diff'])
     return styler
 
-def test_intervention_quality2(ds_out, label_fn, thresh=0.03, take_diff=False, verbose=False, title="Intervention predictive power"):
+def test_intervention_quality2(ds_out, label_fn, thresh=0.03, take_diff=False, verbose=False, title="Intervention predictive power", skip=0, stride=1, model_kwargs={}):
     """
     Check interventions are ordered and different and valid
 
@@ -93,24 +94,25 @@ def test_intervention_quality2(ds_out, label_fn, thresh=0.03, take_diff=False, v
     label = label_fn(ds_out)
 
     # collect hidden states
-    hs_normal = ds_out['end_residual_stream_base']
-    hs_intervene = ds_out['end_residual_stream_adapt']
+    hs_normal = ds_out['end_residual_stream_base'][:, skip::stride]
+    hs_intervene = ds_out['end_residual_stream_adapt'][:, skip::stride]
 
     # print(f"## primary metric: predictive power (of logistic regression on top of intervened hidden states to predict base model Y) [N={len(label)//2}]")
-    s1_baseline = check_lr_intervention_predictive(hs_normal, label)
-    s1_interven = check_lr_intervention_predictive(hs_intervene, label)
-    predictive = s1_interven - s1_baseline# > thresh
-    if verbose: print(f"  - predictive power? {predictive} [i]    = baseline: {s1_baseline:.3f} > {s1_interven:.3f} roc_auc [N={len(label)//2}]")
-    res['predictive'] = dict(roc_auc_baseline=s1_baseline, roc_auc_interven=s1_interven, pred_inc=predictive)
+    s1_baseline = check_lr_intervention_predictive(hs_normal, label, **model_kwargs)
+    s1_interven = check_lr_intervention_predictive(hs_intervene, label, **model_kwargs)
+    predictive = s1_interven['score'] - s1_baseline['score']# > thresh
+    # if verbose: print(f"  - predictive power? {predictive} [i]    = baseline: {s1_baseline:.3f} > {s1_interven:.3f} roc_auc [N={len(label)//2}]")
+    res['residual_{base}'] = dict(roc_auc=s1_baseline['score'], diff=0)
+    res['residual_{adapter}'] = dict(roc_auc=s1_interven['score'], diff=predictive)
 
-    s1_interven = check_lr_intervention_predictive(hs_normal-hs_intervene, label)
-    predictive = s1_interven - s1_baseline# > thresh
+    s1_interven2 = check_lr_intervention_predictive(hs_normal-hs_intervene, label, **model_kwargs)
+    predictive = s1_interven2['score'] - s1_baseline['score']# > thresh
     
-    if verbose: print(f"  - predictive power? {predictive} [i-b]  = baseline: {s1_baseline:.3f} > {s1_interven:.3f} roc_auc")
-    res['predictive_diff'] = dict(roc_auc_baseline=s1_baseline, roc_auc_interven=s1_interven, pred_inc=predictive)
+    # if verbose: print(f"  - predictive power? {predictive} [i-b]  = baseline: {s1_baseline:.3f} > {s1_interven:.3f} roc_auc")
+    res['residual_{base-adapter}'] = dict(roc_auc=s1_interven2['score'], diff=predictive)
 
     df_res = pd.DataFrame(res).T
-    df_res['pass'] = df_res['pred_inc'] > thresh
+    df_res['pass'] = df_res['diff'] > thresh
     df_styled = df_res.style.pipe(make_dfres_pretty, title)
     return df_styled
 

src/eval/labels.py

@@ -12,18 +12,18 @@ def ds2label_model_truth(ds):
     labels_true_ans = ds["label_true_base"] == ans
     return labels_true_ans
 
-def ds2label_model_obey_adapt(ds):
-    """label: whether the model obeyed the instruction to lie"""
-    label_instructed = ds["label_true_adapt"] ^ ds["instructed_to_lie_adapt"]
-    ans = ds["binary_ans_adapt"] > 0.5
-    labels_untruth = label_instructed == ans
-    return labels_untruth
+# def ds2label_model_obey_adapt(ds):
+#     """label: whether the model obeyed the instruction to lie"""
+#     label_instructed = ds["label_true_adapt"] ^ ds["instructed_to_lie_adapt"]
+#     ans = ds["binary_ans_adapt"] > 0.5
+#     labels_untruth = label_instructed == ans
+#     return labels_untruth
 
-def ds2label_model_truth_adapt(ds):
-    """label: whether the model told the truth"""
-    ans = ds["binary_ans_adapt"] > 0.5
-    labels_true_ans = ds["label_true_adapt"] == ans
-    return labels_true_ans
+# def ds2label_model_truth_adapt(ds):
+#     """label: whether the model told the truth"""
+#     ans = ds["binary_ans_adapt"] > 0.5
+#     labels_true_ans = ds["label_true_adapt"] == ans
+#     return labels_true_ans
 
 def ranking_truth_telling(ds):
     """label whether the adapter or the base model were more truthfull."""
@@ -36,6 +36,6 @@ def ranking_instruction_following(ds):
 
 
 LABEL_MAPPING = dict(label_model_truth=ds2label_model_truth, label_model_obey=ds2label_model_obey,
-                        label_model_truth_adapt=ds2label_model_truth_adapt, label_model_obey_adapt=ds2label_model_obey_adapt,
+                        # label_model_truth_adapt=ds2label_model_truth_adapt, label_model_obey_adapt=ds2label_model_obey_adapt,
                         ranking_truth_telling=ranking_truth_telling, ranking_instruction_following=ranking_instruction_following)
  

src/models/phi/model_phi.py

@@ -780,7 +780,7 @@ class ParallelBlock(nn.Module):
 
         hidden_states = attn_outputs + feed_forward_hidden_states + residual
 
-        return feed_forward_hidden_states, attn_outputs
+        return hidden_states, attn_outputs
 
 
 class CausalLMHead(nn.Module):