From 3b4e034ea2662fd8b5077ee017ae8cc4bd730336 Mon Sep 17 00:00:00 2001 From: deep1 <> Date: Thu, 12 Oct 2023 17:06:39 +0800 Subject: [PATCH] 70% acc --- mjc_notes.md | 4 ++ ...train_nanda_probe_w_counterfact_70%.ipynb} | 51 +++++++++---------- 2 files changed, 29 insertions(+), 26 deletions(-) rename notebooks/{027_train_nanda_probe_w_counterfact.ipynb => 027_train_nanda_probe_w_counterfact_70%.ipynb} (99%) diff --git a/mjc_notes.md b/mjc_notes.md index b87b55c..a885ca5 100644 --- a/mjc_notes.md +++ b/mjc_notes.md @@ -1705,3 +1705,7 @@ We use the following linear models during evaluation: 3. K-Means - We take an unlabelled dataset D and perform K-Means clustering with K = 2, hoping to separate high-concept and low-concept samples. We take the difference between the centroids of the two clusters as the concept direction. 4. Mean Difference - We take the difference between the means of high-concept and low- concept samples of the data: 5. Logistic Regression - The weights of logistic regression trained to separate Xhighl and Xlowl on some training data can be used as a concept direction as well. + +# 2023-10-12 13:07:28 + +So I want to try not applying a random but by fitting to a few samples. Probobly mean difference is the easiest. Of course this would be on weights not embeddings. diff --git a/notebooks/027_train_nanda_probe_w_counterfact.ipynb b/notebooks/027_train_nanda_probe_w_counterfact_70%.ipynb similarity index 99% rename from notebooks/027_train_nanda_probe_w_counterfact.ipynb rename to notebooks/027_train_nanda_probe_w_counterfact_70%.ipynb index 4d56f6f..797f72f 100644 --- a/notebooks/027_train_nanda_probe_w_counterfact.ipynb +++ b/notebooks/027_train_nanda_probe_w_counterfact_70%.ipynb @@ -36,7 +36,7 @@ "source": [ "# import your package\n", "%load_ext autoreload\n", - "%autoreload 2" + "%autoreload 2\n" ] }, { @@ -56,7 +56,6 @@ } ], "source": [ - "\n", "import numpy as np\n", "import pandas as pd\n", "from matplotlib import pyplot as plt\n", @@ -90,7 +89,7 @@ "\n", "\n", "\n", - "transformers.__version__" + "transformers.__version__\n" ] }, { @@ -99,7 +98,7 @@ "metadata": {}, "outputs": [], "source": [ - "from src.helpers.lightning import read_metrics_csv" + "from src.helpers.lightning import read_metrics_csv\n" ] }, { @@ -184,7 +183,7 @@ " known_rows_i = df[known_rows].index\n", " \n", " if verbose: print(f\"select rows are {m1.mean():2.2%} based on knowledge\")\n", - " return ds1.select(known_rows_i)" + " return ds1.select(known_rows_i)\n" ] }, { @@ -198,7 +197,7 @@ "# ds.features\n", "# # ds['prompt_truncated'].map(lambda s:s.startswith('<|endoftext|>'))\n", "# ds2 = ds.map(lambda x: {'truncated': x['prompt_truncated'].startswith('<|endoftext|>')})\n", - "# ds2['truncated']" + "# ds2['truncated']\n" ] }, { @@ -222,7 +221,7 @@ "metadata": {}, "outputs": [], "source": [ - "# ds['choice_probs0'][0]" + "# ds['choice_probs0'][0]\n" ] }, { @@ -231,7 +230,7 @@ "metadata": {}, "outputs": [], "source": [ - "# df['ans0'] = ds['ans0'].mean()" + "# df['ans0'] = ds['ans0'].mean()\n" ] }, { @@ -251,7 +250,7 @@ "outputs": [], "source": [ "# df = ds2df(ds)\n", - "# df" + "# df\n" ] }, { @@ -260,7 +259,7 @@ "metadata": {}, "outputs": [], "source": [ - "# ds['txt_ans0'][:5]" + "# ds['txt_ans0'][:5]\n" ] }, { @@ -453,7 +452,7 @@ "dss_known = [filter_ds_to_known(d) for d in dss]\n", "# './.ds/HuggingFaceH4starchat_beta-None-N_8000-ns_3-mc_0.2-2ffc1e'\n", "ds = concatenate_datasets(dss_known)\n", - "ds" + "ds\n" ] }, { @@ -824,7 +823,7 @@ "source": [ "# lets select only the ones where\n", "df = ds2df(ds)\n", - "df" + "df\n" ] }, { @@ -845,7 +844,7 @@ "df2= ds2df(ds)\n", "df_subset_successull_lies = df2.query(\"instructed_to_lie==True & ((llm_ans==1)==label_instructed)\")\n", "print(f\"after filtering we have {len(df_subset_successull_lies)} num successful lies out of {len(df2)} dataset rows\")\n", - "assert len(df_subset_successull_lies)>0, \"there should be successful lies in the dataset\"" + "assert len(df_subset_successull_lies)>0, \"there should be successful lies in the dataset\"\n" ] }, { @@ -888,7 +887,7 @@ "\n", "# # run\n", "# ds = ds.map(normalize_hs, batched=True, input_columns=['hs0', 'hs1'])\n", - "# ds" + "# ds\n" ] }, { @@ -1066,7 +1065,7 @@ ], "source": [ "df = ds2df(ds)\n", - "df.head(4)" + "df.head(4)\n" ] }, { @@ -1083,7 +1082,7 @@ "outputs": [], "source": [ "from src.datasets.dm import imdbHSDataModule\n", - "from einops import reduce, einsum, rearrange" + "from einops import reduce, einsum, rearrange\n" ] }, { @@ -1166,7 +1165,7 @@ "import warnings\n", "warnings.filterwarnings(\"ignore\", \".*does not have many workers.*\")\n", "warnings.filterwarnings(\"ignore\", \".*sampler has shuffling enabled, it is strongly recommended that.*\")\n", - "warnings.filterwarnings(\"ignore\", \".*has been removed as a dependency of.*\")" + "warnings.filterwarnings(\"ignore\", \".*has been removed as a dependency of.*\")\n" ] }, { @@ -1231,7 +1230,7 @@ " if verbose:\n", " print(f\"⭐PRIMARY METRIC⭐ acc={acc:2.2%} from probe\")\n", " print(f\"⭐SECONDARY METRIC⭐ acc_lie_lie={acc_lie_lie:2.2%} from probe\")\n", - " return dict(acc=acc, acc_lie_lie=acc_lie_lie, acc_lie_truth=acc_lie_truth)" + " return dict(acc=acc, acc_lie_lie=acc_lie_lie, acc_lie_truth=acc_lie_truth)\n" ] }, { @@ -1248,7 +1247,7 @@ "def rename(rs):\n", " ks = ['train', 'val', 'test']\n", " rs = {ks[i]: {transform_dl_k(k):v for k,v in rs[i].items()} for i in range(3)}\n", - " return rs" + " return rs\n" ] }, { @@ -1271,7 +1270,7 @@ } ], "source": [ - "ds" + "ds\n" ] }, { @@ -1362,7 +1361,7 @@ "# dl_val = dm.val_dataloader()\n", "# print(len(dl_train), len(dl_val))\n", "# x, y = next(iter(dl_train))\n", - "# x.shape" + "# x.shape\n" ] }, { @@ -1387,7 +1386,7 @@ "source": [ "n = min(max_rows, len(ds))\n", "ds2 = ds.shuffle(42).select(range(n))\n", - "ds2" + "ds2\n" ] }, { @@ -1406,7 +1405,7 @@ "\n", "# TEMP try with the counterfactual residual stream...\n", "dm = imdbHSDataModule2(ds2, batch_size=batch_size)\n", - "dm.setup('train')" + "dm.setup('train')\n" ] }, { @@ -1415,7 +1414,7 @@ "metadata": {}, "outputs": [], "source": [ - "# max_epochs= 10" + "# max_epochs= 10\n" ] }, { @@ -1469,7 +1468,7 @@ " hs=128*4\n", " # x_feats=x_feats\n", " )\n", - "net" + "net\n" ] }, { @@ -3186,7 +3185,7 @@ "rs = rename(rs)\n", "# rs['test'] = {**rs['test'], **test_metrics}\n", "rs['test']['acc_lie_lie'] = testval_metrics['acc_lie_lie']\n", - "rs['testval_metrics'] = rs['test']" + "rs['testval_metrics'] = rs['test']\n" ] }, {