misc

mjc_notes.md

@@ -2151,4 +2151,48 @@ In particular our intervention result in junk.... so no wonder there is nothing
 # Invervention choices direction and magnitude
 
 https://github.com/saprmarks/geometry-of-truth/blob/91b223224699754efe83bbd3cae04d434dda0760/probes.py#L53
-- 
+geometry of truth
+- uses covariance (in some modes)
+- sigmoid(x @ direction) (in other modes)
+
+honest llama
+- uses std of layer vals
+
+a cleaner way?
+
+we need an intervention function
+we need to fit a intervention
+best to use a class.... and pass it around... doesn't need to be a pipeline
+
+but none of them seem to have a reasonable magnitude so.... not sure if any will give valid ones
+
+
+```sh
+export ORIGINAL_ORG=TheBloke
+export NEW_ORG=wassname
+export MODEL_NAME=phi-2-GPTQ
+export NEW_MODEL_NAME=phi-2-GPTQ_w_hidden_states
+# MODEL_NAME=phi-2-GPTQ-hidden_states
+# huggingface-cli login
+huggingface-cli repo create ${NEW_MODEL_NAME} --organization ${NEW_ORG}
+git lfs install --skip-smudge
+git clone https://huggingface.co/$NEW_ORG/$NEW_MODEL_NAME
+cd $NEW_MODEL_NAME
+git remote add upstream https://huggingface.co/$ORIGINAL_ORG/$MODEL_NAME
+git fetch upstream
+git rebase upstream/main
+git push --force-with-lease
+```
+
+
+
+# Phi-2
+
+model = AutoModelForCausalLM.from_pretrained(ckpt_path, torch_dtype=torch.float16, flash_attn=True, flash_rotary=True, fused_dense=True)
+
+I made a version which is quantised and returns hidden states
+
+maybe for padding use 50256? Rather than 0?
+
+"torch_dtype": "float16",
+    "transformers_version": "4.37.0.dev0",

notebooks/104_check_intervention.ipynb

@@ -0,0 +1,259 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# A scratch pad to run model inference manually\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/media/wassname/SGIronWolf/projects5/elk/discovering_latent_knowledge/.venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "1"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "import os\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from matplotlib import pyplot as plt\n",
+    "plt.style.use('ggplot')\n",
+    "\n",
+    "from typing import Optional, List, Dict, Union\n",
+    "\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.functional as F\n",
+    "from torch import Tensor\n",
+    "from torch import optim\n",
+    "from torch.utils.data import random_split, DataLoader, TensorDataset\n",
+    "\n",
+    "from pathlib import Path\n",
+    "import transformers\n",
+    "\n",
+    "\n",
+    "from loguru import logger\n",
+    "logger.add(os.sys.stderr, format=\"{time} {level} {message}\", level=\"INFO\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "CUDA extension not installed.\n",
+      "CUDA extension not installed.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# load my code\n",
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "\n",
+    "from src.extraction.config import ExtractConfig\n",
+    "from src.prompts.prompt_loading import load_preproc_dataset\n",
+    "from src.models.load import load_model\n",
+    "from src.datasets.intervene import create_cache_interventions \n",
+    "from src.prompts.prompt_loading import load_prompt_structure\n",
+    "from src.repe import repe_pipeline_registry\n",
+    "repe_pipeline_registry()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# config transformers\n",
+    "from datasets import set_caching_enabled, disable_caching\n",
+    "disable_caching()\n",
+    "\n",
+    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
+    "\n",
+    "# # cache busting for the transformers map and ds steps\n",
+    "# !rm -rf ~/.cache/huggingface/datasets/generator\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2023-12-16 10:14:43.517\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m24\u001b[0m - \u001b[1mchanging use_cache from True to False\u001b[0m\n",
+      "2023-12-16T10:14:43.517976+0800 INFO changing use_cache from True to False\n",
+      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
+      "\u001b[32m2023-12-16 10:14:43.853\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m24\u001b[0m - \u001b[1mchanging pad_token_id from None to 50256\u001b[0m\n",
+      "2023-12-16T10:14:43.853595+0800 INFO changing pad_token_id from None to 50256\n",
+      "\u001b[32m2023-12-16 10:14:43.854\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m24\u001b[0m - \u001b[1mchanging padding_side from right to left\u001b[0m\n",
+      "2023-12-16T10:14:43.854275+0800 INFO changing padding_side from right to left\n",
+      "\u001b[32m2023-12-16 10:14:43.854\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36msrc.models.load\u001b[0m:\u001b[36mverbose_change_param\u001b[0m:\u001b[36m24\u001b[0m - \u001b[1mchanging truncation_side from right to left\u001b[0m\n",
+      "2023-12-16T10:14:43.854771+0800 INFO changing truncation_side from right to left\n",
+      "Generating train split: 0 examples [00:00, ? examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Extracting 11 variants of each prompt\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Generating train split: 242 examples [00:40,  5.91 examples/s]\n",
+      "format_prompt: 100%|██████████| 242/242 [00:00<00:00, 7027.58 examples/s]\n",
+      "tokenize: 100%|██████████| 242/242 [00:00<00:00, 1284.35 examples/s]\n",
+      "truncated: 100%|██████████| 242/242 [00:00<00:00, 2526.02 examples/s]\n",
+      "truncated: 100%|██████████| 242/242 [00:00<00:00, 2476.92 examples/s]\n",
+      "prompt_truncated: 100%|██████████| 242/242 [00:00<00:00, 307.85 examples/s]\n",
+      "choice_ids: 100%|██████████| 242/242 [00:00<00:00, 6967.33 examples/s]\n",
+      "\u001b[32m2023-12-16 10:15:28.371\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36msrc.prompts.prompt_loading\u001b[0m:\u001b[36mload_preproc_dataset\u001b[0m:\u001b[36m368\u001b[0m - \u001b[1mtruncation rate: 0.0 on amazon_polarity\u001b[0m\n",
+      "2023-12-16T10:15:28.371476+0800 INFO truncation rate: 0.0 on amazon_polarity\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "median token length: 440.0 for amazon_polarity. max_length=1000\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Filter: 100%|██████████| 242/242 [00:00<00:00, 2223.43 examples/s]\n",
+      "Filter: 100%|██████████| 242/242 [00:00<00:00, 2118.60 examples/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "num_rows (after filtering out truncated rows) 242=>242\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "ds_name='amazon_polarity'\n",
+    "cfg = ExtractConfig(max_examples=(40, 40),\n",
+    "                    intervention_fit_examples=10,\n",
+    "                    )\n",
+    "print(cfg)\n",
+    "batch_size = cfg.batch_size\n",
+    "\n",
+    "model, tokenizer = load_model(cfg.model, pad_token_id=cfg.pad_token_id)\n",
+    "print(model)\n",
+    "\n",
+    "N_train, N_test = cfg.max_examples\n",
+    "N=sum(cfg.max_examples)\n",
+    "ds_tokens = load_preproc_dataset(ds_name, tokenizer, N=N, seed=cfg.seed, num_shots=cfg.num_shots, max_length=cfg.max_length, prompt_format=cfg.prompt_format)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "honesty_rep_reader = create_cache_interventions(model, tokenizer, cfg)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Generate\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/make_dataset2.py

@@ -77,7 +77,7 @@ print(cfg)
 
 batch_size = cfg.batch_size
 
-model, tokenizer = load_model(cfg.model)
+model, tokenizer = load_model(cfg.model, pad_token_id=cfg.pad_token_id)
 
 
 tokenizer_args = dict(

pyproject.toml

@@ -13,8 +13,8 @@ torch = {version = "^2.1.0+cu118", source = "pytorch"}
 simple-parsing = "^0.1.4"
 tqdm = "^4.66.1"
 datasets = "^2.14.5"
-transformers = "^4.34.1"
-auto-gptq = "^0.4.2"
+transformers = "^4.36.1"
+auto-gptq = "^0.6.0"
 optimum = "^1.13.2"
 numpy = "^1.26.1"
 pandas = "^2.1.1"

src/extraction/config.py

@@ -15,13 +15,18 @@ class ExtractConfig(Serializable):
     # model: str = "TheBloke/Mistral-7B-Instruct-v0.1-GPTQ" # it wont lie? wtf
     # model: str = "microsoft/phi-2"
     # model: str = "microsoft/phi-2"
-    model: str = "/media/wassname/SGIronWolf/projects5/elk/phi-2"
+    # model: str = "/media/wassname/SGIronWolf/projects5/elk/phi-2"
+    # model: str = "TheBloke/phi-2-GPTQ"
+    model: str = "wassname/phi-2-GPTQ_w_hidden_states"
     
     # model: str = "TheBloke/Llama-2-13B-chat-GPTQ"
     """HF model string identifying the language model to extract hidden states from."""
 
     batch_size: int = 5
 
+    pad_token_id: int = 50256
+    """Token ID to use for padding, most often zero."""
+
     prompt_format: str | None = 'phi'
     """if the tokenizer does not have a chat template you can set a custom one. see src/prompts/templates/prompt_formats/readme.md."""
     

src/models/load.py

@@ -5,8 +5,9 @@ When editing or updating this file check out these resources:
 - [LLM-As-Chatbot](https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/falcon.py)
 - [oobabooga](https://github.com/oobabooga/text-generation-webui/blob/main/modules/models.py#L134)
 """
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase, PreTrainedTokenizer
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM, AutoConfig, PreTrainedTokenizerBase, PreTrainedTokenizer, GPTQConfig
 import torch
+from zmq import has
 from src.datasets.dropout import check_for_dropout
 from loguru import logger
 from typing import Tuple
@@ -14,6 +15,9 @@ from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
 
 def verbose_change_param(tokenizer, path, after):
     
+    if not hasattr(tokenizer, path):
+        logger.info(f"tokenizer does not have {path}")
+        return tokenizer
     before = getattr(tokenizer, path)
     if before!=after:
         setattr(tokenizer, path, after)
@@ -21,7 +25,7 @@ def verbose_change_param(tokenizer, path, after):
     return tokenizer
 
 
-def load_model(model_repo =  "microsoft/phi-2") -> Tuple[AutoModelForCausalLM, PreTrainedTokenizerBase]:
+def load_model(model_repo =  "microsoft/phi-2", pad_token_id=0) -> Tuple[AutoModelForCausalLM, PreTrainedTokenizerBase]:
     """
     A uncensored and large coding ones might be best for lying.
     
@@ -33,19 +37,29 @@ def load_model(model_repo =  "microsoft/phi-2") -> Tuple[AutoModelForCausalLM, P
         torch_dtype=torch.float16, 
         # load_in_8bit=True,
         trust_remote_code=True,
+        # disable_exllama=True,
+        #  flash_attn=True, flash_rotary=True, fused_dense=True
     )
 
     config = AutoConfig.from_pretrained(model_repo, trust_remote_code=True,)
-    # verbose_change_param(config, 'use_cache', False)
+    # config.quantization_config['use_exllama'] = False    
+
+    # disable
+    # quantization_config=GPTQConfig(**dict(**config.quantization_config, disable_exllama=False))
+    # config.quantization_config = quantization_config
+
+    verbose_change_param(config, 'use_cache', False)
     
     tokenizer = AutoTokenizer.from_pretrained(model_repo, use_fast=True, legacy=False)
-    verbose_change_param(tokenizer, 'pad_token_id', 0)
+    verbose_change_param(tokenizer, 'pad_token_id', pad_token_id)
     verbose_change_param(tokenizer, 'padding_side', 'left')
     verbose_change_param(tokenizer, 'truncation_side', 'left')
     
     model = AutoModelForCausalLM.from_pretrained(model_repo, config=config, 
-        # gptq_config=gptq_config,
                                                  **model_options)
+    
+    # from auto_gptq import exllama_set_max_input_length
+    # model = exllama_set_max_input_length(model, max_input_length=5000)
 
     return model, tokenizer
 

src/prompts/prompt_loading.py

@@ -361,8 +361,9 @@ def load_preproc_dataset(ds_name: str, tokenizer: PreTrainedTokenizerBase, N:int
     
     b4 = ds_tokens.num_rows
     # print('num_rows', ds_tokens.num_rows)
+    print(f"median token length: {np.median(ds_tokens['length'])} for {ds_name}. max_length={max_length}")
+    # print(np.histogram(ds_tokens['length']))
     truncation_rate = np.mean(ds_tokens['truncated'])
-    print(np.histogram(ds_tokens['length']))
     assert truncation_rate<0.5, f"truncation rate is too high {truncation_rate}. Try a longer max_length than {max_length}"
     logger.info(f"truncation rate: {truncation_rate} on {ds_name}")
     ds_tokens = ds_tokens.filter(lambda r: r["truncated"] == False)