mirror of
https://github.com/wassname/discovering_latent_knowledge.git
synced 2026-09-09 11:21:22 +08:00
it works, for wizcode, sentiment
This commit is contained in:
deep1
Vendored
+29
-1
@@ -1,4 +1,32 @@
|
||||
{
|
||||
"python.formatting.provider": "black",
|
||||
"python.analysis.typeCheckingMode": "basic"
|
||||
"python.analysis.typeCheckingMode": "basic",
|
||||
"spellright.language": [
|
||||
"en-GB-10-1."
|
||||
],
|
||||
"spellright.documentTypes": [
|
||||
"latex",
|
||||
"plaintext",
|
||||
"markdown"
|
||||
],
|
||||
"workbench.colorCustomizations": {
|
||||
"activityBar.activeBackground": "#65c89b",
|
||||
"activityBar.background": "#65c89b",
|
||||
"activityBar.foreground": "#15202b",
|
||||
"activityBar.inactiveForeground": "#15202b99",
|
||||
"activityBarBadge.background": "#945bc4",
|
||||
"activityBarBadge.foreground": "#e7e7e7",
|
||||
"commandCenter.border": "#15202b99",
|
||||
"sash.hoverBorder": "#65c89b",
|
||||
"statusBar.background": "#42b883",
|
||||
"statusBar.foreground": "#15202b",
|
||||
"statusBarItem.hoverBackground": "#359268",
|
||||
"statusBarItem.remoteBackground": "#42b883",
|
||||
"statusBarItem.remoteForeground": "#15202b",
|
||||
"titleBar.activeBackground": "#42b883",
|
||||
"titleBar.activeForeground": "#15202b",
|
||||
"titleBar.inactiveBackground": "#42b88399",
|
||||
"titleBar.inactiveForeground": "#15202b99"
|
||||
},
|
||||
"peacock.remoteColor": "#42b883"
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,958 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Let's implement CCS from scratch.\n",
|
||||
"This will deliberately be a simple (but less efficient) implementation to make everything as clear as possible."
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"\n",
|
||||
"links:\n",
|
||||
"- [loading](https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/alpaca.py)\n",
|
||||
"- [dict](https://github.com/deep-diver/LLM-As-Chatbot/blob/c79e855a492a968b54bac223e66dc9db448d6eba/model_cards.json#L143)\n",
|
||||
"- [prompt_format](https://github.com/deep-diver/PingPong/blob/main/src/pingpong/alpaca.py)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"import copy\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"from matplotlib import pyplot as plt\n",
|
||||
"\n",
|
||||
"from typing import Optional, List, Dict, Union\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"import torch.nn as nn\n",
|
||||
"import torch.nn.functional as F\n",
|
||||
"from torch import Tensor\n",
|
||||
"from torch import optim\n",
|
||||
"from torch.utils.data import random_split, DataLoader, TensorDataset\n",
|
||||
"\n",
|
||||
"import pickle\n",
|
||||
"import hashlib\n",
|
||||
"from pathlib import Path\n",
|
||||
"\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"import datasets\n",
|
||||
"\n",
|
||||
"from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoModelForMaskedLM, AutoModelForCausalLM, AutoConfig\n",
|
||||
"import transformers\n",
|
||||
"from transformers.models.auto.modeling_auto import AutoModel\n",
|
||||
"from transformers import LogitsProcessorList\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"import lightning.pytorch as pl\n",
|
||||
"from dataclasses import dataclass\n",
|
||||
"\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"# from scipy.stats import zscore\n",
|
||||
"from sklearn.metrics import f1_score, roc_auc_score, accuracy_score\n",
|
||||
"from sklearn.preprocessing import RobustScaler\n",
|
||||
"\n",
|
||||
"from tqdm.auto import tqdm\n",
|
||||
"import gc\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"from loguru import logger\n",
|
||||
"logger.add(os.sys.stderr, format=\"{time} {level} {message}\", level=\"INFO\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"transformers.__version__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Model\n",
|
||||
"\n",
|
||||
"Chosing:\n",
|
||||
"- https://old.reddit.com/r/LocalLLaMA/wiki/models\n",
|
||||
"- https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard\n",
|
||||
"- https://github.com/deep-diver/LLM-As-Chatbot/blob/main/model_cards.json\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"A uncensored and large one might be best for lying."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from peft import PeftModel"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# leaderboard https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard\n",
|
||||
"model_options = dict(\n",
|
||||
" device_map=\"auto\", \n",
|
||||
" # load_in_4bit=True,\n",
|
||||
" # load_in_8bit=True,\n",
|
||||
" torch_dtype=torch.bfloat16,\n",
|
||||
" trust_remote_code=True,\n",
|
||||
" use_safetensors=False,\n",
|
||||
" # use_cache=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# so I need to use either pythia, stablelm, or tiiuae/falcon-7b-instruct to get dropout...\n",
|
||||
"# moel_repo = \"stabilityai/stablelm-tuned-alpha-7b\" # poor performance\n",
|
||||
"\n",
|
||||
"# https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/falcon.py\n",
|
||||
"model_repo = \"tiiuae/falcon-7b-instruct\"\n",
|
||||
"# model_repo = \"tiiuae/falcon-7b\"\n",
|
||||
"# model_repo = \"togethercomputer/RedPajama-INCITE-7B-Instruct\"\n",
|
||||
"model_repo = \"OpenAssis/tant/oasst-sft-4-pythia-12b-epoch-3.5\"\n",
|
||||
"# model_repo = \"OpenAssistant/falcon-7b-sft-top1-696\"\n",
|
||||
"# model_repo = \"openaccess-ai-collective/manticore-13b\"\n",
|
||||
"# model_repo = \"TheBloke/Wizard-Vicuna-13B-Uncensored-HF\"\n",
|
||||
"# model_repo = \"dvruette/llama-13b-pretrained-dropout\"\n",
|
||||
"# model_repo = \"elinas/llama-13b-hf-transformers-4.29\" # no dropout\n",
|
||||
"# lora_repo = \"LLMs/AlpacaGPT4-LoRA-13B-elina\"\n",
|
||||
"lora_repo = None\n",
|
||||
"\n",
|
||||
"config = AutoConfig.from_pretrained(model_repo, trust_remote_code=True,)\n",
|
||||
"print(config)\n",
|
||||
"config.hidden_dropout=0.2\n",
|
||||
"config.attention_dropout=0.2\n",
|
||||
"config.use_cache = False\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_repo)\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(model_repo, config=config, **model_options)\n",
|
||||
"\n",
|
||||
"if lora_repo is not None:\n",
|
||||
" # https://github.com/tloen/alpaca-lora/blob/main/generate.py#L40\n",
|
||||
" from peft import PeftModel\n",
|
||||
" model = PeftModel.from_pretrained(\n",
|
||||
" model,\n",
|
||||
" lora_repo, \n",
|
||||
" torch_dtype=torch.float16,\n",
|
||||
" lora_dropout=0.2,\n",
|
||||
" device_map='auto'\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
"# if not mode_8bit and not mode_4bit:\n",
|
||||
"# model.half()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/falcon.py\n",
|
||||
"print(tokenizer.pad_token_id)\n",
|
||||
"if tokenizer.pad_token_id is None:\n",
|
||||
" tokenizer.pad_token_id = 204 # <unk> https://github.com/deep-diver/LLM-As-Chatbot/blob/main/models/alpaca.py\n",
|
||||
"tokenizer.padding_side = \"left\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# tokenizer.encode(\" \")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Params"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Params\n",
|
||||
"N_SAMPLES = 130\n",
|
||||
"BATCH_SIZE = 10 # 1 for 30B 3 shot. 2 for 30B 1 shot. 4 for 13B. 15 for 7B.\n",
|
||||
"N_SHOTS = 3\n",
|
||||
"USE_MCDROPOUT = 0.3\n",
|
||||
"dataset_n = 200\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" num_layers = len(model.model.layers)\n",
|
||||
" print(num_layers)\n",
|
||||
"except AttributeError:\n",
|
||||
" try:\n",
|
||||
" num_layers = len(model.base_model.model.model.layers)\n",
|
||||
" print(num_layers)\n",
|
||||
" except:\n",
|
||||
" num_layers = 10\n",
|
||||
" \n",
|
||||
"stride = 4\n",
|
||||
"extract_layers = tuple(range(4, num_layers, stride)) + (num_layers,)\n",
|
||||
"extract_layers, num_layers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# get the tokens for 0 and 1, we will use these later...\n",
|
||||
"# note that sentancepeice tokenizers have differen't tokens for No and \\nNo.\n",
|
||||
"token_n = \"negative\"\n",
|
||||
"token_y = \"positive\"\n",
|
||||
"id_n, id_y = tokenizer(f'\\n{token_n}', add_special_tokens=True)['input_ids'][-1], tokenizer(f'\\n{token_y}', add_special_tokens=True)['input_ids'][-1]\n",
|
||||
"assert tokenizer.decode([id_n])==token_n\n",
|
||||
"assert tokenizer.decode([id_y])==token_y\n",
|
||||
"id_n, id_y"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Dataset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Let's just try IMDB for simplicity\n",
|
||||
"dataset = load_dataset(\"amazon_polarity\")\n",
|
||||
"data = dataset['test']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Prompt\n",
|
||||
"\n",
|
||||
"- Lillian Wang's guide https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\n",
|
||||
"- Elvi's guide https://www.promptingguide.ai/techniques/fewshot\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"example_prompts = dataset['train'].shuffle()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"def random_example():\n",
|
||||
" \"\"\"gets a random example from shuffled train and formats it a bit\"\"\"\n",
|
||||
" ex = dict(content=('blah blah '*202))\n",
|
||||
" while len(tokenizer(ex['content']).input_ids) > 400:\n",
|
||||
" i = np.random.randint(example_prompts.num_rows)\n",
|
||||
" ex = example_prompts[i]\n",
|
||||
" input = f\"Title: \\\"{ex['title']}\\\". Content: \\\"{ex['content']}\\\"\"\n",
|
||||
" return input, ex['label']==1\n",
|
||||
"\n",
|
||||
"print(random_example()[0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prefix_lie = prefix_true = prefix = f\"\"\"The classic puzzle of the two guards. It presents a scenario where two guards are involved in a conflict of truths and lies. Here's the setup:\n",
|
||||
"\n",
|
||||
"There are two guards standing at two separate doors. One door leads to safety, while the other leads to danger. One guard always tells the truth, and the other guard always lies. You don't know which guard is which or which door leads to safety.\"\"\"\n",
|
||||
"\n",
|
||||
"char_lie = \"Left Guard\"\n",
|
||||
"char_true = \"Right Guard\"\n",
|
||||
"\n",
|
||||
"def prompt_format_alpaca(input:str, response:str=\"\", include_prefix:bool=False, lie:Optional[bool]=None):\n",
|
||||
" \"\"\"alpaca format\"\"\"\n",
|
||||
" prefix = \"\"\n",
|
||||
" if include_prefix: prefix = (prefix_lie if lie else prefix_true) + \"\\n\\n\"\n",
|
||||
" char = char_lie if lie else char_true\n",
|
||||
" instruction = f'Following the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".'\n",
|
||||
" alpaca_prompt = f'{prefix}### Instruction:\\n{instruction}\\n\\n{input}\\n\\n### {char} Response:\\n{response}'\n",
|
||||
" return alpaca_prompt\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def prompt_format_oa(input:str, response:str=\"\", include_prefix:bool=False, lie:Optional[bool]=None):\n",
|
||||
" \"\"\"alpaca format\"\"\"\n",
|
||||
" prefix = \"\"\n",
|
||||
" if include_prefix: prefix = (prefix_lie if lie else prefix_true) + \"<|endoftext|>\"\n",
|
||||
" char = char_lie if lie else char_true\n",
|
||||
" instruction = f'Following the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".'\n",
|
||||
" if not response==\"\": response+=\"<|endoftext|>\"\n",
|
||||
" alpaca_prompt = f'{prefix}<|prompter|>{instruction}\\n{input}<|endoftext|><|assistant|>{char} Response:\\n{response}'\n",
|
||||
" return alpaca_prompt\n",
|
||||
"\n",
|
||||
"def prompt_format_falcon(input:str, response:str=\"\", include_prefix:bool=False, lie:Optional[bool]=None):\n",
|
||||
" prefix = \"\"\n",
|
||||
" if include_prefix: prefix = \"Instruction:\\n\" + (prefix_lie if lie else prefix_true) + \"\\n\\n\"\n",
|
||||
" char = char_lie if lie else char_true\n",
|
||||
" instruction = f'Following the above instructions, using your own character, classify the sentiment of the given movie review, \"positive\" or \"negative\".'\n",
|
||||
" alpaca_prompt = f'{prefix}Question:\\n{instruction}\\n\\nContext:\\n{input}\\n\\nAnswer:\\n{response}'\n",
|
||||
" return alpaca_prompt\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def prompt_format_vicuna(input:str, question:Optional[bool]=None, response:str=\"\", include_prefix:bool=False, lie:Optional[bool]=None):\n",
|
||||
" \"\"\"\n",
|
||||
" vicuna format\n",
|
||||
" \n",
|
||||
" https://github.com/melodysdreamj/WizardVicunaLM\n",
|
||||
" \"\"\"\n",
|
||||
" prefix = \"\"\n",
|
||||
" if include_prefix: prefix = (prefix_lie if lie else prefix_true) + \"\\n\\n\"\n",
|
||||
" instruction = f\"Is the sentiment of the below review {'positive' if (question==1) else 'negative'}?\"\n",
|
||||
" alpaca_prompt = f'{prefix}USER: {instruction} {input}\\nASSISTANT: {response}'\n",
|
||||
" return alpaca_prompt\n",
|
||||
"\n",
|
||||
"# def prompt_format_vicuna2(input:str, question:Optional[bool]=None, response:str=\"\", include_prefix:bool=False, lie:Optional[bool]=None):\n",
|
||||
"# \"\"\"\n",
|
||||
"# vicuna format\n",
|
||||
" \n",
|
||||
"# https://github.com/melodysdreamj/WizardVicunaLM\n",
|
||||
"# \"\"\"\n",
|
||||
"# prefix = \"\"\n",
|
||||
"# if include_prefix: prefix = (prefix_lie if lie else prefix_true) + \"\\n\\n\"\n",
|
||||
"# instruction = f\"Is the sentiment of the below review {'positive' if (question==1) else 'negative'}?\"\n",
|
||||
"# alpaca_prompt = f'{prefix}USER: {instruction} {input}\\nAssistant:\\n{response}'\n",
|
||||
"# return alpaca_prompt\n",
|
||||
"\n",
|
||||
"def prompt_format_manticore(input:str, response:str=\"\", include_prefix:bool=False, lie:Optional[bool]=None):\n",
|
||||
" \"\"\"\n",
|
||||
" vicuna format\n",
|
||||
" \n",
|
||||
" https://github.com/melodysdreamj/WizardVicunaLM\n",
|
||||
" https://huggingface.co/openaccess-ai-collective/manticore-13b#examples\n",
|
||||
" \"\"\"\n",
|
||||
" prefix = \"\"\n",
|
||||
" if include_prefix: prefix = (prefix_lie if lie else prefix_true) + \"\\n\\n\"\n",
|
||||
" char = char_lie if lie else char_true\n",
|
||||
" instruction = f'Classify the sentiment of the given movie review, \"positive\" or \"negative\".'\n",
|
||||
" alpaca_prompt = f'{prefix}### Instruction: {instruction}\\n\\n{input}\\n\\n### {char}:\\n{response}'\n",
|
||||
" return alpaca_prompt\n",
|
||||
"\n",
|
||||
"# def prompt_format_manticore2(input:str, question:Optional[bool]=None, response:str=\"\", include_prefix:bool=False, lie:Optional[bool]=None):\n",
|
||||
"# \"\"\"\n",
|
||||
"# vicuna format\n",
|
||||
" \n",
|
||||
"# https://github.com/melodysdreamj/WizardVicunaLM\n",
|
||||
"# https://huggingface.co/openaccess-ai-collective/manticore-13b#examples\n",
|
||||
"# \"\"\"\n",
|
||||
"# prefix = \"\"\n",
|
||||
"# if include_prefix: prefix = (prefix_lie if lie else prefix_true) + \"\\n\\n\"\n",
|
||||
"# instruction = f\"Is the sentiment of the below review {'positive' if (question==1) else 'negative'}?\"\n",
|
||||
"# alpaca_prompt = f'{prefix}USER: {instruction} {input}\\nASSISTANT: {response}'\n",
|
||||
"# return alpaca_prompt\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"repo_dict = {\n",
|
||||
" \"TheBloke/Wizard-Vicuna-13B-Uncensored-HF\": 'vicuna',\n",
|
||||
" 'Neko-Institute-of-Science/VicUnLocked-30b-LoRA': 'vicuna',\n",
|
||||
" \"ehartford/Wizard-Vicuna-13B-Uncensored\": 'vicuna',\n",
|
||||
" # 'tiiuae/falcon-7b': 'manticore',\n",
|
||||
" # 'tiiuae/falcon-7b-instruct': 'vicuna',\n",
|
||||
"}\n",
|
||||
"prompt_formats = {\n",
|
||||
" 'vicuna': prompt_format_vicuna,\n",
|
||||
" 'alpaca': prompt_format_alpaca,\n",
|
||||
" 'llama': prompt_format_alpaca,\n",
|
||||
" 'manticore': prompt_format_manticore,\n",
|
||||
" 'falcon': prompt_format_falcon,\n",
|
||||
" \n",
|
||||
"}\n",
|
||||
"def guess_prompt_format(model_repo, lora_repo):\n",
|
||||
" repo = model_repo if (lora_repo is None) else lora_repo\n",
|
||||
" if repo in repo_dict:\n",
|
||||
" prompt_type = repo_dict[repo]\n",
|
||||
" return prompt_formats[prompt_type]\n",
|
||||
" for fmt in prompt_formats:\n",
|
||||
" if fmt in repo.lower():\n",
|
||||
" fn = prompt_formats[fmt]\n",
|
||||
" print(f\"guessing prompt format '{str(fn.__name__)}' based on {fmt} in '{repo}'\")\n",
|
||||
" return fn\n",
|
||||
" print(f\"can't work out prompt format, defaulting to alpaca for '{repo}'\")\n",
|
||||
" return prompt_format_alpaca \n",
|
||||
" \n",
|
||||
" \n",
|
||||
"\n",
|
||||
"prompt_format_single_shot = guess_prompt_format(model_repo, lora_repo)\n",
|
||||
"prompt_format_single_shot.__name__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"rand_bool = lambda : np.random.rand()>0.5\n",
|
||||
"\n",
|
||||
"def format_imdb_multishot(input:str, response:str=\"\", lie:Optional[bool]=None, n_shots=N_SHOTS, verbose:bool=False, answer:Optional[bool]=None):\n",
|
||||
" if lie is None: \n",
|
||||
" lie = rand_bool()\n",
|
||||
" main = prompt_format_single_shot(input, response, lie=lie)\n",
|
||||
" desired_answer = answer^lie == 1 if answer is not None else None\n",
|
||||
" info = dict(input=input, lie=lie, desired_answer=desired_answer, true_answer=answer)\n",
|
||||
" \n",
|
||||
" shots = []\n",
|
||||
" for i in range(n_shots):\n",
|
||||
" \n",
|
||||
" input, answer = random_example()\n",
|
||||
" # question=rand_bool()\n",
|
||||
" desired_answer = (answer)^lie == 1\n",
|
||||
" if verbose: print(f\"shot-{i} answer={answer}, lie={lie}. (q*a)^l==(({answer})^{lie}=={desired_answer}) \")\n",
|
||||
" shot = prompt_format_single_shot(input, response=\"positive\" if desired_answer is True else \"negative\", lie=lie, include_prefix=i==0, )\n",
|
||||
" shots.append(shot)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" return \"\\n\\n\".join(shots+[main]), info\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def none_to_list_of_nones(d, n):\n",
|
||||
" if d is None: return [None]*n\n",
|
||||
" return d\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def format_imdbs_multishot(texts:List[str], response:Optional[str]=\"\", lies:Optional[list]=None, answers:Optional[list]=None):\n",
|
||||
" if response == \"\": response = [\"\"]*len(texts) \n",
|
||||
" lies = none_to_list_of_nones(lies, len(texts))\n",
|
||||
" answers = none_to_list_of_nones(answers, len(texts))\n",
|
||||
" a = [format_imdb_multishot(input=texts[i], lie=lies[i], answer=answers[i]) for i in range(len(texts))]\n",
|
||||
" return [list(a) for a in zip(*a)]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# q, info = format_imdbs_multishot(texts, labels)\n",
|
||||
"# info"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(format_imdb_multishot('test', \"neg\", lie=False, verbose=True)[0])\n",
|
||||
"# format_imdb_multishot('test', 1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(format_imdb_multishot('test', \"True\", lie=True, verbose=True)[0])\n",
|
||||
"# format_imdb_multishot('test', 1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Guess batch size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"def guess_batch_size(model_repo, N_SHOTS):\n",
|
||||
" \"\"\"Some rougth guestimates of batch size. \n",
|
||||
" \n",
|
||||
" Aiming to undershoot rather than crash.\"\"\"\n",
|
||||
" if '7b' in model_repo.lower():\n",
|
||||
" return int(64//(2+N_SHOTS))\n",
|
||||
" elif '13b' in model_repo.lower():\n",
|
||||
" return int(32//(2+N_SHOTS))\n",
|
||||
" elif '30b' in model_repo.lower(): \n",
|
||||
" return int(8//(2+N_SHOTS))\n",
|
||||
" else:\n",
|
||||
" raise NotImplementedError(f\"can't work out size of '{model_repo}'\")\n",
|
||||
" \n",
|
||||
" \n",
|
||||
"BATCH_SIZE = guess_batch_size(model_repo, N_SHOTS)\n",
|
||||
"print(f\"guessing BATCH_SIZE {BATCH_SIZE} for '{model_repo}'\")\n",
|
||||
"\n",
|
||||
"guess_batch_size('7b', N_SHOTS), guess_batch_size('13b', N_SHOTS), guess_batch_size('30b', N_SHOTS)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Check model output"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"see notebook 003"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Cache hidden states"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def clear_mem():\n",
|
||||
" gc.collect()\n",
|
||||
" torch.cuda.empty_cache()\n",
|
||||
" gc.collect()\n",
|
||||
" \n",
|
||||
"clear_mem()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"def enable_dropout(model, USE_MCDROPOUT:Union[float,bool]=True):\n",
|
||||
" \"\"\" Function to enable the dropout layers during test-time \"\"\"\n",
|
||||
" p = 0.2 if USE_MCDROPOUT is True else USE_MCDROPOUT\n",
|
||||
" for m in model.modules():\n",
|
||||
" if m.__class__.__name__.startswith('Dropout'):\n",
|
||||
" m.train()\n",
|
||||
" m.p=p\n",
|
||||
" \n",
|
||||
"def get_hidden_states(model, tokenizer, input_text, layers=extract_layers, truncation_length=900, output_attentions=False):\n",
|
||||
" \"\"\"\n",
|
||||
" Given a decoder model and some texts, gets the hidden states (in a given layer) on that input texts\n",
|
||||
" \"\"\"\n",
|
||||
" if not isinstance(input_text, list):\n",
|
||||
" input_text = [input_text]\n",
|
||||
" input_ids = tokenizer(input_text, \n",
|
||||
" return_tensors=\"pt\",\n",
|
||||
" padding=True,\n",
|
||||
" add_special_tokens=True,\n",
|
||||
" ).input_ids.to(model.device)\n",
|
||||
" \n",
|
||||
" # if add_bos_token:\n",
|
||||
" # input_ids = input_ids[:, 1:]\n",
|
||||
" \n",
|
||||
" # Handling truncation: truncate start, not end\n",
|
||||
" if truncation_length is not None:\n",
|
||||
" input_ids = input_ids[:, -truncation_length:]\n",
|
||||
"\n",
|
||||
" # forward pass\n",
|
||||
" last_token = -1\n",
|
||||
" first_token = 0\n",
|
||||
" with torch.no_grad():\n",
|
||||
" model.train() \n",
|
||||
" if USE_MCDROPOUT: enable_dropout(model)\n",
|
||||
" \n",
|
||||
" # taken from greedy_decode https://github.com/huggingface/transformers/blob/ba695c1efd55091e394eb59c90fb33ac3f9f0d41/src/transformers/generation/utils.py#L2338\n",
|
||||
" logits_processor = LogitsProcessorList()\n",
|
||||
" model_kwargs = dict(use_cache=False)\n",
|
||||
" model_inputs = model.prepare_inputs_for_generation(input_ids, **model_kwargs)\n",
|
||||
" outputs = model.forward(**model_inputs, return_dict=True, output_attentions=output_attentions, output_hidden_states=True)\n",
|
||||
" \n",
|
||||
" next_token_logits = outputs.logits[:, last_token, :]\n",
|
||||
" outputs['scores'] = logits_processor(input_ids, next_token_logits)[:, None,:]\n",
|
||||
" \n",
|
||||
" next_tokens = torch.argmax(outputs['scores'], dim=-1)\n",
|
||||
" outputs['sequences'] = torch.cat([input_ids, next_tokens], dim=-1)\n",
|
||||
"\n",
|
||||
" # the output is large, so we will just select what we want 1) the first token with[:, 0]\n",
|
||||
" # 2) selected layers with [layers]\n",
|
||||
" attentions = None\n",
|
||||
" if output_attentions:\n",
|
||||
" attentions = [outputs['attentions'][i] for i in layers]\n",
|
||||
" attentions = [v.detach().cpu()[:, last_token] for v in attentions]\n",
|
||||
" attentions = torch.concat(attentions).numpy()\n",
|
||||
" \n",
|
||||
" hidden_states = torch.stack([outputs['hidden_states'][i] for i in layers], 1).detach().cpu().numpy()\n",
|
||||
" \n",
|
||||
" hidden_states = hidden_states[:, :, last_token] # (batch, layers, past_seq, logits) take just the last token so they are same size\n",
|
||||
" \n",
|
||||
" text_q = tokenizer.batch_decode(input_ids)\n",
|
||||
" \n",
|
||||
" s = outputs['sequences']\n",
|
||||
" s = [s[i][len(input_ids[i]):] for i in range(len(s))]\n",
|
||||
" text_ans = tokenizer.batch_decode(s)\n",
|
||||
"\n",
|
||||
" scores = outputs['scores'][:, first_token].softmax(-1).detach().cpu().numpy() # for first (and only) token\n",
|
||||
" prob_n, prob_y = scores[:, [id_n, id_y]].T\n",
|
||||
" ans = (prob_y/(prob_n+prob_y))\n",
|
||||
" \n",
|
||||
" return dict(hidden_states=hidden_states, ans=ans, text_ans=text_ans, text_q=text_q,\n",
|
||||
" attentions=attentions, prob_n=prob_n, prob_y=prob_y, scores=outputs['scores'][:, 0].detach().cpu()\n",
|
||||
" )\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Collect pairs\n",
|
||||
"\n",
|
||||
"The idea is this: given two pairs of hidden states, where everything is the same except the random seed or dropout. Then tell me which one is more truthfull? \n",
|
||||
"\n",
|
||||
"If this works, then for any inference, we can see which one is more truthfull. Then we can see if it's the lower or higher probability one, and judge the answer and true or false.\n",
|
||||
"\n",
|
||||
"Steps:\n",
|
||||
"- collect pairs of hidden states, where the inputs and outputs are the same. We modify the random seed and dropout.\n",
|
||||
"- Each pair should have a binary answer. We can get that by comparing the probabilities of two tokens such as Yes and No.\n",
|
||||
"- Train a prob to distinguish the pairs as more and less truthfull\n",
|
||||
"- Test probe to see if it generalizes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# import random\n",
|
||||
"\n",
|
||||
"# # try multi\n",
|
||||
"# hss = {0: [], 1: []}\n",
|
||||
"# infos = {0: [], 1: []}\n",
|
||||
"\n",
|
||||
"# assert BATCH_SIZE>1\n",
|
||||
"\n",
|
||||
"# for i in tqdm(range(N_SAMPLES//BATCH_SIZE//2)):\n",
|
||||
" \n",
|
||||
"# # randomize everything\n",
|
||||
"# lie = rand_bool()\n",
|
||||
"# texts, labels = zip(*[random_example() for _ in range(BATCH_SIZE)])\n",
|
||||
" \n",
|
||||
"# # a pair of passes\n",
|
||||
"# for j in range(2):\n",
|
||||
"# transformers.set_seed(i+j)\n",
|
||||
"# torch.manual_seed(i+j)\n",
|
||||
"# np.random.seed(i+j)\n",
|
||||
"# random.seed(i+j)\n",
|
||||
" \n",
|
||||
"# q, info = format_imdbs_multishot(texts, answers=labels, lies=[lie]*BATCH_SIZE)\n",
|
||||
"# hs = get_hidden_states(model, tokenizer, q)\n",
|
||||
" \n",
|
||||
"# b = len(texts)\n",
|
||||
"# hss[j].append(\n",
|
||||
"# [\n",
|
||||
"# hs[\"hidden_states\"].reshape((b, -1)),\n",
|
||||
"# hs[\"prob_n\"],\n",
|
||||
"# hs[\"prob_y\"],\n",
|
||||
"# ]\n",
|
||||
"# )\n",
|
||||
"# for i in range(BATCH_SIZE):\n",
|
||||
"# infos[j].append(dict(prob_n=hs[\"prob_n\"][i], prob_y=hs[\"prob_y\"][i], **info[i])) \n",
|
||||
" \n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# FIXME, delete, scratch\n",
|
||||
"N_SAMPLES = BATCH_SIZE*4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"\n",
|
||||
"# try multi\n",
|
||||
"hss = {0: [], 1: []}\n",
|
||||
"infos = []\n",
|
||||
"\n",
|
||||
"def set_seeds(n):\n",
|
||||
" transformers.set_seed(n)\n",
|
||||
" torch.manual_seed(n)\n",
|
||||
" np.random.seed(n)\n",
|
||||
" random.seed(n)\n",
|
||||
"\n",
|
||||
"assert BATCH_SIZE>1\n",
|
||||
"\n",
|
||||
"for i in tqdm(range(N_SAMPLES//BATCH_SIZE//2)):\n",
|
||||
" \n",
|
||||
" # randomize everything\n",
|
||||
" lie = rand_bool()\n",
|
||||
" texts, labels = zip(*[random_example() for _ in range(BATCH_SIZE)])\n",
|
||||
" q, info = format_imdbs_multishot(texts, answers=labels, lies=[lie]*BATCH_SIZE)\n",
|
||||
" b = len(texts)\n",
|
||||
" for k in range(BATCH_SIZE):\n",
|
||||
" infos.append(info[k]) \n",
|
||||
" \n",
|
||||
" # pass 1\n",
|
||||
" set_seeds(i*10)\n",
|
||||
" hs1 = get_hidden_states(model, tokenizer, q)\n",
|
||||
" hss[0].append(\n",
|
||||
" [\n",
|
||||
" hs1[\"hidden_states\"].reshape((b, -1)),\n",
|
||||
" hs1[\"prob_n\"],\n",
|
||||
" hs1[\"prob_y\"],\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # pass 2\n",
|
||||
" set_seeds(i*10+1)\n",
|
||||
" hs2 = get_hidden_states(model, tokenizer, q)\n",
|
||||
" hss[1].append(\n",
|
||||
" [\n",
|
||||
" hs2[\"hidden_states\"].reshape((b, -1)),\n",
|
||||
" hs2[\"prob_n\"],\n",
|
||||
" hs2[\"prob_y\"],\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
" assert (hs1[\"prob_y\"]!=hs2[\"prob_y\"]).any(), 'inferences should differ'\n",
|
||||
" if i==0:\n",
|
||||
" # DEBUG\n",
|
||||
" print('text_ans', hs1['text_ans'])\n",
|
||||
" assert ((hs1['prob_y']+hs1['prob_n'])>0.01).all(), 'probability of two main tokens should be above 1%, check your prompt format and the tokens'\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(hs1['text_q'][0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"assert (hs1[\"prob_y\"]!=hs2[\"prob_y\"]).any(), 'inferences should differ'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"hss1b = [np.concatenate(r, 0) for r in zip(*hss[0])]\n",
|
||||
"hss1b\n",
|
||||
"hss2b = [np.concatenate(r, 0) for r in zip(*hss[1])]\n",
|
||||
"hss2b\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(hs1.keys())\n",
|
||||
"hs1['ans']\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_infos2 = pd.DataFrame(infos)\n",
|
||||
"df_infos2[\"model_answer\"] = (df_infos2[\"prob_y\"] > df_infos2[\"prob_n\"])\n",
|
||||
"df_infos2[\"model_conf\"] = (\n",
|
||||
" (df_infos2[\"prob_y\"] + df_infos2[\"prob_n\"])\n",
|
||||
") # total prob should be > 10%\n",
|
||||
"df_infos2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"So the idea here is that we get random pairs. And we try to classify which is more likely to be a lie\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"n = len(df_infos2)\n",
|
||||
"df_infos2['ans'] = (df_infos2['prob_y'])/(df_infos2['prob_y']+df_infos2['prob_n']) # Prob of saying True\n",
|
||||
"y = (df_infos2['ans'][:n//2] - df_infos2['ans'][n//2:].values).values>0 # Prob that right one is more true\n",
|
||||
"X = hss2[0][:n//2]-hss2[0][n//2:]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Try a regression\n",
|
||||
"\n",
|
||||
"# split\n",
|
||||
"n = len(y)\n",
|
||||
"print('split size', n//2)\n",
|
||||
"X_train, X_test = X[:n//2], X[n//2:]\n",
|
||||
"y_train, y_test = y[:n//2], y[n//2:]\n",
|
||||
"\n",
|
||||
"lr = LogisticRegression(class_weight=\"balanced\")\n",
|
||||
"lr.fit(X_train, y_train)\n",
|
||||
"print(\"Logistic regression accuracy: {:2.2f} [TRAIN]\".format(lr.score(X_train, y_train)))\n",
|
||||
"print(\"Logistic regression accuracy: {:2.2f} [TEST]\".format(lr.score(X_test, y_test)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df_info_test = df_infos2.iloc[n//2:].copy()\n",
|
||||
"y_pred = lr.predict(X_test)\n",
|
||||
"df_info_test['inner_truth'] = y_pred\n",
|
||||
"df_info_test"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "dlk2",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.16"
|
||||
},
|
||||
"orig_nbformat": 4
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+2522
-2299
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -5,11 +5,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Lets just do supervised learning\n",
|
||||
"\n",
|
||||
"Since we are looking at pairs with random permuations (from dropout), we can't use CCS. This is because our probabilities do not add to one.\n",
|
||||
"\n",
|
||||
"People question if unsupervised learning bings anything to the table anyway, so lets start with supervised..."
|
||||
"# Lets save our data as a huggingface dataset, so it's quick to reuse"
|
||||
]
|
||||
},
|
||||
{
|
||||
Reference in New Issue
Block a user